Jinemi/thermograph
4
0
Fork 0
Code Issues 2 Pull requests 9 Projects Releases Packages Wiki Activity Actions

promote: main → release #132

Merged
admin_emi merged 56 commits from main into release 2026-08-01 14:48:26 +00:00
Conversation 0 Commits 56 Files changed 128 +1620
15 changed files with 1620 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 6bd07d33cb - Show all commits

9
infra/deploy/env-topology.sh
View file

@ -68,6 +68,14 @@ thermograph_topology() {
TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0
TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0
TG_SSH_HOST=""; TG_SSH_TARGET=""
# Which store render-secrets.sh reads: sops | openbao. Defaults to sops for every
# environment and is overridden per environment below, so a cutover is a one-line
# reviewed change that follows the estate's own dev -> main -> release promotion.
#
# This lives here, and NOT in a host marker file like /etc/thermograph/deploy-mode,
# for the reason that marker was demoted to a fallback in the first place: vps2 runs
# prod and beta side by side, and one host-wide file cannot name two backends.
TG_SECRETS_BACKEND=sops
case "$env_name" in
prod)
@ -220,6 +228,7 @@ thermograph_topology() {
export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
export TG_SECRETS_BACKEND
export TG_SSH_HOST TG_SSH_TARGET
}

239
infra/deploy/render-secrets-openbao.sh Normal file
View file

@ -0,0 +1,239 @@
#!/usr/bin/env bash
# OpenBao source backend for render-secrets.sh. Sourced, never run directly.
#
# Two functions:
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
# thermograph_render_openbao <env> <out> -> source, then write <out>
#
# render_thermograph_secrets in render-secrets.sh dispatches to the second when
# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
# keeps byte-identical behaviour and all new risk is confined to this file. See the
# note above thermograph_render_openbao on why the write path is duplicated here
# rather than shared, and when to consolidate.
#
# WHY OUTPUT SHAPE IS FROZEN: /etc/thermograph.env must stay raw, UNQUOTED KEY=value.
# Centralis' secrets_inventory / secrets_render compare that file textually
# (deploy/secrets/README.md:206-210), compose reads it via `env_file:`, and the Swarm
# stack's env-entrypoint.sh parses it line by line. Quoting it "properly" would be an
# improvement in isolation and a silent break in context.
# thermograph_openbao_source <env_name>
#
# Reads the common set then the per-environment set and merges them with the
# environment winning — the same last-wins semantic as the SOPS concatenation, except
# performed explicitly here because OpenBao returns a map per path rather than a
# stream that can simply be appended.
thermograph_openbao_source() {
local env_name="${1:?env name required}"
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
command -v bao >/dev/null 2>&1 || {
echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
return 1
}
command -v python3 >/dev/null 2>&1 || {
echo "!! python3 required to render dotenv from OpenBao JSON" >&2
return 1
}
# AppRole credentials: role_id and secret_id, one per line, 0400 root — the same
# protection level as /etc/thermograph/age.key today. Read directly if we can, else
# via sudo, mirroring how render-secrets.sh lifts the age key.
local creds
if [ -r "$approle" ]; then
creds=$(cat "$approle")
else
creds=$(sudo cat "$approle" 2>/dev/null || true)
fi
[ -n "$creds" ] || {
echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
return 1
}
local role_id secret_id
role_id=$(printf '%s\n' "$creds" | sed -n '1p')
secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
[ -n "$role_id" ] && [ -n "$secret_id" ] || {
echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
return 1
}
export BAO_ADDR="$addr"
[ -f "$ca" ] && export BAO_CACERT="$ca"
# Exchange the AppRole for a short-lived token. The token TTL is set on the role
# (see bootstrap.sh) and is deliberately measured in minutes: it only has to
# outlive one render.
local token
token=$(bao write -field=token auth/approle/login \
role_id="$role_id" secret_id="$secret_id" 2>/dev/null) || {
echo "!! OpenBao AppRole login failed for env='${env_name}'" >&2
echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
return 1
}
export BAO_TOKEN="$token"
# Fetch. `common` is skipped for dev by the same flag the SOPS path uses — but note
# that on the OpenBao path the flag is belt, not braces: dev's AppRole is bound to
# the tg-host-dev policy, which DENIES thermograph/data/common outright. If this
# flag is ever forgotten at a call site, dev gets a 403 rather than a file full of
# production credentials. That inversion — from "the caller must remember" to "the
# store refuses" — is the main reason for this migration.
local common_json="" env_json
if [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then
common_json=$(bao kv get -format=json -mount="$mount" common 2>/dev/null) || {
echo "!! cannot read ${mount}/common for env='${env_name}'" >&2
echo "!! if this is dev, THERMOGRAPH_SECRETS_SKIP_COMMON=1 was not set and the" >&2
echo "!! tg-host-dev policy correctly refused — that is the guard working." >&2
return 1
}
fi
env_json=$(bao kv get -format=json -mount="$mount" "env/${env_name}" 2>/dev/null) || {
echo "!! cannot read ${mount}/env/${env_name}" >&2
return 1
}
# Merge and emit. Python rather than jq because jq is not installed on these hosts
# and python3 is (the Centralis renderer already relies on it).
#
# This step also enforces the two invariants the dotenv format cannot express:
# * a value containing a newline is REJECTED, because it would silently become
# two lines and the second would parse as a bogus KEY=value or be dropped;
# * a value containing a shell metacharacter is REJECTED, because
# /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
# deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
# `$(...)` or a backtick in a secret is a live command-execution path on the
# deploy host. The SOPS path has always had this hazard and has never tripped it
# only because every current value happens to be alphanumeric. Refusing here
# turns a latent code-execution bug into a loud render failure.
THERMOGRAPH_BAO_COMMON="$common_json" \
THERMOGRAPH_BAO_ENV="$env_json" \
python3 - <<'PY'
import json, os, re, sys
def load(raw):
if not raw:
return {}
doc = json.loads(raw)
return doc.get("data", {}).get("data", {}) or {}
merged = {}
# common first, environment second: last-wins, matching the SOPS concatenation order.
for src in ("THERMOGRAPH_BAO_COMMON", "THERMOGRAPH_BAO_ENV"):
merged.update(load(os.environ.get(src, "")))
KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
# Characters that change meaning when the file is sourced by bash.
UNSAFE = set("`$\\\"'")
bad = []
lines = []
# Validate EVERYTHING before emitting a single byte. Printing as we go would leave
# partial output on stdout when a later key fails, and render-secrets.sh's whole
# discipline is that a failed source never produces a partial env file.
for key in sorted(merged):
if not KEY_RE.match(key):
bad.append(f"{key}: not a valid env var name")
continue
val = merged[key]
if val is None:
val = ""
if not isinstance(val, str):
val = json.dumps(val, separators=(",", ":"))
if "\n" in val or "\r" in val:
bad.append(f"{key}: value contains a newline (would corrupt the dotenv)")
continue
if UNSAFE & set(val):
bad.append(
f"{key}: value contains a shell metacharacter (one of ` $ \\ \" ') and "
f"/etc/thermograph.env is sourced by bash — refusing"
)
continue
lines.append(f"{key}={val}")
if bad:
sys.stderr.write("!! refusing to render; unsafe values:\n")
for line in bad:
sys.stderr.write(f"!! {line}\n")
sys.exit(1)
if not lines:
sys.stderr.write("!! refusing to render: OpenBao returned zero keys\n")
sys.exit(1)
sys.stdout.write("\n".join(lines) + "\n")
PY
}
# thermograph_render_openbao <env_name> <out_file>
#
# The OpenBao equivalent of render_thermograph_secrets: source, then write.
#
# ON THE DUPLICATED WRITE PATH. The write logic below mirrors
# render-secrets.sh:130-154 rather than being factored out and shared. That is a
# deliberate, temporary choice, not an oversight:
#
# render-secrets.sh is on the deploy path for all three environments and there is no
# test suite in front of it (infra/CLAUDE.md: "Shell here runs as root over SSH
# against live hosts with no test suite in front of it"). Extracting the write path
# would mean the SOPS render — which currently works — starts flowing through newly
# moved code that cannot be tested end-to-end from here. An early-return branch keeps
# the SOPS path byte-identical and confines all new risk to the new backend.
#
# CONSOLIDATE THESE after prod has been on the OpenBao backend for a full release
# cycle and the SOPS path is being deleted anyway. Until then, a change to one write
# path must be made in both — which is exactly the cost being accepted here.
thermograph_render_openbao() {
local env_name="${1:?env name required}"
local out="${2:?output path required}"
echo "==> Rendering $out from OpenBao (env=${env_name})"
local tmp; tmp=$(mktemp)
# Holds DECRYPTED secrets. Removed on every exit path. Deliberately not a
# `trap ... RETURN` — see render-secrets.sh:82-91 for why that is actively wrong in
# a sourced function (the trap persists into the caller and re-fires on its next
# `source`, where the local is unset and `set -u` makes it fatal and silent).
if ! thermograph_openbao_source "$env_name" > "$tmp"; then
rm -f "$tmp"
return 1
fi
# An empty render must never reach $out. The source function already refuses a
# zero-key result, so this is a second line of defence rather than the only one:
# a truncated /etc/thermograph.env is the single worst outcome available here,
# because thermograph.service uses `EnvironmentFile=-` (missing is NOT fatal) and
# the app self-generates AUTH_SECRET and the VAPID pair when they are absent. The
# result would be a green deploy that silently invalidated every session and
# deleted every push subscription.
if [ ! -s "$tmp" ]; then
echo "!! OpenBao render produced an empty file; refusing to write $out" >&2
rm -f "$tmp"
return 1
fi
local rc=0
if [ -f "$out" ] && [ -w "$out" ]; then
cat "$tmp" > "$out" || rc=1
elif install -m 0640 "$tmp" "$out" 2>/dev/null; then :
elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then :
else
echo "!! cannot write $out (need file write access or passwordless sudo)" >&2
rc=1
fi
if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then
if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \
|| ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then
echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2
rc=1
fi
fi
rm -f "$tmp"
return "$rc"
}

32
infra/deploy/render-secrets.sh
View file

@ -31,6 +31,38 @@ render_thermograph_secrets() {
local out="${3:-/etc/thermograph.env}"
[ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true)
# Backend dispatch. TG_SECRETS_BACKEND comes from env-topology.sh (per environment,
# because vps2 runs two); THERMOGRAPH_SECRETS_BACKEND overrides it for a by-hand run.
# Default is sops, so an unmigrated host and an unset caller both behave exactly as
# before this branch existed.
#
# The OpenBao path returns early rather than threading a conditional through the
# rest of this function. That is on purpose: everything below — the age-key sudo
# lift, the two-file last-wins concatenation, the three-way write, the mirror — stays
# on precisely the code path it has always been on, so migrating cannot regress the
# backend that is still authoritative for prod.
local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
if [ "$backend" = openbao ]; then
if [ -z "$env_name" ]; then
echo "!! TG_SECRETS_BACKEND=openbao but no environment name was resolved" >&2
return 1
fi
local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
if [ ! -f "$bao_lib" ]; then
echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
return 1
fi
# shellcheck source=/dev/null
. "$bao_lib"
thermograph_render_openbao "$env_name" "$out"
return $?
fi
if [ "$backend" != sops ]; then
echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
return 1
fi
# The per-host file is required; common.yaml is optional so the initial cutover can
# seed each host as an exact copy of its live env (byte-identical render, no value
# changes) and factor out shared secrets into common.yaml later.

254
infra/openbao/README.md Normal file
View file

@ -0,0 +1,254 @@
# OpenBao — the secret store, phase 1
**Status: dormant.** Every file here is committed but nothing is cut over. SOPS is
still authoritative for dev, beta and prod, because `TG_SECRETS_BACKEND` defaults to
`sops` for all three in `infra/deploy/env-topology.sh`. Flipping an environment is a
one-line reviewed change, and it should not happen until `verify-parity.sh` passes.
Verified against **OpenBao v2.6.1** (2026-07-22). The binary is `bao`.
---
## Why do this at all
Not for encryption, and not for audit — though the audit log is a real gain. The
decisive reason is one specific failure class that a file-based vault cannot close.
`infra/CLAUDE.md` states the rule: *"vps1 must never hold prod credentials. It's the
box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of an
isolation boundary."*
Today that rule is enforced by a **shell flag** — `THERMOGRAPH_SECRETS_SKIP_COMMON=1`
— set by the *caller*, at three separate call sites (`env-topology.sh:196` →
`deploy.sh:67`, `deploy-dev.sh:86`, `infra-sync.yml:93-95`). The renderer itself does
not know that `env=dev` means never-common. A fourth call site, or one by-hand
`render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 keypairs
(one read-write on the bucket holding prod's database backups), the VAPID private key
that signs Web Push to real subscribers, `REGISTRY_TOKEN`, and the IndexNow and metrics
tokens onto the CI-runner box — **and exits 0**.
Under `policies/tg-host-dev.hcl` that is not possible to get wrong. dev's identity
cannot read `thermograph/data/common`. A misconfigured caller gets a 403 instead of a
silent success. The failure class is *gone*, not guarded. That is what this migration
buys, and it is worth the cost of running a stateful service.
The estate has already hardened this once by hand (commit `7583258`, "infra-sync:
refuse to render dev's vault onto a host that is not dev"). That is the shape of a
problem that wants an ACL, not another guard.
### What it does *not* buy — stated plainly
- **Rotation of provider-issued credentials.** Roughly half the credential count is
Discord, Contabo S3, the Forgejo registry token, VAPID, IndexNow. OpenBao cannot
rotate any of them; they stay static KV entries. Contabo Object Storage is
S3-compatible but exposes no IAM API, so the AWS secrets engine cannot issue against
it either.
- **Dynamic database credentials.** Inapplicable here for two independent reasons: the
apps read `THERMOGRAPH_DATABASE_URL` once at import and hold a pool, so a TTL'd
credential kills the pool at expiry; and there is no reload path anywhere in the
codebase — no SIGHUP handler, no config re-read.
- **Better review than git.** This is a real regression. Today a secret change is a
PR with a diff, backstopped by `secrets-guard` CI. Under OpenBao it is an
out-of-band API call with no diff and no review. Mitigated by committing a key-name
manifest so CI can still assert no key vanished, plus the audit log and
`bao kv metadata` as the change record — but not fully.
- **Fewer moving parts.** SOPS needs no server. This adds a stateful service to keep
alive, upgrade, back up and TLS-rotate, for ~40 secrets that change rarely.
---
## Shape of the deployment
| Decision | Choice | Why |
|---|---|---|
| Host | **vps2 only** | vps1 must never hold prod credentials, and with a static seal the box also holds the key that decrypts everything. Cost: vps1 needs vps2 up to deploy dev — acceptable, since when vps2 is down prod is down anyway. |
| Process | **native systemd**, not a container | A Swarm service is circular (`deploy-stack.sh` renders secrets to deploy the stack). A plain `docker run` makes the vault depend on the daemon deploys restart, and is one `prune -a` from gone. Native = one dependency, the disk. |
| Storage | **raft**, single node | 2.6.0 **deprecated the `file` backend** for removal in 2.7.0. Raft also has the only backup primitive (`operator raft snapshot save`). A *two*-node raft would be worse than one: quorum of two means losing either node loses writes. |
| Seal | **static auto-unseal** | See below. |
| TLS | self-signed, 10y, on disk | Must **not** come from Caddy/ACME — that would put DNS and the public internet in the boot chain of the secret store. |
| Auth | AppRole per environment, CIDR-bound | 5-minute tokens; secret-ids that only work from the right mesh address. |
| Consumption | keep rendering a dotenv file | Preserves every existing seam and keeps OpenBao a *deploy-time* dependency, never a runtime one. |
### The unseal decision, which is the crux
**Static seal**, key at `/etc/openbao/unseal.key` (0400), plus an off-box copy.
Shamir (`-key-shares=5 -key-threshold=3`) means the vault is sealed after every process
restart — reboot, package upgrade, OOM kill — and a human must type three shares.
`render-secrets.sh` is on the deploy path for all three environments, so a sealed vault
blocks every deploy and every hotfix. With one operator the N-of-M threshold is
theatre: one person holds all the shares, so it buys nothing against the real threat
while costing an availability property the estate has today for free. A Contabo reboot
currently needs no human at all.
OpenBao's docs hedge static seal — *"carefully evaluate"* — but that caveat is aimed at
multi-tenant enterprises. The argument here is that **static seal is not a downgrade
from the status quo**: `/etc/thermograph/age.key` is already one 0400 file per host
that decrypts everything forever with no audit trail. A static seal key is the
identical trust model. What changes is everything layered above it.
**The one new single point of failure:** lose that key and the raft snapshots are
unrestorable. It must exist in ≥2 places, one not on vps2. Hard operator obligation.
A `transit` seal against a second OpenBao on vps1 is genuinely better — vps2's disk
would no longer contain the unseal key — but the vps1 unsealer needs unsealing too, so
the circularity moves rather than dissolves, and it adds "vps1 must be up before vps2's
vault unseals" as a new failure mode. Revisit once the primary migration is boring.
---
## Layout
```
thermograph/data/common the 16 shared beta+prod values (= common.yaml)
thermograph/data/env/prod prod's 16 (= prod.yaml)
thermograph/data/env/beta beta's 8 (= beta.yaml)
thermograph/data/env/dev dev's 12 — inherits NOTHING (= dev.yaml)
thermograph/data/centralis/prod Centralis' 9 (= centralis.prod.yaml)
thermograph/data/ops/backup S3 creds for ops-cron + the backup age recipient
thermograph/data/legacy/age the age PRIVATE key — see "the age key survives"
```
Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
isolation lives in the policy. Today both live in one shell variable.
`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
rule expressible as exactly one path, with no wildcard over `env/*` able to
accidentally re-include it.
**~20 of the 61 keys are not secrets** (`PORT`, `WORKERS`, `APP_CPUS`, `TIMESCALEDB_TAG`,
`THERMOGRAPH_BASE_URL`, the Discord channel IDs, the VAPID *public* key, …). They stay
in the vault **for the cutover**, because byte-identical parity is the entire safety
property and splitting them would destroy it. Moving pure config back into the repo is
a clean follow-on that shrinks the vault-outage blast radius.
---
## The age key survives this migration
**This is the trap that would cause silent data loss.**
`ops-cron.yml:142` and `:197` stream every off-box Postgres and Forgejo dump through
`pg_dump | age -r <recipient> | rclone rcat`, with 30-day S3 retention. Restore uses
`/etc/thermograph/age.key`. So the age keypair is not just the SOPS transport — **it is
the backup encryption key.**
Deleting it along with the SOPS vault files would destroy up to 30 days of database
recoverability, and nothing would notice until someone attempted a restore.
So: keep the age private key at `thermograph/data/legacy/age` *and* in the password
manager, and keep it on disk until either the newest age-encrypted object in S3 has
aged out, or the backup path is re-pointed at a dedicated backup recipient. Retiring
SOPS and retiring age are two different projects.
---
## Runbook
### Stand it up (once, by the operator, on vps2)
```sh
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init
# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
export BAO_TOKEN=<root token>
sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles
bao token revoke -self
```
### Seed and prove (from your own machine — it needs your age key)
```sh
infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing
infra/openbao/seed-from-sops.sh --all
infra/openbao/verify-parity.sh --all # MUST pass before any flip
```
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
inherits that rule.
### Cut an environment over
One PR per hop, following the estate's own promotion model.
```diff
dev)
- TG_SECRETS_BACKEND=sops
+ TG_SECRETS_BACKEND=openbao
```
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
gate before prod: **7 consecutive green parity runs on all three environments**, run
nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
CI any vault access).
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
SOPS path for a full release cycle after prod flips — it is the best mitigation
available for anything unforeseen.
### Rollback
Revert the one-line PR and redeploy. The SOPS path is untouched by this work — verified
by rendering dev (12 keys) and prod (32 keys) through it after the dispatch was added,
matching the live hosts exactly.
---
## Failure modes
| Failure | Effect | Mitigation |
|---|---|---|
| Vault down/sealed at deploy | Render returns 1, deploy aborts, **running stack unaffected** — temp-then-install means `/etc/thermograph.env` is never truncated | `Restart=always` + static seal so a reboot self-heals; alert on `/v1/sys/health`; `TG_SECRETS_BACKEND=sops` is a working revert |
| **Audit device wedges** | ⚠️ OpenBao **stops answering requests entirely** when no enabled audit device can record them — a full `/var` on vps2 blocks every deploy | Two devices (file + syslog); logrotate signals **SIGHUP** or bao writes to an unlinked inode; disk alert on vps2 |
| Static seal key lost | Raft snapshots unrestorable | ≥2 copies, one off-box. The one new SPOF |
| Raft corruption / disk loss | Total vault loss | Nightly `operator raft snapshot save`, age-encrypted to S3 beside the DB dumps. **Verify a restore end-to-end before any consumer depends on it** |
| Cold boot of vps2 | Nothing needs the vault | Swarm restarts from persisted specs; `stack.env` / `thermograph.env` / `centralis.env` all persist. **This property exists because we render a file, and would be destroyed by app-native reads** |
| Secret-id leak | Useless off the mesh | `secret_id_bound_cidrs`; revoke by accessor without knowing the value |
| beta credential reaches prod | Cross-env compromise | Separate AppRoles, separate policies, secret-ids owned by the separate deploy users (`agent` vs `deploy`) — a boundary that does **not** exist today, since both currently reach the same root-owned age key. Honest limit: root on vps2 defeats it, exactly as it defeats the single age key now |
| Upgrade to 2.7.0 | Removes the `file` backend and built-in cloud KMS seals | Already avoided by choosing raft + static. Pin the version |
---
## Deliberately out of scope
For ~40 credentials and one operator the real risk is over-engineering. Not adopted:
HA/multi-node raft, namespaces, dynamic database credentials, `bao agent`, cert auth,
PKI, identity groups, transit seal (revisit later), OIDC for the operator.
**OIDC for CI is out of scope for a reason worth recording.** Forgejo Actions *does*
support OIDC — shipped in **Forgejo v15.0**, needing Runner > v12.5.0, enabled via
`enable-openid-connect: true` rather than GitHub's `permissions: id-token: write`. But
this estate runs `codeberg.org/forgejo/forgejo:9-rootless`
(`infra/deploy/forgejo/docker-stack.yml:59`). That is a six-major-version upgrade, and
coupling it to this migration would mean two large independent migrations at once with
no way to tell which one broke. Revisit after Forgejo reaches v15; then
`role_type=jwt` with `bound_claims` removes the last CI-side bearer credential.
CI keeps its SSH keys and `REGISTRY_TOKEN` as Forgejo secrets. **CI gets no vault
access at all** — the SSH-in architecture already means the *host* renders, never the
runner, which is the least-privilege topology and should be preserved rather than
"upgraded".
### Secret zero, honestly
Nothing eliminates it; you choose where it sits and how little of it there is. After
this phase the chain terminates at **9 Forgejo Actions secrets + 1 static seal key + 4
AppRole secret-ids**, versus today's **13 CI secrets + 1 omnipotent age key per host**.
---
## Verification gaps to close on the box
Flagged rather than guessed, because these could not be checked from a workstation:
1. Whether a raft snapshot is restorable under a *different* seal key. Docs don't say.
**Test in phase 0, before anything depends on it.**
2. Exact `-wrap-ttl` flag spelling on `bao write` — documented in the response-wrapping
concept page but absent from the commands index. Check `bao write -h`.
3. Debian/Ubuntu package repo URL. The install docs claim `.deb` packages exist but
give no repository; `bootstrap.sh` uses the GitHub release tarball with checksum
verification, which is fine and also fixes a standing weakness — `provision-secrets.sh`
installs sops and age with a bare `sudo curl` and no verification at all.
4. Vault-era naming persists inside OpenBao config (`vault { }` stanza,
`X-Vault-Wrap-TTL` header). Expect it; don't "fix" it.

115
infra/openbao/bootstrap-policies.sh Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env bash
# Second half of bootstrap: KV mount, policies, AppRoles, host credentials.
# Run on vps2 with BAO_TOKEN set to a root token. Idempotent — safe to re-run.
#
# export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
# export BAO_TOKEN=<root token>
# sudo -E bash infra/openbao/bootstrap-policies.sh
#
# Split from bootstrap.sh because everything here is safely automatable: none of it
# produces a credential a human has to custody. The one thing that does — the recovery
# material from `bao operator init` — is in bootstrap.sh and stops for the operator.
set -euo pipefail
MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
POLICY_DIR="$SELF_DIR/policies"
: "${BAO_TOKEN:?BAO_TOKEN must be set to a root token}"
: "${BAO_ADDR:=https://127.0.0.1:8200}"
export BAO_ADDR
command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; }
echo "==> KV v2 mount at ${MOUNT}/"
if bao secrets list -format=json | grep -q "\"${MOUNT}/\""; then
echo " already mounted"
else
bao secrets enable -path="$MOUNT" -version=2 kv
fi
echo "==> policies"
for p in "$POLICY_DIR"/*.hcl; do
name="$(basename "$p" .hcl)"
bao policy write "$name" "$p" >/dev/null
echo " wrote $name"
done
echo "==> approle auth"
bao auth list -format=json | grep -q '"approle/"' || bao auth enable approle
# One role per environment, CIDR-bound to the host that legitimately renders it.
#
# secret_id_ttl=0 (never expires) is deliberate: a secret-id that silently lapses
# mid-quarter is a self-inflicted outage. The short lifetime lives on the TOKEN
# instead — it only has to outlive one render.
#
# The CIDR binding is the part that makes a leaked secret-id close to useless: it is
# only accepted from the correct WireGuard address.
add_role() {
local role="$1" policy="$2" cidr="$3"
bao write "auth/approle/role/${role}" \
token_policies="$policy" \
secret_id_bound_cidrs="$cidr" \
token_bound_cidrs="$cidr" \
secret_id_ttl=0 secret_id_num_uses=0 \
token_ttl=5m token_max_ttl=15m token_num_uses=10 >/dev/null
echo " role ${role} -> ${policy} (bound ${cidr})"
}
add_role tg-prod tg-host-prod 10.10.0.1/32
add_role tg-beta tg-host-beta 10.10.0.1/32
add_role tg-dev tg-host-dev 10.10.0.2/32
add_role tg-centralis tg-centralis 10.10.0.1/32
# Install the local (vps2) credentials. prod and beta both render on this box.
#
# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
# creates one.
#
# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
# the one age key. This defends against MISCONFIGURATION, not against intrusion.
install_creds() {
local role="$1" dest="$2" owner="$3"
local rid sid
rid=$(bao read -field=role_id "auth/approle/role/${role}/role-id")
sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
# umask before creation, so the file is never briefly world-readable.
( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
chown "$owner" "$dest"
chmod 0400 "$dest"
echo " installed $dest (0400 $owner)"
}
if [ "$(id -u)" = 0 ]; then
install -d -o root -g root -m 0755 /etc/thermograph
install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent"
install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \
|| echo " (beta creds skipped — no 'deploy' user on this box?)"
install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
else
echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
fi
cat <<EOF
==> done. Policies and AppRoles are in place; the vault is still EMPTY.
dev's credentials must be installed on vps1, not here. From your own machine:
bao read -field=role_id auth/approle/role/tg-dev/role-id
bao write -f -field=secret_id auth/approle/role/tg-dev/secret-id
# then, on vps1, write both lines to /etc/thermograph/openbao-approle (0400 agent)
Prefer response wrapping so the secret-id never lands in shell history or a log:
bao write -f -wrap-ttl=5m auth/approle/role/tg-dev/secret-id # gives a wrapping token
# on vps1: bao unwrap <token> (single-use: a failed unwrap means interception)
Next: seed, then verify parity. Neither cuts anything over.
infra/openbao/seed-from-sops.sh --dry-run
infra/openbao/seed-from-sops.sh --all
infra/openbao/verify-parity.sh --all
EOF

179
infra/openbao/bootstrap.sh Executable file
View file

@ -0,0 +1,179 @@
#!/usr/bin/env bash
# Bring up OpenBao on vps2 from nothing. Run ONCE, as root, ON vps2, BY THE OPERATOR.
#
# sudo bash infra/openbao/bootstrap.sh
#
# ============================================================================
# WHY THIS IS NOT AUTOMATED, AND MUST NOT BE
# ============================================================================
# `bao operator init` mints the credentials that become the new root of trust for the
# entire estate: the recovery keys and the initial root token. Those have to be
# CUSTODIED BY A HUMAN. Any place a script could put them — a file on the box, the
# repo, a CI log, an agent transcript — is worse custody than the age key this
# migration is meant to improve on.
#
# So this script does everything up to and including init, prints the recovery
# material ONCE to the operator's terminal, and then stops and tells you what to do
# with it. It deliberately does not store it, mail it, or push it anywhere.
#
# Everything AFTER custody (policies, AppRoles, seeding, parity checks) is automated,
# because none of it produces a credential a human needs to hold.
set -euo pipefail
BAO_VERSION="${BAO_VERSION:-2.6.1}"
MESH_ADDR="${MESH_ADDR:-10.10.0.1}"
SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
[ "$(id -u)" = 0 ] || { echo "!! run as root (sudo bash $0)" >&2; exit 1; }
hostname_now="$(hostname)"
case "$hostname_now" in
vmi3453260) : ;; # vps2
*)
# Refuse to bootstrap the secret store on the wrong box. Putting it on vps1 would
# invert the estate's central security decision: infra/CLAUDE.md states "vps1 must
# never hold prod credentials. It's the box that runs Forgejo, its CI runner, and
# dev's unreviewed branch — the opposite of an isolation boundary." With a static
# seal, the box also holds the key that decrypts everything.
#
# This mirrors the guard infra-sync.yml already has (commit 7583258, "refuse to
# render dev's vault onto a host that is not dev").
echo "!! this host is '$hostname_now', not vps2 (vmi3453260)." >&2
echo "!! OpenBao belongs on vps2. Refusing." >&2
exit 1
;;
esac
echo "==> 1/7 install the bao binary (v${BAO_VERSION})"
if ! command -v bao >/dev/null 2>&1; then
# NOTE: checksum verification. provision-secrets.sh installs sops and age with a bare
# `sudo curl` and no verification at all, which is a standing weakness (a compromised
# release URL yields a root binary that sees every plaintext). Do not copy that here.
# The checksums file is signed by the release; verify before installing.
tmpd="$(mktemp -d)"
base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
rm -rf "$tmpd"; exit 1
}
tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
rm -rf "$tmpd"
fi
bao version
echo "==> 2/7 user, directories, TLS"
id -u openbao >/dev/null 2>&1 || useradd --system --home /var/lib/openbao --shell /usr/sbin/nologin openbao
install -d -o openbao -g openbao -m 0700 /var/lib/openbao
install -d -o openbao -g openbao -m 0750 /var/log/openbao
install -d -o root -g root -m 0755 /etc/openbao
install -d -o openbao -g openbao -m 0700 /etc/openbao/tls
if [ ! -f /etc/openbao/tls/bao.crt ]; then
# Self-signed, 10 years, SAN on the mesh address. Deliberately NOT from Caddy/ACME:
# that would put DNS and the public internet in the boot chain of the secret store.
openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
-keyout /etc/openbao/tls/bao.key -out /etc/openbao/tls/bao.crt \
-subj "/CN=openbao.thermograph.internal" \
-addext "subjectAltName=IP:${MESH_ADDR},IP:127.0.0.1" >/dev/null 2>&1
chown openbao:openbao /etc/openbao/tls/bao.key /etc/openbao/tls/bao.crt
chmod 0400 /etc/openbao/tls/bao.key
chmod 0444 /etc/openbao/tls/bao.crt
fi
# Consumers need the cert to trust the listener.
install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
echo "==> 3/7 static seal key"
if [ ! -f /etc/openbao/unseal.key ]; then
# 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
chown openbao:openbao /etc/openbao/unseal.key
chmod 0400 /etc/openbao/unseal.key
echo " generated /etc/openbao/unseal.key (0400 openbao)"
echo " ⚠️ COPY THIS OFF THE BOX into your password manager before proceeding."
echo " Without an off-box copy, a raft snapshot is UNRESTORABLE."
else
echo " /etc/openbao/unseal.key already present, leaving alone"
fi
echo "==> 4/7 config + systemd unit"
install -m 0640 -o root -g openbao "$SELF_DIR/config/openbao.hcl" /etc/openbao/config.hcl
install -m 0644 -o root -g root "$SELF_DIR/openbao.service" /etc/systemd/system/openbao.service
# logrotate MUST use SIGHUP: without it bao keeps writing to an unlinked inode and the
# audit log silently stops growing — and a wedged audit device makes OpenBao stop
# answering requests entirely, which would block every deploy on the estate.
cat > /etc/logrotate.d/openbao <<'ROTATE'
/var/log/openbao/audit.log {
daily
rotate 30
compress
missingok
notifempty
create 0600 openbao openbao
postrotate
systemctl kill --signal=SIGHUP openbao.service 2>/dev/null || true
endscript
}
ROTATE
systemctl daemon-reload
systemctl enable --now openbao.service
sleep 3
systemctl is-active --quiet openbao.service || {
echo "!! openbao.service failed to start; check: journalctl -u openbao -n 50" >&2
exit 1
}
export BAO_ADDR="https://127.0.0.1:8200"
export BAO_CACERT=/etc/openbao/tls/bao.crt
echo "==> 5/7 initialise"
if bao status -format=json 2>/dev/null | grep -q '"initialized": *true'; then
echo " already initialised, skipping"
else
echo
echo " ############################################################"
echo " # RECOVERY MATERIAL FOLLOWS. IT IS SHOWN EXACTLY ONCE. #"
echo " # Store the 3 recovery keys in 3 DIFFERENT places. #"
echo " # 2 of 3 are needed to rekey or mint a new root token. #"
echo " ############################################################"
echo
# -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
# yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
# keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
# operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
bao operator init -recovery-shares=3 -recovery-threshold=2
echo
echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
echo " stored off this machine."
read -r _
fi
bao status || true
cat <<EOF
==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
Paste the initial root token (or one minted from recovery keys) and run:
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
export BAO_TOKEN=<root token>
bash ${SELF_DIR}/bootstrap-policies.sh
That script enables the KV mount, writes the policies, creates the AppRoles and
installs each host's credentials. Then, from YOUR OWN machine (it needs your age
key, and per infra/CLAUDE.md a script that reads production secrets is not for an
agent to run):
infra/openbao/seed-from-sops.sh --dry-run # check first
infra/openbao/seed-from-sops.sh --all
infra/openbao/verify-parity.sh --all # must PASS before any flip
Then revoke the root token: bao token revoke -self
NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
until you flip an environment in a reviewed PR — dev first.
EOF

118
infra/openbao/config/openbao.hcl Normal file
View file

@ -0,0 +1,118 @@
// OpenBao server config for the Thermograph estate. Lives at /etc/openbao/config.hcl
// on vps2. Verified against OpenBao v2.6.1 (2026-07-22).
//
// ============================================================================
// WHY THIS RUNS AS A NATIVE SYSTEMD SERVICE AND NOT A CONTAINER
// ============================================================================
// The estate is Docker-native and this is the one thing that must not be. The rule
// is that OpenBao must not depend on anything it secures.
//
// * A Swarm service is fatal: deploy-stack.sh renders secrets in order to deploy
// the stack, so the stack cannot contain the thing that holds them.
// * A standalone `docker run --restart=always` is merely bad: it makes the vault's
// availability a function of the same Docker daemon that deploys restart, and
// leaves it one careless `docker system prune -a` from gone. The root CLAUDE.md
// already lists `prune -a` as a thing that eats the rollback image; it would
// eat this too.
//
// A native binary with Restart=always has exactly one dependency: the local disk.
// See infra/openbao/openbao.service.
//
// Corollary that is easy to get wrong: the listener cert must NOT come from the
// Caddy/ACME path. That would put DNS and the public internet in the boot chain of
// the estate's secret store. It is self-signed, long-lived, and on disk.
// ============================================================================
ui = false
cluster_name = "thermograph"
disable_mlock = true
log_level = "info"
// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
// a migration within a release. Raft also has the only first-class backup primitive
// (`bao operator raft snapshot save`), which is the disaster-recovery story.
//
// A single-node raft is the honest shape for a one-operator estate. Note that a
// TWO-node raft would be strictly worse than one: quorum of two means losing either
// node loses write availability. Three voters would be needed, and the desktop
// (10.10.0.3) is intermittent, so it cannot be a reliable third.
storage "raft" {
path = "/var/lib/openbao"
node_id = "vps2"
}
// Mesh-only. 10.10.0.1 is vps2's WireGuard address. There is deliberately no
// 0.0.0.0 listener and no public DNS record — vps2 is a public VPS, and reaching
// OpenBao should require being on the mesh, exactly like the Swarm control ports
// and the Loki endpoint.
listener "tcp" {
address = "10.10.0.1:8200"
tls_cert_file = "/etc/openbao/tls/bao.crt"
tls_key_file = "/etc/openbao/tls/bao.key"
}
// Loopback, so a by-hand `bao` on the box works without the mesh address.
listener "tcp" {
address = "127.0.0.1:8200"
tls_cert_file = "/etc/openbao/tls/bao.crt"
tls_key_file = "/etc/openbao/tls/bao.key"
}
api_addr = "https://10.10.0.1:8200"
cluster_addr = "https://10.10.0.1:8201"
// ============================================================================
// SEAL: static auto-unseal. This is the crux of the whole design.
// ============================================================================
// The `static` seal (OpenBao v2.4.0+) auto-unseals from a local key with no cloud
// KMS and no HSM. OpenBao's own docs hedge it — "carefully evaluate use of Static
// Key Auto Unseal" — but that caveat is written for multi-tenant enterprises.
//
// The decisive argument here is that static seal is NOT A DOWNGRADE FROM THE STATUS
// QUO. Today /etc/thermograph/age.key is one file, 0400, root-owned, on each
// rendering host, which decrypts every secret in the estate forever with no audit
// trail. A static seal key at 0400 on vps2 is the IDENTICAL trust model: one file on
// one box whose compromise is total compromise. What changes is everything layered
// above it — per-consumer ACLs, an audit log, five-minute tokens, versioned rollback.
//
// WHY NOT SHAMIR: `bao operator init -key-shares=5 -key-threshold=3` means the vault
// is sealed after every process restart — reboot, package upgrade, OOM kill — and a
// human must type three shares. render-secrets.sh is on the deploy path for all three
// environments, so a sealed vault blocks every deploy and every hotfix. With one
// operator the N-of-M threshold is theatre: one person holds all the shares, so it
// buys nothing against the real threat while costing an availability property the
// estate currently has for free. Today a Contabo reboot needs no human at all.
// Introducing a human-in-the-loop requirement that does not exist today is a
// regression this estate cannot absorb.
//
// THE ONE NEW SINGLE POINT OF FAILURE this migration introduces: lose this key and
// the raft snapshots become unrestorable. It MUST exist in at least two places, one
// of them not on vps2. That is a hard operator obligation, not a suggestion.
//
// Rotation is supported n-1 via previous_key / previous_key_id.
seal "static" {
current_key_id = "20260801-1"
current_key = "file:///etc/openbao/unseal.key"
}
// ============================================================================
// AUDIT: two devices, deliberately.
// ============================================================================
// ⚠️ OpenBao STOPS ANSWERING REQUESTS ENTIRELY when no enabled audit device can
// record them. That is correct behaviour for an audit log and a catastrophic way to
// discover a full disk: a full /var on vps2 would block every deploy on the estate.
// Two independent devices mean one wedging is survivable.
//
// Values are HMAC-SHA256'd in the log, so it is safe to ship to Loki through the
// existing Alloy pipeline — which finally makes "who read which secret, when" a
// queryable question. Today reading age.key leaves no trace whatsoever.
//
// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
// unlinked inode and the log silently stops growing.
audit "file" {
file_path = "/var/log/openbao/audit.log"
mode = "0600"
}
audit "syslog" {}

50
infra/openbao/openbao.service Normal file
View file

@ -0,0 +1,50 @@
# OpenBao, as a native systemd service on vps2. Install to
# /etc/systemd/system/openbao.service.
#
# Native, not containerised, on purpose — see the header of config/openbao.hcl. The
# whole point is that this unit's only dependency is the local disk: not the Docker
# daemon that deploys restart, not the Swarm it would otherwise be deployed by, not
# Caddy/ACME for its certificate, and not TimescaleDB.
#
# Note there is currently NO systemd unit of any kind loaded on either host for
# Thermograph (infra/deploy/thermograph.service exists in the repo but is not
# installed), so this is the first. That is deliberate: it is the one component that
# must survive the container layer being broken.
[Unit]
Description=OpenBao secret store
Documentation=https://openbao.org/docs/
# network-online rather than plain network: the mesh listener binds 10.10.0.1, and
# binding a WireGuard address before the interface is up fails the unit outright.
After=network-online.target
Wants=network-online.target
# Explicitly NOT After=docker.service. Adding it would recreate the dependency this
# design exists to avoid.
ConditionFileNotEmpty=/etc/openbao/config.hcl
[Service]
User=openbao
Group=openbao
ExecStart=/usr/local/bin/bao server -config=/etc/openbao/config.hcl
ExecReload=/bin/kill --signal HUP $MAINPID
KillSignal=SIGINT
# Restart always, which together with the static seal is what makes a host reboot
# self-healing. This pair is the reason deploys are not blocked by a reboot.
Restart=always
RestartSec=5
# mlock is disabled in the config (disable_mlock = true), so no IPC_LOCK capability
# is needed. If mlock is ever enabled, add: AmbientCapabilities=CAP_IPC_LOCK
NoNewPrivileges=yes
PrivateTmp=yes
ProtectSystem=full
ProtectHome=yes
# The three paths the service legitimately writes.
ReadWritePaths=/var/lib/openbao /var/log/openbao
# Raft + audit both do a lot of small writes; the default is too low for a store.
LimitNOFILE=65536
LimitCORE=0
[Install]
WantedBy=multi-user.target

67
infra/openbao/policies/tg-centralis.hcl Normal file
View file

@ -0,0 +1,67 @@
// Policy for the Centralis control-plane container on vps2.
//
// Centralis is the tool an operator drives a rotation THROUGH, so it needs more than
// a host render does: it must enumerate key names across every environment to build
// `secrets_inventory`, and write values to perform `secrets_rotate`.
//
// The split below is the important part. Centralis gets:
// * METADATA read/list on every path -> it can build a names-and-versions
// inventory across all three environments WITHOUT being able to read a value.
// * DATA write on every path -> it can rotate.
// * DATA read on NOTHING.
//
// That is a genuine improvement on the SOPS design, not a port of it. Today
// `secrets_inventory` can list key names without a decryption key only because SOPS
// happens to leave key names as plaintext in the YAML — a property of the file
// format, relied on deliberately (tools/secrets.ts:199 `extractKeyNames`). Under
// OpenBao the same "names, never values" guarantee becomes an ENFORCED capability
// boundary instead of a fortunate accident. Centralis cannot print a secret it is
// not able to fetch.
//
// The `read` omission is load-bearing: Centralis holds the Docker socket (root
// -equivalent on prod) and an SSH private key to beta and dev. Granting it data read
// on prod's secrets would make a Centralis compromise equivalent to full estate
// credential disclosure. It already effectively is via the Docker socket — but there
// is no reason to hand it a second, easier path.
// Names and version history across the whole tree — no values.
path "thermograph/metadata/*" {
capabilities = ["read", "list"]
}
// Rotation: write a new version. Note "create" + "update" but NOT "read":
// OpenBao permits a blind write, which is exactly the shape secrets_rotate wants —
// it mints or receives a new value and stores it, and never needs the old one.
path "thermograph/data/*" {
capabilities = ["create", "update"]
}
// Its OWN configuration is the one place Centralis may read, because it must render
// /etc/centralis.env for itself. This is the direct analogue of centralis.prod.yaml.
path "thermograph/data/centralis/prod" {
capabilities = ["read", "create", "update"]
}
// Version rollback, so a bad rotation is recoverable through the control plane
// rather than requiring a shell on the box. This is the capability that replaces
// "revert the PR" in the SOPS model.
path "thermograph/undelete/*" {
capabilities = ["update"]
}
path "thermograph/destroy/*" {
capabilities = ["deny"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}
// Read its own AppRole role-id for self-diagnosis, but never the secret-id.
path "auth/approle/role/tg-centralis/role-id" {
capabilities = ["read"]
}

37
infra/openbao/policies/tg-host-beta.hcl Normal file
View file

@ -0,0 +1,37 @@
// Policy for BETA's render identity on vps2. Mirror of tg-host-prod.hcl.
//
// Beta legitimately reads the shared set: unlike dev, beta is on vps2 and already
// co-resident with prod, so withholding `shared` from it would buy nothing while
// breaking the render (beta's 24 live keys = beta.yaml's 8 + shared's 16).
path "thermograph/data/common" {
capabilities = ["read"]
}
path "thermograph/metadata/common" {
capabilities = ["read", "list"]
}
path "thermograph/data/env/beta" {
capabilities = ["read"]
}
path "thermograph/metadata/env/beta" {
capabilities = ["read", "list"]
}
path "thermograph/data/env/prod" {
capabilities = ["deny"]
}
path "thermograph/data/env/dev" {
capabilities = ["deny"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}

72
infra/openbao/policies/tg-host-dev.hcl Normal file
View file

@ -0,0 +1,72 @@
// Policy for the DEV host's render identity (vps1).
//
// THIS FILE IS THE POINT OF THE MIGRATION.
//
// infra/CLAUDE.md states the rule: "vps1 must never hold prod credentials. It's the
// box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of
// an isolation boundary."
//
// Today that rule is enforced by a SHELL FLAG (THERMOGRAPH_SECRETS_SKIP_COMMON=1) set
// by the *caller*, at three separate call sites: env-topology.sh:196 -> deploy.sh:67,
// deploy-dev.sh:86, and infra-sync.yml:93-95. The renderer itself does not know that
// env=dev means never-common. A fourth call site, or one by-hand
// `render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3
// keypairs (one read-write on the bucket holding prod's database backups), the VAPID
// private key that signs Web Push to real subscribers, REGISTRY_TOKEN, and the
// IndexNow and metrics tokens onto the Forgejo CI-runner box — and exits 0.
//
// Under this policy that is no longer possible to get wrong. dev's identity cannot
// read the common path. A misconfigured caller gets a 403 from OpenBao, not a silent
// success with production credentials on disk. The failure class is gone, not guarded.
// dev's own values. This is the ONLY secret path dev can read.
path "thermograph/data/env/dev" {
capabilities = ["read"]
}
path "thermograph/metadata/env/dev" {
capabilities = ["read", "list"]
}
// EXPLICIT DENY on the shared production credential set.
//
// A deny rule is redundant with OpenBao's default-deny — anything not granted is
// already refused. It is here anyway, and must stay, for two reasons:
// 1. It is documentation that survives refactoring. Someone widening dev's grants
// has to delete an explicit deny to break the rule, which is a much louder edit
// than adding a path to an allow list.
// 2. In OpenBao, a deny on a path beats any grant at that path regardless of rule
// order or specificity. So if dev's identity is ever accidentally given a
// broader policy alongside this one, this still wins.
path "thermograph/data/common" {
capabilities = ["deny"]
}
path "thermograph/metadata/common" {
capabilities = ["deny"]
}
// Deny the other environments outright. Same argument: dev has no business reading
// beta or prod, and being explicit means a widened grant elsewhere cannot silently
// re-open it.
path "thermograph/data/env/prod" {
capabilities = ["deny"]
}
path "thermograph/data/env/beta" {
capabilities = ["deny"]
}
path "thermograph/data/centralis/*" {
capabilities = ["deny"]
}
// Token self-management, so the render can look up and renew its own lease without
// needing a broader grant.
path "auth/token/lookup-self" {
capabilities = ["read"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}

44
infra/openbao/policies/tg-host-prod.hcl Normal file
View file

@ -0,0 +1,44 @@
// Policy for PROD's render identity on vps2.
//
// Reads the shared set plus prod's own overrides — the OpenBao equivalent of
// "common.yaml then prod.yaml, last-wins".
//
// Note what this policy does NOT grant: beta. vps2 runs prod and beta side by side
// with one filesystem and one SSH credential, so the host is emphatically not an
// isolation boundary between them (infra/CLAUDE.md is explicit about this). Separate
// identities with separate policies do not fix that — a foothold on the box can read
// both AppRole credentials — but they do mean a *mistake* cannot cross the line:
// prod's render cannot accidentally pull beta's database password, and the audit log
// attributes every read to one environment or the other.
path "thermograph/data/common" {
capabilities = ["read"]
}
path "thermograph/metadata/common" {
capabilities = ["read", "list"]
}
path "thermograph/data/env/prod" {
capabilities = ["read"]
}
path "thermograph/metadata/env/prod" {
capabilities = ["read", "list"]
}
path "thermograph/data/env/beta" {
capabilities = ["deny"]
}
path "thermograph/data/env/dev" {
capabilities = ["deny"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}

50
infra/openbao/policies/tg-ops-backup.hcl Normal file
View file

@ -0,0 +1,50 @@
// Policy for the ops-cron backup job (.forgejo/workflows/ops-cron.yml).
//
// Exists to DELETE FOUR CI SECRETS. Today S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY and
// S3_SECRET_KEY are Forgejo repo secrets, even though ops-cron only ever uses them
// INSIDE the script it SSHes onto the host — they are passed through `envs:` and
// consumed as RCLONE_CONFIG_ARCHIVE_*. Nothing needs them on the runner. Moving them
// here means the host reads them directly and CI holds four fewer credentials.
//
// It also collapses a genuine duplication: the same values already live in the SOPS
// vault as THERMOGRAPH_S3_* in common.yaml, so there are currently two sources of
// truth for one credential pair, free to drift.
//
// The backup recipient belongs here too — see thermograph/data/ops/backup below.
// ops-cron.yml:142 and :197 currently HARDCODE the age recipient as a literal, in two
// places, and that same public key is duplicated in eleven places across the repo.
path "thermograph/data/ops/backup" {
capabilities = ["read"]
}
path "thermograph/metadata/ops/backup" {
capabilities = ["read", "list"]
}
// ============================================================================
// THE AGE KEY IS NOT RETIRED BY THIS MIGRATION. This path is why.
// ============================================================================
// ops-cron.yml streams every off-box database dump through
// `pg_dump | age -r <recipient> | rclone rcat`, with 30-day S3 retention, and restore
// depends on the private half at /etc/thermograph/age.key. The Forgejo backup path is
// the same shape.
//
// So the age keypair is not merely the SOPS transport — it is the BACKUP ENCRYPTION
// KEY. Deleting it alongside the SOPS vault files would silently destroy up to 30 days
// of database recoverability, and nothing would notice until a restore was attempted.
//
// Therefore: the age private key is stored at thermograph/data/legacy/age (operator
// access only, deliberately NOT granted by this policy) and kept on disk until either
// the newest age-encrypted object in S3 has aged out, or the backup path is re-pointed
// at a dedicated backup recipient. Only then may /etc/thermograph/age.key be removed.
//
// The `deny` is explicit so that widening this policy cannot hand the backup job the
// key to read everything else in the estate.
path "thermograph/data/legacy/age" {
capabilities = ["deny"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}

196
infra/openbao/seed-from-sops.sh Executable file
View file

@ -0,0 +1,196 @@
#!/usr/bin/env bash
# Seed OpenBao from the SOPS vault. One-shot, idempotent, run BY THE OPERATOR.
#
# infra/openbao/seed-from-sops.sh --dry-run # key names only, writes nothing
# infra/openbao/seed-from-sops.sh --env dev # seed one environment
# infra/openbao/seed-from-sops.sh --all # seed everything
#
# ============================================================================
# THIS SCRIPT READS EVERY PRODUCTION SECRET IN THE ESTATE IN PLAINTEXT.
# ============================================================================
# infra/CLAUDE.md and deploy/secrets/README.md both say that
# deploy/secrets/seed-from-live.sh "reads production secrets and is explicitly NOT for
# an agent to run". This script is in the same category and inherits the same rule:
# run it yourself, from your own terminal, with your own age key.
#
# It never prints a value. Every diagnostic is key NAMES and counts, so the output is
# safe to paste into an issue. --dry-run makes that guarantee mechanical: it resolves
# and validates everything, then reports what it WOULD write without contacting
# OpenBao at all.
#
# Why seed from SOPS rather than re-entering values by hand: the safety property of
# this entire migration is that the OpenBao render can be diffed BYTE-FOR-BYTE against
# the SOPS render (see verify-parity.sh). That only works if the values are identical.
# Re-typing 61 values would silently rotate whichever ones were mistyped, and the
# self-generating ones (AUTH_SECRET, the VAPID pair) fail SILENTLY when wrong — the app
# mints a replacement, comes up green, and invalidates every session and push
# subscription. So: copy exactly now, rotate deliberately later.
set -euo pipefail
MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
AGE_KEY="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
DRY_RUN=0
TARGETS=()
# Resolve the repo's infra/ directory from this script's own location, so the script
# works from any cwd. This is the same mistake terraform's remote-exec makes
# (main.tf:256 passes app_dir where app_dir/infra is wanted), so it is worth being
# explicit rather than clever.
SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
VAULT_DIR="$INFRA_DIR/deploy/secrets"
usage() {
cat >&2 <<'EOF'
usage: seed-from-sops.sh [--dry-run] (--all | --env <name> ...)
--dry-run validate and report key names/counts; contact OpenBao not at all
--all seed common, dev, beta, prod and centralis.prod
--env <name> seed one of: common | dev | beta | prod | centralis.prod
Environment:
SOPS_AGE_KEY_FILE age private key (default ~/.config/sops/age/keys.txt)
BAO_ADDR OpenBao address (default https://10.10.0.1:8200)
BAO_TOKEN an operator token with write on <mount>/data/*
THERMOGRAPH_BAO_MOUNT KV v2 mount name (default thermograph)
EOF
exit 2
}
while [ $# -gt 0 ]; do
case "$1" in
--dry-run) DRY_RUN=1; shift ;;
--all) TARGETS=(common dev beta prod centralis.prod); shift ;;
--env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
-h|--help) usage ;;
*) echo "!! unknown argument: $1" >&2; usage ;;
esac
done
[ ${#TARGETS[@]} -gt 0 ] || usage
# The KV v2 path for a given vault file. common.yaml and centralis.prod.yaml are
# special-cased; everything else is an environment overlay.
#
# Keeping `common` at its own top-level path rather than under env/ is what makes the
# dev policy expressible: tg-host-dev.hcl denies exactly one path, and no wildcard
# over env/* can accidentally re-include it.
bao_path_for() {
case "$1" in
common) printf 'common\n' ;;
centralis.prod) printf 'centralis/prod\n' ;;
dev|beta|prod) printf 'env/%s\n' "$1" ;;
*) echo "!! unknown vault target: $1" >&2; return 1 ;;
esac
}
[ -f "$AGE_KEY" ] || { echo "!! age key not found at $AGE_KEY" >&2; exit 1; }
command -v sops >/dev/null 2>&1 || { echo "!! sops not on PATH" >&2; exit 1; }
command -v python3 >/dev/null 2>&1 || { echo "!! python3 not on PATH" >&2; exit 1; }
if [ "$DRY_RUN" = 0 ]; then
command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; }
[ -n "${BAO_TOKEN:-}" ] || { echo "!! BAO_TOKEN is not set" >&2; exit 1; }
export BAO_ADDR="${BAO_ADDR:-https://10.10.0.1:8200}"
fi
rc=0
for target in "${TARGETS[@]}"; do
src="$VAULT_DIR/${target}.yaml"
if [ ! -f "$src" ]; then
echo "!! no vault file at $src" >&2
rc=1
continue
fi
path="$(bao_path_for "$target")" || { rc=1; continue; }
# Decrypt to JSON, not dotenv. JSON is lossless: the dotenv writer flattens a value
# containing a newline into a literal \n that is indistinguishable from a backslash
# followed by n (render-secrets.sh:198 documents this). Seeding through dotenv would
# bake that ambiguity into OpenBao permanently.
plain_json="$(SOPS_AGE_KEY_FILE="$AGE_KEY" \
sops -d --input-type yaml --output-type json "$src" 2>/dev/null)" || {
echo "!! failed to decrypt $src" >&2
rc=1
continue
}
# Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
# values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
# SEED time — before anything depends on it — rather than at deploy time.
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
import json, os, re, sys
doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
if not isinstance(doc, dict) or not doc:
sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
sys.exit(1)
KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
UNSAFE = set("`$\\\"'")
out, bad = {}, []
for key, val in doc.items():
if key == "sops":
continue
if not KEY_RE.match(key):
bad.append(f"{key}: not a valid env var name")
continue
if val is None:
val = ""
if not isinstance(val, str):
val = json.dumps(val, separators=(",", ":"))
if "\n" in val or "\r" in val:
bad.append(f"{key}: contains a newline")
continue
if UNSAFE & set(val):
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
continue
out[key] = val
if bad:
sys.stderr.write("!! unsafe values, refusing:\n")
for line in bad:
sys.stderr.write(f"!! {line}\n")
sys.exit(1)
# stdout line 1: space-separated key NAMES, for the operator-visible log.
# stdout line 2+: the JSON payload, fed to `bao kv put` on stdin.
sys.stdout.write(" ".join(sorted(out)) + "\n")
sys.stdout.write(json.dumps(out) + "\n")
PY
)" || { echo "!! validation failed for $target" >&2; rc=1; continue; }
names="$(printf '%s\n' "$reshaped" | sed -n '1p')"
payload="$(printf '%s\n' "$reshaped" | sed -n '2p')"
count="$(printf '%s\n' "$names" | wc -w | tr -d ' ')"
if [ "$DRY_RUN" = 1 ]; then
echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys, DRY RUN, nothing written)"
printf ' %s\n' "$names"
continue
fi
# `bao kv put -` reads a JSON object from stdin, so no value ever appears in argv
# (where it would be visible in /proc and in the shell's history).
if printf '%s' "$payload" | bao kv put -mount="$MOUNT" "$path" - >/dev/null; then
echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys written)"
else
echo "!! failed writing ${MOUNT}/${path}" >&2
rc=1
fi
done
if [ "$rc" = 0 ] && [ "$DRY_RUN" = 0 ]; then
cat <<EOF
Seeded. NOTHING is cut over yet — SOPS is still authoritative for every
environment, because TG_SECRETS_BACKEND defaults to sops in env-topology.sh.
Next: prove equivalence before flipping anything.
infra/openbao/verify-parity.sh --env dev
EOF
fi
exit "$rc"

158
infra/openbao/verify-parity.sh Executable file
View file

@ -0,0 +1,158 @@
#!/usr/bin/env bash
# Prove that the OpenBao render is EQUIVALENT to the SOPS render, without printing a
# single secret value.
#
# infra/openbao/verify-parity.sh --env dev
# infra/openbao/verify-parity.sh --all
#
# This is the gate for every cutover. The migration's entire safety argument is that
# the two backends produce the same KEY=value set, so flipping TG_SECRETS_BACKEND
# cannot change what any service sees. That argument has to be TESTED, not asserted —
# especially because the failure modes on the other side are silent:
#
# * a missing THERMOGRAPH_AUTH_SECRET makes the app mint a random one per worker
# (accounts/users.py:33), so logins start working intermittently and nothing logs;
# * a half-rendered VAPID pair makes push.py generate a NEW keypair and then delete
# every existing subscription row as "gone" (push.py:180-184);
# * a wrong POSTGRES_PASSWORD makes the data layer swallow the error and refetch from
# Open-Meteo, spending third-party quota, while /healthz stays green.
#
# None of those announce themselves. A byte-level diff beforehand is the only thing
# standing between a flag flip and one of them.
#
# OUTPUT DISCIPLINE: only key NAMES and counts are ever printed. Where values differ,
# the key name is reported and the values are not — the same rule
# deploy/secrets/dry-run-render.sh already follows.
set -euo pipefail
SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
TARGETS=()
usage() {
cat >&2 <<'EOF'
usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)
Renders each environment through BOTH backends and compares the resulting
KEY=value sets. Prints key names and counts only, never values.
Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
Suitable as a CI / cron gate.
EOF
exit 2
}
while [ $# -gt 0 ]; do
case "$1" in
--all) TARGETS=(dev beta prod); shift ;;
--env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
-h|--help) usage ;;
*) echo "!! unknown argument: $1" >&2; usage ;;
esac
done
[ ${#TARGETS[@]} -gt 0 ] || usage
# shellcheck source=/dev/null
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
overall=0
for env_name in "${TARGETS[@]}"; do
echo "== parity check: $env_name"
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
# flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
# the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
# would fail closed with a 403 rather than quietly diverge, which is exactly the
# improvement being verified.
skip_common=0
[ "$env_name" = dev ] && skip_common=1
sops_out=$(mktemp); bao_out=$(mktemp)
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
# trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a
# sourced context persists into the caller's shell and re-fires on the next source).
cleanup() { rm -f "$sops_out" "$bao_out"; }
# --- SOPS side: common (unless dev) then <env>, appended, last-wins ---
key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
[ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
if [ ! -r "$key" ]; then
echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2
cleanup; overall=1; continue
fi
if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then
SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
"$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || {
echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; }
fi
SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
"$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || {
echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; }
# --- OpenBao side ---
if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \
thermograph_openbao_source "$env_name" > "$bao_out"; then
echo "!! OpenBao render failed for $env_name" >&2
cleanup; overall=1; continue
fi
# --- Compare ---
# The SOPS output can legitimately contain the SAME key twice (common then env), and
# every consumer takes the LAST occurrence — `env_file:` and `source` both do. So
# collapsing to last-wins is not a normalisation for convenience, it is what the
# consumers actually see. Comparing raw would report a false mismatch the moment a
# key is overridden.
norm() {
python3 -c '
import sys
seen = {}
order = []
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
line = line.rstrip("\n")
if not line or line.startswith("#") or "=" not in line:
continue
k, _, v = line.partition("=")
if k not in seen:
order.append(k)
seen[k] = v # last occurrence wins
for k in sorted(order):
print(f"{k}={seen[k]}")
' "$1"
}
a=$(mktemp); b=$(mktemp)
norm "$sops_out" > "$a"; norm "$bao_out" > "$b"
n_sops=$(wc -l < "$a" | tr -d ' ')
n_bao=$(wc -l < "$b" | tr -d ' ')
if cmp -s "$a" "$b"; then
echo " PASS — ${n_sops} keys, byte-identical after last-wins collapse"
else
overall=1
echo " FAIL — sops=${n_sops} keys, openbao=${n_bao} keys"
# Report only NAMES. Three buckets, because the fix differs per bucket:
# only-in-sops -> the seed missed a key (re-run seed-from-sops.sh)
# only-in-bao -> OpenBao has something the vault does not (stale/extra write)
# value-differs -> the seed captured a different value; DO NOT flip the backend
comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in SOPS: /'
comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in OpenBao: /'
join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \
| awk -F= '$2 != $3 { print " value differs: " $1 }' | sort -u
fi
rm -f "$a" "$b"
cleanup
done
if [ "$overall" = 0 ]; then
echo
echo "PARITY OK for: ${TARGETS[*]}"
echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
echo "Recommended gate before prod: 7 consecutive green runs on all three."
else
echo
echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
fi
exit "$overall"