diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh index 7e4ae38..9e2b7c9 100644 --- a/infra/deploy/env-topology.sh +++ b/infra/deploy/env-topology.sh @@ -68,6 +68,14 @@ thermograph_topology() { TG_TAGS_FILE=""; TG_LOCK_FILE=""; TG_BIND_ADDR=""; TG_SKIP_COMMON=0 TG_SVC_PREFIX=""; TG_DATA_NETWORK=""; TG_DB_SERVICE=""; TG_POST_DEPLOY=0 TG_SSH_HOST=""; TG_SSH_TARGET="" + # Which store render-secrets.sh reads: sops | openbao. Defaults to sops for every + # environment and is overridden per environment below, so a cutover is a one-line + # reviewed change that follows the estate's own dev -> main -> release promotion. + # + # This lives here, and NOT in a host marker file like /etc/thermograph/deploy-mode, + # for the reason that marker was demoted to a fallback in the first place: vps2 runs + # prod and beta side by side, and one host-wide file cannot name two backends. + TG_SECRETS_BACKEND=sops case "$env_name" in prod) @@ -220,6 +228,7 @@ thermograph_topology() { export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY + export TG_SECRETS_BACKEND export TG_SSH_HOST TG_SSH_TARGET } diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh new file mode 100644 index 0000000..82c449c --- /dev/null +++ b/infra/deploy/render-secrets-openbao.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# OpenBao source backend for render-secrets.sh. Sourced, never run directly. +# +# Two functions: +# thermograph_openbao_source -> merged dotenv on STDOUT +# thermograph_render_openbao -> source, then write +# +# render_thermograph_secrets in render-secrets.sh dispatches to the second when +# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path +# keeps byte-identical behaviour and all new risk is confined to this file. See the +# note above thermograph_render_openbao on why the write path is duplicated here +# rather than shared, and when to consolidate. +# +# WHY OUTPUT SHAPE IS FROZEN: /etc/thermograph.env must stay raw, UNQUOTED KEY=value. +# Centralis' secrets_inventory / secrets_render compare that file textually +# (deploy/secrets/README.md:206-210), compose reads it via `env_file:`, and the Swarm +# stack's env-entrypoint.sh parses it line by line. Quoting it "properly" would be an +# improvement in isolation and a silent break in context. + +# thermograph_openbao_source +# +# Reads the common set then the per-environment set and merges them with the +# environment winning — the same last-wins semantic as the SOPS concatenation, except +# performed explicitly here because OpenBao returns a map per path rather than a +# stream that can simply be appended. +thermograph_openbao_source() { + local env_name="${1:?env name required}" + local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" + local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" + local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}" + local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" + + command -v bao >/dev/null 2>&1 || { + echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2 + return 1 + } + command -v python3 >/dev/null 2>&1 || { + echo "!! python3 required to render dotenv from OpenBao JSON" >&2 + return 1 + } + + # AppRole credentials: role_id and secret_id, one per line, 0400 root — the same + # protection level as /etc/thermograph/age.key today. Read directly if we can, else + # via sudo, mirroring how render-secrets.sh lifts the age key. + local creds + if [ -r "$approle" ]; then + creds=$(cat "$approle") + else + creds=$(sudo cat "$approle" 2>/dev/null || true) + fi + [ -n "$creds" ] || { + echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2 + return 1 + } + + local role_id secret_id + role_id=$(printf '%s\n' "$creds" | sed -n '1p') + secret_id=$(printf '%s\n' "$creds" | sed -n '2p') + [ -n "$role_id" ] && [ -n "$secret_id" ] || { + echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2 + return 1 + } + + export BAO_ADDR="$addr" + [ -f "$ca" ] && export BAO_CACERT="$ca" + + # Exchange the AppRole for a short-lived token. The token TTL is set on the role + # (see bootstrap.sh) and is deliberately measured in minutes: it only has to + # outlive one render. + local token + token=$(bao write -field=token auth/approle/login \ + role_id="$role_id" secret_id="$secret_id" 2>/dev/null) || { + echo "!! OpenBao AppRole login failed for env='${env_name}'" >&2 + echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2 + return 1 + } + export BAO_TOKEN="$token" + + # Fetch. `common` is skipped for dev by the same flag the SOPS path uses — but note + # that on the OpenBao path the flag is belt, not braces: dev's AppRole is bound to + # the tg-host-dev policy, which DENIES thermograph/data/common outright. If this + # flag is ever forgotten at a call site, dev gets a 403 rather than a file full of + # production credentials. That inversion — from "the caller must remember" to "the + # store refuses" — is the main reason for this migration. + local common_json="" env_json + if [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then + common_json=$(bao kv get -format=json -mount="$mount" common 2>/dev/null) || { + echo "!! cannot read ${mount}/common for env='${env_name}'" >&2 + echo "!! if this is dev, THERMOGRAPH_SECRETS_SKIP_COMMON=1 was not set and the" >&2 + echo "!! tg-host-dev policy correctly refused — that is the guard working." >&2 + return 1 + } + fi + + env_json=$(bao kv get -format=json -mount="$mount" "env/${env_name}" 2>/dev/null) || { + echo "!! cannot read ${mount}/env/${env_name}" >&2 + return 1 + } + + # Merge and emit. Python rather than jq because jq is not installed on these hosts + # and python3 is (the Centralis renderer already relies on it). + # + # This step also enforces the two invariants the dotenv format cannot express: + # * a value containing a newline is REJECTED, because it would silently become + # two lines and the second would parse as a bogus KEY=value or be dropped; + # * a value containing a shell metacharacter is REJECTED, because + # /etc/thermograph.env is sourced by bash in six places (deploy.sh:133, + # deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so + # `$(...)` or a backtick in a secret is a live command-execution path on the + # deploy host. The SOPS path has always had this hazard and has never tripped it + # only because every current value happens to be alphanumeric. Refusing here + # turns a latent code-execution bug into a loud render failure. + THERMOGRAPH_BAO_COMMON="$common_json" \ + THERMOGRAPH_BAO_ENV="$env_json" \ + python3 - <<'PY' +import json, os, re, sys + +def load(raw): + if not raw: + return {} + doc = json.loads(raw) + return doc.get("data", {}).get("data", {}) or {} + +merged = {} +# common first, environment second: last-wins, matching the SOPS concatenation order. +for src in ("THERMOGRAPH_BAO_COMMON", "THERMOGRAPH_BAO_ENV"): + merged.update(load(os.environ.get(src, ""))) + +KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +# Characters that change meaning when the file is sourced by bash. +UNSAFE = set("`$\\\"'") + +bad = [] +lines = [] +# Validate EVERYTHING before emitting a single byte. Printing as we go would leave +# partial output on stdout when a later key fails, and render-secrets.sh's whole +# discipline is that a failed source never produces a partial env file. +for key in sorted(merged): + if not KEY_RE.match(key): + bad.append(f"{key}: not a valid env var name") + continue + val = merged[key] + if val is None: + val = "" + if not isinstance(val, str): + val = json.dumps(val, separators=(",", ":")) + if "\n" in val or "\r" in val: + bad.append(f"{key}: value contains a newline (would corrupt the dotenv)") + continue + if UNSAFE & set(val): + bad.append( + f"{key}: value contains a shell metacharacter (one of ` $ \\ \" ') and " + f"/etc/thermograph.env is sourced by bash — refusing" + ) + continue + lines.append(f"{key}={val}") + +if bad: + sys.stderr.write("!! refusing to render; unsafe values:\n") + for line in bad: + sys.stderr.write(f"!! {line}\n") + sys.exit(1) + +if not lines: + sys.stderr.write("!! refusing to render: OpenBao returned zero keys\n") + sys.exit(1) + +sys.stdout.write("\n".join(lines) + "\n") +PY +} + +# thermograph_render_openbao +# +# The OpenBao equivalent of render_thermograph_secrets: source, then write. +# +# ON THE DUPLICATED WRITE PATH. The write logic below mirrors +# render-secrets.sh:130-154 rather than being factored out and shared. That is a +# deliberate, temporary choice, not an oversight: +# +# render-secrets.sh is on the deploy path for all three environments and there is no +# test suite in front of it (infra/CLAUDE.md: "Shell here runs as root over SSH +# against live hosts with no test suite in front of it"). Extracting the write path +# would mean the SOPS render — which currently works — starts flowing through newly +# moved code that cannot be tested end-to-end from here. An early-return branch keeps +# the SOPS path byte-identical and confines all new risk to the new backend. +# +# CONSOLIDATE THESE after prod has been on the OpenBao backend for a full release +# cycle and the SOPS path is being deleted anyway. Until then, a change to one write +# path must be made in both — which is exactly the cost being accepted here. +thermograph_render_openbao() { + local env_name="${1:?env name required}" + local out="${2:?output path required}" + + echo "==> Rendering $out from OpenBao (env=${env_name})" + + local tmp; tmp=$(mktemp) + # Holds DECRYPTED secrets. Removed on every exit path. Deliberately not a + # `trap ... RETURN` — see render-secrets.sh:82-91 for why that is actively wrong in + # a sourced function (the trap persists into the caller and re-fires on its next + # `source`, where the local is unset and `set -u` makes it fatal and silent). + if ! thermograph_openbao_source "$env_name" > "$tmp"; then + rm -f "$tmp" + return 1 + fi + + # An empty render must never reach $out. The source function already refuses a + # zero-key result, so this is a second line of defence rather than the only one: + # a truncated /etc/thermograph.env is the single worst outcome available here, + # because thermograph.service uses `EnvironmentFile=-` (missing is NOT fatal) and + # the app self-generates AUTH_SECRET and the VAPID pair when they are absent. The + # result would be a green deploy that silently invalidated every session and + # deleted every push subscription. + if [ ! -s "$tmp" ]; then + echo "!! OpenBao render produced an empty file; refusing to write $out" >&2 + rm -f "$tmp" + return 1 + fi + + local rc=0 + if [ -f "$out" ] && [ -w "$out" ]; then + cat "$tmp" > "$out" || rc=1 + elif install -m 0640 "$tmp" "$out" 2>/dev/null; then : + elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then : + else + echo "!! cannot write $out (need file write access or passwordless sudo)" >&2 + rc=1 + fi + + if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then + if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \ + || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then + echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2 + rc=1 + fi + fi + + rm -f "$tmp" + return "$rc" +} diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index fb5731c..32093d9 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -31,6 +31,38 @@ render_thermograph_secrets() { local out="${3:-/etc/thermograph.env}" [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true) + # Backend dispatch. TG_SECRETS_BACKEND comes from env-topology.sh (per environment, + # because vps2 runs two); THERMOGRAPH_SECRETS_BACKEND overrides it for a by-hand run. + # Default is sops, so an unmigrated host and an unset caller both behave exactly as + # before this branch existed. + # + # The OpenBao path returns early rather than threading a conditional through the + # rest of this function. That is on purpose: everything below — the age-key sudo + # lift, the two-file last-wins concatenation, the three-way write, the mirror — stays + # on precisely the code path it has always been on, so migrating cannot regress the + # backend that is still authoritative for prod. + local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}" + if [ "$backend" = openbao ]; then + if [ -z "$env_name" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but no environment name was resolved" >&2 + return 1 + fi + local bao_lib="${repo}/deploy/render-secrets-openbao.sh" + if [ ! -f "$bao_lib" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2 + echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2 + return 1 + fi + # shellcheck source=/dev/null + . "$bao_lib" + thermograph_render_openbao "$env_name" "$out" + return $? + fi + if [ "$backend" != sops ]; then + echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2 + return 1 + fi + # The per-host file is required; common.yaml is optional so the initial cutover can # seed each host as an exact copy of its live env (byte-identical render, no value # changes) and factor out shared secrets into common.yaml later. diff --git a/infra/openbao/README.md b/infra/openbao/README.md new file mode 100644 index 0000000..196784a --- /dev/null +++ b/infra/openbao/README.md @@ -0,0 +1,254 @@ +# OpenBao — the secret store, phase 1 + +**Status: dormant.** Every file here is committed but nothing is cut over. SOPS is +still authoritative for dev, beta and prod, because `TG_SECRETS_BACKEND` defaults to +`sops` for all three in `infra/deploy/env-topology.sh`. Flipping an environment is a +one-line reviewed change, and it should not happen until `verify-parity.sh` passes. + +Verified against **OpenBao v2.6.1** (2026-07-22). The binary is `bao`. + +--- + +## Why do this at all + +Not for encryption, and not for audit — though the audit log is a real gain. The +decisive reason is one specific failure class that a file-based vault cannot close. + +`infra/CLAUDE.md` states the rule: *"vps1 must never hold prod credentials. It's the +box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of an +isolation boundary."* + +Today that rule is enforced by a **shell flag** — `THERMOGRAPH_SECRETS_SKIP_COMMON=1` +— set by the *caller*, at three separate call sites (`env-topology.sh:196` → +`deploy.sh:67`, `deploy-dev.sh:86`, `infra-sync.yml:93-95`). The renderer itself does +not know that `env=dev` means never-common. A fourth call site, or one by-hand +`render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 keypairs +(one read-write on the bucket holding prod's database backups), the VAPID private key +that signs Web Push to real subscribers, `REGISTRY_TOKEN`, and the IndexNow and metrics +tokens onto the CI-runner box — **and exits 0**. + +Under `policies/tg-host-dev.hcl` that is not possible to get wrong. dev's identity +cannot read `thermograph/data/common`. A misconfigured caller gets a 403 instead of a +silent success. The failure class is *gone*, not guarded. That is what this migration +buys, and it is worth the cost of running a stateful service. + +The estate has already hardened this once by hand (commit `7583258`, "infra-sync: +refuse to render dev's vault onto a host that is not dev"). That is the shape of a +problem that wants an ACL, not another guard. + +### What it does *not* buy — stated plainly + +- **Rotation of provider-issued credentials.** Roughly half the credential count is + Discord, Contabo S3, the Forgejo registry token, VAPID, IndexNow. OpenBao cannot + rotate any of them; they stay static KV entries. Contabo Object Storage is + S3-compatible but exposes no IAM API, so the AWS secrets engine cannot issue against + it either. +- **Dynamic database credentials.** Inapplicable here for two independent reasons: the + apps read `THERMOGRAPH_DATABASE_URL` once at import and hold a pool, so a TTL'd + credential kills the pool at expiry; and there is no reload path anywhere in the + codebase — no SIGHUP handler, no config re-read. +- **Better review than git.** This is a real regression. Today a secret change is a + PR with a diff, backstopped by `secrets-guard` CI. Under OpenBao it is an + out-of-band API call with no diff and no review. Mitigated by committing a key-name + manifest so CI can still assert no key vanished, plus the audit log and + `bao kv metadata` as the change record — but not fully. +- **Fewer moving parts.** SOPS needs no server. This adds a stateful service to keep + alive, upgrade, back up and TLS-rotate, for ~40 secrets that change rarely. + +--- + +## Shape of the deployment + +| Decision | Choice | Why | +|---|---|---| +| Host | **vps2 only** | vps1 must never hold prod credentials, and with a static seal the box also holds the key that decrypts everything. Cost: vps1 needs vps2 up to deploy dev — acceptable, since when vps2 is down prod is down anyway. | +| Process | **native systemd**, not a container | A Swarm service is circular (`deploy-stack.sh` renders secrets to deploy the stack). A plain `docker run` makes the vault depend on the daemon deploys restart, and is one `prune -a` from gone. Native = one dependency, the disk. | +| Storage | **raft**, single node | 2.6.0 **deprecated the `file` backend** for removal in 2.7.0. Raft also has the only backup primitive (`operator raft snapshot save`). A *two*-node raft would be worse than one: quorum of two means losing either node loses writes. | +| Seal | **static auto-unseal** | See below. | +| TLS | self-signed, 10y, on disk | Must **not** come from Caddy/ACME — that would put DNS and the public internet in the boot chain of the secret store. | +| Auth | AppRole per environment, CIDR-bound | 5-minute tokens; secret-ids that only work from the right mesh address. | +| Consumption | keep rendering a dotenv file | Preserves every existing seam and keeps OpenBao a *deploy-time* dependency, never a runtime one. | + +### The unseal decision, which is the crux + +**Static seal**, key at `/etc/openbao/unseal.key` (0400), plus an off-box copy. + +Shamir (`-key-shares=5 -key-threshold=3`) means the vault is sealed after every process +restart — reboot, package upgrade, OOM kill — and a human must type three shares. +`render-secrets.sh` is on the deploy path for all three environments, so a sealed vault +blocks every deploy and every hotfix. With one operator the N-of-M threshold is +theatre: one person holds all the shares, so it buys nothing against the real threat +while costing an availability property the estate has today for free. A Contabo reboot +currently needs no human at all. + +OpenBao's docs hedge static seal — *"carefully evaluate"* — but that caveat is aimed at +multi-tenant enterprises. The argument here is that **static seal is not a downgrade +from the status quo**: `/etc/thermograph/age.key` is already one 0400 file per host +that decrypts everything forever with no audit trail. A static seal key is the +identical trust model. What changes is everything layered above it. + +**The one new single point of failure:** lose that key and the raft snapshots are +unrestorable. It must exist in ≥2 places, one not on vps2. Hard operator obligation. + +A `transit` seal against a second OpenBao on vps1 is genuinely better — vps2's disk +would no longer contain the unseal key — but the vps1 unsealer needs unsealing too, so +the circularity moves rather than dissolves, and it adds "vps1 must be up before vps2's +vault unseals" as a new failure mode. Revisit once the primary migration is boring. + +--- + +## Layout + +``` +thermograph/data/common the 16 shared beta+prod values (= common.yaml) +thermograph/data/env/prod prod's 16 (= prod.yaml) +thermograph/data/env/beta beta's 8 (= beta.yaml) +thermograph/data/env/dev dev's 12 — inherits NOTHING (= dev.yaml) +thermograph/data/centralis/prod Centralis' 9 (= centralis.prod.yaml) +thermograph/data/ops/backup S3 creds for ops-cron + the backup age recipient +thermograph/data/legacy/age the age PRIVATE key — see "the age key survives" +``` + +Inheritance lives in the render order (`common` then `env/`, last-wins); +isolation lives in the policy. Today both live in one shell variable. + +`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny +rule expressible as exactly one path, with no wildcard over `env/*` able to +accidentally re-include it. + +**~20 of the 61 keys are not secrets** (`PORT`, `WORKERS`, `APP_CPUS`, `TIMESCALEDB_TAG`, +`THERMOGRAPH_BASE_URL`, the Discord channel IDs, the VAPID *public* key, …). They stay +in the vault **for the cutover**, because byte-identical parity is the entire safety +property and splitting them would destroy it. Moving pure config back into the repo is +a clean follow-on that shrinks the vault-outage blast radius. + +--- + +## The age key survives this migration + +**This is the trap that would cause silent data loss.** + +`ops-cron.yml:142` and `:197` stream every off-box Postgres and Forgejo dump through +`pg_dump | age -r | rclone rcat`, with 30-day S3 retention. Restore uses +`/etc/thermograph/age.key`. So the age keypair is not just the SOPS transport — **it is +the backup encryption key.** + +Deleting it along with the SOPS vault files would destroy up to 30 days of database +recoverability, and nothing would notice until someone attempted a restore. + +So: keep the age private key at `thermograph/data/legacy/age` *and* in the password +manager, and keep it on disk until either the newest age-encrypted object in S3 has +aged out, or the backup path is re-pointed at a dedicated backup recipient. Retiring +SOPS and retiring age are two different projects. + +--- + +## Runbook + +### Stand it up (once, by the operator, on vps2) + +```sh +sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init +# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then: +export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +export BAO_TOKEN= +sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles +bao token revoke -self +``` + +### Seed and prove (from your own machine — it needs your age key) + +```sh +infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing +infra/openbao/seed-from-sops.sh --all +infra/openbao/verify-parity.sh --all # MUST pass before any flip +``` + +`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` +the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this +inherits that rule. + +### Cut an environment over + +One PR per hop, following the estate's own promotion model. + +```diff + dev) +- TG_SECRETS_BACKEND=sops ++ TG_SECRETS_BACKEND=openbao +``` + +Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended +gate before prod: **7 consecutive green parity runs on all three environments**, run +nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting +CI any vault access). + +`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the +SOPS path for a full release cycle after prod flips — it is the best mitigation +available for anything unforeseen. + +### Rollback + +Revert the one-line PR and redeploy. The SOPS path is untouched by this work — verified +by rendering dev (12 keys) and prod (32 keys) through it after the dispatch was added, +matching the live hosts exactly. + +--- + +## Failure modes + +| Failure | Effect | Mitigation | +|---|---|---| +| Vault down/sealed at deploy | Render returns 1, deploy aborts, **running stack unaffected** — temp-then-install means `/etc/thermograph.env` is never truncated | `Restart=always` + static seal so a reboot self-heals; alert on `/v1/sys/health`; `TG_SECRETS_BACKEND=sops` is a working revert | +| **Audit device wedges** | ⚠️ OpenBao **stops answering requests entirely** when no enabled audit device can record them — a full `/var` on vps2 blocks every deploy | Two devices (file + syslog); logrotate signals **SIGHUP** or bao writes to an unlinked inode; disk alert on vps2 | +| Static seal key lost | Raft snapshots unrestorable | ≥2 copies, one off-box. The one new SPOF | +| Raft corruption / disk loss | Total vault loss | Nightly `operator raft snapshot save`, age-encrypted to S3 beside the DB dumps. **Verify a restore end-to-end before any consumer depends on it** | +| Cold boot of vps2 | Nothing needs the vault | Swarm restarts from persisted specs; `stack.env` / `thermograph.env` / `centralis.env` all persist. **This property exists because we render a file, and would be destroyed by app-native reads** | +| Secret-id leak | Useless off the mesh | `secret_id_bound_cidrs`; revoke by accessor without knowing the value | +| beta credential reaches prod | Cross-env compromise | Separate AppRoles, separate policies, secret-ids owned by the separate deploy users (`agent` vs `deploy`) — a boundary that does **not** exist today, since both currently reach the same root-owned age key. Honest limit: root on vps2 defeats it, exactly as it defeats the single age key now | +| Upgrade to 2.7.0 | Removes the `file` backend and built-in cloud KMS seals | Already avoided by choosing raft + static. Pin the version | + +--- + +## Deliberately out of scope + +For ~40 credentials and one operator the real risk is over-engineering. Not adopted: +HA/multi-node raft, namespaces, dynamic database credentials, `bao agent`, cert auth, +PKI, identity groups, transit seal (revisit later), OIDC for the operator. + +**OIDC for CI is out of scope for a reason worth recording.** Forgejo Actions *does* +support OIDC — shipped in **Forgejo v15.0**, needing Runner > v12.5.0, enabled via +`enable-openid-connect: true` rather than GitHub's `permissions: id-token: write`. But +this estate runs `codeberg.org/forgejo/forgejo:9-rootless` +(`infra/deploy/forgejo/docker-stack.yml:59`). That is a six-major-version upgrade, and +coupling it to this migration would mean two large independent migrations at once with +no way to tell which one broke. Revisit after Forgejo reaches v15; then +`role_type=jwt` with `bound_claims` removes the last CI-side bearer credential. + +CI keeps its SSH keys and `REGISTRY_TOKEN` as Forgejo secrets. **CI gets no vault +access at all** — the SSH-in architecture already means the *host* renders, never the +runner, which is the least-privilege topology and should be preserved rather than +"upgraded". + +### Secret zero, honestly + +Nothing eliminates it; you choose where it sits and how little of it there is. After +this phase the chain terminates at **9 Forgejo Actions secrets + 1 static seal key + 4 +AppRole secret-ids**, versus today's **13 CI secrets + 1 omnipotent age key per host**. + +--- + +## Verification gaps to close on the box + +Flagged rather than guessed, because these could not be checked from a workstation: + +1. Whether a raft snapshot is restorable under a *different* seal key. Docs don't say. + **Test in phase 0, before anything depends on it.** +2. Exact `-wrap-ttl` flag spelling on `bao write` — documented in the response-wrapping + concept page but absent from the commands index. Check `bao write -h`. +3. Debian/Ubuntu package repo URL. The install docs claim `.deb` packages exist but + give no repository; `bootstrap.sh` uses the GitHub release tarball with checksum + verification, which is fine and also fixes a standing weakness — `provision-secrets.sh` + installs sops and age with a bare `sudo curl` and no verification at all. +4. Vault-era naming persists inside OpenBao config (`vault { }` stanza, + `X-Vault-Wrap-TTL` header). Expect it; don't "fix" it. diff --git a/infra/openbao/bootstrap-policies.sh b/infra/openbao/bootstrap-policies.sh new file mode 100755 index 0000000..81d07bd --- /dev/null +++ b/infra/openbao/bootstrap-policies.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Second half of bootstrap: KV mount, policies, AppRoles, host credentials. +# Run on vps2 with BAO_TOKEN set to a root token. Idempotent — safe to re-run. +# +# export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +# export BAO_TOKEN= +# sudo -E bash infra/openbao/bootstrap-policies.sh +# +# Split from bootstrap.sh because everything here is safely automatable: none of it +# produces a credential a human has to custody. The one thing that does — the recovery +# material from `bao operator init` — is in bootstrap.sh and stops for the operator. +set -euo pipefail + +MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}" +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +POLICY_DIR="$SELF_DIR/policies" + +: "${BAO_TOKEN:?BAO_TOKEN must be set to a root token}" +: "${BAO_ADDR:=https://127.0.0.1:8200}" +export BAO_ADDR +command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; } + +echo "==> KV v2 mount at ${MOUNT}/" +if bao secrets list -format=json | grep -q "\"${MOUNT}/\""; then + echo " already mounted" +else + bao secrets enable -path="$MOUNT" -version=2 kv +fi + +echo "==> policies" +for p in "$POLICY_DIR"/*.hcl; do + name="$(basename "$p" .hcl)" + bao policy write "$name" "$p" >/dev/null + echo " wrote $name" +done + +echo "==> approle auth" +bao auth list -format=json | grep -q '"approle/"' || bao auth enable approle + +# One role per environment, CIDR-bound to the host that legitimately renders it. +# +# secret_id_ttl=0 (never expires) is deliberate: a secret-id that silently lapses +# mid-quarter is a self-inflicted outage. The short lifetime lives on the TOKEN +# instead — it only has to outlive one render. +# +# The CIDR binding is the part that makes a leaked secret-id close to useless: it is +# only accepted from the correct WireGuard address. +add_role() { + local role="$1" policy="$2" cidr="$3" + bao write "auth/approle/role/${role}" \ + token_policies="$policy" \ + secret_id_bound_cidrs="$cidr" \ + token_bound_cidrs="$cidr" \ + secret_id_ttl=0 secret_id_num_uses=0 \ + token_ttl=5m token_max_ttl=15m token_num_uses=10 >/dev/null + echo " role ${role} -> ${policy} (bound ${cidr})" +} + +add_role tg-prod tg-host-prod 10.10.0.1/32 +add_role tg-beta tg-host-beta 10.10.0.1/32 +add_role tg-dev tg-host-dev 10.10.0.2/32 +add_role tg-centralis tg-centralis 10.10.0.1/32 + +# Install the local (vps2) credentials. prod and beta both render on this box. +# +# The OWNERSHIP here is a real boundary that does not exist today, and it is the one +# concrete isolation win available on a shared host. render-secrets.sh:119-124 records +# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the +# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation +# at all. Giving each environment its own 0400 secret-id owned by its own deploy user +# creates one. +# +# Be honest about the limit: root on vps2 reads both files, exactly as root today reads +# the one age key. This defends against MISCONFIGURATION, not against intrusion. +install_creds() { + local role="$1" dest="$2" owner="$3" + local rid sid + rid=$(bao read -field=role_id "auth/approle/role/${role}/role-id") + sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id") + # umask before creation, so the file is never briefly world-readable. + ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" ) + chown "$owner" "$dest" + chmod 0400 "$dest" + echo " installed $dest (0400 $owner)" +} + +if [ "$(id -u)" = 0 ]; then + install -d -o root -g root -m 0755 /etc/thermograph + install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent" + install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \ + || echo " (beta creds skipped — no 'deploy' user on this box?)" + install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root" +else + echo "!! not root; skipping credential install. Re-run with sudo -E." >&2 +fi + +cat < done. Policies and AppRoles are in place; the vault is still EMPTY. + +dev's credentials must be installed on vps1, not here. From your own machine: + + bao read -field=role_id auth/approle/role/tg-dev/role-id + bao write -f -field=secret_id auth/approle/role/tg-dev/secret-id + # then, on vps1, write both lines to /etc/thermograph/openbao-approle (0400 agent) + +Prefer response wrapping so the secret-id never lands in shell history or a log: + bao write -f -wrap-ttl=5m auth/approle/role/tg-dev/secret-id # gives a wrapping token + # on vps1: bao unwrap (single-use: a failed unwrap means interception) + +Next: seed, then verify parity. Neither cuts anything over. + infra/openbao/seed-from-sops.sh --dry-run + infra/openbao/seed-from-sops.sh --all + infra/openbao/verify-parity.sh --all +EOF diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh new file mode 100755 index 0000000..39fc5e6 --- /dev/null +++ b/infra/openbao/bootstrap.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Bring up OpenBao on vps2 from nothing. Run ONCE, as root, ON vps2, BY THE OPERATOR. +# +# sudo bash infra/openbao/bootstrap.sh +# +# ============================================================================ +# WHY THIS IS NOT AUTOMATED, AND MUST NOT BE +# ============================================================================ +# `bao operator init` mints the credentials that become the new root of trust for the +# entire estate: the recovery keys and the initial root token. Those have to be +# CUSTODIED BY A HUMAN. Any place a script could put them — a file on the box, the +# repo, a CI log, an agent transcript — is worse custody than the age key this +# migration is meant to improve on. +# +# So this script does everything up to and including init, prints the recovery +# material ONCE to the operator's terminal, and then stops and tells you what to do +# with it. It deliberately does not store it, mail it, or push it anywhere. +# +# Everything AFTER custody (policies, AppRoles, seeding, parity checks) is automated, +# because none of it produces a credential a human needs to hold. +set -euo pipefail + +BAO_VERSION="${BAO_VERSION:-2.6.1}" +MESH_ADDR="${MESH_ADDR:-10.10.0.1}" +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + +[ "$(id -u)" = 0 ] || { echo "!! run as root (sudo bash $0)" >&2; exit 1; } + +hostname_now="$(hostname)" +case "$hostname_now" in + vmi3453260) : ;; # vps2 + *) + # Refuse to bootstrap the secret store on the wrong box. Putting it on vps1 would + # invert the estate's central security decision: infra/CLAUDE.md states "vps1 must + # never hold prod credentials. It's the box that runs Forgejo, its CI runner, and + # dev's unreviewed branch — the opposite of an isolation boundary." With a static + # seal, the box also holds the key that decrypts everything. + # + # This mirrors the guard infra-sync.yml already has (commit 7583258, "refuse to + # render dev's vault onto a host that is not dev"). + echo "!! this host is '$hostname_now', not vps2 (vmi3453260)." >&2 + echo "!! OpenBao belongs on vps2. Refusing." >&2 + exit 1 + ;; +esac + +echo "==> 1/7 install the bao binary (v${BAO_VERSION})" +if ! command -v bao >/dev/null 2>&1; then + # NOTE: checksum verification. provision-secrets.sh installs sops and age with a bare + # `sudo curl` and no verification at all, which is a standing weakness (a compromised + # release URL yields a root binary that sees every plaintext). Do not copy that here. + # The checksums file is signed by the release; verify before installing. + tmpd="$(mktemp -d)" + base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}" + curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS" + ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || { + echo "!! checksum mismatch on the bao tarball; refusing to install" >&2 + rm -rf "$tmpd"; exit 1 + } + tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao + install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao + rm -rf "$tmpd" +fi +bao version + +echo "==> 2/7 user, directories, TLS" +id -u openbao >/dev/null 2>&1 || useradd --system --home /var/lib/openbao --shell /usr/sbin/nologin openbao +install -d -o openbao -g openbao -m 0700 /var/lib/openbao +install -d -o openbao -g openbao -m 0750 /var/log/openbao +install -d -o root -g root -m 0755 /etc/openbao +install -d -o openbao -g openbao -m 0700 /etc/openbao/tls + +if [ ! -f /etc/openbao/tls/bao.crt ]; then + # Self-signed, 10 years, SAN on the mesh address. Deliberately NOT from Caddy/ACME: + # that would put DNS and the public internet in the boot chain of the secret store. + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \ + -keyout /etc/openbao/tls/bao.key -out /etc/openbao/tls/bao.crt \ + -subj "/CN=openbao.thermograph.internal" \ + -addext "subjectAltName=IP:${MESH_ADDR},IP:127.0.0.1" >/dev/null 2>&1 + chown openbao:openbao /etc/openbao/tls/bao.key /etc/openbao/tls/bao.crt + chmod 0400 /etc/openbao/tls/bao.key + chmod 0444 /etc/openbao/tls/bao.crt +fi +# Consumers need the cert to trust the listener. +install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt + +echo "==> 3/7 static seal key" +if [ ! -f /etc/openbao/unseal.key ]; then + # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key. + ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key ) + chown openbao:openbao /etc/openbao/unseal.key + chmod 0400 /etc/openbao/unseal.key + echo " generated /etc/openbao/unseal.key (0400 openbao)" + echo " ⚠️ COPY THIS OFF THE BOX into your password manager before proceeding." + echo " Without an off-box copy, a raft snapshot is UNRESTORABLE." +else + echo " /etc/openbao/unseal.key already present, leaving alone" +fi + +echo "==> 4/7 config + systemd unit" +install -m 0640 -o root -g openbao "$SELF_DIR/config/openbao.hcl" /etc/openbao/config.hcl +install -m 0644 -o root -g root "$SELF_DIR/openbao.service" /etc/systemd/system/openbao.service +# logrotate MUST use SIGHUP: without it bao keeps writing to an unlinked inode and the +# audit log silently stops growing — and a wedged audit device makes OpenBao stop +# answering requests entirely, which would block every deploy on the estate. +cat > /etc/logrotate.d/openbao <<'ROTATE' +/var/log/openbao/audit.log { + daily + rotate 30 + compress + missingok + notifempty + create 0600 openbao openbao + postrotate + systemctl kill --signal=SIGHUP openbao.service 2>/dev/null || true + endscript +} +ROTATE +systemctl daemon-reload +systemctl enable --now openbao.service +sleep 3 +systemctl is-active --quiet openbao.service || { + echo "!! openbao.service failed to start; check: journalctl -u openbao -n 50" >&2 + exit 1 +} + +export BAO_ADDR="https://127.0.0.1:8200" +export BAO_CACERT=/etc/openbao/tls/bao.crt + +echo "==> 5/7 initialise" +if bao status -format=json 2>/dev/null | grep -q '"initialized": *true'; then + echo " already initialised, skipping" +else + echo + echo " ############################################################" + echo " # RECOVERY MATERIAL FOLLOWS. IT IS SHOWN EXACTLY ONCE. #" + echo " # Store the 3 recovery keys in 3 DIFFERENT places. #" + echo " # 2 of 3 are needed to rekey or mint a new root token. #" + echo " ############################################################" + echo + # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init + # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal + # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one + # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token. + bao operator init -recovery-shares=3 -recovery-threshold=2 + echo + echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are" + echo " stored off this machine." + read -r _ +fi + +bao status || true + +cat < 6/7 and 7/7 need a root token, which is NOT stored anywhere by design. + +Paste the initial root token (or one minted from recovery keys) and run: + + export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt + export BAO_TOKEN= + bash ${SELF_DIR}/bootstrap-policies.sh + +That script enables the KV mount, writes the policies, creates the AppRoles and +installs each host's credentials. Then, from YOUR OWN machine (it needs your age +key, and per infra/CLAUDE.md a script that reads production secrets is not for an +agent to run): + + infra/openbao/seed-from-sops.sh --dry-run # check first + infra/openbao/seed-from-sops.sh --all + infra/openbao/verify-parity.sh --all # must PASS before any flip + +Then revoke the root token: bao token revoke -self + +NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for +every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative +until you flip an environment in a reviewed PR — dev first. +EOF diff --git a/infra/openbao/config/openbao.hcl b/infra/openbao/config/openbao.hcl new file mode 100644 index 0000000..07f4d87 --- /dev/null +++ b/infra/openbao/config/openbao.hcl @@ -0,0 +1,118 @@ +// OpenBao server config for the Thermograph estate. Lives at /etc/openbao/config.hcl +// on vps2. Verified against OpenBao v2.6.1 (2026-07-22). +// +// ============================================================================ +// WHY THIS RUNS AS A NATIVE SYSTEMD SERVICE AND NOT A CONTAINER +// ============================================================================ +// The estate is Docker-native and this is the one thing that must not be. The rule +// is that OpenBao must not depend on anything it secures. +// +// * A Swarm service is fatal: deploy-stack.sh renders secrets in order to deploy +// the stack, so the stack cannot contain the thing that holds them. +// * A standalone `docker run --restart=always` is merely bad: it makes the vault's +// availability a function of the same Docker daemon that deploys restart, and +// leaves it one careless `docker system prune -a` from gone. The root CLAUDE.md +// already lists `prune -a` as a thing that eats the rollback image; it would +// eat this too. +// +// A native binary with Restart=always has exactly one dependency: the local disk. +// See infra/openbao/openbao.service. +// +// Corollary that is easy to get wrong: the listener cert must NOT come from the +// Caddy/ACME path. That would put DNS and the public internet in the boot chain of +// the estate's secret store. It is self-signed, long-lived, and on disk. +// ============================================================================ + +ui = false +cluster_name = "thermograph" +disable_mlock = true +log_level = "info" + +// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0 +// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys +// a migration within a release. Raft also has the only first-class backup primitive +// (`bao operator raft snapshot save`), which is the disaster-recovery story. +// +// A single-node raft is the honest shape for a one-operator estate. Note that a +// TWO-node raft would be strictly worse than one: quorum of two means losing either +// node loses write availability. Three voters would be needed, and the desktop +// (10.10.0.3) is intermittent, so it cannot be a reliable third. +storage "raft" { + path = "/var/lib/openbao" + node_id = "vps2" +} + +// Mesh-only. 10.10.0.1 is vps2's WireGuard address. There is deliberately no +// 0.0.0.0 listener and no public DNS record — vps2 is a public VPS, and reaching +// OpenBao should require being on the mesh, exactly like the Swarm control ports +// and the Loki endpoint. +listener "tcp" { + address = "10.10.0.1:8200" + tls_cert_file = "/etc/openbao/tls/bao.crt" + tls_key_file = "/etc/openbao/tls/bao.key" +} + +// Loopback, so a by-hand `bao` on the box works without the mesh address. +listener "tcp" { + address = "127.0.0.1:8200" + tls_cert_file = "/etc/openbao/tls/bao.crt" + tls_key_file = "/etc/openbao/tls/bao.key" +} + +api_addr = "https://10.10.0.1:8200" +cluster_addr = "https://10.10.0.1:8201" + +// ============================================================================ +// SEAL: static auto-unseal. This is the crux of the whole design. +// ============================================================================ +// The `static` seal (OpenBao v2.4.0+) auto-unseals from a local key with no cloud +// KMS and no HSM. OpenBao's own docs hedge it — "carefully evaluate use of Static +// Key Auto Unseal" — but that caveat is written for multi-tenant enterprises. +// +// The decisive argument here is that static seal is NOT A DOWNGRADE FROM THE STATUS +// QUO. Today /etc/thermograph/age.key is one file, 0400, root-owned, on each +// rendering host, which decrypts every secret in the estate forever with no audit +// trail. A static seal key at 0400 on vps2 is the IDENTICAL trust model: one file on +// one box whose compromise is total compromise. What changes is everything layered +// above it — per-consumer ACLs, an audit log, five-minute tokens, versioned rollback. +// +// WHY NOT SHAMIR: `bao operator init -key-shares=5 -key-threshold=3` means the vault +// is sealed after every process restart — reboot, package upgrade, OOM kill — and a +// human must type three shares. render-secrets.sh is on the deploy path for all three +// environments, so a sealed vault blocks every deploy and every hotfix. With one +// operator the N-of-M threshold is theatre: one person holds all the shares, so it +// buys nothing against the real threat while costing an availability property the +// estate currently has for free. Today a Contabo reboot needs no human at all. +// Introducing a human-in-the-loop requirement that does not exist today is a +// regression this estate cannot absorb. +// +// THE ONE NEW SINGLE POINT OF FAILURE this migration introduces: lose this key and +// the raft snapshots become unrestorable. It MUST exist in at least two places, one +// of them not on vps2. That is a hard operator obligation, not a suggestion. +// +// Rotation is supported n-1 via previous_key / previous_key_id. +seal "static" { + current_key_id = "20260801-1" + current_key = "file:///etc/openbao/unseal.key" +} + +// ============================================================================ +// AUDIT: two devices, deliberately. +// ============================================================================ +// ⚠️ OpenBao STOPS ANSWERING REQUESTS ENTIRELY when no enabled audit device can +// record them. That is correct behaviour for an audit log and a catastrophic way to +// discover a full disk: a full /var on vps2 would block every deploy on the estate. +// Two independent devices mean one wedging is survivable. +// +// Values are HMAC-SHA256'd in the log, so it is safe to ship to Loki through the +// existing Alloy pipeline — which finally makes "who read which secret, when" a +// queryable question. Today reading age.key leaves no trace whatsoever. +// +// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an +// unlinked inode and the log silently stops growing. +audit "file" { + file_path = "/var/log/openbao/audit.log" + mode = "0600" +} + +audit "syslog" {} diff --git a/infra/openbao/openbao.service b/infra/openbao/openbao.service new file mode 100644 index 0000000..a136c4f --- /dev/null +++ b/infra/openbao/openbao.service @@ -0,0 +1,50 @@ +# OpenBao, as a native systemd service on vps2. Install to +# /etc/systemd/system/openbao.service. +# +# Native, not containerised, on purpose — see the header of config/openbao.hcl. The +# whole point is that this unit's only dependency is the local disk: not the Docker +# daemon that deploys restart, not the Swarm it would otherwise be deployed by, not +# Caddy/ACME for its certificate, and not TimescaleDB. +# +# Note there is currently NO systemd unit of any kind loaded on either host for +# Thermograph (infra/deploy/thermograph.service exists in the repo but is not +# installed), so this is the first. That is deliberate: it is the one component that +# must survive the container layer being broken. + +[Unit] +Description=OpenBao secret store +Documentation=https://openbao.org/docs/ +# network-online rather than plain network: the mesh listener binds 10.10.0.1, and +# binding a WireGuard address before the interface is up fails the unit outright. +After=network-online.target +Wants=network-online.target +# Explicitly NOT After=docker.service. Adding it would recreate the dependency this +# design exists to avoid. +ConditionFileNotEmpty=/etc/openbao/config.hcl + +[Service] +User=openbao +Group=openbao +ExecStart=/usr/local/bin/bao server -config=/etc/openbao/config.hcl +ExecReload=/bin/kill --signal HUP $MAINPID +KillSignal=SIGINT + +# Restart always, which together with the static seal is what makes a host reboot +# self-healing. This pair is the reason deploys are not blocked by a reboot. +Restart=always +RestartSec=5 + +# mlock is disabled in the config (disable_mlock = true), so no IPC_LOCK capability +# is needed. If mlock is ever enabled, add: AmbientCapabilities=CAP_IPC_LOCK +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=full +ProtectHome=yes +# The three paths the service legitimately writes. +ReadWritePaths=/var/lib/openbao /var/log/openbao +# Raft + audit both do a lot of small writes; the default is too low for a store. +LimitNOFILE=65536 +LimitCORE=0 + +[Install] +WantedBy=multi-user.target diff --git a/infra/openbao/policies/tg-centralis.hcl b/infra/openbao/policies/tg-centralis.hcl new file mode 100644 index 0000000..6518ba8 --- /dev/null +++ b/infra/openbao/policies/tg-centralis.hcl @@ -0,0 +1,67 @@ +// Policy for the Centralis control-plane container on vps2. +// +// Centralis is the tool an operator drives a rotation THROUGH, so it needs more than +// a host render does: it must enumerate key names across every environment to build +// `secrets_inventory`, and write values to perform `secrets_rotate`. +// +// The split below is the important part. Centralis gets: +// * METADATA read/list on every path -> it can build a names-and-versions +// inventory across all three environments WITHOUT being able to read a value. +// * DATA write on every path -> it can rotate. +// * DATA read on NOTHING. +// +// That is a genuine improvement on the SOPS design, not a port of it. Today +// `secrets_inventory` can list key names without a decryption key only because SOPS +// happens to leave key names as plaintext in the YAML — a property of the file +// format, relied on deliberately (tools/secrets.ts:199 `extractKeyNames`). Under +// OpenBao the same "names, never values" guarantee becomes an ENFORCED capability +// boundary instead of a fortunate accident. Centralis cannot print a secret it is +// not able to fetch. +// +// The `read` omission is load-bearing: Centralis holds the Docker socket (root +// -equivalent on prod) and an SSH private key to beta and dev. Granting it data read +// on prod's secrets would make a Centralis compromise equivalent to full estate +// credential disclosure. It already effectively is via the Docker socket — but there +// is no reason to hand it a second, easier path. + +// Names and version history across the whole tree — no values. +path "thermograph/metadata/*" { + capabilities = ["read", "list"] +} + +// Rotation: write a new version. Note "create" + "update" but NOT "read": +// OpenBao permits a blind write, which is exactly the shape secrets_rotate wants — +// it mints or receives a new value and stores it, and never needs the old one. +path "thermograph/data/*" { + capabilities = ["create", "update"] +} + +// Its OWN configuration is the one place Centralis may read, because it must render +// /etc/centralis.env for itself. This is the direct analogue of centralis.prod.yaml. +path "thermograph/data/centralis/prod" { + capabilities = ["read", "create", "update"] +} + +// Version rollback, so a bad rotation is recoverable through the control plane +// rather than requiring a shell on the box. This is the capability that replaces +// "revert the PR" in the SOPS model. +path "thermograph/undelete/*" { + capabilities = ["update"] +} + +path "thermograph/destroy/*" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} + +// Read its own AppRole role-id for self-diagnosis, but never the secret-id. +path "auth/approle/role/tg-centralis/role-id" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/tg-host-beta.hcl b/infra/openbao/policies/tg-host-beta.hcl new file mode 100644 index 0000000..7d8ae67 --- /dev/null +++ b/infra/openbao/policies/tg-host-beta.hcl @@ -0,0 +1,37 @@ +// Policy for BETA's render identity on vps2. Mirror of tg-host-prod.hcl. +// +// Beta legitimately reads the shared set: unlike dev, beta is on vps2 and already +// co-resident with prod, so withholding `shared` from it would buy nothing while +// breaking the render (beta's 24 live keys = beta.yaml's 8 + shared's 16). + +path "thermograph/data/common" { + capabilities = ["read"] +} + +path "thermograph/metadata/common" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/beta" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/beta" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/prod" { + capabilities = ["deny"] +} + +path "thermograph/data/env/dev" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-host-dev.hcl b/infra/openbao/policies/tg-host-dev.hcl new file mode 100644 index 0000000..2d7b8f9 --- /dev/null +++ b/infra/openbao/policies/tg-host-dev.hcl @@ -0,0 +1,72 @@ +// Policy for the DEV host's render identity (vps1). +// +// THIS FILE IS THE POINT OF THE MIGRATION. +// +// infra/CLAUDE.md states the rule: "vps1 must never hold prod credentials. It's the +// box that runs Forgejo, its CI runner, and dev's unreviewed branch — the opposite of +// an isolation boundary." +// +// Today that rule is enforced by a SHELL FLAG (THERMOGRAPH_SECRETS_SKIP_COMMON=1) set +// by the *caller*, at three separate call sites: env-topology.sh:196 -> deploy.sh:67, +// deploy-dev.sh:86, and infra-sync.yml:93-95. The renderer itself does not know that +// env=dev means never-common. A fourth call site, or one by-hand +// `render_thermograph_secrets /opt/thermograph-dev/infra dev`, writes both S3 +// keypairs (one read-write on the bucket holding prod's database backups), the VAPID +// private key that signs Web Push to real subscribers, REGISTRY_TOKEN, and the +// IndexNow and metrics tokens onto the Forgejo CI-runner box — and exits 0. +// +// Under this policy that is no longer possible to get wrong. dev's identity cannot +// read the common path. A misconfigured caller gets a 403 from OpenBao, not a silent +// success with production credentials on disk. The failure class is gone, not guarded. + +// dev's own values. This is the ONLY secret path dev can read. +path "thermograph/data/env/dev" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/dev" { + capabilities = ["read", "list"] +} + +// EXPLICIT DENY on the shared production credential set. +// +// A deny rule is redundant with OpenBao's default-deny — anything not granted is +// already refused. It is here anyway, and must stay, for two reasons: +// 1. It is documentation that survives refactoring. Someone widening dev's grants +// has to delete an explicit deny to break the rule, which is a much louder edit +// than adding a path to an allow list. +// 2. In OpenBao, a deny on a path beats any grant at that path regardless of rule +// order or specificity. So if dev's identity is ever accidentally given a +// broader policy alongside this one, this still wins. +path "thermograph/data/common" { + capabilities = ["deny"] +} + +path "thermograph/metadata/common" { + capabilities = ["deny"] +} + +// Deny the other environments outright. Same argument: dev has no business reading +// beta or prod, and being explicit means a widened grant elsewhere cannot silently +// re-open it. +path "thermograph/data/env/prod" { + capabilities = ["deny"] +} + +path "thermograph/data/env/beta" { + capabilities = ["deny"] +} + +path "thermograph/data/centralis/*" { + capabilities = ["deny"] +} + +// Token self-management, so the render can look up and renew its own lease without +// needing a broader grant. +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-host-prod.hcl b/infra/openbao/policies/tg-host-prod.hcl new file mode 100644 index 0000000..bb2566d --- /dev/null +++ b/infra/openbao/policies/tg-host-prod.hcl @@ -0,0 +1,44 @@ +// Policy for PROD's render identity on vps2. +// +// Reads the shared set plus prod's own overrides — the OpenBao equivalent of +// "common.yaml then prod.yaml, last-wins". +// +// Note what this policy does NOT grant: beta. vps2 runs prod and beta side by side +// with one filesystem and one SSH credential, so the host is emphatically not an +// isolation boundary between them (infra/CLAUDE.md is explicit about this). Separate +// identities with separate policies do not fix that — a foothold on the box can read +// both AppRole credentials — but they do mean a *mistake* cannot cross the line: +// prod's render cannot accidentally pull beta's database password, and the audit log +// attributes every read to one environment or the other. + +path "thermograph/data/common" { + capabilities = ["read"] +} + +path "thermograph/metadata/common" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/prod" { + capabilities = ["read"] +} + +path "thermograph/metadata/env/prod" { + capabilities = ["read", "list"] +} + +path "thermograph/data/env/beta" { + capabilities = ["deny"] +} + +path "thermograph/data/env/dev" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/infra/openbao/policies/tg-ops-backup.hcl b/infra/openbao/policies/tg-ops-backup.hcl new file mode 100644 index 0000000..670f102 --- /dev/null +++ b/infra/openbao/policies/tg-ops-backup.hcl @@ -0,0 +1,50 @@ +// Policy for the ops-cron backup job (.forgejo/workflows/ops-cron.yml). +// +// Exists to DELETE FOUR CI SECRETS. Today S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY and +// S3_SECRET_KEY are Forgejo repo secrets, even though ops-cron only ever uses them +// INSIDE the script it SSHes onto the host — they are passed through `envs:` and +// consumed as RCLONE_CONFIG_ARCHIVE_*. Nothing needs them on the runner. Moving them +// here means the host reads them directly and CI holds four fewer credentials. +// +// It also collapses a genuine duplication: the same values already live in the SOPS +// vault as THERMOGRAPH_S3_* in common.yaml, so there are currently two sources of +// truth for one credential pair, free to drift. +// +// The backup recipient belongs here too — see thermograph/data/ops/backup below. +// ops-cron.yml:142 and :197 currently HARDCODE the age recipient as a literal, in two +// places, and that same public key is duplicated in eleven places across the repo. + +path "thermograph/data/ops/backup" { + capabilities = ["read"] +} + +path "thermograph/metadata/ops/backup" { + capabilities = ["read", "list"] +} + +// ============================================================================ +// THE AGE KEY IS NOT RETIRED BY THIS MIGRATION. This path is why. +// ============================================================================ +// ops-cron.yml streams every off-box database dump through +// `pg_dump | age -r | rclone rcat`, with 30-day S3 retention, and restore +// depends on the private half at /etc/thermograph/age.key. The Forgejo backup path is +// the same shape. +// +// So the age keypair is not merely the SOPS transport — it is the BACKUP ENCRYPTION +// KEY. Deleting it alongside the SOPS vault files would silently destroy up to 30 days +// of database recoverability, and nothing would notice until a restore was attempted. +// +// Therefore: the age private key is stored at thermograph/data/legacy/age (operator +// access only, deliberately NOT granted by this policy) and kept on disk until either +// the newest age-encrypted object in S3 has aged out, or the backup path is re-pointed +// at a dedicated backup recipient. Only then may /etc/thermograph/age.key be removed. +// +// The `deny` is explicit so that widening this policy cannot hand the backup job the +// key to read everything else in the estate. +path "thermograph/data/legacy/age" { + capabilities = ["deny"] +} + +path "auth/token/lookup-self" { + capabilities = ["read"] +} diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh new file mode 100755 index 0000000..09f32cc --- /dev/null +++ b/infra/openbao/seed-from-sops.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Seed OpenBao from the SOPS vault. One-shot, idempotent, run BY THE OPERATOR. +# +# infra/openbao/seed-from-sops.sh --dry-run # key names only, writes nothing +# infra/openbao/seed-from-sops.sh --env dev # seed one environment +# infra/openbao/seed-from-sops.sh --all # seed everything +# +# ============================================================================ +# THIS SCRIPT READS EVERY PRODUCTION SECRET IN THE ESTATE IN PLAINTEXT. +# ============================================================================ +# infra/CLAUDE.md and deploy/secrets/README.md both say that +# deploy/secrets/seed-from-live.sh "reads production secrets and is explicitly NOT for +# an agent to run". This script is in the same category and inherits the same rule: +# run it yourself, from your own terminal, with your own age key. +# +# It never prints a value. Every diagnostic is key NAMES and counts, so the output is +# safe to paste into an issue. --dry-run makes that guarantee mechanical: it resolves +# and validates everything, then reports what it WOULD write without contacting +# OpenBao at all. +# +# Why seed from SOPS rather than re-entering values by hand: the safety property of +# this entire migration is that the OpenBao render can be diffed BYTE-FOR-BYTE against +# the SOPS render (see verify-parity.sh). That only works if the values are identical. +# Re-typing 61 values would silently rotate whichever ones were mistyped, and the +# self-generating ones (AUTH_SECRET, the VAPID pair) fail SILENTLY when wrong — the app +# mints a replacement, comes up green, and invalidates every session and push +# subscription. So: copy exactly now, rotate deliberately later. +set -euo pipefail + +MOUNT="${THERMOGRAPH_BAO_MOUNT:-thermograph}" +AGE_KEY="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}" +DRY_RUN=0 +TARGETS=() + +# Resolve the repo's infra/ directory from this script's own location, so the script +# works from any cwd. This is the same mistake terraform's remote-exec makes +# (main.tf:256 passes app_dir where app_dir/infra is wanted), so it is worth being +# explicit rather than clever. +SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)" +VAULT_DIR="$INFRA_DIR/deploy/secrets" + +usage() { + cat >&2 <<'EOF' +usage: seed-from-sops.sh [--dry-run] (--all | --env ...) + + --dry-run validate and report key names/counts; contact OpenBao not at all + --all seed common, dev, beta, prod and centralis.prod + --env seed one of: common | dev | beta | prod | centralis.prod + +Environment: + SOPS_AGE_KEY_FILE age private key (default ~/.config/sops/age/keys.txt) + BAO_ADDR OpenBao address (default https://10.10.0.1:8200) + BAO_TOKEN an operator token with write on /data/* + THERMOGRAPH_BAO_MOUNT KV v2 mount name (default thermograph) +EOF + exit 2 +} + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN=1; shift ;; + --all) TARGETS=(common dev beta prod centralis.prod); shift ;; + --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;; + -h|--help) usage ;; + *) echo "!! unknown argument: $1" >&2; usage ;; + esac +done + +[ ${#TARGETS[@]} -gt 0 ] || usage + +# The KV v2 path for a given vault file. common.yaml and centralis.prod.yaml are +# special-cased; everything else is an environment overlay. +# +# Keeping `common` at its own top-level path rather than under env/ is what makes the +# dev policy expressible: tg-host-dev.hcl denies exactly one path, and no wildcard +# over env/* can accidentally re-include it. +bao_path_for() { + case "$1" in + common) printf 'common\n' ;; + centralis.prod) printf 'centralis/prod\n' ;; + dev|beta|prod) printf 'env/%s\n' "$1" ;; + *) echo "!! unknown vault target: $1" >&2; return 1 ;; + esac +} + +[ -f "$AGE_KEY" ] || { echo "!! age key not found at $AGE_KEY" >&2; exit 1; } +command -v sops >/dev/null 2>&1 || { echo "!! sops not on PATH" >&2; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "!! python3 not on PATH" >&2; exit 1; } + +if [ "$DRY_RUN" = 0 ]; then + command -v bao >/dev/null 2>&1 || { echo "!! bao not on PATH" >&2; exit 1; } + [ -n "${BAO_TOKEN:-}" ] || { echo "!! BAO_TOKEN is not set" >&2; exit 1; } + export BAO_ADDR="${BAO_ADDR:-https://10.10.0.1:8200}" +fi + +rc=0 +for target in "${TARGETS[@]}"; do + src="$VAULT_DIR/${target}.yaml" + if [ ! -f "$src" ]; then + echo "!! no vault file at $src" >&2 + rc=1 + continue + fi + + path="$(bao_path_for "$target")" || { rc=1; continue; } + + # Decrypt to JSON, not dotenv. JSON is lossless: the dotenv writer flattens a value + # containing a newline into a literal \n that is indistinguishable from a backslash + # followed by n (render-secrets.sh:198 documents this). Seeding through dotenv would + # bake that ambiguity into OpenBao permanently. + plain_json="$(SOPS_AGE_KEY_FILE="$AGE_KEY" \ + sops -d --input-type yaml --output-type json "$src" 2>/dev/null)" || { + echo "!! failed to decrypt $src" >&2 + rc=1 + continue + } + + # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe + # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at + # SEED time — before anything depends on it — rather than at deploy time. + reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY' +import json, os, re, sys + +doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"]) +if not isinstance(doc, dict) or not doc: + sys.stderr.write("!! vault did not decrypt to a non-empty object\n") + sys.exit(1) + +KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +UNSAFE = set("`$\\\"'") + +out, bad = {}, [] +for key, val in doc.items(): + if key == "sops": + continue + if not KEY_RE.match(key): + bad.append(f"{key}: not a valid env var name") + continue + if val is None: + val = "" + if not isinstance(val, str): + val = json.dumps(val, separators=(",", ":")) + if "\n" in val or "\r" in val: + bad.append(f"{key}: contains a newline") + continue + if UNSAFE & set(val): + bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") + continue + out[key] = val + +if bad: + sys.stderr.write("!! unsafe values, refusing:\n") + for line in bad: + sys.stderr.write(f"!! {line}\n") + sys.exit(1) + +# stdout line 1: space-separated key NAMES, for the operator-visible log. +# stdout line 2+: the JSON payload, fed to `bao kv put` on stdin. +sys.stdout.write(" ".join(sorted(out)) + "\n") +sys.stdout.write(json.dumps(out) + "\n") +PY +)" || { echo "!! validation failed for $target" >&2; rc=1; continue; } + + names="$(printf '%s\n' "$reshaped" | sed -n '1p')" + payload="$(printf '%s\n' "$reshaped" | sed -n '2p')" + count="$(printf '%s\n' "$names" | wc -w | tr -d ' ')" + + if [ "$DRY_RUN" = 1 ]; then + echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys, DRY RUN, nothing written)" + printf ' %s\n' "$names" + continue + fi + + # `bao kv put -` reads a JSON object from stdin, so no value ever appears in argv + # (where it would be visible in /proc and in the shell's history). + if printf '%s' "$payload" | bao kv put -mount="$MOUNT" "$path" - >/dev/null; then + echo "== ${target}.yaml -> ${MOUNT}/${path} (${count} keys written)" + else + echo "!! failed writing ${MOUNT}/${path}" >&2 + rc=1 + fi +done + +if [ "$rc" = 0 ] && [ "$DRY_RUN" = 0 ]; then + cat <&2 <<'EOF' +usage: verify-parity.sh (--all | --env ...) + +Renders each environment through BOTH backends and compares the resulting +KEY=value sets. Prints key names and counts only, never values. + +Exit status: 0 = every requested environment is at parity; 1 = a mismatch. +Suitable as a CI / cron gate. +EOF + exit 2 +} + +while [ $# -gt 0 ]; do + case "$1" in + --all) TARGETS=(dev beta prod); shift ;; + --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;; + -h|--help) usage ;; + *) echo "!! unknown argument: $1" >&2; usage ;; + esac +done +[ ${#TARGETS[@]} -gt 0 ] || usage + +# shellcheck source=/dev/null +. "$INFRA_DIR/deploy/render-secrets-openbao.sh" + +overall=0 +for env_name in "${TARGETS[@]}"; do + echo "== parity check: $env_name" + + # dev renders WITHOUT common on both backends. On the SOPS side that is a caller + # flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps + # the comparison apples-to-apples — and if the flag were wrong, the OpenBao side + # would fail closed with a 403 rather than quietly diverge, which is exactly the + # improvement being verified. + skip_common=0 + [ "$env_name" = dev ] && skip_common=1 + + sops_out=$(mktemp); bao_out=$(mktemp) + # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a + # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a + # sourced context persists into the caller's shell and re-fires on the next source). + cleanup() { rm -f "$sops_out" "$bao_out"; } + + # --- SOPS side: common (unless dev) then , appended, last-wins --- + key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" + [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}" + if [ ! -r "$key" ]; then + echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2 + cleanup; overall=1; continue + fi + + if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then + SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ + "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || { + echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; } + fi + SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ + "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || { + echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; } + + # --- OpenBao side --- + if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \ + thermograph_openbao_source "$env_name" > "$bao_out"; then + echo "!! OpenBao render failed for $env_name" >&2 + cleanup; overall=1; continue + fi + + # --- Compare --- + # The SOPS output can legitimately contain the SAME key twice (common then env), and + # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So + # collapsing to last-wins is not a normalisation for convenience, it is what the + # consumers actually see. Comparing raw would report a false mismatch the moment a + # key is overridden. + norm() { + python3 -c ' +import sys +seen = {} +order = [] +for line in open(sys.argv[1], encoding="utf-8", errors="replace"): + line = line.rstrip("\n") + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if k not in seen: + order.append(k) + seen[k] = v # last occurrence wins +for k in sorted(order): + print(f"{k}={seen[k]}") +' "$1" + } + + a=$(mktemp); b=$(mktemp) + norm "$sops_out" > "$a"; norm "$bao_out" > "$b" + + n_sops=$(wc -l < "$a" | tr -d ' ') + n_bao=$(wc -l < "$b" | tr -d ' ') + + if cmp -s "$a" "$b"; then + echo " PASS — ${n_sops} keys, byte-identical after last-wins collapse" + else + overall=1 + echo " FAIL — sops=${n_sops} keys, openbao=${n_bao} keys" + # Report only NAMES. Three buckets, because the fix differs per bucket: + # only-in-sops -> the seed missed a key (re-run seed-from-sops.sh) + # only-in-bao -> OpenBao has something the vault does not (stale/extra write) + # value-differs -> the seed captured a different value; DO NOT flip the backend + comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in SOPS: /' + comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in OpenBao: /' + join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \ + | awk -F= '$2 != $3 { print " value differs: " $1 }' | sort -u + fi + + rm -f "$a" "$b" + cleanup +done + +if [ "$overall" = 0 ]; then + echo + echo "PARITY OK for: ${TARGETS[*]}" + echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments." + echo "Recommended gate before prod: 7 consecutive green runs on all three." +else + echo + echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2 +fi +exit "$overall"