#!/usr/bin/env bash # Prove that the OpenBao render is EQUIVALENT to the SOPS render, without printing a # single secret value. # # infra/openbao/verify-parity.sh --env dev # infra/openbao/verify-parity.sh --all # # This is the gate for every cutover. The migration's entire safety argument is that # the two backends produce the same KEY=value set, so flipping TG_SECRETS_BACKEND # cannot change what any service sees. That argument has to be TESTED, not asserted — # especially because the failure modes on the other side are silent: # # * a missing THERMOGRAPH_AUTH_SECRET makes the app mint a random one per worker # (accounts/users.py:33), so logins start working intermittently and nothing logs; # * a half-rendered VAPID pair makes push.py generate a NEW keypair and then delete # every existing subscription row as "gone" (push.py:180-184); # * a wrong POSTGRES_PASSWORD makes the data layer swallow the error and refetch from # Open-Meteo, spending third-party quota, while /healthz stays green. # # None of those announce themselves. A byte-level diff beforehand is the only thing # standing between a flag flip and one of them. # # OUTPUT DISCIPLINE: only key NAMES and counts are ever printed. Where values differ, # the key name is reported and the values are not — the same rule # deploy/secrets/dry-run-render.sh already follows. set -euo pipefail SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)" TARGETS=() usage() { cat >&2 <<'EOF' usage: verify-parity.sh (--all | --env ...) Renders each environment through BOTH backends and compares the resulting KEY=value sets. Prints key names and counts only, never values. Exit status: 0 = every requested environment is at parity; 1 = a mismatch. Suitable as a CI / cron gate. EOF exit 2 } while [ $# -gt 0 ]; do case "$1" in --all) TARGETS=(dev beta prod); shift ;; --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;; -h|--help) usage ;; *) echo "!! unknown argument: $1" >&2; usage ;; esac done [ ${#TARGETS[@]} -gt 0 ] || usage # env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets # TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the # bare default — prod's credential — for every environment. On vps2 that made # `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on # thermograph/data/env/beta, so the gate could not verify the one environment whose # isolation it exists to prove. `--all` failed the same way, and dev with it. # # deploy.sh and deploy-stack.sh always sourced this; only the verifier did not, # which is the worst place for the omission to be: the tool whose whole job is to # notice a divergence was itself diverging from the path it verifies. # # shellcheck source=/dev/null . "$INFRA_DIR/deploy/env-topology.sh" # shellcheck source=/dev/null . "$INFRA_DIR/deploy/render-secrets-openbao.sh" overall=0 for env_name in "${TARGETS[@]}"; do echo "== parity check: $env_name" # Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a # filesystem with prod) and TG_SKIP_COMMON. Called inside the loop because --all # walks three environments and the values differ per environment. if ! thermograph_topology "$env_name"; then echo "!! unknown environment: $env_name" >&2 overall=1; continue fi # dev renders WITHOUT common on both backends. On the SOPS side that is a caller # flag; on the OpenBao side the policy also forbids it. Taking the flag from the # topology rather than re-deriving it here keeps the comparison apples-to-apples # AND keeps this script from drifting from deploy.sh:67, which reads the same # variable. If the flag were wrong, the OpenBao side would fail closed with a 403 # rather than quietly diverge — which is exactly the improvement being verified. skip_common="${TG_SKIP_COMMON:-0}" sops_out=$(mktemp); bao_out=$(mktemp) # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a # sourced context persists into the caller's shell and re-fires on the next source). cleanup() { rm -f "$sops_out" "$bao_out"; } # --- SOPS side: common (unless dev) then , appended, last-wins --- key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}" if [ ! -r "$key" ]; then echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2 cleanup; overall=1; continue fi if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || { echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; } fi SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \ "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || { echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; } # --- OpenBao side --- if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \ thermograph_openbao_source "$env_name" > "$bao_out"; then echo "!! OpenBao render failed for $env_name" >&2 cleanup; overall=1; continue fi # --- Compare --- # The SOPS output can legitimately contain the SAME key twice (common then env), and # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So # collapsing to last-wins is not a normalisation for convenience, it is what the # consumers actually see. Comparing raw would report a false mismatch the moment a # key is overridden. norm() { python3 -c ' import sys seen = {} order = [] for line in open(sys.argv[1], encoding="utf-8", errors="replace"): line = line.rstrip("\n") if not line or line.startswith("#") or "=" not in line: continue k, _, v = line.partition("=") if k not in seen: order.append(k) seen[k] = v # last occurrence wins for k in sorted(order): print(f"{k}={seen[k]}") ' "$1" } a=$(mktemp); b=$(mktemp) norm "$sops_out" > "$a"; norm "$bao_out" > "$b" n_sops=$(wc -l < "$a" | tr -d ' ') n_bao=$(wc -l < "$b" | tr -d ' ') if cmp -s "$a" "$b"; then echo " PASS — ${n_sops} keys, byte-identical after last-wins collapse" else overall=1 echo " FAIL — sops=${n_sops} keys, openbao=${n_bao} keys" # Report only NAMES. Three buckets, because the fix differs per bucket: # only-in-sops -> the seed missed a key (re-run seed-from-sops.sh) # only-in-bao -> OpenBao has something the vault does not (stale/extra write) # value-differs -> the seed captured a different value; DO NOT flip the backend comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in SOPS: /' comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/ only in OpenBao: /' join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \ | awk -F= '$2 != $3 { print " value differs: " $1 }' | sort -u fi rm -f "$a" "$b" cleanup done if [ "$overall" = 0 ]; then echo echo "PARITY OK for: ${TARGETS[*]}" echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments." echo "Recommended gate before prod: 7 consecutive green runs on all three." else echo echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2 fi exit "$overall"