thermograph/infra/openbao/verify-parity.sh

#!/usr/bin/env bash
# Prove that the OpenBao render is EQUIVALENT to the SOPS render, without printing a
# single secret value.
#
#   infra/openbao/verify-parity.sh --env dev
#   infra/openbao/verify-parity.sh --all
#
# This is the gate for every cutover. The migration's entire safety argument is that
# the two backends produce the same KEY=value set, so flipping TG_SECRETS_BACKEND
# cannot change what any service sees. That argument has to be TESTED, not asserted —
# especially because the failure modes on the other side are silent:
#
#   * a missing THERMOGRAPH_AUTH_SECRET makes the app mint a random one per worker
#     (accounts/users.py:33), so logins start working intermittently and nothing logs;
#   * a half-rendered VAPID pair makes push.py generate a NEW keypair and then delete
#     every existing subscription row as "gone" (push.py:180-184);
#   * a wrong POSTGRES_PASSWORD makes the data layer swallow the error and refetch from
#     Open-Meteo, spending third-party quota, while /healthz stays green.
#
# None of those announce themselves. A byte-level diff beforehand is the only thing
# standing between a flag flip and one of them.
#
# OUTPUT DISCIPLINE: only key NAMES and counts are ever printed. Where values differ,
# the key name is reported and the values are not — the same rule
# deploy/secrets/dry-run-render.sh already follows.
set -euo pipefail

SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
TARGETS=()

usage() {
  cat >&2 <<'EOF'
usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)

Renders each environment through BOTH backends and compares the resulting
KEY=value sets. Prints key names and counts only, never values.

Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
Suitable as a CI / cron gate.
EOF
  exit 2
}

while [ $# -gt 0 ]; do
  case "$1" in
    --all) TARGETS=(dev beta prod); shift ;;
    --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
    -h|--help) usage ;;
    *) echo "!! unknown argument: $1" >&2; usage ;;
  esac
done
[ ${#TARGETS[@]} -gt 0 ] || usage

# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
# bare default — prod's credential — for every environment. On vps2 that made
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
# thermograph/data/env/beta, so the gate could not verify the one environment whose
# isolation it exists to prove. `--all` failed the same way, and dev with it.
#
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
# which is the worst place for the omission to be: the tool whose whole job is to
# notice a divergence was itself diverging from the path it verifies.
#
# shellcheck source=/dev/null
. "$INFRA_DIR/deploy/env-topology.sh"
# shellcheck source=/dev/null
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"

overall=0
for env_name in "${TARGETS[@]}"; do
  echo "== parity check: $env_name"

  # Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
  # filesystem with prod) and TG_SKIP_COMMON. Called inside the loop because --all
  # walks three environments and the values differ per environment.
  if ! thermograph_topology "$env_name"; then
    echo "!! unknown environment: $env_name" >&2
    overall=1; continue
  fi

  # dev renders WITHOUT common on both backends. On the SOPS side that is a caller
  # flag; on the OpenBao side the policy also forbids it. Taking the flag from the
  # topology rather than re-deriving it here keeps the comparison apples-to-apples
  # AND keeps this script from drifting from deploy.sh:67, which reads the same
  # variable. If the flag were wrong, the OpenBao side would fail closed with a 403
  # rather than quietly diverge — which is exactly the improvement being verified.
  skip_common="${TG_SKIP_COMMON:-0}"

  sops_out=$(mktemp); bao_out=$(mktemp)
  # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
  # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a
  # sourced context persists into the caller's shell and re-fires on the next source).
  cleanup() { rm -f "$sops_out" "$bao_out"; }

  # --- SOPS side: common (unless dev) then <env>, appended, last-wins ---
  key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
  [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
  if [ ! -r "$key" ]; then
    echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2
    cleanup; overall=1; continue
  fi

  if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then
    SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
      "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || {
        echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; }
  fi
  SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
    "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || {
      echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; }

  # --- OpenBao side ---
  if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \
       thermograph_openbao_source "$env_name" > "$bao_out"; then
    echo "!! OpenBao render failed for $env_name" >&2
    cleanup; overall=1; continue
  fi

  # --- Compare ---
  # The SOPS output can legitimately contain the SAME key twice (common then env), and
  # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So
  # collapsing to last-wins is not a normalisation for convenience, it is what the
  # consumers actually see. Comparing raw would report a false mismatch the moment a
  # key is overridden.
  norm() {
    python3 -c '
import sys
seen = {}
order = []
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
    line = line.rstrip("\n")
    if not line or line.startswith("#") or "=" not in line:
        continue
    k, _, v = line.partition("=")
    if k not in seen:
        order.append(k)
    seen[k] = v          # last occurrence wins
for k in sorted(order):
    print(f"{k}={seen[k]}")
' "$1"
  }

  a=$(mktemp); b=$(mktemp)
  norm "$sops_out" > "$a"; norm "$bao_out" > "$b"

  n_sops=$(wc -l < "$a" | tr -d ' ')
  n_bao=$(wc -l < "$b" | tr -d ' ')

  if cmp -s "$a" "$b"; then
    echo "   PASS — ${n_sops} keys, byte-identical after last-wins collapse"
  else
    overall=1
    echo "   FAIL — sops=${n_sops} keys, openbao=${n_bao} keys"
    # Report only NAMES. Three buckets, because the fix differs per bucket:
    #   only-in-sops  -> the seed missed a key (re-run seed-from-sops.sh)
    #   only-in-bao   -> OpenBao has something the vault does not (stale/extra write)
    #   value-differs -> the seed captured a different value; DO NOT flip the backend
    comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in SOPS:    /'
    comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in OpenBao: /'
    join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \
      | awk -F= '$2 != $3 { print "   value differs:   " $1 }' | sort -u
  fi

  rm -f "$a" "$b"
  cleanup
done

if [ "$overall" = 0 ]; then
  echo
  echo "PARITY OK for: ${TARGETS[*]}"
  echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
  echo "Recommended gate before prod: 7 consecutive green runs on all three."
else
  echo
  echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
fi
exit "$overall"