thermograph/infra/openbao/verify-parity.sh

#!/usr/bin/env bash
# Prove that the OpenBao render is EQUIVALENT to the SOPS render, without printing a
# single secret value.
#
#   infra/openbao/verify-parity.sh --env dev
#   infra/openbao/verify-parity.sh --all
#
# This is the gate for every cutover. The migration's entire safety argument is that
# the two backends produce the same KEY=value set, so flipping TG_SECRETS_BACKEND
# cannot change what any service sees. That argument has to be TESTED, not asserted —
# especially because the failure modes on the other side are silent:
#
#   * a missing THERMOGRAPH_AUTH_SECRET makes the app mint a random one per worker
#     (accounts/users.py:33), so logins start working intermittently and nothing logs;
#   * a half-rendered VAPID pair makes push.py generate a NEW keypair and then delete
#     every existing subscription row as "gone" (push.py:180-184);
#   * a wrong POSTGRES_PASSWORD makes the data layer swallow the error and refetch from
#     Open-Meteo, spending third-party quota, while /healthz stays green.
#
# None of those announce themselves. A byte-level diff beforehand is the only thing
# standing between a flag flip and one of them.
#
# OUTPUT DISCIPLINE: only key NAMES and counts are ever printed. Where values differ,
# the key name is reported and the values are not — the same rule
# deploy/secrets/dry-run-render.sh already follows.
set -euo pipefail

SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
INFRA_DIR="$(cd -- "$SELF_DIR/.." && pwd)"
TARGETS=()

usage() {
  cat >&2 <<'EOF'
usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)

Renders each environment through BOTH backends and compares the resulting
KEY=value sets. Prints key names and counts only, never values.

--all means "every environment that lives on THIS host", not all three: dev is
on vps1, beta and prod are on vps2, so no single box can check them all. An
environment with no checkout here is skipped out loud. Skipping every one of
them is an error, not a pass.

Exit status: 0 = every environment checked here is at parity; 1 = a mismatch,
or nothing was checkable. Suitable as a CI / cron gate.
EOF
  exit 2
}

while [ $# -gt 0 ]; do
  case "$1" in
    --all) TARGETS=(dev beta prod); shift ;;
    --env) [ $# -ge 2 ] || usage; TARGETS+=("$2"); shift 2 ;;
    -h|--help) usage ;;
    *) echo "!! unknown argument: $1" >&2; usage ;;
  esac
done
[ ${#TARGETS[@]} -gt 0 ] || usage

# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
# bare default — prod's credential — for every environment. On vps2 that made
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
# thermograph/data/env/beta, so the gate could not verify the one environment whose
# isolation it exists to prove. `--all` failed the same way, and dev with it.
#
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
# which is the worst place for the omission to be: the tool whose whole job is to
# notice a divergence was itself diverging from the path it verifies.
#
# shellcheck source=/dev/null
. "$INFRA_DIR/deploy/env-topology.sh"
# shellcheck source=/dev/null
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"

overall=0
checked=0
for env_name in "${TARGETS[@]}"; do
  # Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
  # filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header
  # line because --all walks three environments and may skip some of them.
  if ! thermograph_topology "$env_name"; then
    echo "!! unknown environment: $env_name" >&2
    overall=1; continue
  fi

  # NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to
  # pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all
  # anywhere therefore guaranteed a failure on whichever environments are elsewhere,
  # and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to
  # /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so
  # prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like
  # a policy bug rather than "wrong box".
  #
  # Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its
  # checkout existing. The first version of this check tested for TG_APP_DIR and was
  # wrong: vps1 still carries a stale /opt/thermograph from before the estate split,
  # so prod looked resident there and was checked anyway. A leftover directory is
  # exactly the kind of thing that outlives the arrangement it belonged to.
  #
  # Skipping is announced, never silent. A gate that quietly narrows what it checked
  # is worse than one that fails, because the run still goes green.
  if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then
    if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
         | grep -qxF "$TG_SSH_HOST"; then
      echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here"
      continue
    fi
  fi

  echo "== parity check: $env_name"
  checked=$((checked + 1))

  # dev renders WITHOUT common on both backends. On the SOPS side that is a caller
  # flag; on the OpenBao side the policy also forbids it. Taking the flag from the
  # topology rather than re-deriving it here keeps the comparison apples-to-apples
  # AND keeps this script from drifting from deploy.sh:67, which reads the same
  # variable. If the flag were wrong, the OpenBao side would fail closed with a 403
  # rather than quietly diverge — which is exactly the improvement being verified.
  skip_common="${TG_SKIP_COMMON:-0}"

  sops_out=$(mktemp); bao_out=$(mktemp)
  # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
  # trap, for the reason documented at render-secrets.sh:82-91 (a RETURN trap set in a
  # sourced context persists into the caller's shell and re-fires on the next source).
  cleanup() { rm -f "$sops_out" "$bao_out"; }

  # --- SOPS side: common (unless dev) then <env>, appended, last-wins ---
  key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
  [ -r "$key" ] || key="${SOPS_AGE_KEY_FILE:-$HOME/.config/sops/age/keys.txt}"
  if [ ! -r "$key" ]; then
    echo "!! no readable age key (tried /etc/thermograph/age.key and \$SOPS_AGE_KEY_FILE)" >&2
    cleanup; overall=1; continue
  fi

  if [ "$skip_common" = 0 ] && [ -f "$INFRA_DIR/deploy/secrets/common.yaml" ]; then
    SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
      "$INFRA_DIR/deploy/secrets/common.yaml" >> "$sops_out" 2>/dev/null || {
        echo "!! SOPS decrypt failed: common.yaml" >&2; cleanup; overall=1; continue; }
  fi
  SOPS_AGE_KEY_FILE="$key" sops -d --input-type yaml --output-type dotenv \
    "$INFRA_DIR/deploy/secrets/${env_name}.yaml" >> "$sops_out" 2>/dev/null || {
      echo "!! SOPS decrypt failed: ${env_name}.yaml" >&2; cleanup; overall=1; continue; }

  # --- OpenBao side ---
  if ! THERMOGRAPH_SECRETS_SKIP_COMMON="$skip_common" \
       thermograph_openbao_source "$env_name" > "$bao_out"; then
    echo "!! OpenBao render failed for $env_name" >&2
    cleanup; overall=1; continue
  fi

  # --- Compare ---
  # The SOPS output can legitimately contain the SAME key twice (common then env), and
  # every consumer takes the LAST occurrence — `env_file:` and `source` both do. So
  # collapsing to last-wins is not a normalisation for convenience, it is what the
  # consumers actually see. Comparing raw would report a false mismatch the moment a
  # key is overridden.
  norm() {
    python3 -c '
import sys
seen = {}
order = []
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
    line = line.rstrip("\n")
    if not line or line.startswith("#") or "=" not in line:
        continue
    k, _, v = line.partition("=")
    if k not in seen:
        order.append(k)
    seen[k] = v          # last occurrence wins
for k in sorted(order):
    print(f"{k}={seen[k]}")
' "$1"
  }

  a=$(mktemp); b=$(mktemp)
  norm "$sops_out" > "$a"; norm "$bao_out" > "$b"

  n_sops=$(wc -l < "$a" | tr -d ' ')
  n_bao=$(wc -l < "$b" | tr -d ' ')

  if cmp -s "$a" "$b"; then
    echo "   PASS — ${n_sops} keys, byte-identical after last-wins collapse"
  else
    overall=1
    echo "   FAIL — sops=${n_sops} keys, openbao=${n_bao} keys"
    # Report only NAMES. Three buckets, because the fix differs per bucket:
    #   only-in-sops  -> the seed missed a key (re-run seed-from-sops.sh)
    #   only-in-bao   -> OpenBao has something the vault does not (stale/extra write)
    #   value-differs -> the seed captured a different value; DO NOT flip the backend
    comm -23 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in SOPS:    /'
    comm -13 <(cut -d= -f1 "$a") <(cut -d= -f1 "$b") | sed 's/^/   only in OpenBao: /'
    join -t= -j1 <(sort -t= -k1,1 "$a") <(sort -t= -k1,1 "$b") -o 0,1.2,2.2 2>/dev/null \
      | awk -F= '$2 != $3 { print "   value differs:   " $1 }' | sort -u
  fi

  rm -f "$a" "$b"
  cleanup
done

# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a
# host that carries none of them — a new box, a renamed checkout, a typo'd --env —
# exits 0 having verified nothing, and a green run is exactly what the 7-day gate
# counts. "Nothing was wrong" and "nothing was examined" must not look alike.
if [ "$checked" -eq 0 ]; then
  echo
  echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2
  echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2
  exit 1
fi

if [ "$overall" = 0 ]; then
  echo
  echo "PARITY OK — ${checked} environment(s) checked on this host."
  echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
  echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments."
  echo "That means both hosts — a green run here says nothing about the other box."
else
  echo
  echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
fi
exit "$overall"