From 75968980f9c7d366f77df66ed33c5f9d204c357a Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 1 Aug 2026 09:18:05 -0700 Subject: [PATCH] openbao: make verify-parity --all host-aware --all expanded to (dev beta prod) and then tried to check all three from wherever it was run. No host carries all three: dev is on vps1, beta and prod are on vps2. So --all could not succeed anywhere, and the way it failed was misleading rather than obvious -- on vps1, prod resolves TG_BAO_APPROLE to /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so prod's check authenticated as tg-dev and took a 403 from tg-host-prod.hcl. That reads like a policy bug. It is "wrong box". --all now means "every environment that lives on THIS host". Residency is decided by the environment's own address being bound here (TG_SSH_HOST against the local interfaces), and skips are announced rather than silent. Residency is deliberately NOT decided by TG_APP_DIR existing. That was the first attempt and it was wrong: vps1 still carries a stale /opt/thermograph from before the estate split (d5357d0, #106), so prod looked resident there and got checked anyway. Verified against both live hosts. A leftover directory is exactly the kind of thing that outlives the arrangement it belonged to. Zero environments checked is now a FAILURE. Previously a run that skipped everything would exit 0, and a green run is precisely what the 7-day cutover gate counts -- "nothing was wrong" and "nothing was examined" must not look alike. The success line reports how many were checked and says plainly that a green run on one host says nothing about the other. Live-verified on both boxes: vps1 --all -> dev PASS, beta and prod skipped, exit 0 vps2 --all -> dev skipped, beta and prod checked (both currently failing on the three known stale common.yaml keys) --- infra/openbao/verify-parity.sh | 59 +++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/infra/openbao/verify-parity.sh b/infra/openbao/verify-parity.sh index c146d07..f76775e 100755 --- a/infra/openbao/verify-parity.sh +++ b/infra/openbao/verify-parity.sh @@ -36,8 +36,13 @@ usage: verify-parity.sh (--all | --env ...) Renders each environment through BOTH backends and compares the resulting KEY=value sets. Prints key names and counts only, never values. -Exit status: 0 = every requested environment is at parity; 1 = a mismatch. -Suitable as a CI / cron gate. +--all means "every environment that lives on THIS host", not all three: dev is +on vps1, beta and prod are on vps2, so no single box can check them all. An +environment with no checkout here is skipped out loud. Skipping every one of +them is an error, not a pass. + +Exit status: 0 = every environment checked here is at parity; 1 = a mismatch, +or nothing was checkable. Suitable as a CI / cron gate. EOF exit 2 } @@ -69,17 +74,43 @@ done . "$INFRA_DIR/deploy/render-secrets-openbao.sh" overall=0 +checked=0 for env_name in "${TARGETS[@]}"; do - echo "== parity check: $env_name" - # Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a - # filesystem with prod) and TG_SKIP_COMMON. Called inside the loop because --all - # walks three environments and the values differ per environment. + # filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header + # line because --all walks three environments and may skip some of them. if ! thermograph_topology "$env_name"; then echo "!! unknown environment: $env_name" >&2 overall=1; continue fi + # NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to + # pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all + # anywhere therefore guaranteed a failure on whichever environments are elsewhere, + # and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to + # /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so + # prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like + # a policy bug rather than "wrong box". + # + # Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its + # checkout existing. The first version of this check tested for TG_APP_DIR and was + # wrong: vps1 still carries a stale /opt/thermograph from before the estate split, + # so prod looked resident there and was checked anyway. A leftover directory is + # exactly the kind of thing that outlives the arrangement it belonged to. + # + # Skipping is announced, never silent. A gate that quietly narrows what it checked + # is worse than one that fails, because the run still goes green. + if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then + if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \ + | grep -qxF "$TG_SSH_HOST"; then + echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here" + continue + fi + fi + + echo "== parity check: $env_name" + checked=$((checked + 1)) + # dev renders WITHOUT common on both backends. On the SOPS side that is a caller # flag; on the OpenBao side the policy also forbids it. Taking the flag from the # topology rather than re-deriving it here keeps the comparison apples-to-apples @@ -167,11 +198,23 @@ for k in sorted(order): cleanup done +# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a +# host that carries none of them — a new box, a renamed checkout, a typo'd --env — +# exits 0 having verified nothing, and a green run is exactly what the 7-day gate +# counts. "Nothing was wrong" and "nothing was examined" must not look alike. +if [ "$checked" -eq 0 ]; then + echo + echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2 + echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2 + exit 1 +fi + if [ "$overall" = 0 ]; then echo - echo "PARITY OK for: ${TARGETS[*]}" + echo "PARITY OK — ${checked} environment(s) checked on this host." echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments." - echo "Recommended gate before prod: 7 consecutive green runs on all three." + echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments." + echo "That means both hosts — a green run here says nothing about the other box." else echo echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2