thermograph/infra/deploy/secrets/verify-centralis-render.sh

#!/usr/bin/env bash
# NON-DESTRUCTIVE go/no-go for the /etc/centralis.env cutover — the Centralis
# analogue of dry-run-render.sh, and the proof obligation for the migration:
# that rendering deploy/secrets/centralis.<env>.yaml produces a file whose
# EFFECTIVE VALUES are identical to the live hand-maintained one.
#
#   deploy/secrets/verify-centralis-render.sh prod agent@169.58.46.181
#
# What it does NOT do: write /etc/centralis.env, restart anything, or print a
# single secret value. It renders to a 0700 scratch directory, sources both
# files in a clean shell, compares the resulting variables, and reports key
# NAMES and a verdict. The scratch directory is removed on every exit path.
#
# "Effective values" is the load-bearing phrase. /etc/centralis.env is consumed
# by `set -a && . /etc/centralis.env`, so the only comparison that means
# anything is between what BASH ends up with from each file — not between the
# two files' text. The live file quotes CENTRALIS_TOKENS and the rendered one
# quotes everything, so a textual diff would report a difference where there is
# none, and (much worse) would report a match if the vault had captured the
# quote characters as part of the value.
set -euo pipefail
cd "$(dirname "$(readlink -f "$0")")/../.."

ENV_NAME="${1:?usage: verify-centralis-render.sh <env> <ssh-target> [ssh-key]}"
SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}"
SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}"
VAULT="deploy/secrets/centralis.${ENV_NAME}.yaml"
[ -f "$VAULT" ] || { echo "!! no $VAULT — run seed-centralis-on-host.sh first" >&2; exit 1; }

SSH=(ssh -i "$SSH_KEY" "$SSH_TARGET")
SCP=(scp -q -i "$SSH_KEY")

# shellcheck disable=SC2016  # single quotes are required: this whole string is
# executed by the REMOTE shell, so $d and the command substitution must survive
# the trip intact. Double-quoting would run mktemp locally and then send a path
# that does not exist on the host.
scratch="$("${SSH[@]}" 'd=$(mktemp -d) && chmod 700 "$d" && mkdir -p "$d/deploy/secrets" && echo "$d"')"
[ -n "$scratch" ] || { echo "!! could not create a scratch dir on the host" >&2; exit 1; }
cleanup() { "${SSH[@]}" "rm -rf -- '$scratch'" >/dev/null 2>&1 || true; }
trap cleanup EXIT

# Only ciphertext and a shell script cross the wire. Nothing decrypted ever
# leaves the box; the render happens there, with the host's own age key.
"${SCP[@]}" deploy/render-secrets.sh "${SSH_TARGET}:${scratch}/deploy/render-secrets.sh"
"${SCP[@]}" "$VAULT" "${SSH_TARGET}:${scratch}/deploy/secrets/centralis.${ENV_NAME}.yaml"

"${SSH[@]}" "SCRATCH='$scratch' bash -s" <<'REMOTE'
set -euo pipefail
umask 077

. "$SCRATCH/deploy/render-secrets.sh"
CENTRALIS_RENDER_DEST="$SCRATCH/rendered.env" render_centralis_secrets "$SCRATCH" >&2

sudo cat /etc/centralis.env > "$SCRATCH/live.env"

# Ask bash what each file means, in a clean environment so no inherited variable
# can stand in for one the file failed to set.
dump() {
  env -i PATH="$PATH" bash -c \
    'set -a; . "$1"; set +a; exec python3 -c "
import json, os, sys
sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$1"
}
dump "$SCRATCH/live.env"     > "$SCRATCH/live.json"
dump "$SCRATCH/rendered.env" > "$SCRATCH/rend.json"

# The names each file actually assigns — so the shell's own PATH/PWD/SHLVL/_ are
# not mistaken for content.
names() {
  grep -oE '^[[:space:]]*(export[[:space:]]+)?[A-Za-z_][A-Za-z0-9_]*=' "$1" \
    | sed -E 's/^[[:space:]]*(export[[:space:]]+)?//; s/=$//' | sort -u
}
names "$SCRATCH/live.env"     > "$SCRATCH/live.names"
names "$SCRATCH/rendered.env" > "$SCRATCH/rend.names"

python3 - "$SCRATCH" <<'PY'
import json, os, sys

d = sys.argv[1]
load = lambda n: json.load(open(os.path.join(d, n), encoding="utf-8"))
names = lambda n: [l.strip() for l in open(os.path.join(d, n), encoding="utf-8") if l.strip()]

live, rend = load("live.json"), load("rend.json")
ln, rn = set(names("live.names")), set(names("rend.names"))

lost, added = sorted(ln - rn), sorted(rn - ln)
changed = sorted(k for k in ln & rn if live.get(k) != rend.get(k))

print()
print("  live /etc/centralis.env : %d keys" % len(ln))
print("  rendered from the vault : %d keys" % len(rn))
print()
for k in sorted(ln & rn):
    print("    %-34s %s" % (k, "same" if live.get(k) == rend.get(k) else "DIFFERENT"))

ok = not (lost or added or changed)
print()
if lost:
    print("  MISSING from the render :", lost)
if added:
    print("  ADDED by the render     :", added)
if changed:
    print("  VALUE CHANGED           :", changed)

# The registry, by subject name. Values are never touched; `sorted(reg)` is a
# list of audit-log subjects, which is what the operator needs to see.
def subjects(env):
    raw = env.get("CENTRALIS_TOKENS", "")
    if not raw:
        return None
    try:
        reg = json.loads(raw)
    except ValueError as exc:
        return "UNPARSEABLE (%s)" % exc
    return sorted(reg) if isinstance(reg, dict) else "not a JSON object"

sl, sr = subjects(live), subjects(rend)
print("  CENTRALIS_TOKENS subjects  live: %s" % (sl,))
print("                          rendered: %s" % (sr,))
if sl != sr:
    ok = False
    print("  !! the token registry does not survive the migration")

print()
print("  VERDICT: %s" % ("PASS — the render is value-identical to the live file"
                         if ok else "FAIL — do NOT cut over"))
sys.exit(0 if ok else 1)
PY

# End-to-end: the two env files must produce the SAME docker compose model. This
# is the claim that actually matters — not "the files agree" but "the container
# Centralis would be started with is the same one it is running now". Compared
# by digest, on the box; the rendered model is full of secrets and is never
# printed, shipped, or kept.
compose=/opt/centralis/deploy/docker-compose.yml
if [ -f "$compose" ] && command -v docker >/dev/null 2>&1; then
  model() {
    env -i PATH="$PATH" HOME="$HOME" bash -c \
      'set -a; . "$1"; set +a; cd /opt/centralis && \
       docker compose -f deploy/docker-compose.yml config' _ "$1" 2>/dev/null \
      | sha256sum | cut -d" " -f1
  }
  a="$(model "$SCRATCH/live.env")"
  b="$(model "$SCRATCH/rendered.env")"
  if [ -z "$a" ] || [ "$a" = "$(printf '' | sha256sum | cut -d' ' -f1)" ]; then
    echo "  compose model : could not be built from the LIVE env — check by hand"
  elif [ "$a" = "$b" ]; then
    echo "  compose model : IDENTICAL (docker compose config digests match)"
  else
    echo "  compose model : DIFFERENT — do NOT cut over"
    exit 1
  fi
else
  echo "  compose model : skipped (no /opt/centralis or no docker here)"
fi
REMOTE