#!/usr/bin/env bash # NON-DESTRUCTIVE go/no-go for the /etc/centralis.env cutover — the Centralis # analogue of dry-run-render.sh, and the proof obligation for the migration: # that rendering deploy/secrets/centralis..yaml produces a file whose # EFFECTIVE VALUES are identical to the live hand-maintained one. # # deploy/secrets/verify-centralis-render.sh prod agent@169.58.46.181 # # What it does NOT do: write /etc/centralis.env, restart anything, or print a # single secret value. It renders to a 0700 scratch directory, sources both # files in a clean shell, compares the resulting variables, and reports key # NAMES and a verdict. The scratch directory is removed on every exit path. # # "Effective values" is the load-bearing phrase. /etc/centralis.env is consumed # by `set -a && . /etc/centralis.env`, so the only comparison that means # anything is between what BASH ends up with from each file — not between the # two files' text. The live file quotes CENTRALIS_TOKENS and the rendered one # quotes everything, so a textual diff would report a difference where there is # none, and (much worse) would report a match if the vault had captured the # quote characters as part of the value. set -euo pipefail cd "$(dirname "$(readlink -f "$0")")/../.." ENV_NAME="${1:?usage: verify-centralis-render.sh [ssh-key]}" SSH_TARGET="${2:?ssh target, e.g. agent@169.58.46.181}" SSH_KEY="${3:-$HOME/.ssh/thermograph_agent_ed25519}" VAULT="deploy/secrets/centralis.${ENV_NAME}.yaml" [ -f "$VAULT" ] || { echo "!! no $VAULT — run seed-centralis-on-host.sh first" >&2; exit 1; } SSH=(ssh -i "$SSH_KEY" "$SSH_TARGET") SCP=(scp -q -i "$SSH_KEY") # shellcheck disable=SC2016 # single quotes are required: this whole string is # executed by the REMOTE shell, so $d and the command substitution must survive # the trip intact. Double-quoting would run mktemp locally and then send a path # that does not exist on the host. scratch="$("${SSH[@]}" 'd=$(mktemp -d) && chmod 700 "$d" && mkdir -p "$d/deploy/secrets" && echo "$d"')" [ -n "$scratch" ] || { echo "!! could not create a scratch dir on the host" >&2; exit 1; } cleanup() { "${SSH[@]}" "rm -rf -- '$scratch'" >/dev/null 2>&1 || true; } trap cleanup EXIT # Only ciphertext and a shell script cross the wire. Nothing decrypted ever # leaves the box; the render happens there, with the host's own age key. "${SCP[@]}" deploy/render-secrets.sh "${SSH_TARGET}:${scratch}/deploy/render-secrets.sh" "${SCP[@]}" "$VAULT" "${SSH_TARGET}:${scratch}/deploy/secrets/centralis.${ENV_NAME}.yaml" "${SSH[@]}" "SCRATCH='$scratch' bash -s" <<'REMOTE' set -euo pipefail umask 077 . "$SCRATCH/deploy/render-secrets.sh" CENTRALIS_RENDER_DEST="$SCRATCH/rendered.env" render_centralis_secrets "$SCRATCH" >&2 sudo cat /etc/centralis.env > "$SCRATCH/live.env" # Ask bash what each file means, in a clean environment so no inherited variable # can stand in for one the file failed to set. dump() { env -i PATH="$PATH" bash -c \ 'set -a; . "$1"; set +a; exec python3 -c " import json, os, sys sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$1" } dump "$SCRATCH/live.env" > "$SCRATCH/live.json" dump "$SCRATCH/rendered.env" > "$SCRATCH/rend.json" # The names each file actually assigns — so the shell's own PATH/PWD/SHLVL/_ are # not mistaken for content. names() { grep -oE '^[[:space:]]*(export[[:space:]]+)?[A-Za-z_][A-Za-z0-9_]*=' "$1" \ | sed -E 's/^[[:space:]]*(export[[:space:]]+)?//; s/=$//' | sort -u } names "$SCRATCH/live.env" > "$SCRATCH/live.names" names "$SCRATCH/rendered.env" > "$SCRATCH/rend.names" python3 - "$SCRATCH" <<'PY' import json, os, sys d = sys.argv[1] load = lambda n: json.load(open(os.path.join(d, n), encoding="utf-8")) names = lambda n: [l.strip() for l in open(os.path.join(d, n), encoding="utf-8") if l.strip()] live, rend = load("live.json"), load("rend.json") ln, rn = set(names("live.names")), set(names("rend.names")) lost, added = sorted(ln - rn), sorted(rn - ln) changed = sorted(k for k in ln & rn if live.get(k) != rend.get(k)) print() print(" live /etc/centralis.env : %d keys" % len(ln)) print(" rendered from the vault : %d keys" % len(rn)) print() for k in sorted(ln & rn): print(" %-34s %s" % (k, "same" if live.get(k) == rend.get(k) else "DIFFERENT")) ok = not (lost or added or changed) print() if lost: print(" MISSING from the render :", lost) if added: print(" ADDED by the render :", added) if changed: print(" VALUE CHANGED :", changed) # The registry, by subject name. Values are never touched; `sorted(reg)` is a # list of audit-log subjects, which is what the operator needs to see. def subjects(env): raw = env.get("CENTRALIS_TOKENS", "") if not raw: return None try: reg = json.loads(raw) except ValueError as exc: return "UNPARSEABLE (%s)" % exc return sorted(reg) if isinstance(reg, dict) else "not a JSON object" sl, sr = subjects(live), subjects(rend) print(" CENTRALIS_TOKENS subjects live: %s" % (sl,)) print(" rendered: %s" % (sr,)) if sl != sr: ok = False print(" !! the token registry does not survive the migration") print() print(" VERDICT: %s" % ("PASS — the render is value-identical to the live file" if ok else "FAIL — do NOT cut over")) sys.exit(0 if ok else 1) PY # End-to-end: the two env files must produce the SAME docker compose model. This # is the claim that actually matters — not "the files agree" but "the container # Centralis would be started with is the same one it is running now". Compared # by digest, on the box; the rendered model is full of secrets and is never # printed, shipped, or kept. compose=/opt/centralis/deploy/docker-compose.yml if [ -f "$compose" ] && command -v docker >/dev/null 2>&1; then model() { env -i PATH="$PATH" HOME="$HOME" bash -c \ 'set -a; . "$1"; set +a; cd /opt/centralis && \ docker compose -f deploy/docker-compose.yml config' _ "$1" 2>/dev/null \ | sha256sum | cut -d" " -f1 } a="$(model "$SCRATCH/live.env")" b="$(model "$SCRATCH/rendered.env")" if [ -z "$a" ] || [ "$a" = "$(printf '' | sha256sum | cut -d' ' -f1)" ]; then echo " compose model : could not be built from the LIVE env — check by hand" elif [ "$a" = "$b" ]; then echo " compose model : IDENTICAL (docker compose config digests match)" else echo " compose model : DIFFERENT — do NOT cut over" exit 1 fi else echo " compose model : skipped (no /opt/centralis or no docker here)" fi REMOTE