thermograph/infra/deploy/secrets/verify-age-quorum.sh

#!/usr/bin/env bash
# Assert that the age recovery quorum is intact.
#
#   infra/deploy/secrets/verify-age-quorum.sh
#
# The age private key is the single recovery root for all five SOPS vaults AND for
# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same
# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE
# two-host quorum, not provisioning residue — see the "age key" section of
# deploy/secrets/README.md. This script is what makes that policy checkable instead of
# merely stated.
#
# It fails if the key is missing on either host, if the two copies have diverged, or
# if either is more permissive than 0440. Divergence matters as much as absence: two
# hosts holding different keys means one of them renders secrets nobody else can
# decrypt, and a restore from backup silently picks the wrong one.
#
# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a
# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is
# never read, echoed or transferred. Nothing here copies the key anywhere.
#
# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate.
set -euo pipefail

VPS1="${TG_VPS1_SSH:-vps1}"
VPS2="${TG_VPS2_SSH:-vps2}"
KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}"
SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}"

# Reads the digest and mode of the key on one host. Uses sudo when the key is not
# directly readable, mirroring how render-secrets.sh lifts it.
probe() {
  local target="$1" out
  # The remote script is a QUOTED heredoc and the key path is passed as $1 to
  # `bash -s`, so nothing expands on this side. Interpolating the path into the
  # command string would work too, but it is the shape that quietly breaks the day
  # a path contains a space, and shellcheck is right to flag it (SC2029).
  # shellcheck disable=SC2086  # SSH_OPTS is a deliberate word-split option list
  out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE'
key="$1"
if [ ! -e "$key" ]; then echo MISSING; exit 0; fi
mode=$(stat -c %a "$key")
owner=$(stat -c %U:%G "$key")
if [ -r "$key" ]; then
  digest=$(sha256sum "$key" | cut -d' ' -f1)
else
  digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1)
fi
[ -n "$digest" ] || { echo UNREADABLE; exit 0; }
echo "$digest $mode $owner"
REMOTE
  ) || { echo UNREACHABLE; return 0; }
  printf '%s\n' "$out"
}

rc=0
declare -A DIGEST

for pair in "vps1:$VPS1" "vps2:$VPS2"; do
  name="${pair%%:*}"
  target="${pair#*:}"
  result="$(probe "$target")"
  case "$result" in
    MISSING)
      echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2
      rc=1
      ;;
    UNREADABLE)
      echo "!! ${name}: key present but unreadable even via sudo" >&2
      rc=1
      ;;
    UNREACHABLE)
      echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2
      rc=1
      ;;
    *)
      digest="${result%% *}"
      rest="${result#* }"
      DIGEST["$name"]="$digest"
      printf '   %-5s %s  mode=%s\n' "$name" "${digest:0:16}…" "$rest"
      mode="${rest%% *}"
      # 0400 or 0440 only. Anything wider means a non-root account on that box can
      # read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner.
      case "$mode" in
        400|440) ;;
        *) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;;
      esac
      # Advisory, not a failure: 0440 with a non-root group means that group can read
      # the key that decrypts prod. On vps1 that is doubly worth knowing, because the
      # same box runs Forgejo, its CI runner and dev's unreviewed branches. Left
      # non-fatal because it is the current provisioned state — a check that fails on
      # day one is a check that gets ignored by day three. See README.md.
      group="${rest#* }"; group="${group#*:}"
      if [ "$mode" = 440 ] && [ "$group" != root ]; then
        printf '         note: group %s can read this key\n' "$group"
      fi
      ;;
  esac
done

if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then
  if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then
    echo "   quorum: 2 copies, identical"
  else
    echo "!! the two copies have DIVERGED — they are different keys" >&2
    echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2
    echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2
    rc=1
  fi
fi

if [ "$rc" -eq 0 ]; then
  echo "   OK — recovery quorum intact"
else
  echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2
fi
exit "$rc"