thermograph/infra/openbao/bootstrap.sh

#!/usr/bin/env bash
# Bring up OpenBao on vps2 from nothing. Run ONCE, as root, ON vps2, BY THE OPERATOR.
#
#   sudo bash infra/openbao/bootstrap.sh
#
# ============================================================================
# WHY THIS IS NOT AUTOMATED, AND MUST NOT BE
# ============================================================================
# `bao operator init` mints the credentials that become the new root of trust for the
# entire estate: the recovery keys and the initial root token. Those have to be
# CUSTODIED BY A HUMAN. Any place a script could put them — a file on the box, the
# repo, a CI log, an agent transcript — is worse custody than the age key this
# migration is meant to improve on.
#
# So this script does everything up to and including init, prints the recovery
# material ONCE to the operator's terminal, and then stops and tells you what to do
# with it. It deliberately does not store it, mail it, or push it anywhere.
#
# Everything AFTER custody (policies, AppRoles, seeding, parity checks) is automated,
# because none of it produces a credential a human needs to hold.
set -euo pipefail

BAO_VERSION="${BAO_VERSION:-2.6.1}"
MESH_ADDR="${MESH_ADDR:-10.10.0.1}"
SELF_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"

[ "$(id -u)" = 0 ] || { echo "!! run as root (sudo bash $0)" >&2; exit 1; }

hostname_now="$(hostname)"
case "$hostname_now" in
  vmi3453260) : ;;   # vps2
  *)
    # Refuse to bootstrap the secret store on the wrong box. Putting it on vps1 would
    # invert the estate's central security decision: infra/CLAUDE.md states "vps1 must
    # never hold prod credentials. It's the box that runs Forgejo, its CI runner, and
    # dev's unreviewed branch — the opposite of an isolation boundary." With a static
    # seal, the box also holds the key that decrypts everything.
    #
    # This mirrors the guard infra-sync.yml already has (commit 7583258, "refuse to
    # render dev's vault onto a host that is not dev").
    echo "!! this host is '$hostname_now', not vps2 (vmi3453260)." >&2
    echo "!! OpenBao belongs on vps2. Refusing." >&2
    exit 1
    ;;
esac

echo "==> 1/7 install the bao binary (v${BAO_VERSION})"
if ! command -v bao >/dev/null 2>&1; then
  # NOTE: checksum verification. provision-secrets.sh installs sops and age with a bare
  # `sudo curl` and no verification at all, which is a standing weakness (a compromised
  # release URL yields a root binary that sees every plaintext). Do not copy that here.
  # The checksums file is signed by the release; verify before installing.
  tmpd="$(mktemp -d)"
  base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
  # Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
  # The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
  tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
  curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
  curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
  # Anchor to end-of-line and save under the real asset name: checksums.txt also lists
  # <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
  # so an unanchored match fails on a file that was never fetched.
  ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
    echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
    rm -rf "$tmpd"; exit 1
  }
  tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
  install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
  rm -rf "$tmpd"
fi
bao version

echo "==> 2/7 user, directories, TLS"
id -u openbao >/dev/null 2>&1 || useradd --system --home /var/lib/openbao --shell /usr/sbin/nologin openbao
install -d -o openbao -g openbao -m 0700 /var/lib/openbao
install -d -o openbao -g openbao -m 0750 /var/log/openbao
install -d -o root    -g root    -m 0755 /etc/openbao
install -d -o openbao -g openbao -m 0700 /etc/openbao/tls

if [ ! -f /etc/openbao/tls/bao.crt ]; then
  # Self-signed, 10 years, SAN on the mesh address. Deliberately NOT from Caddy/ACME:
  # that would put DNS and the public internet in the boot chain of the secret store.
  openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes \
    -keyout /etc/openbao/tls/bao.key -out /etc/openbao/tls/bao.crt \
    -subj "/CN=openbao.thermograph.internal" \
    -addext "subjectAltName=IP:${MESH_ADDR},IP:127.0.0.1" >/dev/null 2>&1
  chown openbao:openbao /etc/openbao/tls/bao.key /etc/openbao/tls/bao.crt
  chmod 0400 /etc/openbao/tls/bao.key
  chmod 0444 /etc/openbao/tls/bao.crt
fi
# Consumers need the cert to trust the listener.
install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt

echo "==> 3/7 static seal key"
if [ ! -f /etc/openbao/unseal.key ]; then
  # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
  # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
  # static seal reads this file RAW. That one trailing byte makes the key neither
  # 32 raw bytes nor clean base64, so bao refuses to start with
  # `Error configuring seal "static": unknown encoding for AES-256 key`.
  ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
  chown openbao:openbao /etc/openbao/unseal.key
  chmod 0400 /etc/openbao/unseal.key
  echo "    generated /etc/openbao/unseal.key (0400 openbao)"
  echo "    ⚠️  COPY THIS OFF THE BOX into your password manager before proceeding."
  echo "        Without an off-box copy, a raft snapshot is UNRESTORABLE."
else
  echo "    /etc/openbao/unseal.key already present, leaving alone"
fi

echo "==> 4/7 config + systemd unit"
install -m 0640 -o root -g openbao "$SELF_DIR/config/openbao.hcl" /etc/openbao/config.hcl
install -m 0644 -o root -g root "$SELF_DIR/openbao.service" /etc/systemd/system/openbao.service
# logrotate MUST use SIGHUP: without it bao keeps writing to an unlinked inode and the
# audit log silently stops growing — and a wedged audit device makes OpenBao stop
# answering requests entirely, which would block every deploy on the estate.
cat > /etc/logrotate.d/openbao <<'ROTATE'
/var/log/openbao/audit.log {
  daily
  rotate 30
  compress
  missingok
  notifempty
  create 0600 openbao openbao
  postrotate
    systemctl kill --signal=SIGHUP openbao.service 2>/dev/null || true
  endscript
}
ROTATE
systemctl daemon-reload
systemctl enable --now openbao.service
sleep 3
systemctl is-active --quiet openbao.service || {
  echo "!! openbao.service failed to start; check: journalctl -u openbao -n 50" >&2
  exit 1
}

# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
# desktop, the phone — has any business reaching the secret store.
if command -v ufw >/dev/null 2>&1; then
  if ufw status | grep -q '8200.*10\.10\.0\.2'; then
    echo "    ufw: 8200/tcp from 10.10.0.2 already allowed"
  else
    ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
      comment 'vps1/dev -> openbao' >/dev/null
    echo "    ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
  fi
else
  echo "    !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
fi

export BAO_ADDR="https://127.0.0.1:8200"
export BAO_CACERT=/etc/openbao/tls/bao.crt

echo "==> 5/7 initialise"
if bao status -format=json 2>/dev/null | grep -q '"initialized": *true'; then
  echo "    already initialised, skipping"
else
  echo
  echo "    ############################################################"
  echo "    #  RECOVERY MATERIAL FOLLOWS. IT IS SHOWN EXACTLY ONCE.    #"
  echo "    #  Store the 3 recovery keys in 3 DIFFERENT places.        #"
  echo "    #  2 of 3 are needed to rekey or mint a new root token.    #"
  echo "    ############################################################"
  echo
  # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
  # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
  # is redundancy against loss, not separation of duty — there is one operator.
  #
  # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
  # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
  # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
  # minting a root token requires a token you already have. The deprecated endpoint is
  # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
  # Consequence: revoking the last root token before another admin identity exists is
  # a one-way door. See the ordering note in the handoff below.
  bao operator init -recovery-shares=3 -recovery-threshold=2
  echo
  echo "    Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
  echo "    stored off this machine."
  read -r _
fi

bao status || true

cat <<EOF

==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.

Paste the initial root token printed above and run:

  export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
  export BAO_TOKEN=<root token>
  bash ${SELF_DIR}/bootstrap-policies.sh

KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
\`bao token revoke -self\` locks you out of an otherwise healthy vault.

That script enables the KV mount, writes the policies, creates the AppRoles and
installs each host's credentials. Then seed — on THIS box, where the age key already
lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
secrets is not for an agent to run:

  export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
  infra/openbao/seed-from-sops.sh --dry-run     # check first: 16/12/8/16 keys
  infra/openbao/seed-from-sops.sh --all

Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
that legitimately renders it, so an environment can only be verified from its own host.

  # here (vps2) -- beta needs its own AppRole, prod uses the default
  infra/openbao/verify-parity.sh --env prod                        # expect 32 keys
  THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\
    infra/openbao/verify-parity.sh --env beta                      # expect 24 keys
  # on vps1
  cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev   # expect 12 keys

Only once all three PASS, and only after you have a second admin identity that is not
the root token, revoke it:  bao token revoke -self

NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
until you flip an environment in a reviewed PR — dev first.
EOF