promote: dev → main #139
4 changed files with 147 additions and 35 deletions
|
|
@ -148,22 +148,53 @@ SOPS and retiring age are two different projects.
|
|||
### Stand it up (once, by the operator, on vps2)
|
||||
|
||||
```sh
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init
|
||||
# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles
|
||||
bao token revoke -self
|
||||
```
|
||||
|
||||
### Seed and prove (from your own machine — it needs your age key)
|
||||
> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
|
||||
> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
|
||||
> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
> minting a root token requires a token you already have. **The recovery keys are not
|
||||
> a way back in**; they rekey, they do not authenticate. Unless
|
||||
> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
|
||||
> the last root token locks you out of an otherwise healthy, unsealed vault, and the
|
||||
> only remedy is re-initialising from scratch. Revoke only once a second admin
|
||||
> identity exists.
|
||||
|
||||
Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
|
||||
`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
|
||||
openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
|
||||
with `certificate signed by unknown authority`.
|
||||
|
||||
### Seed and prove (on vps2 — the age key is already there)
|
||||
|
||||
```sh
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # MUST pass before any flip
|
||||
```
|
||||
|
||||
Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
|
||||
to the host that legitimately renders it, so an environment can only be verified from
|
||||
its own host:
|
||||
|
||||
```sh
|
||||
# on vps2
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
```
|
||||
|
||||
The explicit `THERMOGRAPH_BAO_APPROLE` for beta is a stopgap: `env-topology.sh` does
|
||||
not yet derive the AppRole path per environment, so the renderer falls back to prod's
|
||||
credentials and `tg-host-prod.hcl` correctly denies `env/beta`.
|
||||
|
||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||
inherits that rule.
|
||||
|
|
|
|||
|
|
@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis 10.10.0.1/32
|
|||
|
||||
# Install the local (vps2) credentials. prod and beta both render on this box.
|
||||
#
|
||||
# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
|
||||
# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
|
||||
# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
|
||||
# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
|
||||
# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
|
||||
# creates one.
|
||||
# Both files are owned by `agent`, because that is the single identity both
|
||||
# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
|
||||
# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
|
||||
# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
|
||||
# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
|
||||
# they do not, and the chown simply failed.
|
||||
#
|
||||
# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
|
||||
# the one age key. This defends against MISCONFIGURATION, not against intrusion.
|
||||
# So be accurate about what separate credentials buy on this box: NOT deploy-user
|
||||
# isolation — there is one deploy user, and it can read both files. What they buy is
|
||||
# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
|
||||
# *mistake* crossing between environments: a beta render authenticating with beta's
|
||||
# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
|
||||
# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
|
||||
# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
|
||||
# a separate SSH credential for beta in deploy.yml — a larger change than this script.
|
||||
install_creds() {
|
||||
local role="$1" dest="$2" owner="$3"
|
||||
local rid sid
|
||||
|
|
@ -79,7 +85,17 @@ install_creds() {
|
|||
sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
|
||||
# umask before creation, so the file is never briefly world-readable.
|
||||
( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
|
||||
chown "$owner" "$dest"
|
||||
# Abort the function if chown fails, and remove the half-installed file. The call
|
||||
# site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
|
||||
# so without this check a failed chown falls straight through to chmod and prints a
|
||||
# line asserting an ownership that was never applied. A root-owned credential the
|
||||
# renderer cannot read, reported as installed, is worse than no credential at all:
|
||||
# it fails at deploy time instead of here.
|
||||
if ! chown "$owner" "$dest"; then
|
||||
rm -f "$dest"
|
||||
echo " !! chown $owner failed for $dest — removed, nothing installed" >&2
|
||||
return 1
|
||||
fi
|
||||
chmod 0400 "$dest"
|
||||
echo " installed $dest (0400 $owner)"
|
||||
}
|
||||
|
|
@ -87,8 +103,7 @@ install_creds() {
|
|||
if [ "$(id -u)" = 0 ]; then
|
||||
install -d -o root -g root -m 0755 /etc/thermograph
|
||||
install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \
|
||||
|| echo " (beta creds skipped — no 'deploy' user on this box?)"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent"
|
||||
install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
|
||||
else
|
||||
echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
|
||||
|
|
|
|||
|
|
@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then
|
|||
# The checksums file is signed by the release; verify before installing.
|
||||
tmpd="$(mktemp -d)"
|
||||
base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
|
||||
curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
|
||||
( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
|
||||
# Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
|
||||
# The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
|
||||
tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
|
||||
# Anchor to end-of-line and save under the real asset name: checksums.txt also lists
|
||||
# <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
|
||||
# so an unanchored match fails on a file that was never fetched.
|
||||
( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
|
||||
echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
|
||||
rm -rf "$tmpd"; exit 1
|
||||
}
|
||||
tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
|
||||
tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
|
||||
install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
|
||||
rm -rf "$tmpd"
|
||||
fi
|
||||
|
|
@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
|
|||
echo "==> 3/7 static seal key"
|
||||
if [ ! -f /etc/openbao/unseal.key ]; then
|
||||
# 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
|
||||
( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
|
||||
# tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
|
||||
# static seal reads this file RAW. That one trailing byte makes the key neither
|
||||
# 32 raw bytes nor clean base64, so bao refuses to start with
|
||||
# `Error configuring seal "static": unknown encoding for AES-256 key`.
|
||||
( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
|
||||
chown openbao:openbao /etc/openbao/unseal.key
|
||||
chmod 0400 /etc/openbao/unseal.key
|
||||
echo " generated /etc/openbao/unseal.key (0400 openbao)"
|
||||
|
|
@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
|
||||
# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
|
||||
# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
|
||||
# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
|
||||
# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
|
||||
# desktop, the phone — has any business reaching the secret store.
|
||||
if command -v ufw >/dev/null 2>&1; then
|
||||
if ufw status | grep -q '8200.*10\.10\.0\.2'; then
|
||||
echo " ufw: 8200/tcp from 10.10.0.2 already allowed"
|
||||
else
|
||||
ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
|
||||
comment 'vps1/dev -> openbao' >/dev/null
|
||||
echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
|
||||
fi
|
||||
else
|
||||
echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
|
||||
fi
|
||||
|
||||
export BAO_ADDR="https://127.0.0.1:8200"
|
||||
export BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
|
||||
|
|
@ -140,9 +168,16 @@ else
|
|||
echo " ############################################################"
|
||||
echo
|
||||
# -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
|
||||
# yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
|
||||
# keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
|
||||
# operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
|
||||
# yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
|
||||
# is redundancy against loss, not separation of duty — there is one operator.
|
||||
#
|
||||
# These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
|
||||
# the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
|
||||
# (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
# minting a root token requires a token you already have. The deprecated endpoint is
|
||||
# off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
|
||||
# Consequence: revoking the last root token before another admin identity exists is
|
||||
# a one-way door. See the ordering note in the handoff below.
|
||||
bao operator init -recovery-shares=3 -recovery-threshold=2
|
||||
echo
|
||||
echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
|
||||
|
|
@ -156,22 +191,37 @@ cat <<EOF
|
|||
|
||||
==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
|
||||
|
||||
Paste the initial root token (or one minted from recovery keys) and run:
|
||||
Paste the initial root token printed above and run:
|
||||
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
bash ${SELF_DIR}/bootstrap-policies.sh
|
||||
|
||||
KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
|
||||
cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
|
||||
\`bao token revoke -self\` locks you out of an otherwise healthy vault.
|
||||
|
||||
That script enables the KV mount, writes the policies, creates the AppRoles and
|
||||
installs each host's credentials. Then, from YOUR OWN machine (it needs your age
|
||||
key, and per infra/CLAUDE.md a script that reads production secrets is not for an
|
||||
agent to run):
|
||||
installs each host's credentials. Then seed — on THIS box, where the age key already
|
||||
lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
|
||||
secrets is not for an agent to run:
|
||||
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # must PASS before any flip
|
||||
|
||||
Then revoke the root token: bao token revoke -self
|
||||
Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
|
||||
that legitimately renders it, so an environment can only be verified from its own host.
|
||||
|
||||
# here (vps2) -- beta needs its own AppRole, prod uses the default
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
THERMOGRAPH_BAO_APPROLE=/etc/thermograph/openbao-approle-beta \\
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
|
||||
Only once all three PASS, and only after you have a second admin identity that is not
|
||||
the root token, revoke it: bao token revoke -self
|
||||
|
||||
NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
|
||||
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
|
||||
|
|
|
|||
|
|
@ -25,8 +25,12 @@
|
|||
|
||||
ui = false
|
||||
cluster_name = "thermograph"
|
||||
disable_mlock = true
|
||||
log_level = "info"
|
||||
// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
|
||||
// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
|
||||
// setting the server ignores only trains the operator to skim startup warnings,
|
||||
// which is where a real one will eventually hide. The corresponding note about
|
||||
// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
|
||||
|
||||
// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
|
||||
// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
|
||||
|
|
@ -110,9 +114,21 @@ seal "static" {
|
|||
//
|
||||
// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
|
||||
// unlinked inode and the log silently stops growing.
|
||||
// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
|
||||
// device NAME, not its type — and device settings go in `options`. Written as
|
||||
// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
|
||||
// be specified"); with type+path but a bare file_path it starts but silently
|
||||
// IGNORES the path and mode, which is the worse failure of the two.
|
||||
audit "file" {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
type = "file"
|
||||
path = "file/"
|
||||
options = {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
}
|
||||
|
||||
audit "syslog" {}
|
||||
audit "syslog" {
|
||||
type = "syslog"
|
||||
path = "syslog/"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue