Compare commits
1 commit
7fd364bb8a
...
e60bc66448
| Author | SHA1 | Date | |
|---|---|---|---|
|
e60bc66448 |
8 changed files with 50 additions and 482 deletions
|
|
@ -76,16 +76,6 @@ thermograph_topology() {
|
|||
# for the reason that marker was demoted to a fallback in the first place: vps2 runs
|
||||
# prod and beta side by side, and one host-wide file cannot name two backends.
|
||||
TG_SECRETS_BACKEND=sops
|
||||
# Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
|
||||
# reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
|
||||
# environments, so a single host-wide default cannot serve both. prod and dev each
|
||||
# get the conventional path (dev is alone on vps1, so there is no collision there);
|
||||
# beta is the one that must differ, because it shares a filesystem with prod.
|
||||
#
|
||||
# Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
|
||||
# correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
|
||||
# runs prod.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle
|
||||
|
||||
case "$env_name" in
|
||||
prod)
|
||||
|
|
@ -126,10 +116,6 @@ thermograph_topology() {
|
|||
# therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
|
||||
TG_SSH_HOST=169.58.46.181
|
||||
TG_SSH_TARGET=agent@169.58.46.181
|
||||
# The one environment that cannot use the default AppRole path: it shares a
|
||||
# filesystem with prod, so it needs its own credential file to authenticate as
|
||||
# tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
|
||||
# A SECOND checkout on the same box. Separate from prod's so the two can
|
||||
# sit on different commits of this repo, and so `git reset --hard` in one
|
||||
# deploy can never yank the tree out from under the other's running
|
||||
|
|
@ -242,7 +228,7 @@ thermograph_topology() {
|
|||
export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
|
||||
export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
|
||||
export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
|
||||
export TG_SECRETS_BACKEND TG_BAO_APPROLE
|
||||
export TG_SECRETS_BACKEND
|
||||
export TG_SSH_HOST TG_SSH_TARGET
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
# OpenBao source backend for render-secrets.sh. Sourced, never run directly.
|
||||
#
|
||||
# Functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
# thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
|
||||
#
|
||||
# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
|
||||
# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
|
||||
# a different shape for a different consumer, not an inconsistency.
|
||||
# Two functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
#
|
||||
# render_thermograph_secrets in render-secrets.sh dispatches to the second when
|
||||
# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
|
||||
|
|
@ -32,13 +27,7 @@ thermograph_openbao_source() {
|
|||
local env_name="${1:?env name required}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
# Explicit override first, then the per-environment value env-topology.sh derives,
|
||||
# then the conventional path. Same precedence shape as render-secrets.sh:44's
|
||||
# THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
|
||||
# THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
|
||||
# deploy path sets. Falling straight through to the bare default is what made beta
|
||||
# authenticate as tg-prod and get denied its own path.
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
|
|
@ -118,16 +107,9 @@ thermograph_openbao_source() {
|
|||
# /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
|
||||
# deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
|
||||
# `$(...)` or a backtick in a secret is a live command-execution path on the
|
||||
# deploy host. Refusing here turns a latent code-execution bug into a loud
|
||||
# render failure.
|
||||
#
|
||||
# An earlier version of this comment claimed the SOPS path had never tripped
|
||||
# this "only because every current value happens to be alphanumeric". That is
|
||||
# false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
|
||||
# tripped THESE paths because it is not in them — it lives at
|
||||
# centralis/<env> and renders to /etc/centralis.env, which is single-quoted
|
||||
# precisely because raw dotenv cannot carry it. See
|
||||
# thermograph_render_openbao_centralis at the end of this file.
|
||||
# deploy host. The SOPS path has always had this hazard and has never tripped it
|
||||
# only because every current value happens to be alphanumeric. Refusing here
|
||||
# turns a latent code-execution bug into a loud render failure.
|
||||
THERMOGRAPH_BAO_COMMON="$common_json" \
|
||||
THERMOGRAPH_BAO_ENV="$env_json" \
|
||||
python3 - <<'PY'
|
||||
|
|
@ -255,239 +237,3 @@ thermograph_render_openbao() {
|
|||
rm -f "$tmp"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CENTRALIS
|
||||
# ---------------------------------------------------------------------------
|
||||
# /etc/centralis.env is a different file, with a different consumer, and therefore a
|
||||
# different output shape. render-secrets.sh:206-233 carries the full argument; the
|
||||
# short version is that it is consumed by exactly one thing and that thing is a shell:
|
||||
#
|
||||
# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
|
||||
#
|
||||
# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
|
||||
# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
|
||||
# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
|
||||
# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
|
||||
# by serving a single `shared` identity — indistinguishable from the file never having
|
||||
# been written. That is the 2026-07-24 incident. So this path emits POSIX
|
||||
# single-quoted assignments and PROVES them by sourcing the result in a clean shell
|
||||
# before anything touches $dest.
|
||||
#
|
||||
# This mirrors render_centralis_secrets rather than sharing with it, for the same
|
||||
# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
|
||||
# and all new risk stays in this file. Consolidate the two when the SOPS path is being
|
||||
# deleted anyway, not before.
|
||||
|
||||
# _thermograph_openbao_login <approle_file> <addr> <ca> -> token on STDOUT
|
||||
#
|
||||
# Separate from the login inline in thermograph_openbao_source deliberately: that
|
||||
# function is now exercised by prod and beta parity and is left untouched.
|
||||
_thermograph_openbao_login() {
|
||||
local approle="$1" addr="$2" ca="$3"
|
||||
local creds role_id secret_id
|
||||
if [ -r "$approle" ]; then
|
||||
creds=$(cat "$approle")
|
||||
else
|
||||
creds=$(sudo cat "$approle" 2>/dev/null || true)
|
||||
fi
|
||||
[ -n "$creds" ] || {
|
||||
echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
|
||||
return 1
|
||||
}
|
||||
role_id=$(printf '%s\n' "$creds" | sed -n '1p')
|
||||
secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
|
||||
[ -n "$role_id" ] && [ -n "$secret_id" ] || {
|
||||
echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_ADDR="$addr"
|
||||
[ -f "$ca" ] && export BAO_CACERT="$ca"
|
||||
bao write -field=token auth/approle/login \
|
||||
role_id="$role_id" secret_id="$secret_id" 2>/dev/null
|
||||
}
|
||||
|
||||
# thermograph_render_openbao_centralis <env_name> [dest]
|
||||
thermograph_render_openbao_centralis() {
|
||||
local env_name="${1:?env name required}"
|
||||
local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
|
||||
return 1
|
||||
}
|
||||
command -v python3 >/dev/null 2>&1 || {
|
||||
echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
|
||||
|
||||
local token
|
||||
token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
|
||||
[ -n "$token" ] || {
|
||||
echo "!! OpenBao AppRole login failed for centralis" >&2
|
||||
echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_TOKEN="$token"
|
||||
|
||||
# ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
|
||||
# path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
|
||||
# persists into the caller's shell and re-fires on its next `source`.
|
||||
local work rc=0
|
||||
work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
|
||||
chmod 700 "$work"
|
||||
|
||||
# Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
|
||||
# Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
|
||||
# command that is the left operand of `||`, which this subshell is.
|
||||
(
|
||||
umask 077
|
||||
|
||||
bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
|
||||
> "$work/kv.json" 2>/dev/null || {
|
||||
echo "!! cannot read ${mount}/centralis/${env_name}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
|
||||
import json, sys
|
||||
doc = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
data = doc.get("data", {}).get("data", {}) or {}
|
||||
if not data:
|
||||
sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
|
||||
sys.exit(1)
|
||||
json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
|
||||
UNWRAP
|
||||
|
||||
python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
|
||||
import json, re, sys
|
||||
|
||||
src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
||||
|
||||
|
||||
def shell_quote(v):
|
||||
"""POSIX single-quoting. Inside '...' every byte is literal except ' itself,
|
||||
which is closed, backslash-escaped and reopened. There is no escape sequence
|
||||
to get wrong and no character class to keep up to date: it is total, and
|
||||
total is the only property worth having for a file bash will expand."""
|
||||
return "'" + v.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
def scalar(k, v):
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
if isinstance(v, bool): # before int: bool is an int in Python
|
||||
return "true" if v else "false"
|
||||
if isinstance(v, (int, float)):
|
||||
return json.dumps(v)
|
||||
if v is None:
|
||||
return ""
|
||||
sys.stderr.write(
|
||||
"!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
data = json.load(open(src, encoding="utf-8"))
|
||||
want, lines = {}, []
|
||||
for k, v in data.items():
|
||||
if not NAME.match(k):
|
||||
sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
|
||||
sys.exit(1)
|
||||
want[k] = scalar(k, v)
|
||||
lines.append("%s=%s\n" % (k, shell_quote(want[k])))
|
||||
|
||||
with open(out_env, "w", encoding="utf-8") as fh:
|
||||
fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
|
||||
fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
|
||||
fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
|
||||
fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
|
||||
fh.writelines(lines)
|
||||
with open(want_json, "w", encoding="utf-8") as fh:
|
||||
json.dump(want, fh)
|
||||
sys.stderr.write(" %d keys quoted\n" % len(want))
|
||||
EMIT
|
||||
|
||||
# PROVE IT. Source the rendered file exactly the way the deploy does — clean
|
||||
# environment, `set -a`, nothing inherited that could mask a dropped key — and
|
||||
# read every value back. This is the check the 2026-07-24 file could not pass.
|
||||
# shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded
|
||||
# by the inner `bash -c` against the argument after `_`, not by this shell.
|
||||
env -i PATH="$PATH" bash -c \
|
||||
'set -a; . "$1"; set +a; exec python3 -c "
|
||||
import json, os, sys
|
||||
sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
|
||||
> "$work/got.json" || exit 1
|
||||
|
||||
python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
|
||||
import json, sys
|
||||
|
||||
want = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
got = json.load(open(sys.argv[2], encoding="utf-8"))
|
||||
|
||||
bad = []
|
||||
for k, v in want.items():
|
||||
if k not in got:
|
||||
bad.append("%s: not set at all after sourcing" % k)
|
||||
elif got[k] != v:
|
||||
# NEVER print either value. The difference is the finding; the content is
|
||||
# not, and this runs inside a deploy log.
|
||||
bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
|
||||
% (k, len(v), len(got[k])))
|
||||
if bad:
|
||||
sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
|
||||
for b in bad:
|
||||
sys.stderr.write("!! %s\n" % b)
|
||||
sys.exit(1)
|
||||
|
||||
# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
|
||||
# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
|
||||
# single `shared` identity, which looks exactly like the registry never having been
|
||||
# configured. Refuse to write a file that would do that.
|
||||
raw = got.get("CENTRALIS_TOKENS", "")
|
||||
if raw:
|
||||
try:
|
||||
reg = json.loads(raw)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
|
||||
sys.exit(1)
|
||||
if not isinstance(reg, dict) or not reg:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
|
||||
sys.exit(1)
|
||||
if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
|
||||
sys.exit(1)
|
||||
# Subjects are names on audit lines, not credentials. Printing them is the whole
|
||||
# point: it is how an operator sees at a glance that nobody was lost.
|
||||
sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
|
||||
% (len(reg), ", ".join(sorted(reg))))
|
||||
|
||||
sys.stderr.write(" %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
|
||||
VERIFY
|
||||
) || rc=1
|
||||
|
||||
# Only now, with a file read back through bash and matched value for value, does
|
||||
# anything touch $dest. 0600: four bearer credentials for the estate's control
|
||||
# plane, with no group that needs them.
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
if [ -f "$dest" ] && [ -w "$dest" ]; then
|
||||
cat "$work/out.env" > "$dest" || rc=1
|
||||
elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
else
|
||||
echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "!! render aborted; ${dest} left untouched" >&2
|
||||
fi
|
||||
|
||||
rm -rf "$work"
|
||||
return "$rc"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -272,30 +272,6 @@ render_centralis_secrets() {
|
|||
echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Same early-return dispatch as render_thermograph_secrets, for the same reason
|
||||
# given there: everything below — the age-key sudo lift, the JSON decrypt, the
|
||||
# quote-and-prove pipeline, the three-way write — stays on exactly the code path it
|
||||
# has always been on, so this cannot regress the backend still authoritative for
|
||||
# Centralis. The OpenBao path needs no age key, so it returns before that check.
|
||||
local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
|
||||
if [ "$backend" = openbao ]; then
|
||||
local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
|
||||
if [ ! -f "$bao_lib" ]; then
|
||||
echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
|
||||
echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
|
||||
return 1
|
||||
fi
|
||||
# shellcheck source=/dev/null
|
||||
. "$bao_lib"
|
||||
thermograph_render_openbao_centralis "$env_name" "$dest"
|
||||
return
|
||||
fi
|
||||
if [ "$backend" != sops ]; then
|
||||
echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$key" ]; then
|
||||
echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -112,22 +112,6 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv
|
|||
Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
|
||||
isolation lives in the policy. Today both live in one shell variable.
|
||||
|
||||
**`centralis/<env>` renders differently from everything above it, and must.** The
|
||||
app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
|
||||
is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
|
||||
pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
|
||||
sourcing it in a clean shell and comparing every value back before the file is
|
||||
written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
|
||||
strips its quotes, Centralis fails closed on the malformed result and serves a
|
||||
single `shared` identity, which is indistinguishable from the file never having
|
||||
been written. That is the 2026-07-24 incident, and the round-trip check exists so a
|
||||
render carrying it cannot reach `/etc`.
|
||||
|
||||
Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
|
||||
metacharacters, no newlines) only to `common` and `env/*`. Applying them to
|
||||
`centralis/*` would reject data that renders perfectly well — the validator matches
|
||||
the renderer that will consume the path, not a single global rule.
|
||||
|
||||
`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
|
||||
rule expressible as exactly one path, with no wildcard over `env/*` able to
|
||||
accidentally re-include it.
|
||||
|
|
@ -164,52 +148,22 @@ SOPS and retiring age are two different projects.
|
|||
### Stand it up (once, by the operator, on vps2)
|
||||
|
||||
```sh
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init
|
||||
# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles
|
||||
bao token revoke -self
|
||||
```
|
||||
|
||||
> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
|
||||
> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
|
||||
> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
> minting a root token requires a token you already have. **The recovery keys are not
|
||||
> a way back in**; they rekey, they do not authenticate. Unless
|
||||
> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
|
||||
> the last root token locks you out of an otherwise healthy, unsealed vault, and the
|
||||
> only remedy is re-initialising from scratch. Revoke only once a second admin
|
||||
> identity exists.
|
||||
|
||||
Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
|
||||
`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
|
||||
openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
|
||||
with `certificate signed by unknown authority`.
|
||||
|
||||
### Seed and prove (on vps2 — the age key is already there)
|
||||
### Seed and prove (from your own machine — it needs your age key)
|
||||
|
||||
```sh
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # MUST pass before any flip
|
||||
```
|
||||
|
||||
Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
|
||||
to the host that legitimately renders it, so an environment can only be verified from
|
||||
its own host:
|
||||
|
||||
```sh
|
||||
# on vps2
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
```
|
||||
|
||||
Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
|
||||
environment, so beta authenticates as `tg-beta` rather than falling back to prod's
|
||||
credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||
|
||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||
inherits that rule.
|
||||
|
|
|
|||
|
|
@ -63,21 +63,15 @@ add_role tg-centralis tg-centralis 10.10.0.1/32
|
|||
|
||||
# Install the local (vps2) credentials. prod and beta both render on this box.
|
||||
#
|
||||
# Both files are owned by `agent`, because that is the single identity both
|
||||
# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
|
||||
# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
|
||||
# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
|
||||
# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
|
||||
# they do not, and the chown simply failed.
|
||||
# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
|
||||
# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
|
||||
# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
|
||||
# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
|
||||
# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
|
||||
# creates one.
|
||||
#
|
||||
# So be accurate about what separate credentials buy on this box: NOT deploy-user
|
||||
# isolation — there is one deploy user, and it can read both files. What they buy is
|
||||
# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
|
||||
# *mistake* crossing between environments: a beta render authenticating with beta's
|
||||
# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
|
||||
# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
|
||||
# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
|
||||
# a separate SSH credential for beta in deploy.yml — a larger change than this script.
|
||||
# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
|
||||
# the one age key. This defends against MISCONFIGURATION, not against intrusion.
|
||||
install_creds() {
|
||||
local role="$1" dest="$2" owner="$3"
|
||||
local rid sid
|
||||
|
|
@ -85,17 +79,7 @@ install_creds() {
|
|||
sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
|
||||
# umask before creation, so the file is never briefly world-readable.
|
||||
( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
|
||||
# Abort the function if chown fails, and remove the half-installed file. The call
|
||||
# site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
|
||||
# so without this check a failed chown falls straight through to chmod and prints a
|
||||
# line asserting an ownership that was never applied. A root-owned credential the
|
||||
# renderer cannot read, reported as installed, is worse than no credential at all:
|
||||
# it fails at deploy time instead of here.
|
||||
if ! chown "$owner" "$dest"; then
|
||||
rm -f "$dest"
|
||||
echo " !! chown $owner failed for $dest — removed, nothing installed" >&2
|
||||
return 1
|
||||
fi
|
||||
chown "$owner" "$dest"
|
||||
chmod 0400 "$dest"
|
||||
echo " installed $dest (0400 $owner)"
|
||||
}
|
||||
|
|
@ -103,7 +87,8 @@ install_creds() {
|
|||
if [ "$(id -u)" = 0 ]; then
|
||||
install -d -o root -g root -m 0755 /etc/thermograph
|
||||
install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \
|
||||
|| echo " (beta creds skipped — no 'deploy' user on this box?)"
|
||||
install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
|
||||
else
|
||||
echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
|
||||
|
|
|
|||
|
|
@ -52,19 +52,13 @@ if ! command -v bao >/dev/null 2>&1; then
|
|||
# The checksums file is signed by the release; verify before installing.
|
||||
tmpd="$(mktemp -d)"
|
||||
base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
|
||||
# Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
|
||||
# The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
|
||||
tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
|
||||
# Anchor to end-of-line and save under the real asset name: checksums.txt also lists
|
||||
# <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
|
||||
# so an unanchored match fails on a file that was never fetched.
|
||||
( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
|
||||
curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
|
||||
( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
|
||||
echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
|
||||
rm -rf "$tmpd"; exit 1
|
||||
}
|
||||
tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
|
||||
tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
|
||||
install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
|
||||
rm -rf "$tmpd"
|
||||
fi
|
||||
|
|
@ -94,11 +88,7 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
|
|||
echo "==> 3/7 static seal key"
|
||||
if [ ! -f /etc/openbao/unseal.key ]; then
|
||||
# 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
|
||||
# tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
|
||||
# static seal reads this file RAW. That one trailing byte makes the key neither
|
||||
# 32 raw bytes nor clean base64, so bao refuses to start with
|
||||
# `Error configuring seal "static": unknown encoding for AES-256 key`.
|
||||
( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
|
||||
( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
|
||||
chown openbao:openbao /etc/openbao/unseal.key
|
||||
chmod 0400 /etc/openbao/unseal.key
|
||||
echo " generated /etc/openbao/unseal.key (0400 openbao)"
|
||||
|
|
@ -135,24 +125,6 @@ systemctl is-active --quiet openbao.service || {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
|
||||
# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
|
||||
# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
|
||||
# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
|
||||
# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
|
||||
# desktop, the phone — has any business reaching the secret store.
|
||||
if command -v ufw >/dev/null 2>&1; then
|
||||
if ufw status | grep -q '8200.*10\.10\.0\.2'; then
|
||||
echo " ufw: 8200/tcp from 10.10.0.2 already allowed"
|
||||
else
|
||||
ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
|
||||
comment 'vps1/dev -> openbao' >/dev/null
|
||||
echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
|
||||
fi
|
||||
else
|
||||
echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
|
||||
fi
|
||||
|
||||
export BAO_ADDR="https://127.0.0.1:8200"
|
||||
export BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
|
||||
|
|
@ -168,16 +140,9 @@ else
|
|||
echo " ############################################################"
|
||||
echo
|
||||
# -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
|
||||
# yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
|
||||
# is redundancy against loss, not separation of duty — there is one operator.
|
||||
#
|
||||
# These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
|
||||
# the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
|
||||
# (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
# minting a root token requires a token you already have. The deprecated endpoint is
|
||||
# off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
|
||||
# Consequence: revoking the last root token before another admin identity exists is
|
||||
# a one-way door. See the ordering note in the handoff below.
|
||||
# yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
|
||||
# keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
|
||||
# operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
|
||||
bao operator init -recovery-shares=3 -recovery-threshold=2
|
||||
echo
|
||||
echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
|
||||
|
|
@ -191,36 +156,22 @@ cat <<EOF
|
|||
|
||||
==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
|
||||
|
||||
Paste the initial root token printed above and run:
|
||||
Paste the initial root token (or one minted from recovery keys) and run:
|
||||
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
bash ${SELF_DIR}/bootstrap-policies.sh
|
||||
|
||||
KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
|
||||
cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
|
||||
\`bao token revoke -self\` locks you out of an otherwise healthy vault.
|
||||
|
||||
That script enables the KV mount, writes the policies, creates the AppRoles and
|
||||
installs each host's credentials. Then seed — on THIS box, where the age key already
|
||||
lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
|
||||
secrets is not for an agent to run:
|
||||
installs each host's credentials. Then, from YOUR OWN machine (it needs your age
|
||||
key, and per infra/CLAUDE.md a script that reads production secrets is not for an
|
||||
agent to run):
|
||||
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # must PASS before any flip
|
||||
|
||||
Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
|
||||
that legitimately renders it, so an environment can only be verified from its own host.
|
||||
|
||||
# here (vps2)
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
|
||||
Only once all three PASS, and only after you have a second admin identity that is not
|
||||
the root token, revoke it: bao token revoke -self
|
||||
Then revoke the root token: bao token revoke -self
|
||||
|
||||
NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
|
||||
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
|
||||
|
|
|
|||
|
|
@ -25,12 +25,8 @@
|
|||
|
||||
ui = false
|
||||
cluster_name = "thermograph"
|
||||
disable_mlock = true
|
||||
log_level = "info"
|
||||
// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
|
||||
// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
|
||||
// setting the server ignores only trains the operator to skim startup warnings,
|
||||
// which is where a real one will eventually hide. The corresponding note about
|
||||
// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
|
||||
|
||||
// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
|
||||
// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
|
||||
|
|
@ -114,21 +110,9 @@ seal "static" {
|
|||
//
|
||||
// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
|
||||
// unlinked inode and the log silently stops growing.
|
||||
// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
|
||||
// device NAME, not its type — and device settings go in `options`. Written as
|
||||
// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
|
||||
// be specified"); with type+path but a bare file_path it starts but silently
|
||||
// IGNORES the path and mode, which is the worse failure of the two.
|
||||
audit "file" {
|
||||
type = "file"
|
||||
path = "file/"
|
||||
options = {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
|
||||
audit "syslog" {
|
||||
type = "syslog"
|
||||
path = "syslog/"
|
||||
}
|
||||
audit "syslog" {}
|
||||
|
|
|
|||
|
|
@ -119,17 +119,7 @@ for target in "${TARGETS[@]}"; do
|
|||
# Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
|
||||
# values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
|
||||
# SEED time — before anything depends on it — rather than at deploy time.
|
||||
# Which renderer will consume this path decides which values are legal, so the
|
||||
# validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
|
||||
# KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
|
||||
# there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
|
||||
# render_centralis_secrets and its OpenBao counterpart, where every byte inside
|
||||
# '...' is literal — so applying the dotenv rules to it rejects data that renders
|
||||
# perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
|
||||
quoted=0
|
||||
case "$target" in centralis.*) quoted=1 ;; esac
|
||||
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
|
||||
import json, os, re, sys
|
||||
|
||||
doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
|
||||
|
|
@ -137,9 +127,6 @@ if not isinstance(doc, dict) or not doc:
|
|||
sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
|
||||
sys.exit(1)
|
||||
|
||||
# 1 => destined for a single-quoted env file; 0 => raw dotenv.
|
||||
QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
|
||||
|
||||
KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
UNSAFE = set("`$\\\"'")
|
||||
|
||||
|
|
@ -154,13 +141,12 @@ for key, val in doc.items():
|
|||
val = ""
|
||||
if not isinstance(val, str):
|
||||
val = json.dumps(val, separators=(",", ":"))
|
||||
if not QUOTED:
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
out[key] = val
|
||||
|
||||
if bad:
|
||||
|
|
|
|||
Loading…
Reference in a new issue