diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh index aaf606d..518f999 100644 --- a/infra/deploy/render-secrets-openbao.sh +++ b/infra/deploy/render-secrets-openbao.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # OpenBao source backend for render-secrets.sh. Sourced, never run directly. # -# Two functions: -# thermograph_openbao_source -> merged dotenv on STDOUT -# thermograph_render_openbao -> source, then write +# Functions: +# thermograph_openbao_source -> merged dotenv on STDOUT +# thermograph_render_openbao -> source, then write +# thermograph_render_openbao_centralis [out] -> quoted /etc/centralis.env +# +# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The +# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted — +# a different shape for a different consumer, not an inconsistency. # # render_thermograph_secrets in render-secrets.sh dispatches to the second when # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path @@ -113,9 +118,16 @@ thermograph_openbao_source() { # /etc/thermograph.env is sourced by bash in six places (deploy.sh:133, # deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so # `$(...)` or a backtick in a secret is a live command-execution path on the - # deploy host. The SOPS path has always had this hazard and has never tripped it - # only because every current value happens to be alphanumeric. Refusing here - # turns a latent code-execution bug into a loud render failure. + # deploy host. Refusing here turns a latent code-execution bug into a loud + # render failure. + # + # An earlier version of this comment claimed the SOPS path had never tripped + # this "only because every current value happens to be alphanumeric". That is + # false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never + # tripped THESE paths because it is not in them — it lives at + # centralis/ and renders to /etc/centralis.env, which is single-quoted + # precisely because raw dotenv cannot carry it. See + # thermograph_render_openbao_centralis at the end of this file. THERMOGRAPH_BAO_COMMON="$common_json" \ THERMOGRAPH_BAO_ENV="$env_json" \ python3 - <<'PY' @@ -243,3 +255,239 @@ thermograph_render_openbao() { rm -f "$tmp" return "$rc" } + +# --------------------------------------------------------------------------- +# CENTRALIS +# --------------------------------------------------------------------------- +# /etc/centralis.env is a different file, with a different consumer, and therefore a +# different output shape. render-secrets.sh:206-233 carries the full argument; the +# short version is that it is consumed by exactly one thing and that thing is a shell: +# +# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up' +# +# `set -a; . file` runs every line through bash's whole expansion pipeline — quote +# removal, parameter expansion, command substitution, field splitting. Raw dotenv is +# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered +# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result +# by serving a single `shared` identity — indistinguishable from the file never having +# been written. That is the 2026-07-24 incident. So this path emits POSIX +# single-quoted assignments and PROVES them by sourcing the result in a clean shell +# before anything touches $dest. +# +# This mirrors render_centralis_secrets rather than sharing with it, for the same +# reason given above thermograph_render_openbao: the SOPS path stays byte-identical +# and all new risk stays in this file. Consolidate the two when the SOPS path is being +# deleted anyway, not before. + +# _thermograph_openbao_login -> token on STDOUT +# +# Separate from the login inline in thermograph_openbao_source deliberately: that +# function is now exercised by prod and beta parity and is left untouched. +_thermograph_openbao_login() { + local approle="$1" addr="$2" ca="$3" + local creds role_id secret_id + if [ -r "$approle" ]; then + creds=$(cat "$approle") + else + creds=$(sudo cat "$approle" 2>/dev/null || true) + fi + [ -n "$creds" ] || { + echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2 + return 1 + } + role_id=$(printf '%s\n' "$creds" | sed -n '1p') + secret_id=$(printf '%s\n' "$creds" | sed -n '2p') + [ -n "$role_id" ] && [ -n "$secret_id" ] || { + echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2 + return 1 + } + export BAO_ADDR="$addr" + [ -f "$ca" ] && export BAO_CACERT="$ca" + bao write -field=token auth/approle/login \ + role_id="$role_id" secret_id="$secret_id" 2>/dev/null +} + +# thermograph_render_openbao_centralis [dest] +thermograph_render_openbao_centralis() { + local env_name="${1:?env name required}" + local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}" + local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" + local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" + local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" + local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}" + + command -v bao >/dev/null 2>&1 || { + echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2 + return 1 + } + command -v python3 >/dev/null 2>&1 || { + echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2 + return 1 + } + + echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})" + + local token + token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1 + [ -n "$token" ] || { + echo "!! OpenBao AppRole login failed for centralis" >&2 + echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2 + return 1 + } + export BAO_TOKEN="$token" + + # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every + # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap + # persists into the caller's shell and re-fires on its next `source`. + local work rc=0 + work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; } + chmod 700 "$work" + + # Subshell so no failure path can skip the cleanup, and so umask cannot leak out. + # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound + # command that is the left operand of `||`, which this subshell is. + ( + umask 077 + + bao kv get -format=json -mount="$mount" "centralis/${env_name}" \ + > "$work/kv.json" 2>/dev/null || { + echo "!! cannot read ${mount}/centralis/${env_name}" >&2 + exit 1 + } + + python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1 +import json, sys +doc = json.load(open(sys.argv[1], encoding="utf-8")) +data = doc.get("data", {}).get("data", {}) or {} +if not data: + sys.stderr.write("!! OpenBao returned zero keys for centralis\n") + sys.exit(1) +json.dump(data, open(sys.argv[2], "w", encoding="utf-8")) +UNWRAP + + python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1 +import json, re, sys + +src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3] +NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +def shell_quote(v): + """POSIX single-quoting. Inside '...' every byte is literal except ' itself, + which is closed, backslash-escaped and reopened. There is no escape sequence + to get wrong and no character class to keep up to date: it is total, and + total is the only property worth having for a file bash will expand.""" + return "'" + v.replace("'", "'\\''") + "'" + + +def scalar(k, v): + if isinstance(v, str): + return v + if isinstance(v, bool): # before int: bool is an int in Python + return "true" if v else "false" + if isinstance(v, (int, float)): + return json.dumps(v) + if v is None: + return "" + sys.stderr.write( + "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__)) + sys.exit(1) + + +data = json.load(open(src, encoding="utf-8")) +want, lines = {}, [] +for k, v in data.items(): + if not NAME.match(k): + sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,)) + sys.exit(1) + want[k] = scalar(k, v) + lines.append("%s=%s\n" % (k, shell_quote(want[k]))) + +with open(out_env, "w", encoding="utf-8") as fh: + fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n") + fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n") + fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n") + fh.write("# Rotate with `bao kv put` against thermograph/centralis/ instead.\n") + fh.writelines(lines) +with open(want_json, "w", encoding="utf-8") as fh: + json.dump(want, fh) +sys.stderr.write(" %d keys quoted\n" % len(want)) +EMIT + + # PROVE IT. Source the rendered file exactly the way the deploy does — clean + # environment, `set -a`, nothing inherited that could mask a dropped key — and + # read every value back. This is the check the 2026-07-24 file could not pass. + # shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded + # by the inner `bash -c` against the argument after `_`, not by this shell. + env -i PATH="$PATH" bash -c \ + 'set -a; . "$1"; set +a; exec python3 -c " +import json, os, sys +sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \ + > "$work/got.json" || exit 1 + + python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1 +import json, sys + +want = json.load(open(sys.argv[1], encoding="utf-8")) +got = json.load(open(sys.argv[2], encoding="utf-8")) + +bad = [] +for k, v in want.items(): + if k not in got: + bad.append("%s: not set at all after sourcing" % k) + elif got[k] != v: + # NEVER print either value. The difference is the finding; the content is + # not, and this runs inside a deploy log. + bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)" + % (k, len(v), len(got[k]))) +if bad: + sys.stderr.write("!! the rendered file does not round-trip through bash:\n") + for b in bad: + sys.stderr.write("!! %s\n" % b) + sys.exit(1) + +# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes, +# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the +# single `shared` identity, which looks exactly like the registry never having been +# configured. Refuse to write a file that would do that. +raw = got.get("CENTRALIS_TOKENS", "") +if raw: + try: + reg = json.loads(raw) + except ValueError as exc: + sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc) + sys.exit(1) + if not isinstance(reg, dict) or not reg: + sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n") + sys.exit(1) + if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()): + sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n") + sys.exit(1) + # Subjects are names on audit lines, not credentials. Printing them is the whole + # point: it is how an operator sees at a glance that nobody was lost. + sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n" + % (len(reg), ", ".join(sorted(reg)))) + +sys.stderr.write(" %d keys survive `set -a; . ` byte-for-byte\n" % len(want)) +VERIFY + ) || rc=1 + + # Only now, with a file read back through bash and matched value for value, does + # anything touch $dest. 0600: four bearer credentials for the estate's control + # plane, with no group that needs them. + if [ "$rc" -eq 0 ]; then + if [ -f "$dest" ] && [ -w "$dest" ]; then + cat "$work/out.env" > "$dest" || rc=1 + elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then : + elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then : + else + echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2 + rc=1 + fi + else + echo "!! render aborted; ${dest} left untouched" >&2 + fi + + rm -rf "$work" + return "$rc" +} diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index 32093d9..ea7de00 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -272,6 +272,30 @@ render_centralis_secrets() { echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2 return 1 fi + + # Same early-return dispatch as render_thermograph_secrets, for the same reason + # given there: everything below — the age-key sudo lift, the JSON decrypt, the + # quote-and-prove pipeline, the three-way write — stays on exactly the code path it + # has always been on, so this cannot regress the backend still authoritative for + # Centralis. The OpenBao path needs no age key, so it returns before that check. + local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}" + if [ "$backend" = openbao ]; then + local bao_lib="${repo}/deploy/render-secrets-openbao.sh" + if [ ! -f "$bao_lib" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2 + echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2 + return 1 + fi + # shellcheck source=/dev/null + . "$bao_lib" + thermograph_render_openbao_centralis "$env_name" "$dest" + return + fi + if [ "$backend" != sops ]; then + echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2 + return 1 + fi + if [ ! -f "$key" ]; then echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2 return 1 diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 855e4d7..14c6ac8 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -112,6 +112,22 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv Inheritance lives in the render order (`common` then `env/`, last-wins); isolation lives in the policy. Today both live in one shell variable. +**`centralis/` renders differently from everything above it, and must.** The +app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env` +is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion +pipeline, so it is emitted as POSIX single-quoted assignments and then verified by +sourcing it in a clean shell and comparing every value back before the file is +written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash +strips its quotes, Centralis fails closed on the malformed result and serves a +single `shared` identity, which is indistinguishable from the file never having +been written. That is the 2026-07-24 incident, and the round-trip check exists so a +render carrying it cannot reach `/etc`. + +Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell +metacharacters, no newlines) only to `common` and `env/*`. Applying them to +`centralis/*` would reject data that renders perfectly well — the validator matches +the renderer that will consume the path, not a single global rule. + `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny rule expressible as exactly one path, with no wildcard over `env/*` able to accidentally re-include it. diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh index 09f32cc..3988ca7 100755 --- a/infra/openbao/seed-from-sops.sh +++ b/infra/openbao/seed-from-sops.sh @@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at # SEED time — before anything depends on it — rather than at deploy time. - reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY' + # Which renderer will consume this path decides which values are legal, so the + # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED + # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter + # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by + # render_centralis_secrets and its OpenBao counterpart, where every byte inside + # '...' is literal — so applying the dotenv rules to it rejects data that renders + # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them. + quoted=0 + case "$target" in centralis.*) quoted=1 ;; esac + + reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY' import json, os, re, sys doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"]) @@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc: sys.stderr.write("!! vault did not decrypt to a non-empty object\n") sys.exit(1) +# 1 => destined for a single-quoted env file; 0 => raw dotenv. +QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1" + KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") UNSAFE = set("`$\\\"'") @@ -141,12 +154,13 @@ for key, val in doc.items(): val = "" if not isinstance(val, str): val = json.dumps(val, separators=(",", ":")) - if "\n" in val or "\r" in val: - bad.append(f"{key}: contains a newline") - continue - if UNSAFE & set(val): - bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") - continue + if not QUOTED: + if "\n" in val or "\r" in val: + bad.append(f"{key}: contains a newline") + continue + if UNSAFE & set(val): + bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") + continue out[key] = val if bad: