diff --git a/infra/deploy/env-topology.sh b/infra/deploy/env-topology.sh index 9e2b7c9..d57d148 100644 --- a/infra/deploy/env-topology.sh +++ b/infra/deploy/env-topology.sh @@ -76,6 +76,16 @@ thermograph_topology() { # for the reason that marker was demoted to a fallback in the first place: vps2 runs # prod and beta side by side, and one host-wide file cannot name two backends. TG_SECRETS_BACKEND=sops + # Which AppRole credential file render-secrets-openbao.sh authenticates with. Same + # reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO + # environments, so a single host-wide default cannot serve both. prod and dev each + # get the conventional path (dev is alone on vps1, so there is no collision there); + # beta is the one that must differ, because it shares a filesystem with prod. + # + # Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl + # correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that + # runs prod. + TG_BAO_APPROLE=/etc/thermograph/openbao-approle case "$env_name" in prod) @@ -116,6 +126,10 @@ thermograph_topology() { # therefore prod's blast radius too (see .claude/hooks/prod-guard.sh). TG_SSH_HOST=169.58.46.181 TG_SSH_TARGET=agent@169.58.46.181 + # The one environment that cannot use the default AppRole path: it shares a + # filesystem with prod, so it needs its own credential file to authenticate as + # tg-beta rather than tg-prod. bootstrap-policies.sh installs it here. + TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta # A SECOND checkout on the same box. Separate from prod's so the two can # sit on different commits of this repo, and so `git reset --hard` in one # deploy can never yank the tree out from under the other's running @@ -228,7 +242,7 @@ thermograph_topology() { export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY - export TG_SECRETS_BACKEND + export TG_SECRETS_BACKEND TG_BAO_APPROLE export TG_SSH_HOST TG_SSH_TARGET } diff --git a/infra/deploy/render-secrets-openbao.sh b/infra/deploy/render-secrets-openbao.sh index 82c449c..518f999 100644 --- a/infra/deploy/render-secrets-openbao.sh +++ b/infra/deploy/render-secrets-openbao.sh @@ -1,9 +1,14 @@ #!/usr/bin/env bash # OpenBao source backend for render-secrets.sh. Sourced, never run directly. # -# Two functions: -# thermograph_openbao_source -> merged dotenv on STDOUT -# thermograph_render_openbao -> source, then write +# Functions: +# thermograph_openbao_source -> merged dotenv on STDOUT +# thermograph_render_openbao -> source, then write +# thermograph_render_openbao_centralis [out] -> quoted /etc/centralis.env +# +# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The +# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted — +# a different shape for a different consumer, not an inconsistency. # # render_thermograph_secrets in render-secrets.sh dispatches to the second when # TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path @@ -27,7 +32,13 @@ thermograph_openbao_source() { local env_name="${1:?env name required}" local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" - local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}" + # Explicit override first, then the per-environment value env-topology.sh derives, + # then the conventional path. Same precedence shape as render-secrets.sh:44's + # THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the + # THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the + # deploy path sets. Falling straight through to the bare default is what made beta + # authenticate as tg-prod and get denied its own path. + local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}" local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" command -v bao >/dev/null 2>&1 || { @@ -107,9 +118,16 @@ thermograph_openbao_source() { # /etc/thermograph.env is sourced by bash in six places (deploy.sh:133, # deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so # `$(...)` or a backtick in a secret is a live command-execution path on the - # deploy host. The SOPS path has always had this hazard and has never tripped it - # only because every current value happens to be alphanumeric. Refusing here - # turns a latent code-execution bug into a loud render failure. + # deploy host. Refusing here turns a latent code-execution bug into a loud + # render failure. + # + # An earlier version of this comment claimed the SOPS path had never tripped + # this "only because every current value happens to be alphanumeric". That is + # false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never + # tripped THESE paths because it is not in them — it lives at + # centralis/ and renders to /etc/centralis.env, which is single-quoted + # precisely because raw dotenv cannot carry it. See + # thermograph_render_openbao_centralis at the end of this file. THERMOGRAPH_BAO_COMMON="$common_json" \ THERMOGRAPH_BAO_ENV="$env_json" \ python3 - <<'PY' @@ -237,3 +255,239 @@ thermograph_render_openbao() { rm -f "$tmp" return "$rc" } + +# --------------------------------------------------------------------------- +# CENTRALIS +# --------------------------------------------------------------------------- +# /etc/centralis.env is a different file, with a different consumer, and therefore a +# different output shape. render-secrets.sh:206-233 carries the full argument; the +# short version is that it is consumed by exactly one thing and that thing is a shell: +# +# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up' +# +# `set -a; . file` runs every line through bash's whole expansion pipeline — quote +# removal, parameter expansion, command substitution, field splitting. Raw dotenv is +# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered +# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result +# by serving a single `shared` identity — indistinguishable from the file never having +# been written. That is the 2026-07-24 incident. So this path emits POSIX +# single-quoted assignments and PROVES them by sourcing the result in a clean shell +# before anything touches $dest. +# +# This mirrors render_centralis_secrets rather than sharing with it, for the same +# reason given above thermograph_render_openbao: the SOPS path stays byte-identical +# and all new risk stays in this file. Consolidate the two when the SOPS path is being +# deleted anyway, not before. + +# _thermograph_openbao_login -> token on STDOUT +# +# Separate from the login inline in thermograph_openbao_source deliberately: that +# function is now exercised by prod and beta parity and is left untouched. +_thermograph_openbao_login() { + local approle="$1" addr="$2" ca="$3" + local creds role_id secret_id + if [ -r "$approle" ]; then + creds=$(cat "$approle") + else + creds=$(sudo cat "$approle" 2>/dev/null || true) + fi + [ -n "$creds" ] || { + echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2 + return 1 + } + role_id=$(printf '%s\n' "$creds" | sed -n '1p') + secret_id=$(printf '%s\n' "$creds" | sed -n '2p') + [ -n "$role_id" ] && [ -n "$secret_id" ] || { + echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2 + return 1 + } + export BAO_ADDR="$addr" + [ -f "$ca" ] && export BAO_CACERT="$ca" + bao write -field=token auth/approle/login \ + role_id="$role_id" secret_id="$secret_id" 2>/dev/null +} + +# thermograph_render_openbao_centralis [dest] +thermograph_render_openbao_centralis() { + local env_name="${1:?env name required}" + local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}" + local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}" + local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}" + local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}" + local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}" + + command -v bao >/dev/null 2>&1 || { + echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2 + return 1 + } + command -v python3 >/dev/null 2>&1 || { + echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2 + return 1 + } + + echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})" + + local token + token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1 + [ -n "$token" ] || { + echo "!! OpenBao AppRole login failed for centralis" >&2 + echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2 + return 1 + } + export BAO_TOKEN="$token" + + # ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every + # path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap + # persists into the caller's shell and re-fires on its next `source`. + local work rc=0 + work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; } + chmod 700 "$work" + + # Subshell so no failure path can skip the cleanup, and so umask cannot leak out. + # Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound + # command that is the left operand of `||`, which this subshell is. + ( + umask 077 + + bao kv get -format=json -mount="$mount" "centralis/${env_name}" \ + > "$work/kv.json" 2>/dev/null || { + echo "!! cannot read ${mount}/centralis/${env_name}" >&2 + exit 1 + } + + python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1 +import json, sys +doc = json.load(open(sys.argv[1], encoding="utf-8")) +data = doc.get("data", {}).get("data", {}) or {} +if not data: + sys.stderr.write("!! OpenBao returned zero keys for centralis\n") + sys.exit(1) +json.dump(data, open(sys.argv[2], "w", encoding="utf-8")) +UNWRAP + + python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1 +import json, re, sys + +src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3] +NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +def shell_quote(v): + """POSIX single-quoting. Inside '...' every byte is literal except ' itself, + which is closed, backslash-escaped and reopened. There is no escape sequence + to get wrong and no character class to keep up to date: it is total, and + total is the only property worth having for a file bash will expand.""" + return "'" + v.replace("'", "'\\''") + "'" + + +def scalar(k, v): + if isinstance(v, str): + return v + if isinstance(v, bool): # before int: bool is an int in Python + return "true" if v else "false" + if isinstance(v, (int, float)): + return json.dumps(v) + if v is None: + return "" + sys.stderr.write( + "!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__)) + sys.exit(1) + + +data = json.load(open(src, encoding="utf-8")) +want, lines = {}, [] +for k, v in data.items(): + if not NAME.match(k): + sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,)) + sys.exit(1) + want[k] = scalar(k, v) + lines.append("%s=%s\n" % (k, shell_quote(want[k]))) + +with open(out_env, "w", encoding="utf-8") as fh: + fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n") + fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n") + fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n") + fh.write("# Rotate with `bao kv put` against thermograph/centralis/ instead.\n") + fh.writelines(lines) +with open(want_json, "w", encoding="utf-8") as fh: + json.dump(want, fh) +sys.stderr.write(" %d keys quoted\n" % len(want)) +EMIT + + # PROVE IT. Source the rendered file exactly the way the deploy does — clean + # environment, `set -a`, nothing inherited that could mask a dropped key — and + # read every value back. This is the check the 2026-07-24 file could not pass. + # shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded + # by the inner `bash -c` against the argument after `_`, not by this shell. + env -i PATH="$PATH" bash -c \ + 'set -a; . "$1"; set +a; exec python3 -c " +import json, os, sys +sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \ + > "$work/got.json" || exit 1 + + python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1 +import json, sys + +want = json.load(open(sys.argv[1], encoding="utf-8")) +got = json.load(open(sys.argv[2], encoding="utf-8")) + +bad = [] +for k, v in want.items(): + if k not in got: + bad.append("%s: not set at all after sourcing" % k) + elif got[k] != v: + # NEVER print either value. The difference is the finding; the content is + # not, and this runs inside a deploy log. + bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)" + % (k, len(v), len(got[k]))) +if bad: + sys.stderr.write("!! the rendered file does not round-trip through bash:\n") + for b in bad: + sys.stderr.write("!! %s\n" % b) + sys.exit(1) + +# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes, +# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the +# single `shared` identity, which looks exactly like the registry never having been +# configured. Refuse to write a file that would do that. +raw = got.get("CENTRALIS_TOKENS", "") +if raw: + try: + reg = json.loads(raw) + except ValueError as exc: + sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc) + sys.exit(1) + if not isinstance(reg, dict) or not reg: + sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n") + sys.exit(1) + if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()): + sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n") + sys.exit(1) + # Subjects are names on audit lines, not credentials. Printing them is the whole + # point: it is how an operator sees at a glance that nobody was lost. + sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n" + % (len(reg), ", ".join(sorted(reg)))) + +sys.stderr.write(" %d keys survive `set -a; . ` byte-for-byte\n" % len(want)) +VERIFY + ) || rc=1 + + # Only now, with a file read back through bash and matched value for value, does + # anything touch $dest. 0600: four bearer credentials for the estate's control + # plane, with no group that needs them. + if [ "$rc" -eq 0 ]; then + if [ -f "$dest" ] && [ -w "$dest" ]; then + cat "$work/out.env" > "$dest" || rc=1 + elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then : + elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then : + else + echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2 + rc=1 + fi + else + echo "!! render aborted; ${dest} left untouched" >&2 + fi + + rm -rf "$work" + return "$rc" +} diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index 32093d9..ea7de00 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -272,6 +272,30 @@ render_centralis_secrets() { echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2 return 1 fi + + # Same early-return dispatch as render_thermograph_secrets, for the same reason + # given there: everything below — the age-key sudo lift, the JSON decrypt, the + # quote-and-prove pipeline, the three-way write — stays on exactly the code path it + # has always been on, so this cannot regress the backend still authoritative for + # Centralis. The OpenBao path needs no age key, so it returns before that check. + local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}" + if [ "$backend" = openbao ]; then + local bao_lib="${repo}/deploy/render-secrets-openbao.sh" + if [ ! -f "$bao_lib" ]; then + echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2 + echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2 + return 1 + fi + # shellcheck source=/dev/null + . "$bao_lib" + thermograph_render_openbao_centralis "$env_name" "$dest" + return + fi + if [ "$backend" != sops ]; then + echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2 + return 1 + fi + if [ ! -f "$key" ]; then echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2 return 1 diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md index 4b85e59..7ec4bb3 100644 --- a/infra/deploy/secrets/README.md +++ b/infra/deploy/secrets/README.md @@ -251,17 +251,44 @@ files), so layering would only push the VAPID private key, both S3 keypairs and - `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time. - **Back it up** in the password manager. The public recipient is in `.sops.yaml`. -On **vps1** (dev) those two can end up being the same file. `render-secrets.sh` -falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes -the account driving a CI-triggered dev deploy has no passwordless sudo and no -tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) — -inherited from the old desktop setup, where that was true of the whole box. -Whether it's still true of vps1's CI-runner account specifically (as opposed -to the `agent` login, which does have passwordless sudo there per `ACCESS.md`) -wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY` -at the operator's keyring as a fallback either way, so the box keeps **one** -copy of the recovery root rather than two regardless of which account ends up -mattering. +### The two on-host copies are a deliberate quorum + +`/etc/thermograph/age.key` on **vps1** and **vps2** is not provisioning residue and +must not be tidied away. Together they are the recovery quorum for the entire +estate: five SOPS vaults, and every off-box backup, since `ops-cron.yml:142,197` +pipe those dumps through `age -r` to this same recipient. + +That is not theoretical. On **2026-07-30** the operator's copy was shredded from the +desktop *before* it had been copied to its replacement machine, leaving zero +operator-side copies. These two were the only surviving material, and the key was +restored from vps2. The ordering rule below exists because of that hour. + +- **Never remove the last operator-side copy** until its replacement has been + verified against all five vaults (`sops -d … | grep -c '='` → 16 / 12 / 8 / 16 / 9). + Deleting first and verifying second is unrecoverable if the copy is bad. +- **Both hosts must hold byte-identical copies.** Divergence is as bad as absence: + two different keys means one host renders secrets the other cannot read, and a + restore silently picks the wrong one. +- **Verify it, don't assume it** — `deploy/secrets/verify-age-quorum.sh` asserts + presence, permissions and hash equality across both hosts. It prints SHA-256 + digests and modes only, never the key, so it is safe in a CI log and suitable as + a cron gate. Deletion and divergence are both silent failures; nothing else in the + estate would notice either until a restore. + +Verified 2026-07-30: vps1 `0440 root:deploy`, vps2 `0400 root:root`, digests equal. +Note the asymmetry — on vps1 the group `deploy` can read the key that decrypts +`common.yaml` and `prod.yaml`, on the box that also runs Forgejo, its CI runner and +dev's unreviewed branches. The dev render does not need that group bit: it runs as +`agent`, which is not in group `deploy` and reaches the key through its passwordless +`sudo` instead (`/opt/thermograph-dev` is `agent:agent`). Tightening vps1 to `0400 +root:root` is therefore expected to be safe and is the obvious follow-up; it is +called out rather than done here because it changes a live host on the deploy path. + +This weakens — in practice, not in intent — the rule in `infra/CLAUDE.md` that "vps1 +must never hold prod credentials". Withholding `common.yaml` from dev's *render* does +not withhold the key that decrypts it from the *box*, because every vault is +encrypted to a single recipient. Fixing that properly means a second age identity for +dev, not a policy line. ## Prerequisites (once per machine) diff --git a/infra/deploy/secrets/common.yaml b/infra/deploy/secrets/common.yaml index 83ec094..247789d 100644 --- a/infra/deploy/secrets/common.yaml +++ b/infra/deploy/secrets/common.yaml @@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7 THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str] THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str] THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str] -THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str] +THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str] THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str] THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str] THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str] THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str] -THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str] -THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str] +THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str] +THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str] THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str] THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str] TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str] @@ -25,7 +25,7 @@ sops: bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-25T00:38:24Z" - mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str] + lastmodified: "2026-07-31T04:23:12Z" + mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/secrets/verify-age-quorum.sh b/infra/deploy/secrets/verify-age-quorum.sh new file mode 100755 index 0000000..8bb36ba --- /dev/null +++ b/infra/deploy/secrets/verify-age-quorum.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Assert that the age recovery quorum is intact. +# +# infra/deploy/secrets/verify-age-quorum.sh +# +# The age private key is the single recovery root for all five SOPS vaults AND for +# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same +# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE +# two-host quorum, not provisioning residue — see the "age key" section of +# deploy/secrets/README.md. This script is what makes that policy checkable instead of +# merely stated. +# +# It fails if the key is missing on either host, if the two copies have diverged, or +# if either is more permissive than 0440. Divergence matters as much as absence: two +# hosts holding different keys means one of them renders secrets nobody else can +# decrypt, and a restore from backup silently picks the wrong one. +# +# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a +# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is +# never read, echoed or transferred. Nothing here copies the key anywhere. +# +# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate. +set -euo pipefail + +VPS1="${TG_VPS1_SSH:-vps1}" +VPS2="${TG_VPS2_SSH:-vps2}" +KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}" +SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}" + +# Reads the digest and mode of the key on one host. Uses sudo when the key is not +# directly readable, mirroring how render-secrets.sh lifts it. +probe() { + local target="$1" out + # The remote script is a QUOTED heredoc and the key path is passed as $1 to + # `bash -s`, so nothing expands on this side. Interpolating the path into the + # command string would work too, but it is the shape that quietly breaks the day + # a path contains a space, and shellcheck is right to flag it (SC2029). + # shellcheck disable=SC2086 # SSH_OPTS is a deliberate word-split option list + out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE' +key="$1" +if [ ! -e "$key" ]; then echo MISSING; exit 0; fi +mode=$(stat -c %a "$key") +owner=$(stat -c %U:%G "$key") +if [ -r "$key" ]; then + digest=$(sha256sum "$key" | cut -d' ' -f1) +else + digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1) +fi +[ -n "$digest" ] || { echo UNREADABLE; exit 0; } +echo "$digest $mode $owner" +REMOTE + ) || { echo UNREACHABLE; return 0; } + printf '%s\n' "$out" +} + +rc=0 +declare -A DIGEST + +for pair in "vps1:$VPS1" "vps2:$VPS2"; do + name="${pair%%:*}" + target="${pair#*:}" + result="$(probe "$target")" + case "$result" in + MISSING) + echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2 + rc=1 + ;; + UNREADABLE) + echo "!! ${name}: key present but unreadable even via sudo" >&2 + rc=1 + ;; + UNREACHABLE) + echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2 + rc=1 + ;; + *) + digest="${result%% *}" + rest="${result#* }" + DIGEST["$name"]="$digest" + printf ' %-5s %s mode=%s\n' "$name" "${digest:0:16}…" "$rest" + mode="${rest%% *}" + # 0400 or 0440 only. Anything wider means a non-root account on that box can + # read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner. + case "$mode" in + 400|440) ;; + *) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;; + esac + # Advisory, not a failure: 0440 with a non-root group means that group can read + # the key that decrypts prod. On vps1 that is doubly worth knowing, because the + # same box runs Forgejo, its CI runner and dev's unreviewed branches. Left + # non-fatal because it is the current provisioned state — a check that fails on + # day one is a check that gets ignored by day three. See README.md. + group="${rest#* }"; group="${group#*:}" + if [ "$mode" = 440 ] && [ "$group" != root ]; then + printf ' note: group %s can read this key\n' "$group" + fi + ;; + esac +done + +if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then + if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then + echo " quorum: 2 copies, identical" + else + echo "!! the two copies have DIVERGED — they are different keys" >&2 + echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2 + echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2 + rc=1 + fi +fi + +if [ "$rc" -eq 0 ]; then + echo " OK — recovery quorum intact" +else + echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2 +fi +exit "$rc" diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 196784a..14c6ac8 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -112,6 +112,22 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv Inheritance lives in the render order (`common` then `env/`, last-wins); isolation lives in the policy. Today both live in one shell variable. +**`centralis/` renders differently from everything above it, and must.** The +app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env` +is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion +pipeline, so it is emitted as POSIX single-quoted assignments and then verified by +sourcing it in a clean shell and comparing every value back before the file is +written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash +strips its quotes, Centralis fails closed on the malformed result and serves a +single `shared` identity, which is indistinguishable from the file never having +been written. That is the 2026-07-24 incident, and the round-trip check exists so a +render carrying it cannot reach `/etc`. + +Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell +metacharacters, no newlines) only to `common` and `env/*`. Applying them to +`centralis/*` would reject data that renders perfectly well — the validator matches +the renderer that will consume the path, not a single global rule. + `common` is a top-level path rather than `env/common` on purpose: it makes the dev deny rule expressible as exactly one path, with no wildcard over `env/*` able to accidentally re-include it. @@ -148,22 +164,52 @@ SOPS and retiring age are two different projects. ### Stand it up (once, by the operator, on vps2) ```sh -sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init +sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init # custody the recovery keys + /etc/openbao/unseal.key OFF the box, then: -export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt +export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt export BAO_TOKEN= sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles -bao token revoke -self ``` -### Seed and prove (from your own machine — it needs your age key) +> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated +> `/sys/generate-root` with an authenticated `/sys/generate-root-token` +> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so +> minting a root token requires a token you already have. **The recovery keys are not +> a way back in**; they rekey, they do not authenticate. Unless +> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking +> the last root token locks you out of an otherwise healthy, unsealed vault, and the +> only remedy is re-initialising from scratch. Revoke only once a second admin +> identity exists. + +Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than +`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700 +openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails +with `certificate signed by unknown authority`. + +### Seed and prove (on vps2 — the age key is already there) ```sh +export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing infra/openbao/seed-from-sops.sh --all -infra/openbao/verify-parity.sh --all # MUST pass before any flip ``` +Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs` +to the host that legitimately renders it, so an environment can only be verified from +its own host: + +```sh +# on vps2 +infra/openbao/verify-parity.sh --env prod # expect 32 keys +infra/openbao/verify-parity.sh --env beta # expect 24 keys +# on vps1 +cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys +``` + +Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per +environment, so beta authenticates as `tg-beta` rather than falling back to prod's +credentials and being denied its own path by `tg-host-prod.hcl`. + `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this inherits that rule. diff --git a/infra/openbao/bootstrap-policies.sh b/infra/openbao/bootstrap-policies.sh index 81d07bd..2104e51 100755 --- a/infra/openbao/bootstrap-policies.sh +++ b/infra/openbao/bootstrap-policies.sh @@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis 10.10.0.1/32 # Install the local (vps2) credentials. prod and beta both render on this box. # -# The OWNERSHIP here is a real boundary that does not exist today, and it is the one -# concrete isolation win available on a shared host. render-secrets.sh:119-124 records -# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the -# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation -# at all. Giving each environment its own 0400 secret-id owned by its own deploy user -# creates one. +# Both files are owned by `agent`, because that is the single identity both +# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for +# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no +# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's +# file to `deploy:deploy` on the theory that beta and prod deploy as different users; +# they do not, and the chown simply failed. # -# Be honest about the limit: root on vps2 reads both files, exactly as root today reads -# the one age key. This defends against MISCONFIGURATION, not against intrusion. +# So be accurate about what separate credentials buy on this box: NOT deploy-user +# isolation — there is one deploy user, and it can read both files. What they buy is +# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a +# *mistake* crossing between environments: a beta render authenticating with beta's +# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong +# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's +# database password. Real deploy-user isolation would need a `deploy` user on vps2 and +# a separate SSH credential for beta in deploy.yml — a larger change than this script. install_creds() { local role="$1" dest="$2" owner="$3" local rid sid @@ -79,7 +85,17 @@ install_creds() { sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id") # umask before creation, so the file is never briefly world-readable. ( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" ) - chown "$owner" "$dest" + # Abort the function if chown fails, and remove the half-installed file. The call + # site wraps this in `|| echo`, which suppresses `set -e` for everything inside — + # so without this check a failed chown falls straight through to chmod and prints a + # line asserting an ownership that was never applied. A root-owned credential the + # renderer cannot read, reported as installed, is worse than no credential at all: + # it fails at deploy time instead of here. + if ! chown "$owner" "$dest"; then + rm -f "$dest" + echo " !! chown $owner failed for $dest — removed, nothing installed" >&2 + return 1 + fi chmod 0400 "$dest" echo " installed $dest (0400 $owner)" } @@ -87,8 +103,7 @@ install_creds() { if [ "$(id -u)" = 0 ]; then install -d -o root -g root -m 0755 /etc/thermograph install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent" - install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \ - || echo " (beta creds skipped — no 'deploy' user on this box?)" + install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent" install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root" else echo "!! not root; skipping credential install. Re-run with sudo -E." >&2 diff --git a/infra/openbao/bootstrap.sh b/infra/openbao/bootstrap.sh index 39fc5e6..6fa199d 100755 --- a/infra/openbao/bootstrap.sh +++ b/infra/openbao/bootstrap.sh @@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then # The checksums file is signed by the release; verify before installing. tmpd="$(mktemp -d)" base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}" - curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz" - curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS" - ( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || { + # Asset names are `openbao__...`, and the checksums file is `checksums.txt`. + # The previous `bao__...` / `bao__SHA256SUMS` names both 404. + tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}" + curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt" + # Anchor to end-of-line and save under the real asset name: checksums.txt also lists + # .sbom.json, and `sha256sum -c` resolves each line by the filename IN it, + # so an unanchored match fails on a file that was never fetched. + ( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || { echo "!! checksum mismatch on the bao tarball; refusing to install" >&2 rm -rf "$tmpd"; exit 1 } - tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao + tar -xzf "$tmpd/$tarball" -C "$tmpd" bao install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao rm -rf "$tmpd" fi @@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt echo "==> 3/7 static seal key" if [ ! -f /etc/openbao/unseal.key ]; then # 32 random bytes, base64. Same protection level as /etc/thermograph/age.key. - ( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key ) + # tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the + # static seal reads this file RAW. That one trailing byte makes the key neither + # 32 raw bytes nor clean base64, so bao refuses to start with + # `Error configuring seal "static": unknown encoding for AES-256 key`. + ( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key ) chown openbao:openbao /etc/openbao/unseal.key chmod 0400 /etc/openbao/unseal.key echo " generated /etc/openbao/unseal.key (0400 openbao)" @@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || { exit 1 } +# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but +# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without +# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao +# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the +# AppRole CIDR bindings are the real control, and nothing else on the mesh — the +# desktop, the phone — has any business reaching the secret store. +if command -v ufw >/dev/null 2>&1; then + if ufw status | grep -q '8200.*10\.10\.0\.2'; then + echo " ufw: 8200/tcp from 10.10.0.2 already allowed" + else + ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \ + comment 'vps1/dev -> openbao' >/dev/null + echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)" + fi +else + echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2 +fi + export BAO_ADDR="https://127.0.0.1:8200" export BAO_CACERT=/etc/openbao/tls/bao.crt @@ -140,9 +168,16 @@ else echo " ############################################################" echo # -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init - # yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal - # keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one - # operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token. + # yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here + # is redundancy against loss, not separation of duty — there is one operator. + # + # These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced + # the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token + # (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so + # minting a root token requires a token you already have. The deprecated endpoint is + # off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl. + # Consequence: revoking the last root token before another admin identity exists is + # a one-way door. See the ordering note in the handoff below. bao operator init -recovery-shares=3 -recovery-threshold=2 echo echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are" @@ -156,22 +191,36 @@ cat < 6/7 and 7/7 need a root token, which is NOT stored anywhere by design. -Paste the initial root token (or one minted from recovery keys) and run: +Paste the initial root token printed above and run: export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt export BAO_TOKEN= bash ${SELF_DIR}/bootstrap-policies.sh +KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys +cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature +\`bao token revoke -self\` locks you out of an otherwise healthy vault. + That script enables the KV mount, writes the policies, creates the AppRoles and -installs each host's credentials. Then, from YOUR OWN machine (it needs your age -key, and per infra/CLAUDE.md a script that reads production secrets is not for an -agent to run): +installs each host's credentials. Then seed — on THIS box, where the age key already +lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production +secrets is not for an agent to run: - infra/openbao/seed-from-sops.sh --dry-run # check first + export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key + infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys infra/openbao/seed-from-sops.sh --all - infra/openbao/verify-parity.sh --all # must PASS before any flip -Then revoke the root token: bao token revoke -self +Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host +that legitimately renders it, so an environment can only be verified from its own host. + + # here (vps2) + infra/openbao/verify-parity.sh --env prod # expect 32 keys + infra/openbao/verify-parity.sh --env beta # expect 24 keys + # on vps1 + cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys + +Only once all three PASS, and only after you have a second admin identity that is not +the root token, revoke it: bao token revoke -self NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative diff --git a/infra/openbao/config/openbao.hcl b/infra/openbao/config/openbao.hcl index 07f4d87..b9175de 100644 --- a/infra/openbao/config/openbao.hcl +++ b/infra/openbao/config/openbao.hcl @@ -25,8 +25,12 @@ ui = false cluster_name = "thermograph" -disable_mlock = true log_level = "info" +// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and +// logs `unknown or unsupported field disable_mlock` on every start. Carrying a +// setting the server ignores only trains the operator to skim startup warnings, +// which is where a real one will eventually hide. The corresponding note about +// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason. // STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0 // DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys @@ -110,9 +114,21 @@ seal "static" { // // logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an // unlinked inode and the log silently stops growing. +// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the +// device NAME, not its type — and device settings go in `options`. Written as +// `audit "file" { file_path = ... }` the server refuses to start ("audit type must +// be specified"); with type+path but a bare file_path it starts but silently +// IGNORES the path and mode, which is the worse failure of the two. audit "file" { - file_path = "/var/log/openbao/audit.log" - mode = "0600" + type = "file" + path = "file/" + options = { + file_path = "/var/log/openbao/audit.log" + mode = "0600" + } } -audit "syslog" {} +audit "syslog" { + type = "syslog" + path = "syslog/" +} diff --git a/infra/openbao/seed-from-sops.sh b/infra/openbao/seed-from-sops.sh index 09f32cc..3988ca7 100755 --- a/infra/openbao/seed-from-sops.sh +++ b/infra/openbao/seed-from-sops.sh @@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do # Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe # values render-secrets-openbao.sh refuses, so an unrenderable value is caught at # SEED time — before anything depends on it — rather than at deploy time. - reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY' + # Which renderer will consume this path decides which values are legal, so the + # validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED + # KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter + # there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by + # render_centralis_secrets and its OpenBao counterpart, where every byte inside + # '...' is literal — so applying the dotenv rules to it rejects data that renders + # perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them. + quoted=0 + case "$target" in centralis.*) quoted=1 ;; esac + + reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY' import json, os, re, sys doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"]) @@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc: sys.stderr.write("!! vault did not decrypt to a non-empty object\n") sys.exit(1) +# 1 => destined for a single-quoted env file; 0 => raw dotenv. +QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1" + KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") UNSAFE = set("`$\\\"'") @@ -141,12 +154,13 @@ for key, val in doc.items(): val = "" if not isinstance(val, str): val = json.dumps(val, separators=(",", ":")) - if "\n" in val or "\r" in val: - bad.append(f"{key}: contains a newline") - continue - if UNSAFE & set(val): - bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") - continue + if not QUOTED: + if "\n" in val or "\r" in val: + bad.append(f"{key}: contains a newline") + continue + if UNSAFE & set(val): + bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')") + continue out[key] = val if bad: