#!/usr/bin/env bash # Render /etc/thermograph.env from the SOPS-encrypted source of truth # (deploy/secrets/.yaml, committed encrypted) — sourced by deploy.sh and # deploy-dev.sh, not run directly. # # The encrypted files are the single source of truth; /etc/thermograph.env becomes a # rendered artifact that the existing seams (docker-compose `env_file:`, the systemd # unit's EnvironmentFile, and entrypoint.sh's /run/secrets shim) keep consuming # unchanged. Common secrets + the per-host overrides are concatenated with the host # file LAST, so a host value wins (env_file and `source` both take the last # occurrence of a duplicate key). # # Presence-detected: a host is "configured for SOPS" only when it has an age key at # /etc/thermograph/age.key AND an environment name in /etc/thermograph/secrets-env # (prod|beta|dev). A host without them keeps whatever /etc/thermograph.env it already # has — so merging this is a safe no-op until a host is deliberately migrated. See # deploy/secrets/README.md. # render_thermograph_secrets [env_name] [out_file] # # env_name and out_file are explicit since vps2 began running two environments. # A host-wide marker file cannot name two environments, and one output path # cannot hold two renders: on vps2 prod renders prod.yaml to /etc/thermograph.env # while beta renders beta.yaml to /etc/thermograph-beta.env. Both arguments keep # their pre-split defaults (marker file, /etc/thermograph.env), so a # single-environment host and a by-hand run behave exactly as before. render_thermograph_secrets() { local repo="${1:-.}" # repo root holding deploy/secrets local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" local env_name="${2:-}" local out="${3:-/etc/thermograph.env}" [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true) # The per-host file is required; common.yaml is optional so the initial cutover can # seed each host as an exact copy of its live env (byte-identical render, no value # changes) and factor out shared secrets into common.yaml later. # # These are two different situations and used to share one `return 0`. # # (a) Not configured for SOPS at all — no env marker, or no age key. The LAN # dev box is legitimately in this state. Saying so and succeeding is right. if [ -z "$env_name" ] || [ ! -f "$key" ]; then echo "==> SOPS secrets not configured here; using existing $out" return 0 fi # (b) Configured — there is a key and a named environment — but the vault for # that environment is not where we looked. That is a failure, and it used # to return 0: a silent no-op wearing a success code. A deploy would report # success having rendered nothing, and the host would quietly keep serving # whatever /etc/thermograph.env already held, including after a rotation. # # The likely cause is the caller passing the wrong root. This function # wants the directory CONTAINING deploy/secrets — on the hosts that is # /opt/thermograph/infra, not /opt/thermograph. Getting it wrong looked # exactly like "not configured here", which is why it went unnoticed. if [ ! -f "$repo/deploy/secrets/${env_name}.yaml" ]; then echo "!! this host is configured for SOPS (env='${env_name}', age key present)" >&2 echo "!! but no vault was found at ${repo}/deploy/secrets/${env_name}.yaml" >&2 echo "!! pass the directory containing deploy/secrets — on the hosts that is" >&2 echo "!! /opt/thermograph/infra — or add ${env_name}.yaml to the vault." >&2 return 1 fi if ! command -v sops >/dev/null 2>&1; then echo "!! sops not installed but this host is configured for SOPS secrets" >&2 return 1 fi echo "==> Rendering $out from deploy/secrets (common + ${env_name})" # The age private key is root-owned (0400). Read it directly if we can, else via # sudo into SOPS_AGE_KEY — so the key never has to be readable by the deploy user. # (The render needs sudo to write /etc/thermograph.env below anyway.) local key_env=() if [ -r "$key" ]; then key_env=("SOPS_AGE_KEY_FILE=$key") else local keymat; keymat=$(sudo cat "$key" 2>/dev/null | grep '^AGE-SECRET-KEY-' || true) [ -n "$keymat" ] || { echo "!! cannot read age key at $key (need sudo)" >&2; return 1; } key_env=("SOPS_AGE_KEY=$keymat") fi local tmp; tmp=$(mktemp) # The tmp file holds DECRYPTED secrets, so every exit path below removes it. # # Deliberately NOT a `trap ... RETURN`, which looks like the tidier way to make # that unconditional: a RETURN trap set inside a SOURCED function persists in # the CALLER's shell after the function returns, and a RETURN trap also fires # when a `.`/source completes. deploy.sh sources /etc/thermograph.env a few # lines after calling us, which would re-fire the trap at top level where `tmp` # (function-local) is unset -- fatal under deploy.sh's `set -u`, and silent, # because that line already redirects stderr to /dev/null. Explicit removal on # each path is duller and correct. : > "$tmp" || { rm -f "$tmp"; return 1; } # Decrypt failures return 1 explicitly, so a partial env is never written and # correctness doesn't depend on the caller's shell options. common first, host # second, so a host value overrides a shared one (last-wins). # # THERMOGRAPH_SECRETS_SKIP_COMMON=1 renders the host file ALONE. deploy-dev.sh # sets it, and the reason is what common.yaml contains: eleven of its sixteen # values are live production credentials — both S3 keypairs (one is read-write # on the bucket holding prod's database backups), the VAPID private key that # signs Web Push to real subscribers, REGISTRY_TOKEN, the IndexNow and metrics # tokens. This render is a plaintext concatenation consumed via `env_file:`, so # layering common on dev would put every one of those into the environment of # every container on a desktop that is also the Forgejo CI runner, executing # unreviewed dev-branch code with the docker socket mounted. # # An override in dev.yaml would not fix it: last-wins governs *consumers*, but # the production value is still physically a line in the file. if [ -f "$repo/deploy/secrets/common.yaml" ] \ && [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \ "$repo/deploy/secrets/common.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; } fi env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \ "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; } # Write /etc/thermograph.env. Prefer an in-place write when the existing file is # writable by us (e.g. a group-writable 0660 root: on a box whose CI # deploy user isn't root and has no broad sudo — beta's `deploy`), since that needs # only file write, not /etc dir write or sudo. Else install; else sudo install (a # root/agent deploy) -- and on that sudo path CHOWN the result to the invoking # deploy user (-o/-g $(id -un/-gn)), or the file lands root:root 0640 and the very # next line of deploy.sh (`. /etc/thermograph.env` as that non-root user) can't read # it, so POSTGRES_PASSWORD never enters the env and `docker compose` dies on # interpolation. Fail loudly rather than deploy against stale secrets. # rc + a single cleanup point: the plaintext tmp must go whichever branch runs, # but the old trailing `rm` was also the function's last command, so it masked a # failed in-place `cat` write to status 0. Capture the status instead, so a # half-written /etc/thermograph.env fails loudly rather than deploying stale. local rc=0 if [ -f "$out" ] && [ -w "$out" ]; then cat "$tmp" > "$out" || rc=1 elif install -m 0640 "$tmp" "$out" 2>/dev/null; then : elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then : else echo "!! cannot write $out (need file write access or passwordless sudo)" >&2 rc=1 fi # Optional second copy under $HOME, for a host whose docker CLI cannot read # /etc/thermograph.env at all -- the LAN dev box's snap-packaged Docker is # confined to $HOME (plus a short allowlist) and silently treats anything # under /etc as nonexistent, so env_file: /etc/thermograph.env resolves to # nothing for every container even though the file is right there and the # shell that rendered it can read it fine. deploy-dev.sh sets this; # prod/beta (apt-installed Docker, no snap confinement) leave it unset and # this is a no-op for them. if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \ || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2 rc=1 fi fi rm -f "$tmp" return "$rc" } # ============================================================================ # Centralis — render /etc/centralis.env from deploy/secrets/centralis..yaml # ============================================================================ # # Centralis (the estate's control plane, prod-only) keeps its own env file, # deliberately separate from the app stack's /etc/thermograph.env. This function # brings that file under the same vault, so its four credentials # (CENTRALIS_AUTH_TOKEN, CENTRALIS_TOKENS, CENTRALIS_FORGEJO_TOKEN, # DISCORD_TOKEN) stop being hand-edited on the box. # # --------------------------------------------------------------------------- # WHY THIS IS A SEPARATE FUNCTION, NOT AN ARGUMENT TO THE ONE ABOVE # --------------------------------------------------------------------------- # The two files have DIFFERENT OUTPUT CONTRACTS, and that is the whole reason # this migration is being done. # # /etc/thermograph.env is consumed by docker-compose `env_file:`, by the systemd # unit's EnvironmentFile=, and by the stack containers' `. /host/thermograph.env`. # It is written as raw `KEY=value` — sops' own dotenv output, unquoted. # # /etc/centralis.env is consumed by EXACTLY ONE thing, and it is a shell: # # sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up' # # `set -a; . file` runs every line through bash's full word-expansion pipeline: # quote removal, parameter expansion, command substitution, field splitting. # Raw dotenv output is therefore NOT a safe representation for this file, and # sops cannot make it one — `sops -d --output-type dotenv` emits values verbatim. # On a value of {"emi":"..."} that yields # # CENTRALIS_TOKENS={"emi":"..."} -> bash strips the quotes -> {emi:...} # # which is exactly the hand-edit that took the token registry out on 2026-07-24: # Centralis rejected the malformed JSON, failed closed as designed, and served a # single `shared` identity — indistinguishable from the file never having been # written. A value containing a space would truncate; one containing a backtick # or $( ) would EXECUTE. So this function does not reuse the dotenv path at all. # It decrypts to JSON (lossless, unambiguous — dotenv escapes a newline to a # literal \n and cannot be told apart from a literal backslash-n) and emits # POSIX single-quoted assignments, then PROVES the result by sourcing it in a # clean shell and comparing every value back against the plaintext before the # file is allowed anywhere near /etc. # # Making render_thermograph_secrets take an output format would put a flag on # the app stack's deploy path whose wrong value is a production outage, to save # a few lines. It also could not be changed to emit quoted values for BOTH # files: /etc/thermograph.env's consumers include Centralis's own secrets tools, # which compare rendered values against the live file textually, so quoting it # would report all 32 keys as value-changed. Two consumers, two formats. # # --------------------------------------------------------------------------- # WHY common.yaml IS NOT READ HERE # --------------------------------------------------------------------------- # Not an oversight. Centralis shares no key with the app stack (verified: zero # overlap between the two live env files). Concatenating common.yaml would put # the VAPID private key, both S3 keypairs and REGISTRY_TOKEN into # /etc/centralis.env and into the shell that runs Centralis's compose — 16 # credentials handed to a service that wants none of them, for no benefit. # # --------------------------------------------------------------------------- # WHY THIS ONE FAILS LOUDLY WITH NO "not configured here" PATH # --------------------------------------------------------------------------- # render_thermograph_secrets is called unconditionally by every deploy on every # box, so it needs the (a) branch for hosts that have not been migrated. This # function is called only by Centralis's own deploy, on the one host that runs # Centralis. A caller asking for it has already asserted it should happen, so # every missing prerequisite is an error. There is no silent no-op path, because # a silent no-op here means deploying against a stale hand-edited file — the # state this exists to end. render_centralis_secrets() { local repo="${1:-.}" # dir CONTAINING deploy/secrets local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}" local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}" # Overridable so the pre-cutover dry run can render somewhere harmless and diff # it against the live file. Nothing in the deploy path sets it. local dest="${CENTRALIS_RENDER_DEST:-/etc/centralis.env}" local env_name vault env_name=$(cat "$marker" 2>/dev/null || true) if [ -z "$env_name" ]; then echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2 return 1 fi if [ ! -f "$key" ]; then echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2 return 1 fi # Scoped by environment even though Centralis runs only on prod today. The # alternative (one unscoped centralis.yaml) would render prod's bearer tokens # onto any host that ran this, which is the wrong failure to leave lying around # for the day Centralis gains a second home. Adding one is then a new file, # not a redesign. vault="$repo/deploy/secrets/centralis.${env_name}.yaml" if [ ! -f "$vault" ]; then echo "!! this host is configured for SOPS (env='${env_name}', age key present)" >&2 echo "!! but no Centralis vault was found at ${vault}" >&2 echo "!! pass the directory containing deploy/secrets — on the hosts that is" >&2 echo "!! /opt/thermograph/infra — or add centralis.${env_name}.yaml to the vault." >&2 return 1 fi command -v sops >/dev/null 2>&1 || { echo "!! sops not installed but this host is configured for SOPS secrets" >&2; return 1; } command -v python3 >/dev/null 2>&1 || { echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2; return 1; } echo "==> Rendering ${dest} from ${vault}" # Same age-key handling as render_thermograph_secrets: read the 0400 root key # directly when we can, else lift it through sudo into SOPS_AGE_KEY, so the key # never has to be made readable by the deploy user. local key_env=() if [ -r "$key" ]; then key_env=("SOPS_AGE_KEY_FILE=$key") else local keymat; keymat=$(sudo cat "$key" 2>/dev/null | grep '^AGE-SECRET-KEY-' || true) [ -n "$keymat" ] || { echo "!! cannot read age key at $key (need sudo)" >&2; return 1; } key_env=("SOPS_AGE_KEY=$keymat") fi # ONE temp directory, 0700, holding plaintext — so cleanup is a single rm on # every path. Deliberately NOT a `trap ... RETURN`: this file is SOURCED, and a # RETURN trap set inside a sourced function persists into the caller's shell and # re-fires when its next `source` completes (see the note in the function above). local work rc=0 work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; } chmod 700 "$work" # The work happens in a subshell so that no failure path can skip the cleanup # below, and so umask/cwd changes cannot leak into deploy.sh. # # Every step carries its own `|| exit 1` rather than relying on `set -e`. That # is not belt-and-braces: `set -e` is DISABLED inside a compound command that # is the left operand of `||`, which this subshell is (`( ... ) || rc=1`, needed # so a failure here cannot abort a `set -e` caller mid-deploy). A `set -e` in # here reads as protection and provides none — caught by a test that fed the # renderer a nested mapping and watched it sail past the rejection. ( umask 077 # JSON, not dotenv. Lossless and unambiguous — see the header. env "${key_env[@]}" sops -d --input-type yaml --output-type json "$vault" \ > "$work/plain.json" || exit 1 python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1 import json, re, sys src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3] NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") def shell_quote(v): """POSIX single-quoting. Inside '...' every byte is literal except ' itself, which is closed, backslash-escaped and reopened. There is no escape sequence to get wrong and no character class to keep up to date: it is total, and total is the only property worth having for a file bash will expand.""" return "'" + v.replace("'", "'\\''") + "'" def scalar(k, v): if isinstance(v, str): return v if isinstance(v, bool): # before int: bool is an int in Python return "true" if v else "false" if isinstance(v, (int, float)): return json.dumps(v) if v is None: return "" sys.stderr.write( "!! %s decrypts to a %s, but an env file holds only scalars\n" % (k, type(v).__name__)) sys.exit(1) data = json.load(open(src, encoding="utf-8")) if not isinstance(data, dict): sys.stderr.write("!! the vault did not decrypt to a mapping\n") sys.exit(1) want, lines = {}, [] for k, v in data.items(): if not NAME.match(k): sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,)) sys.exit(1) want[k] = scalar(k, v) lines.append("%s=%s\n" % (k, shell_quote(want[k]))) with open(out_env, "w", encoding="utf-8") as fh: fh.write("# Rendered by infra/deploy/render-secrets.sh from the SOPS vault.\n") fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n") fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n") fh.write("# Rotate by editing infra/deploy/secrets/centralis..yaml instead.\n") fh.writelines(lines) with open(want_json, "w", encoding="utf-8") as fh: json.dump(want, fh) sys.stderr.write(" %d keys quoted\n" % len(want)) EMIT # PROVE IT. Source the rendered file the way the deploy does — in a clean # environment, `set -a`, no inherited variables to mask a dropped key — and # read every value back out. This is the test that the file cannot pass while # carrying the 2026-07-24 bug, and it runs on every render, not once. # shellcheck disable=SC2016 # single quotes are the point: "$1" must be # expanded by the inner `bash -c`, against the argument passed after `_`, # not by this shell. Double-quoting would interpolate the OUTER $1 — the # function's own first argument, the repo root — and the check would then # verify the wrong file, or nothing at all, while still reporting success. env -i PATH="$PATH" bash -c \ 'set -a; . "$1"; set +a; exec python3 -c " import json, os, sys sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \ > "$work/got.json" || exit 1 python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1 import json, sys want = json.load(open(sys.argv[1], encoding="utf-8")) got = json.load(open(sys.argv[2], encoding="utf-8")) bad = [] for k, v in want.items(): if k not in got: bad.append("%s: not set at all after sourcing" % k) elif got[k] != v: # NEVER print either value. The difference is the finding; the content is # not, and this runs inside a deploy log. bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)" % (k, len(v), len(got[k]))) if bad: sys.stderr.write("!! the rendered file does not round-trip through bash:\n") for b in bad: sys.stderr.write("!! %s\n" % b) sys.exit(1) # CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double # quotes, in a file bash expands. Centralis fails CLOSED on malformed JSON — it # drops to the single `shared` identity, which looks exactly like the registry # having never been configured. Refuse to write a file that would do that, # rather than discovering it from a dashboard that is missing two people. raw = got.get("CENTRALIS_TOKENS", "") if raw: try: reg = json.loads(raw) except ValueError as exc: sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc) sys.exit(1) if not isinstance(reg, dict) or not reg: sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n") sys.exit(1) if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()): sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n") sys.exit(1) # Subjects are names on audit lines, not credentials. Printing them is the # whole point: it is how an operator sees at a glance that nobody was lost. sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n" % (len(reg), ", ".join(sorted(reg)))) sys.stderr.write(" %d keys survive `set -a; . ` byte-for-byte\n" % len(want)) VERIFY ) || rc=1 # Only now, with a file that has been read back through bash and matched value # for value, does anything touch $dest. if [ "$rc" -eq 0 ]; then # 0600, not 0640: this file is four bearer credentials for the estate's # control plane and has no group that needs it. Prefer an in-place write so # the existing owner/mode survive; fall back the same way the app renderer # does, chowning on the sudo path so a non-root deploy user can still source # what it just rendered. if [ -f "$dest" ] && [ -w "$dest" ]; then cat "$work/out.env" > "$dest" || rc=1 elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then : elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then : else echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2 rc=1 fi else echo "!! render aborted; ${dest} left untouched" >&2 fi rm -rf "$work" return "$rc" }