thermograph/infra/deploy/render-secrets.sh

#!/usr/bin/env bash
# Render /etc/thermograph.env from the SOPS-encrypted source of truth
# (deploy/secrets/<env>.yaml, committed encrypted) — sourced by deploy.sh and
# deploy-dev.sh, not run directly.
#
# The encrypted files are the single source of truth; /etc/thermograph.env becomes a
# rendered artifact that the existing seams (docker-compose `env_file:`, the systemd
# unit's EnvironmentFile, and entrypoint.sh's /run/secrets shim) keep consuming
# unchanged. Common secrets + the per-host overrides are concatenated with the host
# file LAST, so a host value wins (env_file and `source` both take the last
# occurrence of a duplicate key).
#
# Presence-detected: a host is "configured for SOPS" only when it has an age key at
# /etc/thermograph/age.key AND an environment name in /etc/thermograph/secrets-env
# (prod|beta|dev). A host without them keeps whatever /etc/thermograph.env it already
# has — so merging this is a safe no-op until a host is deliberately migrated. See
# deploy/secrets/README.md.
# render_thermograph_secrets <repo> [env_name] [out_file]
#
# env_name and out_file are explicit since vps2 began running two environments.
# A host-wide marker file cannot name two environments, and one output path
# cannot hold two renders: on vps2 prod renders prod.yaml to /etc/thermograph.env
# while beta renders beta.yaml to /etc/thermograph-beta.env. Both arguments keep
# their pre-split defaults (marker file, /etc/thermograph.env), so a
# single-environment host and a by-hand run behave exactly as before.
render_thermograph_secrets() {
  local repo="${1:-.}"                                   # repo root holding deploy/secrets
  local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
  local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}"
  local env_name="${2:-}"
  local out="${3:-/etc/thermograph.env}"
  [ -n "$env_name" ] || env_name=$(cat "$marker" 2>/dev/null || true)

  # The per-host file is required; common.yaml is optional so the initial cutover can
  # seed each host as an exact copy of its live env (byte-identical render, no value
  # changes) and factor out shared secrets into common.yaml later.
  #
  # These are two different situations and used to share one `return 0`.
  #
  # (a) Not configured for SOPS at all — no env marker, or no age key. The LAN
  #     dev box is legitimately in this state. Saying so and succeeding is right.
  if [ -z "$env_name" ] || [ ! -f "$key" ]; then
    echo "==> SOPS secrets not configured here; using existing $out"
    return 0
  fi

  # (b) Configured — there is a key and a named environment — but the vault for
  #     that environment is not where we looked. That is a failure, and it used
  #     to return 0: a silent no-op wearing a success code. A deploy would report
  #     success having rendered nothing, and the host would quietly keep serving
  #     whatever /etc/thermograph.env already held, including after a rotation.
  #
  #     The likely cause is the caller passing the wrong root. This function
  #     wants the directory CONTAINING deploy/secrets — on the hosts that is
  #     /opt/thermograph/infra, not /opt/thermograph. Getting it wrong looked
  #     exactly like "not configured here", which is why it went unnoticed.
  if [ ! -f "$repo/deploy/secrets/${env_name}.yaml" ]; then
    echo "!! this host is configured for SOPS (env='${env_name}', age key present)" >&2
    echo "!! but no vault was found at ${repo}/deploy/secrets/${env_name}.yaml" >&2
    echo "!! pass the directory containing deploy/secrets — on the hosts that is" >&2
    echo "!!   /opt/thermograph/infra — or add ${env_name}.yaml to the vault." >&2
    return 1
  fi
  if ! command -v sops >/dev/null 2>&1; then
    echo "!! sops not installed but this host is configured for SOPS secrets" >&2
    return 1
  fi

  echo "==> Rendering $out from deploy/secrets (common + ${env_name})"
  # The age private key is root-owned (0400). Read it directly if we can, else via
  # sudo into SOPS_AGE_KEY — so the key never has to be readable by the deploy user.
  # (The render needs sudo to write /etc/thermograph.env below anyway.)
  local key_env=()
  if [ -r "$key" ]; then
    key_env=("SOPS_AGE_KEY_FILE=$key")
  else
    local keymat; keymat=$(sudo cat "$key" 2>/dev/null | grep '^AGE-SECRET-KEY-' || true)
    [ -n "$keymat" ] || { echo "!! cannot read age key at $key (need sudo)" >&2; return 1; }
    key_env=("SOPS_AGE_KEY=$keymat")
  fi
  local tmp; tmp=$(mktemp)
  # The tmp file holds DECRYPTED secrets, so every exit path below removes it.
  #
  # Deliberately NOT a `trap ... RETURN`, which looks like the tidier way to make
  # that unconditional: a RETURN trap set inside a SOURCED function persists in
  # the CALLER's shell after the function returns, and a RETURN trap also fires
  # when a `.`/source completes. deploy.sh sources /etc/thermograph.env a few
  # lines after calling us, which would re-fire the trap at top level where `tmp`
  # (function-local) is unset -- fatal under deploy.sh's `set -u`, and silent,
  # because that line already redirects stderr to /dev/null. Explicit removal on
  # each path is duller and correct.
  : > "$tmp" || { rm -f "$tmp"; return 1; }
  # Decrypt failures return 1 explicitly, so a partial env is never written and
  # correctness doesn't depend on the caller's shell options. common first, host
  # second, so a host value overrides a shared one (last-wins).
  #
  # THERMOGRAPH_SECRETS_SKIP_COMMON=1 renders the host file ALONE. deploy-dev.sh
  # sets it, and the reason is what common.yaml contains: eleven of its sixteen
  # values are live production credentials — both S3 keypairs (one is read-write
  # on the bucket holding prod's database backups), the VAPID private key that
  # signs Web Push to real subscribers, REGISTRY_TOKEN, the IndexNow and metrics
  # tokens. This render is a plaintext concatenation consumed via `env_file:`, so
  # layering common on dev would put every one of those into the environment of
  # every container on a desktop that is also the Forgejo CI runner, executing
  # unreviewed dev-branch code with the docker socket mounted.
  #
  # An override in dev.yaml would not fix it: last-wins governs *consumers*, but
  # the production value is still physically a line in the file.
  if [ -f "$repo/deploy/secrets/common.yaml" ] \
     && [ "${THERMOGRAPH_SECRETS_SKIP_COMMON:-0}" != 1 ]; then
    env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \
      "$repo/deploy/secrets/common.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; }
  fi
  env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \
    "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; }

  # Write /etc/thermograph.env. Prefer an in-place write when the existing file is
  # writable by us (e.g. a group-writable 0660 root:<deploygroup> on a box whose CI
  # deploy user isn't root and has no broad sudo — beta's `deploy`), since that needs
  # only file write, not /etc dir write or sudo. Else install; else sudo install (a
  # root/agent deploy) -- and on that sudo path CHOWN the result to the invoking
  # deploy user (-o/-g $(id -un/-gn)), or the file lands root:root 0640 and the very
  # next line of deploy.sh (`. /etc/thermograph.env` as that non-root user) can't read
  # it, so POSTGRES_PASSWORD never enters the env and `docker compose` dies on
  # interpolation. Fail loudly rather than deploy against stale secrets.
  # rc + a single cleanup point: the plaintext tmp must go whichever branch runs,
  # but the old trailing `rm` was also the function's last command, so it masked a
  # failed in-place `cat` write to status 0. Capture the status instead, so a
  # half-written /etc/thermograph.env fails loudly rather than deploying stale.
  local rc=0
  if [ -f "$out" ] && [ -w "$out" ]; then
    cat "$tmp" > "$out" || rc=1
  elif install -m 0640 "$tmp" "$out" 2>/dev/null; then :
  elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" "$out" 2>/dev/null; then :
  else
    echo "!! cannot write $out (need file write access or passwordless sudo)" >&2
    rc=1
  fi

  # Optional second copy under $HOME, for a host whose docker CLI cannot read
  # /etc/thermograph.env at all -- the LAN dev box's snap-packaged Docker is
  # confined to $HOME (plus a short allowlist) and silently treats anything
  # under /etc as nonexistent, so env_file: /etc/thermograph.env resolves to
  # nothing for every container even though the file is right there and the
  # shell that rendered it can read it fine. deploy-dev.sh sets this;
  # prod/beta (apt-installed Docker, no snap confinement) leave it unset and
  # this is a no-op for them.
  if [ "$rc" = 0 ] && [ -n "${THERMOGRAPH_SECRETS_ENV_FILE_MIRROR:-}" ]; then
    if ! mkdir -p "$(dirname "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR")" \
       || ! install -m 0600 "$tmp" "$THERMOGRAPH_SECRETS_ENV_FILE_MIRROR"; then
      echo "!! cannot write mirror at $THERMOGRAPH_SECRETS_ENV_FILE_MIRROR" >&2
      rc=1
    fi
  fi

  rm -f "$tmp"
  return "$rc"
}

# ============================================================================
# Centralis — render /etc/centralis.env from deploy/secrets/centralis.<env>.yaml
# ============================================================================
#
# Centralis (the estate's control plane, prod-only) keeps its own env file,
# deliberately separate from the app stack's /etc/thermograph.env. This function
# brings that file under the same vault, so its four credentials
# (CENTRALIS_AUTH_TOKEN, CENTRALIS_TOKENS, CENTRALIS_FORGEJO_TOKEN,
# DISCORD_TOKEN) stop being hand-edited on the box.
#
# ---------------------------------------------------------------------------
# WHY THIS IS A SEPARATE FUNCTION, NOT AN ARGUMENT TO THE ONE ABOVE
# ---------------------------------------------------------------------------
# The two files have DIFFERENT OUTPUT CONTRACTS, and that is the whole reason
# this migration is being done.
#
# /etc/thermograph.env is consumed by docker-compose `env_file:`, by the systemd
# unit's EnvironmentFile=, and by the stack containers' `. /host/thermograph.env`.
# It is written as raw `KEY=value` — sops' own dotenv output, unquoted.
#
# /etc/centralis.env is consumed by EXACTLY ONE thing, and it is a shell:
#
#     sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
#
# `set -a; . file` runs every line through bash's full word-expansion pipeline:
# quote removal, parameter expansion, command substitution, field splitting.
# Raw dotenv output is therefore NOT a safe representation for this file, and
# sops cannot make it one — `sops -d --output-type dotenv` emits values verbatim.
# On a value of {"emi":"..."} that yields
#
#     CENTRALIS_TOKENS={"emi":"..."}     ->  bash strips the quotes  ->  {emi:...}
#
# which is exactly the hand-edit that took the token registry out on 2026-07-24:
# Centralis rejected the malformed JSON, failed closed as designed, and served a
# single `shared` identity — indistinguishable from the file never having been
# written. A value containing a space would truncate; one containing a backtick
# or $( ) would EXECUTE. So this function does not reuse the dotenv path at all.
# It decrypts to JSON (lossless, unambiguous — dotenv escapes a newline to a
# literal \n and cannot be told apart from a literal backslash-n) and emits
# POSIX single-quoted assignments, then PROVES the result by sourcing it in a
# clean shell and comparing every value back against the plaintext before the
# file is allowed anywhere near /etc.
#
# Making render_thermograph_secrets take an output format would put a flag on
# the app stack's deploy path whose wrong value is a production outage, to save
# a few lines. It also could not be changed to emit quoted values for BOTH
# files: /etc/thermograph.env's consumers include Centralis's own secrets tools,
# which compare rendered values against the live file textually, so quoting it
# would report all 32 keys as value-changed. Two consumers, two formats.
#
# ---------------------------------------------------------------------------
# WHY common.yaml IS NOT READ HERE
# ---------------------------------------------------------------------------
# Not an oversight. Centralis shares no key with the app stack (verified: zero
# overlap between the two live env files). Concatenating common.yaml would put
# the VAPID private key, both S3 keypairs and REGISTRY_TOKEN into
# /etc/centralis.env and into the shell that runs Centralis's compose — 16
# credentials handed to a service that wants none of them, for no benefit.
#
# ---------------------------------------------------------------------------
# WHY THIS ONE FAILS LOUDLY WITH NO "not configured here" PATH
# ---------------------------------------------------------------------------
# render_thermograph_secrets is called unconditionally by every deploy on every
# box, so it needs the (a) branch for hosts that have not been migrated. This
# function is called only by Centralis's own deploy, on the one host that runs
# Centralis. A caller asking for it has already asserted it should happen, so
# every missing prerequisite is an error. There is no silent no-op path, because
# a silent no-op here means deploying against a stale hand-edited file — the
# state this exists to end.
render_centralis_secrets() {
  local repo="${1:-.}"                                   # dir CONTAINING deploy/secrets
  local key="${THERMOGRAPH_AGE_KEY:-/etc/thermograph/age.key}"
  local marker="${THERMOGRAPH_SECRETS_ENV_FILE:-/etc/thermograph/secrets-env}"
  # Overridable so the pre-cutover dry run can render somewhere harmless and diff
  # it against the live file. Nothing in the deploy path sets it.
  local dest="${CENTRALIS_RENDER_DEST:-/etc/centralis.env}"
  local env_name vault
  env_name=$(cat "$marker" 2>/dev/null || true)

  if [ -z "$env_name" ]; then
    echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
    return 1
  fi
  if [ ! -f "$key" ]; then
    echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
    return 1
  fi
  # Scoped by environment even though Centralis runs only on prod today. The
  # alternative (one unscoped centralis.yaml) would render prod's bearer tokens
  # onto any host that ran this, which is the wrong failure to leave lying around
  # for the day Centralis gains a second home. Adding one is then a new file,
  # not a redesign.
  vault="$repo/deploy/secrets/centralis.${env_name}.yaml"
  if [ ! -f "$vault" ]; then
    echo "!! this host is configured for SOPS (env='${env_name}', age key present)" >&2
    echo "!! but no Centralis vault was found at ${vault}" >&2
    echo "!! pass the directory containing deploy/secrets — on the hosts that is" >&2
    echo "!!   /opt/thermograph/infra — or add centralis.${env_name}.yaml to the vault." >&2
    return 1
  fi
  command -v sops >/dev/null 2>&1 || {
    echo "!! sops not installed but this host is configured for SOPS secrets" >&2; return 1; }
  command -v python3 >/dev/null 2>&1 || {
    echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2; return 1; }

  echo "==> Rendering ${dest} from ${vault}"
  # Same age-key handling as render_thermograph_secrets: read the 0400 root key
  # directly when we can, else lift it through sudo into SOPS_AGE_KEY, so the key
  # never has to be made readable by the deploy user.
  local key_env=()
  if [ -r "$key" ]; then
    key_env=("SOPS_AGE_KEY_FILE=$key")
  else
    local keymat; keymat=$(sudo cat "$key" 2>/dev/null | grep '^AGE-SECRET-KEY-' || true)
    [ -n "$keymat" ] || { echo "!! cannot read age key at $key (need sudo)" >&2; return 1; }
    key_env=("SOPS_AGE_KEY=$keymat")
  fi

  # ONE temp directory, 0700, holding plaintext — so cleanup is a single rm on
  # every path. Deliberately NOT a `trap ... RETURN`: this file is SOURCED, and a
  # RETURN trap set inside a sourced function persists into the caller's shell and
  # re-fires when its next `source` completes (see the note in the function above).
  local work rc=0
  work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
  chmod 700 "$work"

  # The work happens in a subshell so that no failure path can skip the cleanup
  # below, and so umask/cwd changes cannot leak into deploy.sh.
  #
  # Every step carries its own `|| exit 1` rather than relying on `set -e`. That
  # is not belt-and-braces: `set -e` is DISABLED inside a compound command that
  # is the left operand of `||`, which this subshell is (`( ... ) || rc=1`, needed
  # so a failure here cannot abort a `set -e` caller mid-deploy). A `set -e` in
  # here reads as protection and provides none — caught by a test that fed the
  # renderer a nested mapping and watched it sail past the rejection.
  (
    umask 077

    # JSON, not dotenv. Lossless and unambiguous — see the header.
    env "${key_env[@]}" sops -d --input-type yaml --output-type json "$vault" \
      > "$work/plain.json" || exit 1

    python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
import json, re, sys

src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")


def shell_quote(v):
    """POSIX single-quoting. Inside '...' every byte is literal except ' itself,
    which is closed, backslash-escaped and reopened. There is no escape sequence
    to get wrong and no character class to keep up to date: it is total, and
    total is the only property worth having for a file bash will expand."""
    return "'" + v.replace("'", "'\\''") + "'"


def scalar(k, v):
    if isinstance(v, str):
        return v
    if isinstance(v, bool):          # before int: bool is an int in Python
        return "true" if v else "false"
    if isinstance(v, (int, float)):
        return json.dumps(v)
    if v is None:
        return ""
    sys.stderr.write(
        "!! %s decrypts to a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
    sys.exit(1)


data = json.load(open(src, encoding="utf-8"))
if not isinstance(data, dict):
    sys.stderr.write("!! the vault did not decrypt to a mapping\n")
    sys.exit(1)

want, lines = {}, []
for k, v in data.items():
    if not NAME.match(k):
        sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
        sys.exit(1)
    want[k] = scalar(k, v)
    lines.append("%s=%s\n" % (k, shell_quote(want[k])))

with open(out_env, "w", encoding="utf-8") as fh:
    fh.write("# Rendered by infra/deploy/render-secrets.sh from the SOPS vault.\n")
    fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
    fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
    fh.write("# Rotate by editing infra/deploy/secrets/centralis.<env>.yaml instead.\n")
    fh.writelines(lines)
with open(want_json, "w", encoding="utf-8") as fh:
    json.dump(want, fh)
sys.stderr.write("    %d keys quoted\n" % len(want))
EMIT

    # PROVE IT. Source the rendered file the way the deploy does — in a clean
    # environment, `set -a`, no inherited variables to mask a dropped key — and
    # read every value back out. This is the test that the file cannot pass while
    # carrying the 2026-07-24 bug, and it runs on every render, not once.
    # shellcheck disable=SC2016  # single quotes are the point: "$1" must be
    # expanded by the inner `bash -c`, against the argument passed after `_`,
    # not by this shell. Double-quoting would interpolate the OUTER $1 — the
    # function's own first argument, the repo root — and the check would then
    # verify the wrong file, or nothing at all, while still reporting success.
    env -i PATH="$PATH" bash -c \
      'set -a; . "$1"; set +a; exec python3 -c "
import json, os, sys
sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
      > "$work/got.json" || exit 1

    python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
import json, sys

want = json.load(open(sys.argv[1], encoding="utf-8"))
got = json.load(open(sys.argv[2], encoding="utf-8"))

bad = []
for k, v in want.items():
    if k not in got:
        bad.append("%s: not set at all after sourcing" % k)
    elif got[k] != v:
        # NEVER print either value. The difference is the finding; the content is
        # not, and this runs inside a deploy log.
        bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
                   % (k, len(v), len(got[k])))
if bad:
    sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
    for b in bad:
        sys.stderr.write("!!   %s\n" % b)
    sys.exit(1)

# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double
# quotes, in a file bash expands. Centralis fails CLOSED on malformed JSON — it
# drops to the single `shared` identity, which looks exactly like the registry
# having never been configured. Refuse to write a file that would do that,
# rather than discovering it from a dashboard that is missing two people.
raw = got.get("CENTRALIS_TOKENS", "")
if raw:
    try:
        reg = json.loads(raw)
    except ValueError as exc:
        sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
        sys.exit(1)
    if not isinstance(reg, dict) or not reg:
        sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
        sys.exit(1)
    if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
        sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
        sys.exit(1)
    # Subjects are names on audit lines, not credentials. Printing them is the
    # whole point: it is how an operator sees at a glance that nobody was lost.
    sys.stderr.write("    CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
                     % (len(reg), ", ".join(sorted(reg))))

sys.stderr.write("    %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
VERIFY
  ) || rc=1

  # Only now, with a file that has been read back through bash and matched value
  # for value, does anything touch $dest.
  if [ "$rc" -eq 0 ]; then
    # 0600, not 0640: this file is four bearer credentials for the estate's
    # control plane and has no group that needs it. Prefer an in-place write so
    # the existing owner/mode survive; fall back the same way the app renderer
    # does, chowning on the sudo path so a non-root deploy user can still source
    # what it just rendered.
    if [ -f "$dest" ] && [ -w "$dest" ]; then
      cat "$work/out.env" > "$dest" || rc=1
    elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
    elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
    else
      echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
      rc=1
    fi
  else
    echo "!! render aborted; ${dest} left untouched" >&2
  fi

  rm -rf "$work"
  return "$rc"
}