From b62eb601218238c7b1dca0fc14d8b6a64a5a5b3e Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 1 Aug 2026 09:10:34 -0700 Subject: [PATCH] infra-sync: render /etc/centralis.env from the vault render_centralis_secrets has existed since the OpenBao work and had no caller on any deploy path -- only verify-centralis-render.sh, a check. A renderer nothing calls is not a migration, it is a file. /etc/centralis.env has stayed hand-maintained the whole time, with the vault holding a copy nothing consumed. That gap is why turning on Google sign-in for Centralis had no safe home: the only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file, which is precisely what render-secrets.sh's header warns against. THE GUARD IS THE POINT. On 2026-07-24 this estate lost Centralis' token registry by writing a file missing a key the live one had. The renderer proves its own output round-trips through bash, but it cannot know about a key that exists only in the live file. So this job renders to a scratch path first, compares KEY SETS (names only, both sides sourced in `env -i` so the comparison is over what bash actually ends up with), and refuses to write -- failing loudly -- if the live file has anything the vault does not. Rendering turns from a way to lose a hand-edit into the thing that catches one. Verified the guard against a live file carrying a hand-added CENTRALIS_GOOGLE_CLIENT_ID: correctly identified as a key that would be deleted, write refused. Also verified the empty/unreadable-live-file path, which needs `|| true` to survive pipefail. It recreates the container where the other three jobs deliberately roll nothing, and the difference is real rather than an exception: Centralis' compose file interpolates /etc/centralis.env at PARSE time, so rendering without recreating changes nothing -- the silent no-op this estate keeps getting bitten by. `docker restart` would not do it either. One container, no replicas, a two-second recreate. It also skips the recreate entirely when the rendered file is byte-identical to the live one, so a no-op push does not bounce a healthy container. --- .forgejo/workflows/infra-sync.yml | 102 ++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index a3af20e..1827354 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -163,3 +163,105 @@ jobs: render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env fi echo "synced to $(git log --oneline -1)" + + sync-centralis: + # Centralis' /etc/centralis.env has been HAND-MAINTAINED. The vault has held a + # copy since deploy/secrets/centralis.prod.yaml was seeded, and + # render_centralis_secrets has existed since the OpenBao work -- but nothing + # on any deploy path ever called it. A renderer with no caller is not a + # migration, it is a file. + # + # That gap is why enabling Google sign-in on Centralis had no safe home: the + # only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file, + # which is exactly what render-secrets.sh's header warns against. + # + # WHY THIS JOB RECREATES A CONTAINER WHEN THE OTHER THREE DELIBERATELY DO NOT. + # This workflow's header says it never rolls a service, because image tags are + # the app domains' axis. Centralis is different in kind: its compose file + # interpolates /etc/centralis.env at PARSE time (`set -a && . /etc/centralis.env + # && docker compose up -d`), so rendering the file without recreating the + # container changes nothing at all. Rendering and not recreating would be the + # silent no-op this estate keeps getting bitten by. It is one container with no + # replicas -- a two-second recreate, not a deploy. + if: github.ref_name == 'main' + runs-on: docker + concurrency: + group: infra-sync-centralis + cancel-in-progress: false + steps: + - name: Render /etc/centralis.env from the vault, then recreate Centralis + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + script: | + set -euo pipefail + umask 077 + cd /opt/thermograph + [ -f infra/deploy/secrets/centralis.prod.yaml ] || { echo "no Centralis vault — nothing to do"; exit 0; } + + . infra/deploy/env-topology.sh + thermograph_topology prod + . infra/deploy/render-secrets.sh + + scratch=$(mktemp -d); chmod 700 "$scratch" + trap 'rm -rf -- "$scratch"' EXIT + + # Render to scratch FIRST. CENTRALIS_RENDER_DEST is the override the + # renderer already carries for exactly this: look before writing. + CENTRALIS_RENDER_DEST="$scratch/rendered.env" \ + render_centralis_secrets /opt/thermograph/infra + + # THE GUARD. On 2026-07-24 this estate lost Centralis' token registry by + # writing a file that was missing a key the live one had. The renderer + # proves its OWN output round-trips through bash, but it cannot know + # about a key that exists only in the live file -- someone hand-editing + # /etc/centralis.env to add, say, CENTRALIS_GOOGLE_CLIENT_ID is invisible + # to it. So: the vault must be a SUPERSET of what is live, or nothing is + # written and the job fails loudly. + # + # Names only. Both files are sourced in a clean environment so the + # comparison is over what bash actually ends up with, not over text. + # The `|| true` is load-bearing under `set -o pipefail`: grep exits 1 when + # a file yields no matching keys, which would otherwise abort the job on + # the exact edge case this guard exists to handle -- an empty or + # unreadable live file, i.e. a first run. + keys() { env -i bash -c 'set -a; . "$1" >/dev/null 2>&1; set +a; compgen -v' _ "$1" \ + | { grep -E '^(CENTRALIS_|DISCORD_)' || true; } | sort -u; } + keys "$scratch/rendered.env" > "$scratch/new.keys" + if [ -r /etc/centralis.env ]; then keys /etc/centralis.env > "$scratch/live.keys"; else : > "$scratch/live.keys"; fi + + missing=$(comm -23 "$scratch/live.keys" "$scratch/new.keys" || true) + if [ -n "$missing" ]; then + echo "!! REFUSING to write /etc/centralis.env." + echo "!! The live file has keys the vault does not, so rendering would DELETE them:" + printf '!! %s\n' $missing + echo "!! Someone hand-edited the live file. Put those keys in the vault instead:" + echo "!! sops edit infra/deploy/secrets/centralis.prod.yaml" + exit 1 + fi + + added=$(comm -13 "$scratch/live.keys" "$scratch/new.keys" || true) + [ -n "$added" ] && printf ' new key from the vault: %s\n' $added + + # Same file, byte for byte? Then there is nothing to do and no reason to + # bounce a healthy container. + if [ -r /etc/centralis.env ] && cmp -s "$scratch/rendered.env" /etc/centralis.env; then + echo "==> /etc/centralis.env already matches the vault — not recreating" + exit 0 + fi + + install -m 0600 -o "$(stat -c %U /etc/centralis.env 2>/dev/null || echo "$USER")" \ + -g "$(stat -c %G /etc/centralis.env 2>/dev/null || echo "$USER")" \ + "$scratch/rendered.env" /etc/centralis.env 2>/dev/null \ + || sudo install -m 0600 -o agent -g agent "$scratch/rendered.env" /etc/centralis.env + echo "==> wrote /etc/centralis.env from the vault" + + # Recreate so compose re-reads it. `docker restart` would NOT: the values + # are interpolated when the compose file is parsed, not read at runtime. + cd /opt/centralis/deploy + set -a; . /etc/centralis.env; set +a + docker compose up -d + docker compose ps --format '{{.Name}}\t{{.State}}\t{{.Status}}'