Compare commits
2 commits
df409f88b3
...
e75b46cba8
| Author | SHA1 | Date | |
|---|---|---|---|
|
e75b46cba8 | ||
|
|
e60bc66448 |
19 changed files with 87 additions and 1092 deletions
|
|
@ -86,15 +86,6 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# env-topology.sh FIRST: it is what sets TG_SECRETS_BACKEND, and
|
||||
# render-secrets.sh:44 defaults to `sops` when it is unset. Without
|
||||
# this line the cutover would be only half-applied — deploy.sh would
|
||||
# read the new backend while THIS workflow, which runs on every push
|
||||
# touching infra/**, kept re-rendering the same file from SOPS. That
|
||||
# is silent while parity holds and a hard failure the moment the SOPS
|
||||
# files are retired.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology dev
|
||||
. infra/deploy/render-secrets.sh
|
||||
# SKIP_COMMON is dev's standing rule, not a preference: common.yaml
|
||||
# is the fleet's shared production credential set, and vps1 also
|
||||
|
|
@ -123,13 +114,6 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev: env-topology.sh is what sets
|
||||
# TG_SECRETS_BACKEND, and for beta it also sets TG_BAO_APPROLE to
|
||||
# the -beta credential. Beta shares a filesystem with prod, so the
|
||||
# bare default is prod's file and tg-host-prod.hcl denies beta's
|
||||
# own path — a 403 at render time on the box that runs prod.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology beta
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env
|
||||
fi
|
||||
|
|
@ -155,113 +139,7 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev — without env-topology.sh this job keeps
|
||||
# rendering from SOPS regardless of what the backend selector says.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology prod
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env
|
||||
fi
|
||||
echo "synced to $(git log --oneline -1)"
|
||||
|
||||
sync-centralis:
|
||||
# Centralis' /etc/centralis.env has been HAND-MAINTAINED. The vault has held a
|
||||
# copy since deploy/secrets/centralis.prod.yaml was seeded, and
|
||||
# render_centralis_secrets has existed since the OpenBao work -- but nothing
|
||||
# on any deploy path ever called it. A renderer with no caller is not a
|
||||
# migration, it is a file.
|
||||
#
|
||||
# That gap is why enabling Google sign-in on Centralis had no safe home: the
|
||||
# only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file,
|
||||
# which is exactly what render-secrets.sh's header warns against.
|
||||
#
|
||||
# WHY THIS JOB RECREATES A CONTAINER WHEN THE OTHER THREE DELIBERATELY DO NOT.
|
||||
# This workflow's header says it never rolls a service, because image tags are
|
||||
# the app domains' axis. Centralis is different in kind: its compose file
|
||||
# interpolates /etc/centralis.env at PARSE time (`set -a && . /etc/centralis.env
|
||||
# && docker compose up -d`), so rendering the file without recreating the
|
||||
# container changes nothing at all. Rendering and not recreating would be the
|
||||
# silent no-op this estate keeps getting bitten by. It is one container with no
|
||||
# replicas -- a two-second recreate, not a deploy.
|
||||
if: github.ref_name == 'main'
|
||||
runs-on: docker
|
||||
concurrency:
|
||||
group: infra-sync-centralis
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Render /etc/centralis.env from the vault, then recreate Centralis
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
||||
username: ${{ secrets.VPS2_SSH_USER }}
|
||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
cd /opt/thermograph
|
||||
[ -f infra/deploy/secrets/centralis.prod.yaml ] || { echo "no Centralis vault — nothing to do"; exit 0; }
|
||||
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology prod
|
||||
. infra/deploy/render-secrets.sh
|
||||
|
||||
scratch=$(mktemp -d); chmod 700 "$scratch"
|
||||
trap 'rm -rf -- "$scratch"' EXIT
|
||||
|
||||
# Render to scratch FIRST. CENTRALIS_RENDER_DEST is the override the
|
||||
# renderer already carries for exactly this: look before writing.
|
||||
CENTRALIS_RENDER_DEST="$scratch/rendered.env" \
|
||||
render_centralis_secrets /opt/thermograph/infra
|
||||
|
||||
# THE GUARD. On 2026-07-24 this estate lost Centralis' token registry by
|
||||
# writing a file that was missing a key the live one had. The renderer
|
||||
# proves its OWN output round-trips through bash, but it cannot know
|
||||
# about a key that exists only in the live file -- someone hand-editing
|
||||
# /etc/centralis.env to add, say, CENTRALIS_GOOGLE_CLIENT_ID is invisible
|
||||
# to it. So: the vault must be a SUPERSET of what is live, or nothing is
|
||||
# written and the job fails loudly.
|
||||
#
|
||||
# Names only. Both files are sourced in a clean environment so the
|
||||
# comparison is over what bash actually ends up with, not over text.
|
||||
# The `|| true` is load-bearing under `set -o pipefail`: grep exits 1 when
|
||||
# a file yields no matching keys, which would otherwise abort the job on
|
||||
# the exact edge case this guard exists to handle -- an empty or
|
||||
# unreadable live file, i.e. a first run.
|
||||
keys() { env -i bash -c 'set -a; . "$1" >/dev/null 2>&1; set +a; compgen -v' _ "$1" \
|
||||
| { grep -E '^(CENTRALIS_|DISCORD_)' || true; } | sort -u; }
|
||||
keys "$scratch/rendered.env" > "$scratch/new.keys"
|
||||
if [ -r /etc/centralis.env ]; then keys /etc/centralis.env > "$scratch/live.keys"; else : > "$scratch/live.keys"; fi
|
||||
|
||||
missing=$(comm -23 "$scratch/live.keys" "$scratch/new.keys" || true)
|
||||
if [ -n "$missing" ]; then
|
||||
echo "!! REFUSING to write /etc/centralis.env."
|
||||
echo "!! The live file has keys the vault does not, so rendering would DELETE them:"
|
||||
printf '!! %s\n' $missing
|
||||
echo "!! Someone hand-edited the live file. Put those keys in the vault instead:"
|
||||
echo "!! sops edit infra/deploy/secrets/centralis.prod.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
added=$(comm -13 "$scratch/live.keys" "$scratch/new.keys" || true)
|
||||
[ -n "$added" ] && printf ' new key from the vault: %s\n' $added
|
||||
|
||||
# Same file, byte for byte? Then there is nothing to do and no reason to
|
||||
# bounce a healthy container.
|
||||
if [ -r /etc/centralis.env ] && cmp -s "$scratch/rendered.env" /etc/centralis.env; then
|
||||
echo "==> /etc/centralis.env already matches the vault — not recreating"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
install -m 0600 -o "$(stat -c %U /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
||||
-g "$(stat -c %G /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
||||
"$scratch/rendered.env" /etc/centralis.env 2>/dev/null \
|
||||
|| sudo install -m 0600 -o agent -g agent "$scratch/rendered.env" /etc/centralis.env
|
||||
echo "==> wrote /etc/centralis.env from the vault"
|
||||
|
||||
# Recreate so compose re-reads it. `docker restart` would NOT: the values
|
||||
# are interpolated when the compose file is parsed, not read at runtime.
|
||||
cd /opt/centralis/deploy
|
||||
set -a; . /etc/centralis.env; set +a
|
||||
docker compose up -d
|
||||
docker compose ps --format '{{.Name}}\t{{.State}}\t{{.Status}}'
|
||||
|
|
|
|||
|
|
@ -8,15 +8,8 @@ name: Ops cron (backup + IndexNow)
|
|||
#
|
||||
# The jobs SSH into a host and run inside the already-running stack (docker
|
||||
# exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new
|
||||
# network exposure, no separate dependency install. Runs on the `docker` label,
|
||||
# served by the vps2 runner (infra/deploy/forgejo/runner-vps2/) and, when it is
|
||||
# up, the desktop.
|
||||
#
|
||||
# THIS WORKFLOW IS THE ESTATE'S ONLY BACKUP, so its dependency on a runner is
|
||||
# load-bearing. On 2026-07-31 the desktop was the only registered runner and
|
||||
# went offline for 21 hours; this schedule did not fire at 03:00Z. Forgejo held
|
||||
# the run in `waiting` and it completed on reconnect, so nothing was lost that
|
||||
# time. The vps2 runner exists so the next outage is not a coin flip.
|
||||
# network exposure, no separate dependency install. Runs on the `docker` label
|
||||
# (the always-on Swarm-hosted runner deploy/forgejo/ stood up).
|
||||
#
|
||||
# WHICH BOX EACH JOB TALKS TO, and why the secret names say so:
|
||||
#
|
||||
|
|
@ -216,65 +209,6 @@ jobs:
|
|||
rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true
|
||||
rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true
|
||||
|
||||
secrets-parity:
|
||||
name: OpenBao/SOPS parity gate
|
||||
runs-on: docker
|
||||
# The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive
|
||||
# green parity runs" -- and which, until now, nothing implemented. A gate
|
||||
# that exists only in prose gets satisfied by assertion.
|
||||
#
|
||||
# verify-parity.sh renders each environment through BOTH backends and diffs
|
||||
# the KEY=value sets after a last-wins collapse. It prints key NAMES and
|
||||
# counts only, never values, so these logs are safe to read and to paste.
|
||||
#
|
||||
# A mismatch FAILS the job, deliberately and loudly. The failure modes on
|
||||
# the other side of a bad cutover are all silent: a missing
|
||||
# THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start
|
||||
# working intermittently; a half-rendered VAPID pair makes push.py generate a
|
||||
# new keypair and delete every subscription row as "gone". Red here is the
|
||||
# only warning that arrives before those.
|
||||
#
|
||||
# Runs under sudo because the age key is 0400 root and the AppRole files are
|
||||
# 0400 agent/root -- the SSH user cannot read them directly. This grants CI
|
||||
# no vault access of its own: the HOST renders, CI only asks it to.
|
||||
#
|
||||
# The run history in Forgejo IS the record of consecutive green days. Do not
|
||||
# count a day this job was skipped or the runner was down as green.
|
||||
concurrency:
|
||||
group: ops-secrets-parity
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: prod + beta (vps2)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
||||
username: ${{ secrets.VPS2_SSH_USER }}
|
||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# Each environment is checked from its OWN checkout. The trees are
|
||||
# identical, but running beta's from beta's directory also proves
|
||||
# that checkout is current rather than assuming it.
|
||||
sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod
|
||||
sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta
|
||||
|
||||
- name: dev (vps1)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS1_SSH_HOST }}
|
||||
username: ${{ secrets.VPS1_SSH_USER }}
|
||||
key: ${{ secrets.VPS1_SSH_KEY }}
|
||||
port: ${{ secrets.VPS1_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# dev renders dev.yaml ALONE -- verify-parity.sh takes that from
|
||||
# env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the
|
||||
# comparison matches what deploy-dev.sh actually does. vps1 must
|
||||
# never see common.yaml: it runs Forgejo, its CI, and dev's
|
||||
# unreviewed branch.
|
||||
sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev
|
||||
|
||||
indexnow:
|
||||
name: IndexNow ping
|
||||
runs-on: docker
|
||||
|
|
|
|||
|
|
@ -76,16 +76,6 @@ thermograph_topology() {
|
|||
# for the reason that marker was demoted to a fallback in the first place: vps2 runs
|
||||
# prod and beta side by side, and one host-wide file cannot name two backends.
|
||||
TG_SECRETS_BACKEND=sops
|
||||
# Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
|
||||
# reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
|
||||
# environments, so a single host-wide default cannot serve both. prod and dev each
|
||||
# get the conventional path (dev is alone on vps1, so there is no collision there);
|
||||
# beta is the one that must differ, because it shares a filesystem with prod.
|
||||
#
|
||||
# Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
|
||||
# correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
|
||||
# runs prod.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle
|
||||
|
||||
case "$env_name" in
|
||||
prod)
|
||||
|
|
@ -126,10 +116,6 @@ thermograph_topology() {
|
|||
# therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
|
||||
TG_SSH_HOST=169.58.46.181
|
||||
TG_SSH_TARGET=agent@169.58.46.181
|
||||
# The one environment that cannot use the default AppRole path: it shares a
|
||||
# filesystem with prod, so it needs its own credential file to authenticate as
|
||||
# tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
|
||||
# A SECOND checkout on the same box. Separate from prod's so the two can
|
||||
# sit on different commits of this repo, and so `git reset --hard` in one
|
||||
# deploy can never yank the tree out from under the other's running
|
||||
|
|
@ -242,7 +228,7 @@ thermograph_topology() {
|
|||
export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
|
||||
export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
|
||||
export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
|
||||
export TG_SECRETS_BACKEND TG_BAO_APPROLE
|
||||
export TG_SECRETS_BACKEND
|
||||
export TG_SSH_HOST TG_SSH_TARGET
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,20 +7,12 @@ deploy`s that happen to run on the manager node, vps2 — see
|
|||
(`75.119.132.91`) via the `role=forge` label from
|
||||
`deploy/swarm/label-forge-node.sh`.
|
||||
|
||||
The Actions **runner** is deliberately *not* part of this stack: a
|
||||
Swarm-scheduled runner cannot redeploy the Swarm that schedules it, so CI would
|
||||
be unavailable exactly when the cluster is what needs repairing. An earlier
|
||||
revision did run it as a Swarm-scheduled Docker-in-Docker sidecar pinned to the
|
||||
Forgejo node; that is gone.
|
||||
|
||||
Runners live in two places, and only one of them counts:
|
||||
|
||||
- **`runner-vps2/`** — the always-on runner, a plain `restart: always` container
|
||||
on vps2. This is the one CI depends on.
|
||||
- **`register-lan-runner.sh`** — the desktop's systemd runner. Extra capacity.
|
||||
**Nothing may assume it is up.** It was the only registered runner in the
|
||||
estate until 2026-08-01, and its 21-hour outage on 2026-07-31 froze every
|
||||
merge and deploy and stopped the nightly backup from firing.
|
||||
The Actions **runner** is deliberately *not* part of this stack — see
|
||||
`DEPLOY-DEV.md` and the note in `register-lan-runner.sh`'s own header for
|
||||
where it runs today. That's the canonical placement per
|
||||
`thermograph-docs/runbooks/implementation-handoff.md` Track B step 5; an
|
||||
earlier revision of this stack ran the runner as a Swarm-scheduled
|
||||
Docker-in-Docker sidecar pinned to the Forgejo node, which is gone now.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
|
|
|||
|
|
@ -3,16 +3,11 @@
|
|||
# from the Terraform-managed docker-compose.yml that runs the Thermograph app
|
||||
# itself: this stack's only job is Forgejo and its container registry.
|
||||
#
|
||||
# The Actions RUNNER is deliberately NOT a service in this stack, and that is a
|
||||
# reasoned choice rather than an omission: a Swarm-scheduled runner cannot
|
||||
# redeploy the Swarm that schedules it, so CI would be gone exactly when the
|
||||
# cluster is the thing that is broken.
|
||||
#
|
||||
# The runner that CI depends on runs on vps2 as a plain restart:always container
|
||||
# — deploy/forgejo/runner-vps2/. The desktop's systemd runner
|
||||
# (deploy/forgejo/register-lan-runner.sh) stays registered as extra capacity.
|
||||
# Nothing may assume the desktop is up: it was the ONLY runner until 2026-08-01,
|
||||
# and its 21-hour outage on 2026-07-31 froze every merge, deploy and backup.
|
||||
# The Actions RUNNER is deliberately NOT a service in this stack. Per
|
||||
# thermograph-docs/runbooks/implementation-handoff.md (Track B step 5), the canonical
|
||||
# design registers the runner on the desktop node as a plain systemd service
|
||||
# (deploy/forgejo/register-lan-runner.sh) — the same place the pre-Forgejo
|
||||
# GitHub self-hosted runner already lived, not a Swarm-scheduled container.
|
||||
#
|
||||
# No Traefik here. Forgejo is pinned to vps1 (node.labels.role == forge)
|
||||
# because that's what was chosen, and vps1 is ALSO the emigriffith.dev portfolio
|
||||
|
|
|
|||
|
|
@ -18,16 +18,11 @@
|
|||
# breaks by keeping it registered — no workflow requests it — but do not build
|
||||
# anything new on it.
|
||||
#
|
||||
# This machine keeps serving the `docker` label as EXTRA capacity, and that is
|
||||
# now a settled call rather than an open one. The claim it used to make here —
|
||||
# that a Swarm-hosted runner was always-on, so the estate no longer depended on
|
||||
# this box — was false: no such runner existed. This was the only registered
|
||||
# runner in the estate until 2026-08-01, and when it went offline on 2026-07-31
|
||||
# every merge, deploy and backup stopped for 21 hours.
|
||||
#
|
||||
# The runner the estate depends on is deploy/forgejo/runner-vps2/. Keeping this
|
||||
# one is worthwhile (a second runner means a queue drains instead of stalling),
|
||||
# but nothing may be built on the assumption that this box is up.
|
||||
# TODO(cutover): whether this machine keeps serving the `docker` label at all is
|
||||
# an open call. It is no longer the only runner (the Swarm-hosted one is
|
||||
# always-on), and the box's new job is AI model hosting plus flex Swarm-worker
|
||||
# capacity. Keeping it is fine — it is real CI capacity — but the estate no
|
||||
# longer depends on it, so decide deliberately rather than by inertia.
|
||||
#
|
||||
# bash deploy/forgejo/register-lan-runner.sh <forgejo_url> <registration_token>
|
||||
#
|
||||
|
|
|
|||
3
infra/deploy/forgejo/runner-vps2/.gitignore
vendored
3
infra/deploy/forgejo/runner-vps2/.gitignore
vendored
|
|
@ -1,3 +0,0 @@
|
|||
# The runner registration (data/.runner) holds a long-lived runner token, and
|
||||
# the daemon writes caches and job workspaces beside it. None of it belongs in git.
|
||||
data/
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
# The always-on Actions runner (vps2)
|
||||
|
||||
The runner CI actually depends on. The desktop runner
|
||||
(`../register-lan-runner.sh`) stays registered as extra capacity, but nothing in
|
||||
the estate may assume it is up.
|
||||
|
||||
Labels: `docker` only. `thermograph-lan` is deliberately **not** registered here
|
||||
— it was the host-native LAN-deploy label, dev moved to vps1 and is deployed
|
||||
over SSH like beta and prod, and no workflow requests it any more.
|
||||
|
||||
## Why this exists
|
||||
|
||||
On 2026-07-31 16:31Z the desktop — then the estate's **only** registered runner
|
||||
— went offline. For the next 21 hours nothing merged (every protected branch
|
||||
requires a check that only a runner can produce), nothing deployed, and the
|
||||
03:00Z `ops-cron` did not run. Forgejo held that scheduled run in `waiting`
|
||||
rather than dropping it, so all three jobs completed on reconnect and no backup
|
||||
was actually lost. A longer outage would not have been so kind.
|
||||
|
||||
`docker-stack.yml` and this directory's parent README both described an
|
||||
"always-on Swarm-hosted runner" during that entire window. There wasn't one; an
|
||||
early revision had it as a Docker-in-Docker sidecar and it was removed. That is
|
||||
why the single point of failure went unnoticed for weeks, and it is the reason
|
||||
the root `CLAUDE.md` treats a stale doc as a correctness bug rather than a
|
||||
documentation one.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- vps2's docker daemon is logged in to `git.thermograph.org` (it is —
|
||||
`/root/.docker/config.json`), so the daemon can pull the job image.
|
||||
- `git.thermograph.org` resolves to the **mesh** address from vps2 for registry
|
||||
traffic. Job containers get this via `--add-host` in `config.yaml`; the host
|
||||
daemon already has it.
|
||||
- The job image exists: `git.thermograph.org/jinemi/thermograph/ci-runner:v2`.
|
||||
Built from `../ci-runner/Dockerfile`; see that file for when to rebuild.
|
||||
|
||||
## Where it is deployed, and why not from the checkout
|
||||
|
||||
**`/opt/forgejo-runner/` on vps2**, holding a copy of `docker-compose.yml` and
|
||||
`config.yaml` from this directory, plus a `data/` the runner owns.
|
||||
|
||||
Deliberately *not* run in place from `/opt/thermograph/…/runner-vps2/`. That
|
||||
checkout is `git reset --hard`'d on every prod deploy; untracked files do
|
||||
survive that (it is the same property `deploy/.image-tags.env` relies on), but a
|
||||
single `git clean -fdx` during troubleshooting would delete `data/.runner` and
|
||||
de-register the runner. Putting the thing that repairs the estate inside the
|
||||
tree the estate rewrites is the same mistake as scheduling it on the Swarm it
|
||||
deploys.
|
||||
|
||||
So this directory is the source of truth; vps2 gets a copy:
|
||||
|
||||
```bash
|
||||
install -d -m 0755 /opt/forgejo-runner
|
||||
cp /opt/thermograph/infra/deploy/forgejo/runner-vps2/docker-compose.yml \
|
||||
/opt/thermograph/infra/deploy/forgejo/runner-vps2/config.yaml \
|
||||
/opt/forgejo-runner/
|
||||
```
|
||||
|
||||
Re-copy after changing either file here, then `docker compose up -d`. There is
|
||||
no automation for that step and there should not be: a workflow that redeploys
|
||||
the runner runs *on* the runner.
|
||||
|
||||
## One-time registration
|
||||
|
||||
The registration token is short-lived and single-purpose. Fetch it from the
|
||||
Forgejo UI (**Site Administration → Actions → Runners → Create new Runner**) or
|
||||
the API, and keep it out of shell history — it is a credential, however
|
||||
briefly.
|
||||
|
||||
```bash
|
||||
cd /opt/forgejo-runner
|
||||
mkdir -p data
|
||||
cp config.yaml data/config.yaml
|
||||
|
||||
read -rs REG_TOKEN && export REG_TOKEN # paste; not echoed, not in history
|
||||
|
||||
docker run --rm -it \
|
||||
-v "$PWD/data:/data" -w /data --user root \
|
||||
code.forgejo.org/forgejo/runner:6.3.1 \
|
||||
forgejo-runner register --no-interactive \
|
||||
--instance https://git.thermograph.org \
|
||||
--token "$REG_TOKEN" \
|
||||
--name thermograph-vps2 \
|
||||
--labels 'docker:docker://git.thermograph.org/jinemi/thermograph/ci-runner:v2'
|
||||
|
||||
unset REG_TOKEN
|
||||
```
|
||||
|
||||
That writes `data/.runner`, which holds the **long-lived** runner token. It is
|
||||
`0600` and must stay out of git — `data/` is gitignored for exactly this reason.
|
||||
Re-running `register` with a fresh registration token replaces it; the old
|
||||
registration then shows offline in the UI and should be deleted there.
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
docker compose logs -f --tail=50 runner # expect "runner: ... successfully activated"
|
||||
```
|
||||
|
||||
Confirm Forgejo agrees, rather than trusting the log:
|
||||
|
||||
```bash
|
||||
# on vps1, where Forgejo's database lives
|
||||
fdb=$(docker ps -qf name=forgejo_db | head -1)
|
||||
docker exec "$fdb" sh -c 'psql -U $POSTGRES_USER -d $POSTGRES_DB -c \
|
||||
"select name, to_timestamp(last_online) as last_online, agent_labels from action_runner order by id;"'
|
||||
```
|
||||
|
||||
Two rows, both recent. `last_online` is the only honest liveness signal — a
|
||||
runner process that is up but cannot reach Forgejo looks healthy locally and is
|
||||
useless.
|
||||
|
||||
## Upgrading
|
||||
|
||||
Pin changes go in `docker-compose.yml` (`image:`) and are applied with
|
||||
`docker compose up -d`. The runner token survives an image change; it lives in
|
||||
`data/.runner`, not in the image.
|
||||
|
||||
Changing the **job** image (the `docker://…ci-runner:vN` label) is a
|
||||
re-registration or a hand-edit of `data/.runner`'s `labels` array — the same
|
||||
caveat `../ci-runner/Dockerfile` records for the desktop runner. Keep the two
|
||||
runners on the same job image or a workflow's behaviour starts depending on
|
||||
which runner happened to pick it up.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Jobs queue forever with both runners online.** The workflow asked for a label
|
||||
nobody registered. `runs-on: docker` is the only label this estate uses.
|
||||
|
||||
**`docker pull` fails inside a job, works on the host.** The `--add-host` line
|
||||
in `config.yaml` is missing or wrong. vps1's Caddy serves `/v2/*` to the mesh
|
||||
only, and the job container has no other way to learn the mesh address.
|
||||
|
||||
**Jobs fail with permission denied on the socket.** `user: root` was dropped
|
||||
from the compose file.
|
||||
|
||||
**The runner is up but `last_online` is stale.** It cannot reach
|
||||
`https://git.thermograph.org` — check egress from vps2 and that Forgejo on vps1
|
||||
is serving.
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Forgejo Actions runner config for the vps2 runner. Mounted at /data/config.yaml
|
||||
# by runner-vps2/docker-compose.yml.
|
||||
#
|
||||
# Values that differ from `forgejo-runner generate-config` defaults are the only
|
||||
# ones written out below; everything else is left at its default deliberately, so
|
||||
# a future runner upgrade inherits new defaults instead of a frozen copy of the
|
||||
# old ones.
|
||||
|
||||
log:
|
||||
level: info
|
||||
job_level: info
|
||||
|
||||
runner:
|
||||
file: .runner
|
||||
|
||||
# ONE job at a time. The desktop runner uses capacity 8, which is right for a
|
||||
# box whose whole job is CI. This runner shares a host with prod, so the
|
||||
# binding constraint is not throughput, it is that a CI burst must never be
|
||||
# able to contend with thermograph_web for CPU. One job at a time also makes
|
||||
# the resource ceiling below exact rather than a per-job estimate multiplied
|
||||
# by an unknown.
|
||||
capacity: 1
|
||||
|
||||
timeout: 3h
|
||||
shutdown_timeout: 3h
|
||||
|
||||
container:
|
||||
# 1) --add-host: git.thermograph.org resolves publicly to vps1, but vps1's
|
||||
# Caddy rejects the /v2/* registry API from off-mesh. Job containers that
|
||||
# docker-pull or docker-push therefore need the MESH address, and DNS will
|
||||
# not give it to them. Same line the desktop runner carries; the value is
|
||||
# the same from vps2 because both boxes are on 10.10.0.0/24.
|
||||
#
|
||||
# 2) --cpus/--memory: vps2 runs prod. Without a ceiling a runaway job (a
|
||||
# `docker build` that fans out, a test that leaks) competes with the
|
||||
# production app for the same 6 cores. These bound the JOB container, which
|
||||
# is a sibling of the runner rather than its child, so the runner's own
|
||||
# compose limits do not cover it -- this is the only lever that does.
|
||||
options: --add-host=git.thermograph.org:10.10.0.2 --cpus=2 --memory=4g
|
||||
|
||||
# Jobs may mount NOTHING from the host. The workflows here do not need to:
|
||||
# they check out into the job container's own workspace and reach the estate
|
||||
# over SSH. An empty list is the difference between "a job can read
|
||||
# /etc/thermograph.env" and "a job cannot", on the box where that file is
|
||||
# prod's.
|
||||
valid_volumes: []
|
||||
|
||||
privileged: false
|
||||
|
||||
# Mounts the host docker socket into the job container. build-push.yml runs
|
||||
# `docker build`, so it needs a daemon; this points it at vps2's. See the
|
||||
# README for what that does and does not mean for trust.
|
||||
docker_host: "automount"
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
# The always-on Forgejo Actions runner, on vps2.
|
||||
#
|
||||
# cd /opt/forgejo-runner # a COPY of this file, not the checkout
|
||||
# docker compose up -d
|
||||
#
|
||||
# It runs from /opt/forgejo-runner rather than in place, because the checkout is
|
||||
# git reset --hard'd on every prod deploy and one `git clean -fdx` there would
|
||||
# delete data/.runner. See README.md.
|
||||
#
|
||||
# Registration is a one-off and is NOT in this file, because it takes a
|
||||
# short-lived token a human fetches. See README.md.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHY A SECOND RUNNER EXISTS AT ALL
|
||||
# ============================================================================
|
||||
# Until 2026-08-01 the estate had exactly one registered runner, on the desktop.
|
||||
# It went offline at 2026-07-31 16:31Z and everything stopped for 21 hours: no
|
||||
# merge could satisfy a required check, no deploy could run, and the 03:00Z
|
||||
# ops-cron -- THE backup for both application databases and for Forgejo -- did
|
||||
# not fire. Forgejo queued that run rather than dropping it, so it completed on
|
||||
# reconnect, but a longer outage would have meant real backup gaps.
|
||||
#
|
||||
# So this runner is not extra capacity. It is the one that has to be up.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHY A PLAIN CONTAINER AND NOT A SWARM SERVICE
|
||||
# ============================================================================
|
||||
# Same argument as infra/openbao/config/openbao.hcl makes for OpenBao: the thing
|
||||
# that repairs the estate must not depend on the estate. A Swarm-scheduled
|
||||
# runner cannot redeploy the Swarm it is scheduled by, and if the cluster is the
|
||||
# thing that is broken, CI is gone exactly when it is needed. `restart: always`
|
||||
# on the local daemon has one dependency: the local daemon.
|
||||
#
|
||||
# An earlier revision of deploy/forgejo/docker-stack.yml did run a runner as a
|
||||
# Swarm-scheduled Docker-in-Docker sidecar. It was removed, and the docs kept
|
||||
# claiming an "always-on Swarm-hosted runner" for weeks afterwards -- which is
|
||||
# how the single point of failure went unnoticed.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHAT MOUNTING THE DOCKER SOCKET ON THE PROD HOST MEANS
|
||||
# ============================================================================
|
||||
# Be plain about it: /var/run/docker.sock is root-equivalent on vps2, and vps2
|
||||
# runs prod. A workflow that can start a container can mount / and read
|
||||
# /etc/thermograph.env. This runner does not create that exposure from nothing
|
||||
# -- Forgejo's own database already stores VPS2_SSH_KEY, which is root on this
|
||||
# box -- but it does move the exposure from "a secret at rest" to "a secret a
|
||||
# job can reach", and that is a real difference.
|
||||
#
|
||||
# What bounds it, in order of how much each is worth:
|
||||
# 1. Only this repo's workflows run here, and only the owner can push to it.
|
||||
# 2. config.yaml sets valid_volumes: [] -- jobs cannot bind-mount host paths.
|
||||
# 3. capacity: 1 and --cpus/--memory keep a job from contending with prod.
|
||||
# 4. This compose project joins no thermograph network, so a job container
|
||||
# reaches prod's services no more easily than any other host process.
|
||||
#
|
||||
# It is defence against accident, not against a hostile workflow author. On a
|
||||
# two-person estate where both people can already SSH to this box as root, that
|
||||
# is the honest boundary.
|
||||
|
||||
name: forgejo-runner
|
||||
|
||||
services:
|
||||
runner:
|
||||
image: code.forgejo.org/forgejo/runner:6.3.1
|
||||
restart: always
|
||||
# The socket is root:docker on the host; the image's default user is not in
|
||||
# that group. Running as root is what the socket mount requires.
|
||||
user: root
|
||||
working_dir: /data
|
||||
command: forgejo-runner daemon --config /data/config.yaml
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# Bounds the DAEMON. Job containers are siblings started through the socket,
|
||||
# not children, so they are unaffected by these -- container.options in
|
||||
# config.yaml is what bounds those.
|
||||
cpus: 1.0
|
||||
mem_limit: 1g
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
|
@ -1,14 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
# OpenBao source backend for render-secrets.sh. Sourced, never run directly.
|
||||
#
|
||||
# Functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
# thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
|
||||
#
|
||||
# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
|
||||
# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
|
||||
# a different shape for a different consumer, not an inconsistency.
|
||||
# Two functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
#
|
||||
# render_thermograph_secrets in render-secrets.sh dispatches to the second when
|
||||
# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
|
||||
|
|
@ -32,13 +27,7 @@ thermograph_openbao_source() {
|
|||
local env_name="${1:?env name required}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
# Explicit override first, then the per-environment value env-topology.sh derives,
|
||||
# then the conventional path. Same precedence shape as render-secrets.sh:44's
|
||||
# THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
|
||||
# THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
|
||||
# deploy path sets. Falling straight through to the bare default is what made beta
|
||||
# authenticate as tg-prod and get denied its own path.
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
|
|
@ -118,16 +107,9 @@ thermograph_openbao_source() {
|
|||
# /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
|
||||
# deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
|
||||
# `$(...)` or a backtick in a secret is a live command-execution path on the
|
||||
# deploy host. Refusing here turns a latent code-execution bug into a loud
|
||||
# render failure.
|
||||
#
|
||||
# An earlier version of this comment claimed the SOPS path had never tripped
|
||||
# this "only because every current value happens to be alphanumeric". That is
|
||||
# false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
|
||||
# tripped THESE paths because it is not in them — it lives at
|
||||
# centralis/<env> and renders to /etc/centralis.env, which is single-quoted
|
||||
# precisely because raw dotenv cannot carry it. See
|
||||
# thermograph_render_openbao_centralis at the end of this file.
|
||||
# deploy host. The SOPS path has always had this hazard and has never tripped it
|
||||
# only because every current value happens to be alphanumeric. Refusing here
|
||||
# turns a latent code-execution bug into a loud render failure.
|
||||
THERMOGRAPH_BAO_COMMON="$common_json" \
|
||||
THERMOGRAPH_BAO_ENV="$env_json" \
|
||||
python3 - <<'PY'
|
||||
|
|
@ -255,239 +237,3 @@ thermograph_render_openbao() {
|
|||
rm -f "$tmp"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CENTRALIS
|
||||
# ---------------------------------------------------------------------------
|
||||
# /etc/centralis.env is a different file, with a different consumer, and therefore a
|
||||
# different output shape. render-secrets.sh:206-233 carries the full argument; the
|
||||
# short version is that it is consumed by exactly one thing and that thing is a shell:
|
||||
#
|
||||
# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
|
||||
#
|
||||
# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
|
||||
# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
|
||||
# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
|
||||
# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
|
||||
# by serving a single `shared` identity — indistinguishable from the file never having
|
||||
# been written. That is the 2026-07-24 incident. So this path emits POSIX
|
||||
# single-quoted assignments and PROVES them by sourcing the result in a clean shell
|
||||
# before anything touches $dest.
|
||||
#
|
||||
# This mirrors render_centralis_secrets rather than sharing with it, for the same
|
||||
# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
|
||||
# and all new risk stays in this file. Consolidate the two when the SOPS path is being
|
||||
# deleted anyway, not before.
|
||||
|
||||
# _thermograph_openbao_login <approle_file> <addr> <ca> -> token on STDOUT
|
||||
#
|
||||
# Separate from the login inline in thermograph_openbao_source deliberately: that
|
||||
# function is now exercised by prod and beta parity and is left untouched.
|
||||
_thermograph_openbao_login() {
|
||||
local approle="$1" addr="$2" ca="$3"
|
||||
local creds role_id secret_id
|
||||
if [ -r "$approle" ]; then
|
||||
creds=$(cat "$approle")
|
||||
else
|
||||
creds=$(sudo cat "$approle" 2>/dev/null || true)
|
||||
fi
|
||||
[ -n "$creds" ] || {
|
||||
echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
|
||||
return 1
|
||||
}
|
||||
role_id=$(printf '%s\n' "$creds" | sed -n '1p')
|
||||
secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
|
||||
[ -n "$role_id" ] && [ -n "$secret_id" ] || {
|
||||
echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_ADDR="$addr"
|
||||
[ -f "$ca" ] && export BAO_CACERT="$ca"
|
||||
bao write -field=token auth/approle/login \
|
||||
role_id="$role_id" secret_id="$secret_id" 2>/dev/null
|
||||
}
|
||||
|
||||
# thermograph_render_openbao_centralis <env_name> [dest]
|
||||
thermograph_render_openbao_centralis() {
|
||||
local env_name="${1:?env name required}"
|
||||
local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
|
||||
return 1
|
||||
}
|
||||
command -v python3 >/dev/null 2>&1 || {
|
||||
echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
|
||||
|
||||
local token
|
||||
token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
|
||||
[ -n "$token" ] || {
|
||||
echo "!! OpenBao AppRole login failed for centralis" >&2
|
||||
echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_TOKEN="$token"
|
||||
|
||||
# ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
|
||||
# path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
|
||||
# persists into the caller's shell and re-fires on its next `source`.
|
||||
local work rc=0
|
||||
work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
|
||||
chmod 700 "$work"
|
||||
|
||||
# Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
|
||||
# Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
|
||||
# command that is the left operand of `||`, which this subshell is.
|
||||
(
|
||||
umask 077
|
||||
|
||||
bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
|
||||
> "$work/kv.json" 2>/dev/null || {
|
||||
echo "!! cannot read ${mount}/centralis/${env_name}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
|
||||
import json, sys
|
||||
doc = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
data = doc.get("data", {}).get("data", {}) or {}
|
||||
if not data:
|
||||
sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
|
||||
sys.exit(1)
|
||||
json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
|
||||
UNWRAP
|
||||
|
||||
python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
|
||||
import json, re, sys
|
||||
|
||||
src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
||||
|
||||
|
||||
def shell_quote(v):
|
||||
"""POSIX single-quoting. Inside '...' every byte is literal except ' itself,
|
||||
which is closed, backslash-escaped and reopened. There is no escape sequence
|
||||
to get wrong and no character class to keep up to date: it is total, and
|
||||
total is the only property worth having for a file bash will expand."""
|
||||
return "'" + v.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
def scalar(k, v):
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
if isinstance(v, bool): # before int: bool is an int in Python
|
||||
return "true" if v else "false"
|
||||
if isinstance(v, (int, float)):
|
||||
return json.dumps(v)
|
||||
if v is None:
|
||||
return ""
|
||||
sys.stderr.write(
|
||||
"!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
data = json.load(open(src, encoding="utf-8"))
|
||||
want, lines = {}, []
|
||||
for k, v in data.items():
|
||||
if not NAME.match(k):
|
||||
sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
|
||||
sys.exit(1)
|
||||
want[k] = scalar(k, v)
|
||||
lines.append("%s=%s\n" % (k, shell_quote(want[k])))
|
||||
|
||||
with open(out_env, "w", encoding="utf-8") as fh:
|
||||
fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
|
||||
fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
|
||||
fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
|
||||
fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
|
||||
fh.writelines(lines)
|
||||
with open(want_json, "w", encoding="utf-8") as fh:
|
||||
json.dump(want, fh)
|
||||
sys.stderr.write(" %d keys quoted\n" % len(want))
|
||||
EMIT
|
||||
|
||||
# PROVE IT. Source the rendered file exactly the way the deploy does — clean
|
||||
# environment, `set -a`, nothing inherited that could mask a dropped key — and
|
||||
# read every value back. This is the check the 2026-07-24 file could not pass.
|
||||
# shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded
|
||||
# by the inner `bash -c` against the argument after `_`, not by this shell.
|
||||
env -i PATH="$PATH" bash -c \
|
||||
'set -a; . "$1"; set +a; exec python3 -c "
|
||||
import json, os, sys
|
||||
sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
|
||||
> "$work/got.json" || exit 1
|
||||
|
||||
python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
|
||||
import json, sys
|
||||
|
||||
want = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
got = json.load(open(sys.argv[2], encoding="utf-8"))
|
||||
|
||||
bad = []
|
||||
for k, v in want.items():
|
||||
if k not in got:
|
||||
bad.append("%s: not set at all after sourcing" % k)
|
||||
elif got[k] != v:
|
||||
# NEVER print either value. The difference is the finding; the content is
|
||||
# not, and this runs inside a deploy log.
|
||||
bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
|
||||
% (k, len(v), len(got[k])))
|
||||
if bad:
|
||||
sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
|
||||
for b in bad:
|
||||
sys.stderr.write("!! %s\n" % b)
|
||||
sys.exit(1)
|
||||
|
||||
# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
|
||||
# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
|
||||
# single `shared` identity, which looks exactly like the registry never having been
|
||||
# configured. Refuse to write a file that would do that.
|
||||
raw = got.get("CENTRALIS_TOKENS", "")
|
||||
if raw:
|
||||
try:
|
||||
reg = json.loads(raw)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
|
||||
sys.exit(1)
|
||||
if not isinstance(reg, dict) or not reg:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
|
||||
sys.exit(1)
|
||||
if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
|
||||
sys.exit(1)
|
||||
# Subjects are names on audit lines, not credentials. Printing them is the whole
|
||||
# point: it is how an operator sees at a glance that nobody was lost.
|
||||
sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
|
||||
% (len(reg), ", ".join(sorted(reg))))
|
||||
|
||||
sys.stderr.write(" %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
|
||||
VERIFY
|
||||
) || rc=1
|
||||
|
||||
# Only now, with a file read back through bash and matched value for value, does
|
||||
# anything touch $dest. 0600: four bearer credentials for the estate's control
|
||||
# plane, with no group that needs them.
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
if [ -f "$dest" ] && [ -w "$dest" ]; then
|
||||
cat "$work/out.env" > "$dest" || rc=1
|
||||
elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
else
|
||||
echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "!! render aborted; ${dest} left untouched" >&2
|
||||
fi
|
||||
|
||||
rm -rf "$work"
|
||||
return "$rc"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -272,30 +272,6 @@ render_centralis_secrets() {
|
|||
echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Same early-return dispatch as render_thermograph_secrets, for the same reason
|
||||
# given there: everything below — the age-key sudo lift, the JSON decrypt, the
|
||||
# quote-and-prove pipeline, the three-way write — stays on exactly the code path it
|
||||
# has always been on, so this cannot regress the backend still authoritative for
|
||||
# Centralis. The OpenBao path needs no age key, so it returns before that check.
|
||||
local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
|
||||
if [ "$backend" = openbao ]; then
|
||||
local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
|
||||
if [ ! -f "$bao_lib" ]; then
|
||||
echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
|
||||
echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
|
||||
return 1
|
||||
fi
|
||||
# shellcheck source=/dev/null
|
||||
. "$bao_lib"
|
||||
thermograph_render_openbao_centralis "$env_name" "$dest"
|
||||
return
|
||||
fi
|
||||
if [ "$backend" != sops ]; then
|
||||
echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$key" ]; then
|
||||
echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7
|
|||
THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str]
|
||||
THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str]
|
||||
THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str]
|
||||
THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str]
|
||||
THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str]
|
||||
THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str]
|
||||
THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str]
|
||||
THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str]
|
||||
THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str]
|
||||
THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str]
|
||||
THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str]
|
||||
THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str]
|
||||
TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str]
|
||||
|
|
@ -25,7 +25,7 @@ sops:
|
|||
bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
|
||||
lastmodified: "2026-07-31T04:23:12Z"
|
||||
mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str]
|
||||
lastmodified: "2026-07-25T00:38:24Z"
|
||||
mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
|
|
|
|||
|
|
@ -112,22 +112,6 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv
|
|||
Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
|
||||
isolation lives in the policy. Today both live in one shell variable.
|
||||
|
||||
**`centralis/<env>` renders differently from everything above it, and must.** The
|
||||
app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
|
||||
is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
|
||||
pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
|
||||
sourcing it in a clean shell and comparing every value back before the file is
|
||||
written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
|
||||
strips its quotes, Centralis fails closed on the malformed result and serves a
|
||||
single `shared` identity, which is indistinguishable from the file never having
|
||||
been written. That is the 2026-07-24 incident, and the round-trip check exists so a
|
||||
render carrying it cannot reach `/etc`.
|
||||
|
||||
Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
|
||||
metacharacters, no newlines) only to `common` and `env/*`. Applying them to
|
||||
`centralis/*` would reject data that renders perfectly well — the validator matches
|
||||
the renderer that will consume the path, not a single global rule.
|
||||
|
||||
`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
|
||||
rule expressible as exactly one path, with no wildcard over `env/*` able to
|
||||
accidentally re-include it.
|
||||
|
|
@ -164,59 +148,22 @@ SOPS and retiring age are two different projects.
|
|||
### Stand it up (once, by the operator, on vps2)
|
||||
|
||||
```sh
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init
|
||||
# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles
|
||||
bao token revoke -self
|
||||
```
|
||||
|
||||
> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
|
||||
> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
|
||||
> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
> minting a root token requires a token you already have. **The recovery keys are not
|
||||
> a way back in**; they rekey, they do not authenticate. Unless
|
||||
> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
|
||||
> the last root token locks you out of an otherwise healthy, unsealed vault, and the
|
||||
> only remedy is re-initialising from scratch. Revoke only once a second admin
|
||||
> identity exists.
|
||||
|
||||
Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
|
||||
`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
|
||||
openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
|
||||
with `certificate signed by unknown authority`.
|
||||
|
||||
### Seed and prove (on vps2 — the age key is already there)
|
||||
### Seed and prove (from your own machine — it needs your age key)
|
||||
|
||||
```sh
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # MUST pass before any flip
|
||||
```
|
||||
|
||||
Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
|
||||
to the host that legitimately renders it, so an environment can only be verified from
|
||||
its own host:
|
||||
|
||||
```sh
|
||||
# on vps2
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
```
|
||||
|
||||
Beta needs no special handling: `verify-parity.sh` sources `env-topology.sh` and calls
|
||||
`thermograph_topology` per environment, so `TG_BAO_APPROLE` points at
|
||||
`/etc/thermograph/openbao-approle-beta` and beta authenticates as `tg-beta` rather than
|
||||
falling back to prod's credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||
|
||||
That sourcing is load-bearing and was missing until 2026-08-01: this document asserted
|
||||
the property while the verifier alone did not have it, so `--env beta` and `--all` both
|
||||
took a 403 and the gate could not check the one environment whose isolation it exists
|
||||
to prove. If you refactor `verify-parity.sh`, the `thermograph_topology` call is not
|
||||
boilerplate.
|
||||
|
||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||
inherits that rule.
|
||||
|
|
@ -231,27 +178,10 @@ One PR per hop, following the estate's own promotion model.
|
|||
+ TG_SECRETS_BACKEND=openbao
|
||||
```
|
||||
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. The gate before
|
||||
prod is **7 consecutive green parity runs on all three environments**. That runs
|
||||
nightly from the `secrets-parity` job in `ops-cron.yml` over SSH, which gives
|
||||
continuous evidence without granting CI any vault access — the host renders, CI only
|
||||
asks it to. Forgejo's run history for that job is the record; a night the job was
|
||||
skipped, or the runner was down, does not count as green.
|
||||
|
||||
Measured 2026-08-01, immediately after the verifier was fixed:
|
||||
|
||||
| env | keys | result |
|
||||
|---|---|---|
|
||||
| dev | 12 | **PASS** — byte-identical |
|
||||
| beta | 24 | FAIL — 3 values differ |
|
||||
| prod | 32 | FAIL — 3 values differ |
|
||||
|
||||
Identical three keys on both: `THERMOGRAPH_S3_SECRET_KEY`,
|
||||
`THERMOGRAPH_LAKE_S3_SECRET_KEY`, `THERMOGRAPH_VAPID_CONTACT`. All three live in
|
||||
`common.yaml`, which was seeded on 2026-07-31 at 02:30Z — *before* the Contabo
|
||||
rotation landed. dev passes because dev never layers `common`. One
|
||||
`seed-from-sops.sh --env common` fixes beta and prod together; the 7-day clock starts
|
||||
after that, not before.
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
|
||||
gate before prod: **7 consecutive green parity runs on all three environments**, run
|
||||
nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
|
||||
CI any vault access).
|
||||
|
||||
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
|
||||
SOPS path for a full release cycle after prod flips — it is the best mitigation
|
||||
|
|
|
|||
|
|
@ -63,21 +63,15 @@ add_role tg-centralis tg-centralis 10.10.0.1/32
|
|||
|
||||
# Install the local (vps2) credentials. prod and beta both render on this box.
|
||||
#
|
||||
# Both files are owned by `agent`, because that is the single identity both
|
||||
# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
|
||||
# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
|
||||
# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
|
||||
# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
|
||||
# they do not, and the chown simply failed.
|
||||
# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
|
||||
# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
|
||||
# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
|
||||
# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
|
||||
# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
|
||||
# creates one.
|
||||
#
|
||||
# So be accurate about what separate credentials buy on this box: NOT deploy-user
|
||||
# isolation — there is one deploy user, and it can read both files. What they buy is
|
||||
# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
|
||||
# *mistake* crossing between environments: a beta render authenticating with beta's
|
||||
# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
|
||||
# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
|
||||
# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
|
||||
# a separate SSH credential for beta in deploy.yml — a larger change than this script.
|
||||
# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
|
||||
# the one age key. This defends against MISCONFIGURATION, not against intrusion.
|
||||
install_creds() {
|
||||
local role="$1" dest="$2" owner="$3"
|
||||
local rid sid
|
||||
|
|
@ -85,17 +79,7 @@ install_creds() {
|
|||
sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
|
||||
# umask before creation, so the file is never briefly world-readable.
|
||||
( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
|
||||
# Abort the function if chown fails, and remove the half-installed file. The call
|
||||
# site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
|
||||
# so without this check a failed chown falls straight through to chmod and prints a
|
||||
# line asserting an ownership that was never applied. A root-owned credential the
|
||||
# renderer cannot read, reported as installed, is worse than no credential at all:
|
||||
# it fails at deploy time instead of here.
|
||||
if ! chown "$owner" "$dest"; then
|
||||
rm -f "$dest"
|
||||
echo " !! chown $owner failed for $dest — removed, nothing installed" >&2
|
||||
return 1
|
||||
fi
|
||||
chown "$owner" "$dest"
|
||||
chmod 0400 "$dest"
|
||||
echo " installed $dest (0400 $owner)"
|
||||
}
|
||||
|
|
@ -103,7 +87,8 @@ install_creds() {
|
|||
if [ "$(id -u)" = 0 ]; then
|
||||
install -d -o root -g root -m 0755 /etc/thermograph
|
||||
install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \
|
||||
|| echo " (beta creds skipped — no 'deploy' user on this box?)"
|
||||
install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
|
||||
else
|
||||
echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
|
||||
|
|
|
|||
|
|
@ -52,19 +52,13 @@ if ! command -v bao >/dev/null 2>&1; then
|
|||
# The checksums file is signed by the release; verify before installing.
|
||||
tmpd="$(mktemp -d)"
|
||||
base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
|
||||
# Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
|
||||
# The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
|
||||
tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
|
||||
# Anchor to end-of-line and save under the real asset name: checksums.txt also lists
|
||||
# <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
|
||||
# so an unanchored match fails on a file that was never fetched.
|
||||
( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
|
||||
curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
|
||||
( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
|
||||
echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
|
||||
rm -rf "$tmpd"; exit 1
|
||||
}
|
||||
tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
|
||||
tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
|
||||
install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
|
||||
rm -rf "$tmpd"
|
||||
fi
|
||||
|
|
@ -94,11 +88,7 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
|
|||
echo "==> 3/7 static seal key"
|
||||
if [ ! -f /etc/openbao/unseal.key ]; then
|
||||
# 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
|
||||
# tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
|
||||
# static seal reads this file RAW. That one trailing byte makes the key neither
|
||||
# 32 raw bytes nor clean base64, so bao refuses to start with
|
||||
# `Error configuring seal "static": unknown encoding for AES-256 key`.
|
||||
( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
|
||||
( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
|
||||
chown openbao:openbao /etc/openbao/unseal.key
|
||||
chmod 0400 /etc/openbao/unseal.key
|
||||
echo " generated /etc/openbao/unseal.key (0400 openbao)"
|
||||
|
|
@ -135,24 +125,6 @@ systemctl is-active --quiet openbao.service || {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
|
||||
# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
|
||||
# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
|
||||
# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
|
||||
# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
|
||||
# desktop, the phone — has any business reaching the secret store.
|
||||
if command -v ufw >/dev/null 2>&1; then
|
||||
if ufw status | grep -q '8200.*10\.10\.0\.2'; then
|
||||
echo " ufw: 8200/tcp from 10.10.0.2 already allowed"
|
||||
else
|
||||
ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
|
||||
comment 'vps1/dev -> openbao' >/dev/null
|
||||
echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
|
||||
fi
|
||||
else
|
||||
echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
|
||||
fi
|
||||
|
||||
export BAO_ADDR="https://127.0.0.1:8200"
|
||||
export BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
|
||||
|
|
@ -168,16 +140,9 @@ else
|
|||
echo " ############################################################"
|
||||
echo
|
||||
# -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
|
||||
# yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
|
||||
# is redundancy against loss, not separation of duty — there is one operator.
|
||||
#
|
||||
# These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
|
||||
# the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
|
||||
# (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
# minting a root token requires a token you already have. The deprecated endpoint is
|
||||
# off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
|
||||
# Consequence: revoking the last root token before another admin identity exists is
|
||||
# a one-way door. See the ordering note in the handoff below.
|
||||
# yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
|
||||
# keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
|
||||
# operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
|
||||
bao operator init -recovery-shares=3 -recovery-threshold=2
|
||||
echo
|
||||
echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
|
||||
|
|
@ -191,36 +156,22 @@ cat <<EOF
|
|||
|
||||
==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
|
||||
|
||||
Paste the initial root token printed above and run:
|
||||
Paste the initial root token (or one minted from recovery keys) and run:
|
||||
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
bash ${SELF_DIR}/bootstrap-policies.sh
|
||||
|
||||
KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
|
||||
cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
|
||||
\`bao token revoke -self\` locks you out of an otherwise healthy vault.
|
||||
|
||||
That script enables the KV mount, writes the policies, creates the AppRoles and
|
||||
installs each host's credentials. Then seed — on THIS box, where the age key already
|
||||
lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
|
||||
secrets is not for an agent to run:
|
||||
installs each host's credentials. Then, from YOUR OWN machine (it needs your age
|
||||
key, and per infra/CLAUDE.md a script that reads production secrets is not for an
|
||||
agent to run):
|
||||
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # must PASS before any flip
|
||||
|
||||
Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
|
||||
that legitimately renders it, so an environment can only be verified from its own host.
|
||||
|
||||
# here (vps2)
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
|
||||
Only once all three PASS, and only after you have a second admin identity that is not
|
||||
the root token, revoke it: bao token revoke -self
|
||||
Then revoke the root token: bao token revoke -self
|
||||
|
||||
NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
|
||||
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
|
||||
|
|
|
|||
|
|
@ -25,12 +25,8 @@
|
|||
|
||||
ui = false
|
||||
cluster_name = "thermograph"
|
||||
disable_mlock = true
|
||||
log_level = "info"
|
||||
// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
|
||||
// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
|
||||
// setting the server ignores only trains the operator to skim startup warnings,
|
||||
// which is where a real one will eventually hide. The corresponding note about
|
||||
// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
|
||||
|
||||
// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
|
||||
// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
|
||||
|
|
@ -114,21 +110,9 @@ seal "static" {
|
|||
//
|
||||
// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
|
||||
// unlinked inode and the log silently stops growing.
|
||||
// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
|
||||
// device NAME, not its type — and device settings go in `options`. Written as
|
||||
// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
|
||||
// be specified"); with type+path but a bare file_path it starts but silently
|
||||
// IGNORES the path and mode, which is the worse failure of the two.
|
||||
audit "file" {
|
||||
type = "file"
|
||||
path = "file/"
|
||||
options = {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
|
||||
audit "syslog" {
|
||||
type = "syslog"
|
||||
path = "syslog/"
|
||||
}
|
||||
audit "syslog" {}
|
||||
|
|
|
|||
|
|
@ -119,17 +119,7 @@ for target in "${TARGETS[@]}"; do
|
|||
# Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
|
||||
# values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
|
||||
# SEED time — before anything depends on it — rather than at deploy time.
|
||||
# Which renderer will consume this path decides which values are legal, so the
|
||||
# validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
|
||||
# KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
|
||||
# there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
|
||||
# render_centralis_secrets and its OpenBao counterpart, where every byte inside
|
||||
# '...' is literal — so applying the dotenv rules to it rejects data that renders
|
||||
# perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
|
||||
quoted=0
|
||||
case "$target" in centralis.*) quoted=1 ;; esac
|
||||
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
|
||||
import json, os, re, sys
|
||||
|
||||
doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
|
||||
|
|
@ -137,9 +127,6 @@ if not isinstance(doc, dict) or not doc:
|
|||
sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
|
||||
sys.exit(1)
|
||||
|
||||
# 1 => destined for a single-quoted env file; 0 => raw dotenv.
|
||||
QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
|
||||
|
||||
KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
UNSAFE = set("`$\\\"'")
|
||||
|
||||
|
|
@ -154,13 +141,12 @@ for key, val in doc.items():
|
|||
val = ""
|
||||
if not isinstance(val, str):
|
||||
val = json.dumps(val, separators=(",", ":"))
|
||||
if not QUOTED:
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
out[key] = val
|
||||
|
||||
if bad:
|
||||
|
|
|
|||
|
|
@ -36,13 +36,8 @@ usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)
|
|||
Renders each environment through BOTH backends and compares the resulting
|
||||
KEY=value sets. Prints key names and counts only, never values.
|
||||
|
||||
--all means "every environment that lives on THIS host", not all three: dev is
|
||||
on vps1, beta and prod are on vps2, so no single box can check them all. An
|
||||
environment with no checkout here is skipped out loud. Skipping every one of
|
||||
them is an error, not a pass.
|
||||
|
||||
Exit status: 0 = every environment checked here is at parity; 1 = a mismatch,
|
||||
or nothing was checkable. Suitable as a CI / cron gate.
|
||||
Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
|
||||
Suitable as a CI / cron gate.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
|
@ -57,67 +52,20 @@ while [ $# -gt 0 ]; do
|
|||
done
|
||||
[ ${#TARGETS[@]} -gt 0 ] || usage
|
||||
|
||||
# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
|
||||
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
|
||||
# bare default — prod's credential — for every environment. On vps2 that made
|
||||
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
|
||||
# thermograph/data/env/beta, so the gate could not verify the one environment whose
|
||||
# isolation it exists to prove. `--all` failed the same way, and dev with it.
|
||||
#
|
||||
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
|
||||
# which is the worst place for the omission to be: the tool whose whole job is to
|
||||
# notice a divergence was itself diverging from the path it verifies.
|
||||
#
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/env-topology.sh"
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
|
||||
|
||||
overall=0
|
||||
checked=0
|
||||
for env_name in "${TARGETS[@]}"; do
|
||||
# Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
|
||||
# filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header
|
||||
# line because --all walks three environments and may skip some of them.
|
||||
if ! thermograph_topology "$env_name"; then
|
||||
echo "!! unknown environment: $env_name" >&2
|
||||
overall=1; continue
|
||||
fi
|
||||
|
||||
# NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to
|
||||
# pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all
|
||||
# anywhere therefore guaranteed a failure on whichever environments are elsewhere,
|
||||
# and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to
|
||||
# /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so
|
||||
# prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like
|
||||
# a policy bug rather than "wrong box".
|
||||
#
|
||||
# Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its
|
||||
# checkout existing. The first version of this check tested for TG_APP_DIR and was
|
||||
# wrong: vps1 still carries a stale /opt/thermograph from before the estate split,
|
||||
# so prod looked resident there and was checked anyway. A leftover directory is
|
||||
# exactly the kind of thing that outlives the arrangement it belonged to.
|
||||
#
|
||||
# Skipping is announced, never silent. A gate that quietly narrows what it checked
|
||||
# is worse than one that fails, because the run still goes green.
|
||||
if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then
|
||||
if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
|
||||
| grep -qxF "$TG_SSH_HOST"; then
|
||||
echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "== parity check: $env_name"
|
||||
checked=$((checked + 1))
|
||||
|
||||
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
|
||||
# flag; on the OpenBao side the policy also forbids it. Taking the flag from the
|
||||
# topology rather than re-deriving it here keeps the comparison apples-to-apples
|
||||
# AND keeps this script from drifting from deploy.sh:67, which reads the same
|
||||
# variable. If the flag were wrong, the OpenBao side would fail closed with a 403
|
||||
# rather than quietly diverge — which is exactly the improvement being verified.
|
||||
skip_common="${TG_SKIP_COMMON:-0}"
|
||||
# flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
|
||||
# the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
|
||||
# would fail closed with a 403 rather than quietly diverge, which is exactly the
|
||||
# improvement being verified.
|
||||
skip_common=0
|
||||
[ "$env_name" = dev ] && skip_common=1
|
||||
|
||||
sops_out=$(mktemp); bao_out=$(mktemp)
|
||||
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
|
||||
|
|
@ -198,23 +146,11 @@ for k in sorted(order):
|
|||
cleanup
|
||||
done
|
||||
|
||||
# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a
|
||||
# host that carries none of them — a new box, a renamed checkout, a typo'd --env —
|
||||
# exits 0 having verified nothing, and a green run is exactly what the 7-day gate
|
||||
# counts. "Nothing was wrong" and "nothing was examined" must not look alike.
|
||||
if [ "$checked" -eq 0 ]; then
|
||||
echo
|
||||
echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2
|
||||
echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$overall" = 0 ]; then
|
||||
echo
|
||||
echo "PARITY OK — ${checked} environment(s) checked on this host."
|
||||
echo "PARITY OK for: ${TARGETS[*]}"
|
||||
echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
|
||||
echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments."
|
||||
echo "That means both hosts — a green run here says nothing about the other box."
|
||||
echo "Recommended gate before prod: 7 consecutive green runs on all three."
|
||||
else
|
||||
echo
|
||||
echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
|
||||
|
|
|
|||
Loading…
Reference in a new issue