Compare commits
2 commits
2194aaa1c4
...
7b3ea69ef9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7b3ea69ef9 | ||
|
|
7b14b9062c |
6 changed files with 254 additions and 13 deletions
|
|
@ -86,6 +86,15 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# env-topology.sh FIRST: it is what sets TG_SECRETS_BACKEND, and
|
||||
# render-secrets.sh:44 defaults to `sops` when it is unset. Without
|
||||
# this line the cutover would be only half-applied — deploy.sh would
|
||||
# read the new backend while THIS workflow, which runs on every push
|
||||
# touching infra/**, kept re-rendering the same file from SOPS. That
|
||||
# is silent while parity holds and a hard failure the moment the SOPS
|
||||
# files are retired.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology dev
|
||||
. infra/deploy/render-secrets.sh
|
||||
# SKIP_COMMON is dev's standing rule, not a preference: common.yaml
|
||||
# is the fleet's shared production credential set, and vps1 also
|
||||
|
|
@ -114,6 +123,13 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev: env-topology.sh is what sets
|
||||
# TG_SECRETS_BACKEND, and for beta it also sets TG_BAO_APPROLE to
|
||||
# the -beta credential. Beta shares a filesystem with prod, so the
|
||||
# bare default is prod's file and tg-host-prod.hcl denies beta's
|
||||
# own path — a 403 at render time on the box that runs prod.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology beta
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env
|
||||
fi
|
||||
|
|
@ -139,6 +155,10 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev — without env-topology.sh this job keeps
|
||||
# rendering from SOPS regardless of what the backend selector says.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology prod
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -216,6 +216,65 @@ jobs:
|
|||
rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true
|
||||
rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true
|
||||
|
||||
secrets-parity:
|
||||
name: OpenBao/SOPS parity gate
|
||||
runs-on: docker
|
||||
# The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive
|
||||
# green parity runs" -- and which, until now, nothing implemented. A gate
|
||||
# that exists only in prose gets satisfied by assertion.
|
||||
#
|
||||
# verify-parity.sh renders each environment through BOTH backends and diffs
|
||||
# the KEY=value sets after a last-wins collapse. It prints key NAMES and
|
||||
# counts only, never values, so these logs are safe to read and to paste.
|
||||
#
|
||||
# A mismatch FAILS the job, deliberately and loudly. The failure modes on
|
||||
# the other side of a bad cutover are all silent: a missing
|
||||
# THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start
|
||||
# working intermittently; a half-rendered VAPID pair makes push.py generate a
|
||||
# new keypair and delete every subscription row as "gone". Red here is the
|
||||
# only warning that arrives before those.
|
||||
#
|
||||
# Runs under sudo because the age key is 0400 root and the AppRole files are
|
||||
# 0400 agent/root -- the SSH user cannot read them directly. This grants CI
|
||||
# no vault access of its own: the HOST renders, CI only asks it to.
|
||||
#
|
||||
# The run history in Forgejo IS the record of consecutive green days. Do not
|
||||
# count a day this job was skipped or the runner was down as green.
|
||||
concurrency:
|
||||
group: ops-secrets-parity
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: prod + beta (vps2)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
||||
username: ${{ secrets.VPS2_SSH_USER }}
|
||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# Each environment is checked from its OWN checkout. The trees are
|
||||
# identical, but running beta's from beta's directory also proves
|
||||
# that checkout is current rather than assuming it.
|
||||
sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod
|
||||
sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta
|
||||
|
||||
- name: dev (vps1)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS1_SSH_HOST }}
|
||||
username: ${{ secrets.VPS1_SSH_USER }}
|
||||
key: ${{ secrets.VPS1_SSH_KEY }}
|
||||
port: ${{ secrets.VPS1_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# dev renders dev.yaml ALONE -- verify-parity.sh takes that from
|
||||
# env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the
|
||||
# comparison matches what deploy-dev.sh actually does. vps1 must
|
||||
# never see common.yaml: it runs Forgejo, its CI, and dev's
|
||||
# unreviewed branch.
|
||||
sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev
|
||||
|
||||
indexnow:
|
||||
name: IndexNow ping
|
||||
runs-on: docker
|
||||
|
|
|
|||
|
|
@ -468,6 +468,60 @@ if raw:
|
|||
sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
|
||||
% (len(reg), ", ".join(sorted(reg))))
|
||||
|
||||
# CENTRALIS_TEAM: same hazard, same shape of guard. Kept in step with the SOPS
|
||||
# renderer deliberately — the parity argument this whole migration rests on is that
|
||||
# the two backends produce the same file, and a validation that exists on only one
|
||||
# side means one backend can write a file the other would have refused.
|
||||
#
|
||||
# It is the Google sign-in allowlist, and buildOAuth() needs a Google client AND at
|
||||
# least one member, so a malformed team turns OAuth off rather than failing. Worse,
|
||||
# centralis' parseTeam() SKIPS a bad entry and carries on, so a typo'd domain drops
|
||||
# exactly one person behind a log line nobody reads.
|
||||
raw_team = got.get("CENTRALIS_TEAM", "")
|
||||
if raw_team:
|
||||
try:
|
||||
team = json.loads(raw_team)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM is not valid JSON after sourcing: %s\n" % exc)
|
||||
sys.exit(1)
|
||||
|
||||
# The three shapes centralis' parseTeam accepts, in src/config.ts.
|
||||
rows = []
|
||||
if isinstance(team, dict):
|
||||
for email, value in team.items():
|
||||
role = value if isinstance(value, str) else (value or {}).get("role") or "member"
|
||||
rows.append((email, role))
|
||||
elif isinstance(team, list):
|
||||
for row in team:
|
||||
row = row or {}
|
||||
rows.append((str(row.get("email", "")), row.get("role") or "member"))
|
||||
else:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM must be a JSON object or array\n")
|
||||
sys.exit(1)
|
||||
|
||||
bad_team = []
|
||||
seen_emails = set()
|
||||
for email, role in rows:
|
||||
key = str(email).strip().lower()
|
||||
if key == "" or "@" not in key:
|
||||
bad_team.append("%r is not an email address — centralis would skip it" % email)
|
||||
elif key in seen_emails:
|
||||
bad_team.append("%s appears twice — centralis would ignore the second" % key)
|
||||
else:
|
||||
seen_emails.add(key)
|
||||
if bad_team:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM has entries centralis would silently drop:\n")
|
||||
for b in bad_team:
|
||||
sys.stderr.write("!! %s\n" % b)
|
||||
sys.exit(1)
|
||||
if not seen_emails:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM is empty — that disables Google sign-in entirely\n")
|
||||
sys.exit(1)
|
||||
|
||||
sys.stderr.write(" CENTRALIS_TEAM parses: %d member(s) — %s\n"
|
||||
% (len(rows), ", ".join("%s=%s" % (e.strip().lower(), r)
|
||||
for e, r in sorted(rows))))
|
||||
|
||||
sys.stderr.write(" %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
|
||||
VERIFY
|
||||
) || rc=1
|
||||
|
|
|
|||
|
|
@ -467,6 +467,69 @@ if raw:
|
|||
sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
|
||||
% (len(reg), ", ".join(sorted(reg))))
|
||||
|
||||
# CENTRALIS_TEAM is the same hazard as CENTRALIS_TOKENS and had none of the same
|
||||
# guard: JSON, full of double quotes, in a file bash expands. It is the allowlist
|
||||
# for Google sign-in, and centralis' buildOAuth() requires a Google client AND at
|
||||
# least one team member — so an empty or malformed team does not fail loudly, it
|
||||
# turns OAuth OFF and leaves the server bearer-token-only. That looks identical to
|
||||
# never having configured it.
|
||||
#
|
||||
# Worse than the tokens case: centralis' parseTeam() SKIPS a bad entry with a
|
||||
# console.error and carries on. An address with a typo'd domain does not break
|
||||
# anything visibly; it silently drops exactly one person, and the log line saying
|
||||
# so scrolls past on a container start nobody watches. So entries parseTeam would
|
||||
# skip are refused HERE, where a human is reading the output.
|
||||
raw_team = got.get("CENTRALIS_TEAM", "")
|
||||
if raw_team:
|
||||
try:
|
||||
team = json.loads(raw_team)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM is not valid JSON after sourcing: %s\n" % exc)
|
||||
sys.exit(1)
|
||||
|
||||
# The three shapes centralis' parseTeam accepts, in src/config.ts:
|
||||
# {"emi@x.com": "owner"} email -> role
|
||||
# {"jin@x.com": {"subject": ..., "role": ...}} email -> object
|
||||
# [{"email": ..., "subject": ..., "role": ...}] array of objects
|
||||
rows = []
|
||||
if isinstance(team, dict):
|
||||
for email, value in team.items():
|
||||
role = value if isinstance(value, str) else (value or {}).get("role") or "member"
|
||||
rows.append((email, role))
|
||||
elif isinstance(team, list):
|
||||
for row in team:
|
||||
row = row or {}
|
||||
rows.append((str(row.get("email", "")), row.get("role") or "member"))
|
||||
else:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM must be a JSON object or array\n")
|
||||
sys.exit(1)
|
||||
|
||||
bad_team = []
|
||||
seen_emails = set()
|
||||
for email, role in rows:
|
||||
key = str(email).strip().lower()
|
||||
if key == "" or "@" not in key:
|
||||
bad_team.append("%r is not an email address — centralis would skip it" % email)
|
||||
elif key in seen_emails:
|
||||
bad_team.append("%s appears twice — centralis would ignore the second" % key)
|
||||
else:
|
||||
seen_emails.add(key)
|
||||
if bad_team:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM has entries centralis would silently drop:\n")
|
||||
for b in bad_team:
|
||||
sys.stderr.write("!! %s\n" % b)
|
||||
sys.exit(1)
|
||||
if not seen_emails:
|
||||
sys.stderr.write("!! CENTRALIS_TEAM is empty — that disables Google sign-in entirely\n")
|
||||
sys.exit(1)
|
||||
|
||||
# Addresses, not credentials, and the same argument as the tokens line above:
|
||||
# seeing the list is how an operator notices a person is missing or a domain
|
||||
# is misspelled, which is the whole failure this check exists to catch.
|
||||
sys.stderr.write(" CENTRALIS_TEAM parses: %d member(s) — %s\n"
|
||||
% (len(rows), ", ".join("%s=%s" % (e.strip().lower(), r)
|
||||
for e, r in sorted(rows))))
|
||||
|
||||
sys.stderr.write(" %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
|
||||
VERIFY
|
||||
) || rc=1
|
||||
|
|
|
|||
|
|
@ -206,9 +206,16 @@ infra/openbao/verify-parity.sh --env beta # expect 24 k
|
|||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
```
|
||||
|
||||
Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
|
||||
environment, so beta authenticates as `tg-beta` rather than falling back to prod's
|
||||
credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||
Beta needs no special handling: `verify-parity.sh` sources `env-topology.sh` and calls
|
||||
`thermograph_topology` per environment, so `TG_BAO_APPROLE` points at
|
||||
`/etc/thermograph/openbao-approle-beta` and beta authenticates as `tg-beta` rather than
|
||||
falling back to prod's credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||
|
||||
That sourcing is load-bearing and was missing until 2026-08-01: this document asserted
|
||||
the property while the verifier alone did not have it, so `--env beta` and `--all` both
|
||||
took a 403 and the gate could not check the one environment whose isolation it exists
|
||||
to prove. If you refactor `verify-parity.sh`, the `thermograph_topology` call is not
|
||||
boilerplate.
|
||||
|
||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||
|
|
@ -224,10 +231,27 @@ One PR per hop, following the estate's own promotion model.
|
|||
+ TG_SECRETS_BACKEND=openbao
|
||||
```
|
||||
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
|
||||
gate before prod: **7 consecutive green parity runs on all three environments**, run
|
||||
nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
|
||||
CI any vault access).
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. The gate before
|
||||
prod is **7 consecutive green parity runs on all three environments**. That runs
|
||||
nightly from the `secrets-parity` job in `ops-cron.yml` over SSH, which gives
|
||||
continuous evidence without granting CI any vault access — the host renders, CI only
|
||||
asks it to. Forgejo's run history for that job is the record; a night the job was
|
||||
skipped, or the runner was down, does not count as green.
|
||||
|
||||
Measured 2026-08-01, immediately after the verifier was fixed:
|
||||
|
||||
| env | keys | result |
|
||||
|---|---|---|
|
||||
| dev | 12 | **PASS** — byte-identical |
|
||||
| beta | 24 | FAIL — 3 values differ |
|
||||
| prod | 32 | FAIL — 3 values differ |
|
||||
|
||||
Identical three keys on both: `THERMOGRAPH_S3_SECRET_KEY`,
|
||||
`THERMOGRAPH_LAKE_S3_SECRET_KEY`, `THERMOGRAPH_VAPID_CONTACT`. All three live in
|
||||
`common.yaml`, which was seeded on 2026-07-31 at 02:30Z — *before* the Contabo
|
||||
rotation landed. dev passes because dev never layers `common`. One
|
||||
`seed-from-sops.sh --env common` fixes beta and prod together; the 7-day clock starts
|
||||
after that, not before.
|
||||
|
||||
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
|
||||
SOPS path for a full release cycle after prod flips — it is the best mitigation
|
||||
|
|
|
|||
|
|
@ -52,6 +52,19 @@ while [ $# -gt 0 ]; do
|
|||
done
|
||||
[ ${#TARGETS[@]} -gt 0 ] || usage
|
||||
|
||||
# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
|
||||
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
|
||||
# bare default — prod's credential — for every environment. On vps2 that made
|
||||
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
|
||||
# thermograph/data/env/beta, so the gate could not verify the one environment whose
|
||||
# isolation it exists to prove. `--all` failed the same way, and dev with it.
|
||||
#
|
||||
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
|
||||
# which is the worst place for the omission to be: the tool whose whole job is to
|
||||
# notice a divergence was itself diverging from the path it verifies.
|
||||
#
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/env-topology.sh"
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
|
||||
|
||||
|
|
@ -59,13 +72,21 @@ overall=0
|
|||
for env_name in "${TARGETS[@]}"; do
|
||||
echo "== parity check: $env_name"
|
||||
|
||||
# Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
|
||||
# filesystem with prod) and TG_SKIP_COMMON. Called inside the loop because --all
|
||||
# walks three environments and the values differ per environment.
|
||||
if ! thermograph_topology "$env_name"; then
|
||||
echo "!! unknown environment: $env_name" >&2
|
||||
overall=1; continue
|
||||
fi
|
||||
|
||||
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
|
||||
# flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
|
||||
# the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
|
||||
# would fail closed with a 403 rather than quietly diverge, which is exactly the
|
||||
# improvement being verified.
|
||||
skip_common=0
|
||||
[ "$env_name" = dev ] && skip_common=1
|
||||
# flag; on the OpenBao side the policy also forbids it. Taking the flag from the
|
||||
# topology rather than re-deriving it here keeps the comparison apples-to-apples
|
||||
# AND keeps this script from drifting from deploy.sh:67, which reads the same
|
||||
# variable. If the flag were wrong, the OpenBao side would fail closed with a 403
|
||||
# rather than quietly diverge — which is exactly the improvement being verified.
|
||||
skip_common="${TG_SKIP_COMMON:-0}"
|
||||
|
||||
sops_out=$(mktemp); bao_out=$(mktemp)
|
||||
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
|
||||
|
|
|
|||
Loading…
Reference in a new issue