Compare commits
No commits in common. "000a8a1048cd3d66574d9c5c56d0f429930ea850" and "5b5f28560d5955b5b5083c4e1b0c25d6262e7059" have entirely different histories.
000a8a1048
...
5b5f28560d
6 changed files with 17 additions and 325 deletions
|
|
@ -86,15 +86,6 @@ jobs:
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||||
# env-topology.sh FIRST: it is what sets TG_SECRETS_BACKEND, and
|
|
||||||
# render-secrets.sh:44 defaults to `sops` when it is unset. Without
|
|
||||||
# this line the cutover would be only half-applied — deploy.sh would
|
|
||||||
# read the new backend while THIS workflow, which runs on every push
|
|
||||||
# touching infra/**, kept re-rendering the same file from SOPS. That
|
|
||||||
# is silent while parity holds and a hard failure the moment the SOPS
|
|
||||||
# files are retired.
|
|
||||||
. infra/deploy/env-topology.sh
|
|
||||||
thermograph_topology dev
|
|
||||||
. infra/deploy/render-secrets.sh
|
. infra/deploy/render-secrets.sh
|
||||||
# SKIP_COMMON is dev's standing rule, not a preference: common.yaml
|
# SKIP_COMMON is dev's standing rule, not a preference: common.yaml
|
||||||
# is the fleet's shared production credential set, and vps1 also
|
# is the fleet's shared production credential set, and vps1 also
|
||||||
|
|
@ -123,13 +114,6 @@ jobs:
|
||||||
git fetch --prune origin main
|
git fetch --prune origin main
|
||||||
git reset --hard origin/main
|
git reset --hard origin/main
|
||||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||||
# See the note in sync-dev: env-topology.sh is what sets
|
|
||||||
# TG_SECRETS_BACKEND, and for beta it also sets TG_BAO_APPROLE to
|
|
||||||
# the -beta credential. Beta shares a filesystem with prod, so the
|
|
||||||
# bare default is prod's file and tg-host-prod.hcl denies beta's
|
|
||||||
# own path — a 403 at render time on the box that runs prod.
|
|
||||||
. infra/deploy/env-topology.sh
|
|
||||||
thermograph_topology beta
|
|
||||||
. infra/deploy/render-secrets.sh
|
. infra/deploy/render-secrets.sh
|
||||||
render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env
|
render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env
|
||||||
fi
|
fi
|
||||||
|
|
@ -155,113 +139,7 @@ jobs:
|
||||||
git fetch --prune origin main
|
git fetch --prune origin main
|
||||||
git reset --hard origin/main
|
git reset --hard origin/main
|
||||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||||
# See the note in sync-dev — without env-topology.sh this job keeps
|
|
||||||
# rendering from SOPS regardless of what the backend selector says.
|
|
||||||
. infra/deploy/env-topology.sh
|
|
||||||
thermograph_topology prod
|
|
||||||
. infra/deploy/render-secrets.sh
|
. infra/deploy/render-secrets.sh
|
||||||
render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env
|
render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env
|
||||||
fi
|
fi
|
||||||
echo "synced to $(git log --oneline -1)"
|
echo "synced to $(git log --oneline -1)"
|
||||||
|
|
||||||
sync-centralis:
|
|
||||||
# Centralis' /etc/centralis.env has been HAND-MAINTAINED. The vault has held a
|
|
||||||
# copy since deploy/secrets/centralis.prod.yaml was seeded, and
|
|
||||||
# render_centralis_secrets has existed since the OpenBao work -- but nothing
|
|
||||||
# on any deploy path ever called it. A renderer with no caller is not a
|
|
||||||
# migration, it is a file.
|
|
||||||
#
|
|
||||||
# That gap is why enabling Google sign-in on Centralis had no safe home: the
|
|
||||||
# only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file,
|
|
||||||
# which is exactly what render-secrets.sh's header warns against.
|
|
||||||
#
|
|
||||||
# WHY THIS JOB RECREATES A CONTAINER WHEN THE OTHER THREE DELIBERATELY DO NOT.
|
|
||||||
# This workflow's header says it never rolls a service, because image tags are
|
|
||||||
# the app domains' axis. Centralis is different in kind: its compose file
|
|
||||||
# interpolates /etc/centralis.env at PARSE time (`set -a && . /etc/centralis.env
|
|
||||||
# && docker compose up -d`), so rendering the file without recreating the
|
|
||||||
# container changes nothing at all. Rendering and not recreating would be the
|
|
||||||
# silent no-op this estate keeps getting bitten by. It is one container with no
|
|
||||||
# replicas -- a two-second recreate, not a deploy.
|
|
||||||
if: github.ref_name == 'main'
|
|
||||||
runs-on: docker
|
|
||||||
concurrency:
|
|
||||||
group: infra-sync-centralis
|
|
||||||
cancel-in-progress: false
|
|
||||||
steps:
|
|
||||||
- name: Render /etc/centralis.env from the vault, then recreate Centralis
|
|
||||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
|
||||||
with:
|
|
||||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
|
||||||
username: ${{ secrets.VPS2_SSH_USER }}
|
|
||||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
|
||||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
|
||||||
script: |
|
|
||||||
set -euo pipefail
|
|
||||||
umask 077
|
|
||||||
cd /opt/thermograph
|
|
||||||
[ -f infra/deploy/secrets/centralis.prod.yaml ] || { echo "no Centralis vault — nothing to do"; exit 0; }
|
|
||||||
|
|
||||||
. infra/deploy/env-topology.sh
|
|
||||||
thermograph_topology prod
|
|
||||||
. infra/deploy/render-secrets.sh
|
|
||||||
|
|
||||||
scratch=$(mktemp -d); chmod 700 "$scratch"
|
|
||||||
trap 'rm -rf -- "$scratch"' EXIT
|
|
||||||
|
|
||||||
# Render to scratch FIRST. CENTRALIS_RENDER_DEST is the override the
|
|
||||||
# renderer already carries for exactly this: look before writing.
|
|
||||||
CENTRALIS_RENDER_DEST="$scratch/rendered.env" \
|
|
||||||
render_centralis_secrets /opt/thermograph/infra
|
|
||||||
|
|
||||||
# THE GUARD. On 2026-07-24 this estate lost Centralis' token registry by
|
|
||||||
# writing a file that was missing a key the live one had. The renderer
|
|
||||||
# proves its OWN output round-trips through bash, but it cannot know
|
|
||||||
# about a key that exists only in the live file -- someone hand-editing
|
|
||||||
# /etc/centralis.env to add, say, CENTRALIS_GOOGLE_CLIENT_ID is invisible
|
|
||||||
# to it. So: the vault must be a SUPERSET of what is live, or nothing is
|
|
||||||
# written and the job fails loudly.
|
|
||||||
#
|
|
||||||
# Names only. Both files are sourced in a clean environment so the
|
|
||||||
# comparison is over what bash actually ends up with, not over text.
|
|
||||||
# The `|| true` is load-bearing under `set -o pipefail`: grep exits 1 when
|
|
||||||
# a file yields no matching keys, which would otherwise abort the job on
|
|
||||||
# the exact edge case this guard exists to handle -- an empty or
|
|
||||||
# unreadable live file, i.e. a first run.
|
|
||||||
keys() { env -i bash -c 'set -a; . "$1" >/dev/null 2>&1; set +a; compgen -v' _ "$1" \
|
|
||||||
| { grep -E '^(CENTRALIS_|DISCORD_)' || true; } | sort -u; }
|
|
||||||
keys "$scratch/rendered.env" > "$scratch/new.keys"
|
|
||||||
if [ -r /etc/centralis.env ]; then keys /etc/centralis.env > "$scratch/live.keys"; else : > "$scratch/live.keys"; fi
|
|
||||||
|
|
||||||
missing=$(comm -23 "$scratch/live.keys" "$scratch/new.keys" || true)
|
|
||||||
if [ -n "$missing" ]; then
|
|
||||||
echo "!! REFUSING to write /etc/centralis.env."
|
|
||||||
echo "!! The live file has keys the vault does not, so rendering would DELETE them:"
|
|
||||||
printf '!! %s\n' $missing
|
|
||||||
echo "!! Someone hand-edited the live file. Put those keys in the vault instead:"
|
|
||||||
echo "!! sops edit infra/deploy/secrets/centralis.prod.yaml"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
added=$(comm -13 "$scratch/live.keys" "$scratch/new.keys" || true)
|
|
||||||
[ -n "$added" ] && printf ' new key from the vault: %s\n' $added
|
|
||||||
|
|
||||||
# Same file, byte for byte? Then there is nothing to do and no reason to
|
|
||||||
# bounce a healthy container.
|
|
||||||
if [ -r /etc/centralis.env ] && cmp -s "$scratch/rendered.env" /etc/centralis.env; then
|
|
||||||
echo "==> /etc/centralis.env already matches the vault — not recreating"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
install -m 0600 -o "$(stat -c %U /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
|
||||||
-g "$(stat -c %G /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
|
||||||
"$scratch/rendered.env" /etc/centralis.env 2>/dev/null \
|
|
||||||
|| sudo install -m 0600 -o agent -g agent "$scratch/rendered.env" /etc/centralis.env
|
|
||||||
echo "==> wrote /etc/centralis.env from the vault"
|
|
||||||
|
|
||||||
# Recreate so compose re-reads it. `docker restart` would NOT: the values
|
|
||||||
# are interpolated when the compose file is parsed, not read at runtime.
|
|
||||||
cd /opt/centralis/deploy
|
|
||||||
set -a; . /etc/centralis.env; set +a
|
|
||||||
docker compose up -d
|
|
||||||
docker compose ps --format '{{.Name}}\t{{.State}}\t{{.Status}}'
|
|
||||||
|
|
|
||||||
|
|
@ -216,65 +216,6 @@ jobs:
|
||||||
rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true
|
rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true
|
||||||
rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true
|
rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true
|
||||||
|
|
||||||
secrets-parity:
|
|
||||||
name: OpenBao/SOPS parity gate
|
|
||||||
runs-on: docker
|
|
||||||
# The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive
|
|
||||||
# green parity runs" -- and which, until now, nothing implemented. A gate
|
|
||||||
# that exists only in prose gets satisfied by assertion.
|
|
||||||
#
|
|
||||||
# verify-parity.sh renders each environment through BOTH backends and diffs
|
|
||||||
# the KEY=value sets after a last-wins collapse. It prints key NAMES and
|
|
||||||
# counts only, never values, so these logs are safe to read and to paste.
|
|
||||||
#
|
|
||||||
# A mismatch FAILS the job, deliberately and loudly. The failure modes on
|
|
||||||
# the other side of a bad cutover are all silent: a missing
|
|
||||||
# THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start
|
|
||||||
# working intermittently; a half-rendered VAPID pair makes push.py generate a
|
|
||||||
# new keypair and delete every subscription row as "gone". Red here is the
|
|
||||||
# only warning that arrives before those.
|
|
||||||
#
|
|
||||||
# Runs under sudo because the age key is 0400 root and the AppRole files are
|
|
||||||
# 0400 agent/root -- the SSH user cannot read them directly. This grants CI
|
|
||||||
# no vault access of its own: the HOST renders, CI only asks it to.
|
|
||||||
#
|
|
||||||
# The run history in Forgejo IS the record of consecutive green days. Do not
|
|
||||||
# count a day this job was skipped or the runner was down as green.
|
|
||||||
concurrency:
|
|
||||||
group: ops-secrets-parity
|
|
||||||
cancel-in-progress: false
|
|
||||||
steps:
|
|
||||||
- name: prod + beta (vps2)
|
|
||||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
|
||||||
with:
|
|
||||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
|
||||||
username: ${{ secrets.VPS2_SSH_USER }}
|
|
||||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
|
||||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
|
||||||
script: |
|
|
||||||
set -euo pipefail
|
|
||||||
# Each environment is checked from its OWN checkout. The trees are
|
|
||||||
# identical, but running beta's from beta's directory also proves
|
|
||||||
# that checkout is current rather than assuming it.
|
|
||||||
sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod
|
|
||||||
sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta
|
|
||||||
|
|
||||||
- name: dev (vps1)
|
|
||||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
|
||||||
with:
|
|
||||||
host: ${{ secrets.VPS1_SSH_HOST }}
|
|
||||||
username: ${{ secrets.VPS1_SSH_USER }}
|
|
||||||
key: ${{ secrets.VPS1_SSH_KEY }}
|
|
||||||
port: ${{ secrets.VPS1_SSH_PORT }}
|
|
||||||
script: |
|
|
||||||
set -euo pipefail
|
|
||||||
# dev renders dev.yaml ALONE -- verify-parity.sh takes that from
|
|
||||||
# env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the
|
|
||||||
# comparison matches what deploy-dev.sh actually does. vps1 must
|
|
||||||
# never see common.yaml: it runs Forgejo, its CI, and dev's
|
|
||||||
# unreviewed branch.
|
|
||||||
sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev
|
|
||||||
|
|
||||||
indexnow:
|
indexnow:
|
||||||
name: IndexNow ping
|
name: IndexNow ping
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
|
|
|
||||||
|
|
@ -13,34 +13,6 @@
|
||||||
{
|
{
|
||||||
auto_https off
|
auto_https off
|
||||||
admin off
|
admin off
|
||||||
|
|
||||||
# WITHOUT THIS, EVERY OAUTH REDIRECT URI THE BACKEND BUILDS IS http://.
|
|
||||||
#
|
|
||||||
# There are two Caddy hops in front of the app: the HOST Caddy terminates TLS
|
|
||||||
# and sets X-Forwarded-Proto: https, then proxies to 127.0.0.1:8137 over plain
|
|
||||||
# HTTP; this LB is the second hop. Caddy only preserves an incoming
|
|
||||||
# X-Forwarded-* header when the immediate peer is a TRUSTED proxy — otherwise
|
|
||||||
# it overwrites the header with the scheme of the connection it just received,
|
|
||||||
# which here is http. So the app saw http:// no matter how the user arrived.
|
|
||||||
#
|
|
||||||
# accounts/oauth.py:_redirect_uri builds the callback from x-forwarded-proto,
|
|
||||||
# and that URI must match what is registered in the provider's console.
|
|
||||||
# Discord tolerates the http:// form; GOOGLE REJECTS a non-HTTPS redirect URI
|
|
||||||
# for a web client outright, so Google sign-in could not work at all until
|
|
||||||
# this was fixed — independently of whether the credentials were configured.
|
|
||||||
#
|
|
||||||
# Measured on vps2 before the fix, same request to each hop:
|
|
||||||
# direct to web, XFP=https -> https://thermograph.org/api/v2/...
|
|
||||||
# through this LB, XFP=https -> http://thermograph.org/api/v2/...
|
|
||||||
#
|
|
||||||
# private_ranges covers 127.0.0.1/8 and the Docker bridge ranges, which is
|
|
||||||
# where the host Caddy reaches this container from. Nothing outside the box
|
|
||||||
# can connect here — the LB is bound to loopback precisely so it cannot be
|
|
||||||
# reached un-fronted (hazard #6, above) — so trusting the local peer does not
|
|
||||||
# widen anything: the only party that can set these headers is the host Caddy.
|
|
||||||
servers {
|
|
||||||
trusted_proxies static private_ranges
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
:8137 {
|
:8137 {
|
||||||
|
|
|
||||||
|
|
@ -17,17 +17,6 @@
|
||||||
{
|
{
|
||||||
auto_https off
|
auto_https off
|
||||||
admin off
|
admin off
|
||||||
|
|
||||||
# Same reason as ./Caddyfile — see the long note there. Without it this second
|
|
||||||
# Caddy hop overwrites the host Caddy's X-Forwarded-Proto: https with http,
|
|
||||||
# and every OAuth redirect URI the backend builds comes out non-HTTPS, which
|
|
||||||
# Google rejects outright for a web client.
|
|
||||||
#
|
|
||||||
# Beta needs it as much as prod does, and arguably first: beta is where an
|
|
||||||
# OAuth provider change gets tested before it reaches thermograph.org.
|
|
||||||
servers {
|
|
||||||
trusted_proxies static private_ranges
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
:8137 {
|
:8137 {
|
||||||
|
|
|
||||||
|
|
@ -206,16 +206,9 @@ infra/openbao/verify-parity.sh --env beta # expect 24 k
|
||||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||||
```
|
```
|
||||||
|
|
||||||
Beta needs no special handling: `verify-parity.sh` sources `env-topology.sh` and calls
|
Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per
|
||||||
`thermograph_topology` per environment, so `TG_BAO_APPROLE` points at
|
environment, so beta authenticates as `tg-beta` rather than falling back to prod's
|
||||||
`/etc/thermograph/openbao-approle-beta` and beta authenticates as `tg-beta` rather than
|
credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||||
falling back to prod's credentials and being denied its own path by `tg-host-prod.hcl`.
|
|
||||||
|
|
||||||
That sourcing is load-bearing and was missing until 2026-08-01: this document asserted
|
|
||||||
the property while the verifier alone did not have it, so `--env beta` and `--all` both
|
|
||||||
took a 403 and the gate could not check the one environment whose isolation it exists
|
|
||||||
to prove. If you refactor `verify-parity.sh`, the `thermograph_topology` call is not
|
|
||||||
boilerplate.
|
|
||||||
|
|
||||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||||
|
|
@ -231,27 +224,10 @@ One PR per hop, following the estate's own promotion model.
|
||||||
+ TG_SECRETS_BACKEND=openbao
|
+ TG_SECRETS_BACKEND=openbao
|
||||||
```
|
```
|
||||||
|
|
||||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. The gate before
|
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
|
||||||
prod is **7 consecutive green parity runs on all three environments**. That runs
|
gate before prod: **7 consecutive green parity runs on all three environments**, run
|
||||||
nightly from the `secrets-parity` job in `ops-cron.yml` over SSH, which gives
|
nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
|
||||||
continuous evidence without granting CI any vault access — the host renders, CI only
|
CI any vault access).
|
||||||
asks it to. Forgejo's run history for that job is the record; a night the job was
|
|
||||||
skipped, or the runner was down, does not count as green.
|
|
||||||
|
|
||||||
Measured 2026-08-01, immediately after the verifier was fixed:
|
|
||||||
|
|
||||||
| env | keys | result |
|
|
||||||
|---|---|---|
|
|
||||||
| dev | 12 | **PASS** — byte-identical |
|
|
||||||
| beta | 24 | FAIL — 3 values differ |
|
|
||||||
| prod | 32 | FAIL — 3 values differ |
|
|
||||||
|
|
||||||
Identical three keys on both: `THERMOGRAPH_S3_SECRET_KEY`,
|
|
||||||
`THERMOGRAPH_LAKE_S3_SECRET_KEY`, `THERMOGRAPH_VAPID_CONTACT`. All three live in
|
|
||||||
`common.yaml`, which was seeded on 2026-07-31 at 02:30Z — *before* the Contabo
|
|
||||||
rotation landed. dev passes because dev never layers `common`. One
|
|
||||||
`seed-from-sops.sh --env common` fixes beta and prod together; the 7-day clock starts
|
|
||||||
after that, not before.
|
|
||||||
|
|
||||||
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
|
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
|
||||||
SOPS path for a full release cycle after prod flips — it is the best mitigation
|
SOPS path for a full release cycle after prod flips — it is the best mitigation
|
||||||
|
|
|
||||||
|
|
@ -36,13 +36,8 @@ usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)
|
||||||
Renders each environment through BOTH backends and compares the resulting
|
Renders each environment through BOTH backends and compares the resulting
|
||||||
KEY=value sets. Prints key names and counts only, never values.
|
KEY=value sets. Prints key names and counts only, never values.
|
||||||
|
|
||||||
--all means "every environment that lives on THIS host", not all three: dev is
|
Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
|
||||||
on vps1, beta and prod are on vps2, so no single box can check them all. An
|
Suitable as a CI / cron gate.
|
||||||
environment with no checkout here is skipped out loud. Skipping every one of
|
|
||||||
them is an error, not a pass.
|
|
||||||
|
|
||||||
Exit status: 0 = every environment checked here is at parity; 1 = a mismatch,
|
|
||||||
or nothing was checkable. Suitable as a CI / cron gate.
|
|
||||||
EOF
|
EOF
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
@ -57,67 +52,20 @@ while [ $# -gt 0 ]; do
|
||||||
done
|
done
|
||||||
[ ${#TARGETS[@]} -gt 0 ] || usage
|
[ ${#TARGETS[@]} -gt 0 ] || usage
|
||||||
|
|
||||||
# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
|
|
||||||
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
|
|
||||||
# bare default — prod's credential — for every environment. On vps2 that made
|
|
||||||
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
|
|
||||||
# thermograph/data/env/beta, so the gate could not verify the one environment whose
|
|
||||||
# isolation it exists to prove. `--all` failed the same way, and dev with it.
|
|
||||||
#
|
|
||||||
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
|
|
||||||
# which is the worst place for the omission to be: the tool whose whole job is to
|
|
||||||
# notice a divergence was itself diverging from the path it verifies.
|
|
||||||
#
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
. "$INFRA_DIR/deploy/env-topology.sh"
|
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
|
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
|
||||||
|
|
||||||
overall=0
|
overall=0
|
||||||
checked=0
|
|
||||||
for env_name in "${TARGETS[@]}"; do
|
for env_name in "${TARGETS[@]}"; do
|
||||||
# Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
|
|
||||||
# filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header
|
|
||||||
# line because --all walks three environments and may skip some of them.
|
|
||||||
if ! thermograph_topology "$env_name"; then
|
|
||||||
echo "!! unknown environment: $env_name" >&2
|
|
||||||
overall=1; continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
# NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to
|
|
||||||
# pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all
|
|
||||||
# anywhere therefore guaranteed a failure on whichever environments are elsewhere,
|
|
||||||
# and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to
|
|
||||||
# /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so
|
|
||||||
# prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like
|
|
||||||
# a policy bug rather than "wrong box".
|
|
||||||
#
|
|
||||||
# Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its
|
|
||||||
# checkout existing. The first version of this check tested for TG_APP_DIR and was
|
|
||||||
# wrong: vps1 still carries a stale /opt/thermograph from before the estate split,
|
|
||||||
# so prod looked resident there and was checked anyway. A leftover directory is
|
|
||||||
# exactly the kind of thing that outlives the arrangement it belonged to.
|
|
||||||
#
|
|
||||||
# Skipping is announced, never silent. A gate that quietly narrows what it checked
|
|
||||||
# is worse than one that fails, because the run still goes green.
|
|
||||||
if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then
|
|
||||||
if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
|
|
||||||
| grep -qxF "$TG_SSH_HOST"; then
|
|
||||||
echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "== parity check: $env_name"
|
echo "== parity check: $env_name"
|
||||||
checked=$((checked + 1))
|
|
||||||
|
|
||||||
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
|
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
|
||||||
# flag; on the OpenBao side the policy also forbids it. Taking the flag from the
|
# flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
|
||||||
# topology rather than re-deriving it here keeps the comparison apples-to-apples
|
# the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
|
||||||
# AND keeps this script from drifting from deploy.sh:67, which reads the same
|
# would fail closed with a 403 rather than quietly diverge, which is exactly the
|
||||||
# variable. If the flag were wrong, the OpenBao side would fail closed with a 403
|
# improvement being verified.
|
||||||
# rather than quietly diverge — which is exactly the improvement being verified.
|
skip_common=0
|
||||||
skip_common="${TG_SKIP_COMMON:-0}"
|
[ "$env_name" = dev ] && skip_common=1
|
||||||
|
|
||||||
sops_out=$(mktemp); bao_out=$(mktemp)
|
sops_out=$(mktemp); bao_out=$(mktemp)
|
||||||
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
|
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
|
||||||
|
|
@ -198,23 +146,11 @@ for k in sorted(order):
|
||||||
cleanup
|
cleanup
|
||||||
done
|
done
|
||||||
|
|
||||||
# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a
|
|
||||||
# host that carries none of them — a new box, a renamed checkout, a typo'd --env —
|
|
||||||
# exits 0 having verified nothing, and a green run is exactly what the 7-day gate
|
|
||||||
# counts. "Nothing was wrong" and "nothing was examined" must not look alike.
|
|
||||||
if [ "$checked" -eq 0 ]; then
|
|
||||||
echo
|
|
||||||
echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2
|
|
||||||
echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$overall" = 0 ]; then
|
if [ "$overall" = 0 ]; then
|
||||||
echo
|
echo
|
||||||
echo "PARITY OK — ${checked} environment(s) checked on this host."
|
echo "PARITY OK for: ${TARGETS[*]}"
|
||||||
echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
|
echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
|
||||||
echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments."
|
echo "Recommended gate before prod: 7 consecutive green runs on all three."
|
||||||
echo "That means both hosts — a green run here says nothing about the other box."
|
|
||||||
else
|
else
|
||||||
echo
|
echo
|
||||||
echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
|
echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue