Compare commits
12 commits
e75b46cba8
...
df409f88b3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df409f88b3 | ||
|
|
75968980f9 | ||
|
|
b62eb60121 | ||
|
|
7b14b9062c | ||
|
|
001e6b1365 | ||
|
|
4ae50ff759 | ||
|
|
b2077b5294 | ||
|
|
b32c7b6b11 | ||
|
|
11f1275998 | ||
|
|
657e27ba9f | ||
|
|
5001269b37 | ||
|
|
702bcd920c |
54 changed files with 1329 additions and 180 deletions
|
|
@ -4,8 +4,8 @@ name: Build + push images (Forgejo registry)
|
|||
# They were identical apart from the domain string: the paths filter, the
|
||||
# concurrency group, IMAGE_PATH, the tag key and the build context.
|
||||
#
|
||||
# Images stay separate and independently deployable -- emi/thermograph/backend
|
||||
# and emi/thermograph/frontend -- exactly as before. Build-once,
|
||||
# Images stay separate and independently deployable -- jinemi/thermograph/backend
|
||||
# and jinemi/thermograph/frontend -- exactly as before. Build-once,
|
||||
# deploy-everywhere; nothing about the published artefacts changes here.
|
||||
#
|
||||
# SEMVER EXCEPTION, preserved: a v*.*.* tag push carries no paths context, so a
|
||||
|
|
@ -116,7 +116,7 @@ jobs:
|
|||
# independent of any permission setting). Needs a manually provisioned
|
||||
# PAT with write:package scope, stored as REGISTRY_TOKEN.
|
||||
echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "${{ steps.image.outputs.host }}" \
|
||||
--username emi --password-stdin
|
||||
--username admin_emi --password-stdin
|
||||
|
||||
- name: Build
|
||||
if: steps.image.outputs.changed == 'true'
|
||||
|
|
|
|||
|
|
@ -86,6 +86,15 @@ jobs:
|
|||
exit 0
|
||||
fi
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# env-topology.sh FIRST: it is what sets TG_SECRETS_BACKEND, and
|
||||
# render-secrets.sh:44 defaults to `sops` when it is unset. Without
|
||||
# this line the cutover would be only half-applied — deploy.sh would
|
||||
# read the new backend while THIS workflow, which runs on every push
|
||||
# touching infra/**, kept re-rendering the same file from SOPS. That
|
||||
# is silent while parity holds and a hard failure the moment the SOPS
|
||||
# files are retired.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology dev
|
||||
. infra/deploy/render-secrets.sh
|
||||
# SKIP_COMMON is dev's standing rule, not a preference: common.yaml
|
||||
# is the fleet's shared production credential set, and vps1 also
|
||||
|
|
@ -114,6 +123,13 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev: env-topology.sh is what sets
|
||||
# TG_SECRETS_BACKEND, and for beta it also sets TG_BAO_APPROLE to
|
||||
# the -beta credential. Beta shares a filesystem with prod, so the
|
||||
# bare default is prod's file and tg-host-prod.hcl denies beta's
|
||||
# own path — a 403 at render time on the box that runs prod.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology beta
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env
|
||||
fi
|
||||
|
|
@ -139,7 +155,113 @@ jobs:
|
|||
git fetch --prune origin main
|
||||
git reset --hard origin/main
|
||||
if [ -f infra/deploy/render-secrets.sh ]; then
|
||||
# See the note in sync-dev — without env-topology.sh this job keeps
|
||||
# rendering from SOPS regardless of what the backend selector says.
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology prod
|
||||
. infra/deploy/render-secrets.sh
|
||||
render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env
|
||||
fi
|
||||
echo "synced to $(git log --oneline -1)"
|
||||
|
||||
sync-centralis:
|
||||
# Centralis' /etc/centralis.env has been HAND-MAINTAINED. The vault has held a
|
||||
# copy since deploy/secrets/centralis.prod.yaml was seeded, and
|
||||
# render_centralis_secrets has existed since the OpenBao work -- but nothing
|
||||
# on any deploy path ever called it. A renderer with no caller is not a
|
||||
# migration, it is a file.
|
||||
#
|
||||
# That gap is why enabling Google sign-in on Centralis had no safe home: the
|
||||
# only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file,
|
||||
# which is exactly what render-secrets.sh's header warns against.
|
||||
#
|
||||
# WHY THIS JOB RECREATES A CONTAINER WHEN THE OTHER THREE DELIBERATELY DO NOT.
|
||||
# This workflow's header says it never rolls a service, because image tags are
|
||||
# the app domains' axis. Centralis is different in kind: its compose file
|
||||
# interpolates /etc/centralis.env at PARSE time (`set -a && . /etc/centralis.env
|
||||
# && docker compose up -d`), so rendering the file without recreating the
|
||||
# container changes nothing at all. Rendering and not recreating would be the
|
||||
# silent no-op this estate keeps getting bitten by. It is one container with no
|
||||
# replicas -- a two-second recreate, not a deploy.
|
||||
if: github.ref_name == 'main'
|
||||
runs-on: docker
|
||||
concurrency:
|
||||
group: infra-sync-centralis
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Render /etc/centralis.env from the vault, then recreate Centralis
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
||||
username: ${{ secrets.VPS2_SSH_USER }}
|
||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
cd /opt/thermograph
|
||||
[ -f infra/deploy/secrets/centralis.prod.yaml ] || { echo "no Centralis vault — nothing to do"; exit 0; }
|
||||
|
||||
. infra/deploy/env-topology.sh
|
||||
thermograph_topology prod
|
||||
. infra/deploy/render-secrets.sh
|
||||
|
||||
scratch=$(mktemp -d); chmod 700 "$scratch"
|
||||
trap 'rm -rf -- "$scratch"' EXIT
|
||||
|
||||
# Render to scratch FIRST. CENTRALIS_RENDER_DEST is the override the
|
||||
# renderer already carries for exactly this: look before writing.
|
||||
CENTRALIS_RENDER_DEST="$scratch/rendered.env" \
|
||||
render_centralis_secrets /opt/thermograph/infra
|
||||
|
||||
# THE GUARD. On 2026-07-24 this estate lost Centralis' token registry by
|
||||
# writing a file that was missing a key the live one had. The renderer
|
||||
# proves its OWN output round-trips through bash, but it cannot know
|
||||
# about a key that exists only in the live file -- someone hand-editing
|
||||
# /etc/centralis.env to add, say, CENTRALIS_GOOGLE_CLIENT_ID is invisible
|
||||
# to it. So: the vault must be a SUPERSET of what is live, or nothing is
|
||||
# written and the job fails loudly.
|
||||
#
|
||||
# Names only. Both files are sourced in a clean environment so the
|
||||
# comparison is over what bash actually ends up with, not over text.
|
||||
# The `|| true` is load-bearing under `set -o pipefail`: grep exits 1 when
|
||||
# a file yields no matching keys, which would otherwise abort the job on
|
||||
# the exact edge case this guard exists to handle -- an empty or
|
||||
# unreadable live file, i.e. a first run.
|
||||
keys() { env -i bash -c 'set -a; . "$1" >/dev/null 2>&1; set +a; compgen -v' _ "$1" \
|
||||
| { grep -E '^(CENTRALIS_|DISCORD_)' || true; } | sort -u; }
|
||||
keys "$scratch/rendered.env" > "$scratch/new.keys"
|
||||
if [ -r /etc/centralis.env ]; then keys /etc/centralis.env > "$scratch/live.keys"; else : > "$scratch/live.keys"; fi
|
||||
|
||||
missing=$(comm -23 "$scratch/live.keys" "$scratch/new.keys" || true)
|
||||
if [ -n "$missing" ]; then
|
||||
echo "!! REFUSING to write /etc/centralis.env."
|
||||
echo "!! The live file has keys the vault does not, so rendering would DELETE them:"
|
||||
printf '!! %s\n' $missing
|
||||
echo "!! Someone hand-edited the live file. Put those keys in the vault instead:"
|
||||
echo "!! sops edit infra/deploy/secrets/centralis.prod.yaml"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
added=$(comm -13 "$scratch/live.keys" "$scratch/new.keys" || true)
|
||||
[ -n "$added" ] && printf ' new key from the vault: %s\n' $added
|
||||
|
||||
# Same file, byte for byte? Then there is nothing to do and no reason to
|
||||
# bounce a healthy container.
|
||||
if [ -r /etc/centralis.env ] && cmp -s "$scratch/rendered.env" /etc/centralis.env; then
|
||||
echo "==> /etc/centralis.env already matches the vault — not recreating"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
install -m 0600 -o "$(stat -c %U /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
||||
-g "$(stat -c %G /etc/centralis.env 2>/dev/null || echo "$USER")" \
|
||||
"$scratch/rendered.env" /etc/centralis.env 2>/dev/null \
|
||||
|| sudo install -m 0600 -o agent -g agent "$scratch/rendered.env" /etc/centralis.env
|
||||
echo "==> wrote /etc/centralis.env from the vault"
|
||||
|
||||
# Recreate so compose re-reads it. `docker restart` would NOT: the values
|
||||
# are interpolated when the compose file is parsed, not read at runtime.
|
||||
cd /opt/centralis/deploy
|
||||
set -a; . /etc/centralis.env; set +a
|
||||
docker compose up -d
|
||||
docker compose ps --format '{{.Name}}\t{{.State}}\t{{.Status}}'
|
||||
|
|
|
|||
|
|
@ -8,8 +8,15 @@ name: Ops cron (backup + IndexNow)
|
|||
#
|
||||
# The jobs SSH into a host and run inside the already-running stack (docker
|
||||
# exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new
|
||||
# network exposure, no separate dependency install. Runs on the `docker` label
|
||||
# (the always-on Swarm-hosted runner deploy/forgejo/ stood up).
|
||||
# network exposure, no separate dependency install. Runs on the `docker` label,
|
||||
# served by the vps2 runner (infra/deploy/forgejo/runner-vps2/) and, when it is
|
||||
# up, the desktop.
|
||||
#
|
||||
# THIS WORKFLOW IS THE ESTATE'S ONLY BACKUP, so its dependency on a runner is
|
||||
# load-bearing. On 2026-07-31 the desktop was the only registered runner and
|
||||
# went offline for 21 hours; this schedule did not fire at 03:00Z. Forgejo held
|
||||
# the run in `waiting` and it completed on reconnect, so nothing was lost that
|
||||
# time. The vps2 runner exists so the next outage is not a coin flip.
|
||||
#
|
||||
# WHICH BOX EACH JOB TALKS TO, and why the secret names say so:
|
||||
#
|
||||
|
|
@ -209,6 +216,65 @@ jobs:
|
|||
rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true
|
||||
rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true
|
||||
|
||||
secrets-parity:
|
||||
name: OpenBao/SOPS parity gate
|
||||
runs-on: docker
|
||||
# The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive
|
||||
# green parity runs" -- and which, until now, nothing implemented. A gate
|
||||
# that exists only in prose gets satisfied by assertion.
|
||||
#
|
||||
# verify-parity.sh renders each environment through BOTH backends and diffs
|
||||
# the KEY=value sets after a last-wins collapse. It prints key NAMES and
|
||||
# counts only, never values, so these logs are safe to read and to paste.
|
||||
#
|
||||
# A mismatch FAILS the job, deliberately and loudly. The failure modes on
|
||||
# the other side of a bad cutover are all silent: a missing
|
||||
# THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start
|
||||
# working intermittently; a half-rendered VAPID pair makes push.py generate a
|
||||
# new keypair and delete every subscription row as "gone". Red here is the
|
||||
# only warning that arrives before those.
|
||||
#
|
||||
# Runs under sudo because the age key is 0400 root and the AppRole files are
|
||||
# 0400 agent/root -- the SSH user cannot read them directly. This grants CI
|
||||
# no vault access of its own: the HOST renders, CI only asks it to.
|
||||
#
|
||||
# The run history in Forgejo IS the record of consecutive green days. Do not
|
||||
# count a day this job was skipped or the runner was down as green.
|
||||
concurrency:
|
||||
group: ops-secrets-parity
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: prod + beta (vps2)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS2_SSH_HOST }}
|
||||
username: ${{ secrets.VPS2_SSH_USER }}
|
||||
key: ${{ secrets.VPS2_SSH_KEY }}
|
||||
port: ${{ secrets.VPS2_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# Each environment is checked from its OWN checkout. The trees are
|
||||
# identical, but running beta's from beta's directory also proves
|
||||
# that checkout is current rather than assuming it.
|
||||
sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod
|
||||
sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta
|
||||
|
||||
- name: dev (vps1)
|
||||
uses: https://github.com/appleboy/ssh-action@v1.2.0
|
||||
with:
|
||||
host: ${{ secrets.VPS1_SSH_HOST }}
|
||||
username: ${{ secrets.VPS1_SSH_USER }}
|
||||
key: ${{ secrets.VPS1_SSH_KEY }}
|
||||
port: ${{ secrets.VPS1_SSH_PORT }}
|
||||
script: |
|
||||
set -euo pipefail
|
||||
# dev renders dev.yaml ALONE -- verify-parity.sh takes that from
|
||||
# env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the
|
||||
# comparison matches what deploy-dev.sh actually does. vps1 must
|
||||
# never see common.yaml: it runs Forgejo, its CI, and dev's
|
||||
# unreviewed branch.
|
||||
sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev
|
||||
|
||||
indexnow:
|
||||
name: IndexNow ping
|
||||
runs-on: docker
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ cannot describe vps2 alone, since vps2 runs beta and prod side by side.
|
|||
- Each service's live tag is persisted host-side, so a single-service roll never
|
||||
disturbs the sibling's running tag.
|
||||
|
||||
Images are `emi/thermograph/backend` and `emi/thermograph/frontend`, tagged
|
||||
Images are `jinemi/thermograph/backend` and `jinemi/thermograph/frontend`, tagged
|
||||
`sha-<12hex>` on every push and by semver on `v*.*.*` tags. They build and deploy
|
||||
**independently** — a backend change ships without a frontend deploy and vice
|
||||
versa. That independence is the point of the FE/BE split and survived
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ Forgejo only reads root workflows, but dead copies are a trap):**
|
|||
|
||||
- `backend-build-push.yml` / `frontend-build-push.yml` — the old per-repo
|
||||
build-push, path-filtered (`paths: ['backend/**']` etc.), image paths now
|
||||
**explicit**: `emi/thermograph/backend|frontend` (the old
|
||||
**explicit**: `jinemi/thermograph/backend|frontend` (the old
|
||||
`${{ github.repository }}/app` derivation collides in a monorepo). Per-domain
|
||||
concurrency groups. Caveat: `v*.*.*` tag pushes carry no paths context, so a
|
||||
version tag builds **both** images (intended: a release names the set).
|
||||
|
|
@ -65,7 +65,7 @@ Forgejo only reads root workflows, but dead copies are a trap):**
|
|||
name, so no project-name concern there).
|
||||
|
||||
**Verified here:** every workflow + compose file parses (PyYAML); all three
|
||||
deploy scripts pass `bash -n`; no `emi/thermograph-{backend,frontend}/app`
|
||||
deploy scripts pass `bash -n`; no `admin_emi/thermograph-{backend,frontend}/app`
|
||||
stragglers in sh/yml. **Not verified (impossible locally):** Forgejo actually
|
||||
honoring `on.push.paths` and `workflow_call` inputs on our version; any live
|
||||
deploy.
|
||||
|
|
@ -118,7 +118,7 @@ Checked every split-repo branch by dry-run merge; migrated everything real:
|
|||
|
||||
## Cutover runbook
|
||||
|
||||
1. Create `emi/thermograph` on Forgejo (a **new** repo — do not resurrect the
|
||||
1. Create `Jinemi/thermograph` on Forgejo (a **new** repo — do not resurrect the
|
||||
archived monorepo) and push `main`.
|
||||
2. Actions config, once: secrets `REGISTRY_TOKEN`, `SSH_HOST/USER/KEY/PORT`
|
||||
(beta), `PROD_SSH_*` (prod); variable `REGISTRY_HOST=git.thermograph.org`.
|
||||
|
|
@ -132,7 +132,7 @@ Checked every split-repo branch by dry-run merge; migrated everything real:
|
|||
`/etc/thermograph*` files stay as-is.
|
||||
5. LAN box: re-point `~/thermograph-dev` at the monorepo the same way.
|
||||
6. First push builds the **new** image paths — old
|
||||
`emi/thermograph-{backend,frontend}/app` packages stay in the registry for
|
||||
`admin_emi/thermograph-{backend,frontend}/app` packages stay in the registry for
|
||||
rollback; GC after burn-in. deploy.sh's tag prune already targets the new
|
||||
paths only.
|
||||
7. One `workflow_dispatch` of `backend-deploy.yml` after its build-push
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**.
|
|||
|
||||
| Dir | What | CI |
|
||||
|---|---|---|
|
||||
| `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `emi/thermograph/backend`; `deploy` |
|
||||
| `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `emi/thermograph/frontend` |
|
||||
| `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `jinemi/thermograph/backend`; `deploy` |
|
||||
| `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `jinemi/thermograph/frontend` |
|
||||
| `infra/` | Compose (dev, on vps1) + two Swarm stacks co-resident on vps2 (beta, prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` |
|
||||
| `observability/` | Loki + Grafana + Alloy stack | `observability-validate` |
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ which docs in this repo are currently stale.
|
|||
|
||||
Every workflow in `.forgejo/workflows/` is **path-filtered to its domain**: a
|
||||
push touching only `frontend/**` builds/deploys nothing else. Images stay
|
||||
separate (`emi/thermograph/backend`, `emi/thermograph/frontend`, each tagged
|
||||
separate (`jinemi/thermograph/backend`, `jinemi/thermograph/frontend`, each tagged
|
||||
`sha-<12hex>`), deploys stay per-service (`infra/deploy/deploy.sh
|
||||
SERVICE=backend|frontend|all`), and the API version contract
|
||||
(`GET /api/version`, `PAYLOAD_VER`) still lets FE and BE ship out of lockstep.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
The Thermograph API service: grades recent local weather against ~45 years of
|
||||
climate history, and hosts the accounts, notification, and SSR-content
|
||||
back-end that the rest of the split Thermograph stack (`thermograph-frontend`)
|
||||
talks to over HTTP. Split from the `emi/thermograph` monorepo — this repo owns
|
||||
talks to over HTTP. Split from the `Jinemi/thermograph` monorepo — this repo owns
|
||||
the API/DB/accounts/notifications layer only; it renders no HTML/CSS/JS of its
|
||||
own.
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ cities_flavor.json bundled reference data (generated by gen_cities.py /
|
|||
`/content/...`; it negotiates compatibility via `GET /api/version`.
|
||||
- **`thermograph-infra`** owns the deploy/Compose/Terraform layer — this repo
|
||||
only builds and publishes its own container image
|
||||
(`git.thermograph.org/emi/thermograph-backend/app`) and hands infra a tag to
|
||||
(`git.thermograph.org/admin_emi/thermograph-backend/app`) and hands infra a tag to
|
||||
roll out (`SERVICE=backend` + `BACKEND_IMAGE_TAG` into infra's
|
||||
`deploy/deploy.sh`).
|
||||
- **`thermograph-docs`** holds the cross-repo architecture/runbook docs; this
|
||||
|
|
|
|||
|
|
@ -22,11 +22,11 @@ services:
|
|||
backend:
|
||||
# Defaults to a locally-built `:local` image (scripts/smoke.sh builds it). In CI,
|
||||
# set BACKEND_IMAGE_TAG=sha-<12hex> to smoke the exact published image.
|
||||
# The path must match what build-push.yml publishes -- emi/thermograph/backend.
|
||||
# It read emi/thermograph-backend/app (the retired split-era path) until this
|
||||
# The path must match what build-push.yml publishes -- jinemi/thermograph/backend.
|
||||
# It read admin_emi/thermograph-backend/app (the retired split-era path) until this
|
||||
# was corrected; harmless for the default `:local` build, wrong the moment
|
||||
# BACKEND_IMAGE_TAG names a real published tag.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-smoke}"
|
|||
# The image to smoke, pulled from the registry (no local build). CI passes the just-
|
||||
# pushed sha (BACKEND_IMAGE_TAG=sha-<12hex>); locally it defaults to a published tag.
|
||||
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-v0.0.2-split-ci}"
|
||||
IMG="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG}"
|
||||
IMG="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-admin_emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG}"
|
||||
export SMOKE_HOST_PORT="${SMOKE_HOST_PORT:-18137}"
|
||||
COMPOSE=(docker compose -f docker-compose.test.yml)
|
||||
PORT_URL="http://127.0.0.1:${SMOKE_HOST_PORT}"
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ thermograph/
|
|||
├── README.md domain table + how CI stays decoupled
|
||||
├── .forgejo/workflows/ ALL CI. Nine files. Never add workflows elsewhere.
|
||||
├── .claude/ settings.json + enforcement hooks
|
||||
├── backend/ FastAPI API + Go daemon → emi/thermograph/backend
|
||||
├── frontend/ Go SSR + static assets → emi/thermograph/frontend
|
||||
├── backend/ FastAPI API + Go daemon → jinemi/thermograph/backend
|
||||
├── frontend/ Go SSR + static assets → jinemi/thermograph/frontend
|
||||
├── infra/ compose, Swarm, deploy, secrets, terraform, ops
|
||||
├── observability/ Loki + Grafana + Alloy
|
||||
└── docs/onboarding/ you are here
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ alembic + `/healthz` 200 + a real `/api/v2/place` 200.
|
|||
|
||||
Triggers on pushes to `dev`/`main`/`release` touching `backend/**` or
|
||||
`frontend/**`, and on `v*.*.*` tags. Publishes
|
||||
`git.thermograph.org/emi/thermograph/{backend,frontend}:sha-<12hex>`.
|
||||
`git.thermograph.org/Jinemi/thermograph/{backend,frontend}:sha-<12hex>`.
|
||||
|
||||
- **Serialised** (`max-parallel: 1`): one physical runner, and two whole image
|
||||
builds racing each other buys nothing.
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ pages nobody. **The only proof is a message actually arriving in
|
|||
|
||||
> ⚠️ **Merging is not deploying — and this one has already bitten.**
|
||||
> `/opt/observability` on vps1 and vps2 is a checkout of the **archived**
|
||||
> `emi/thermograph-observability` repo. It can never `git pull` again; the
|
||||
> `admin_emi/thermograph-observability` repo. It can never `git pull` again; the
|
||||
> content now lives here under `observability/`. A previous observability PR
|
||||
> merged and had **zero live effect** for exactly this reason.
|
||||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ mentally translate host names.
|
|||
### `backend/README.md` is written for the split-repo era
|
||||
|
||||
It describes `thermograph-backend` as its own repo with sibling repos, uses the
|
||||
retired image path `git.thermograph.org/emi/thermograph-backend/app`, and says
|
||||
retired image path `git.thermograph.org/admin_emi/thermograph-backend/app`, and says
|
||||
*"there is no Makefile in this repo yet"* — there is (`make test`, `make smoke`,
|
||||
`make clean-venv`).
|
||||
|
||||
|
|
@ -106,7 +106,7 @@ it — that's the standing instruction in the root `CLAUDE.md`.
|
|||
### `observability/README.md`: merging is not deploying
|
||||
|
||||
`/opt/observability` on vps1 and vps2 is a checkout of the **archived**
|
||||
`emi/thermograph-observability` repo. It can never `git pull` again. A previous
|
||||
`admin_emi/thermograph-observability` repo. It can never `git pull` again. A previous
|
||||
observability PR merged and had **zero live effect** for exactly this reason.
|
||||
The README documents the hand-ship procedure; until the checkouts are re-pointed
|
||||
at the monorepo, that's the only path. Flagged again in
|
||||
|
|
@ -118,7 +118,7 @@ at the monorepo, that's the only path. Flagged again in
|
|||
|
||||
### ~~The `docker-compose.test.yml` files pull retired image paths~~ — fixed
|
||||
|
||||
**Fixed.** Both now use `emi/thermograph/backend`, the path `build-push.yml`
|
||||
**Fixed.** Both now use `jinemi/thermograph/backend`, the path `build-push.yml`
|
||||
actually publishes.
|
||||
|
||||
The frontend harness had the worse half of this: it also pinned
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Onboarding — becoming a full contributor to Thermograph
|
||||
|
||||
This is the developer onboarding path for the `emi/thermograph` monorepo: what
|
||||
This is the developer onboarding path for the `Jinemi/thermograph` monorepo: what
|
||||
the product is, how the code is shaped, how to run it, what will break silently
|
||||
if you get it wrong, and how a change actually travels from your editor to
|
||||
`thermograph.org`.
|
||||
|
|
@ -35,10 +35,10 @@ readings.
|
|||
|
||||
- **`backend/`** — Python 3.12 / FastAPI. Owns all climate data, grading, the
|
||||
API, accounts, notifications, plus a Go daemon (`backend/daemon/`) that owns
|
||||
the Discord gateway and recurring timers. Ships as `emi/thermograph/backend`.
|
||||
the Discord gateway and recurring timers. Ships as `jinemi/thermograph/backend`.
|
||||
- **`frontend/`** — **Go** SSR service (`frontend/server/`) plus every static
|
||||
asset. No climate data, no database, no compute. Ships as
|
||||
`emi/thermograph/frontend`. (The Python implementation at `frontend/*.py` is
|
||||
`jinemi/thermograph/frontend`. (The Python implementation at `frontend/*.py` is
|
||||
the superseded original — see [traps](11-traps.md).)
|
||||
- **`infra/`** — compose and Swarm files, deploy scripts, the SOPS secrets
|
||||
vault, Terraform, ops query tooling.
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ names.
|
|||
## The live service is Go — `server/`
|
||||
|
||||
`server/` (module `thermograph/frontend`, Go 1.26) is what builds, tests, ships
|
||||
and runs. It is the only thing in the `emi/thermograph/frontend` image.
|
||||
and runs. It is the only thing in the `jinemi/thermograph/frontend` image.
|
||||
|
||||
The Python files at this level — `app.py`, `content.py`, `api_client.py`,
|
||||
`format.py`, `content_loader.py`, `paths.py`, `templates/*.html.j2` — are the
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ static asset for [Thermograph](https://thermograph.org) — grades recent local
|
|||
weather against ~45 years of climate history. This domain holds **no climate
|
||||
data and does no compute**; it renders pages and serves the JS/CSS/image assets
|
||||
that call the backend's API from the browser. It builds its own image
|
||||
(`emi/thermograph/frontend`) and deploys independently of the backend.
|
||||
(`jinemi/thermograph/frontend`) and deploys independently of the backend.
|
||||
|
||||
See [`CLAUDE.md`](CLAUDE.md) for the agent-facing detail on the deploy contract
|
||||
and the cross-service contracts, and [`DESIGN.md`](DESIGN.md) for the visual
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@ services:
|
|||
# domain-keyed rule build-push.yml and deploy.yml both use -- so the harness
|
||||
# tracks the tree instead of drifting behind a hardcoded pin. (It was pinned
|
||||
# to the split-era v0.0.2-split-ci, under the retired
|
||||
# emi/thermograph-backend/app path, long after both were superseded.)
|
||||
# admin_emi/thermograph-backend/app path, long after both were superseded.)
|
||||
# Override THERMOGRAPH_BACKEND_TEST_TAG to pin a specific build.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${THERMOGRAPH_BACKEND_TEST_TAG:?set it, or run via scripts/backend-for-tests.sh which derives it}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${THERMOGRAPH_BACKEND_TEST_TAG:?set it, or run via scripts/backend-for-tests.sh which derives it}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -55,5 +55,5 @@ binary; static assets and the YAML copy are read from disk.
|
|||
go build ./... && go vet ./... && go test ./...
|
||||
|
||||
The deployed binary is `/usr/local/bin/thermograph-frontend` inside the
|
||||
`emi/thermograph/frontend` image; the image name, `frontend-*` CI workflows and
|
||||
`jinemi/thermograph/frontend` image; the image name, `frontend-*` CI workflows and
|
||||
deploy path are unchanged from the Python service.
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ THERMOGRAPH_INTERNAL_TOKEN=
|
|||
# :local, which is the default here -- only set these to pull a specific
|
||||
# published build instead of building locally.
|
||||
# REGISTRY_HOST=git.thermograph.org
|
||||
# BACKEND_IMAGE_PATH=emi/thermograph-backend/app
|
||||
# BACKEND_IMAGE_PATH=admin_emi/thermograph-backend/app
|
||||
# BACKEND_IMAGE_TAG=local
|
||||
# FRONTEND_IMAGE_PATH=emi/thermograph-frontend/app
|
||||
# FRONTEND_IMAGE_PATH=admin_emi/thermograph-frontend/app
|
||||
# FRONTEND_IMAGE_TAG=local
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Infrastructure for [Thermograph](https://thermograph.org): Terraform host
|
||||
provisioning, the SOPS+age secrets vault, WireGuard/Swarm networking, Forgejo,
|
||||
Caddy, mail, and the deploy scripts that run the already-built app images on each
|
||||
host. This is a domain of the `emi/thermograph` monorepo — a host's checkout
|
||||
host. This is a domain of the `Jinemi/thermograph` monorepo — a host's checkout
|
||||
(`/opt/thermograph`, `/opt/thermograph-beta`, or `/opt/thermograph-dev`) is a
|
||||
checkout of the whole monorepo, and `infra/` never builds app source; it only
|
||||
runs published images.
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ or 8 is done wrong, which is what the verification lines are for.
|
|||
|
||||
```
|
||||
ssh agent@169.58.46.181
|
||||
sudo git clone --branch main http://10.10.0.2:3080/emi/thermograph.git /opt/thermograph-beta
|
||||
sudo git clone --branch main http://10.10.0.2:3080/Jinemi/thermograph.git /opt/thermograph-beta
|
||||
sudo chown -R agent:agent /opt/thermograph-beta
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
# ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha-<a> FRONTEND_IMAGE_TAG=sha-<b> /opt/thermograph/infra/deploy/deploy.sh'
|
||||
#
|
||||
# FE/BE CI-CD split: backend and frontend are published from separate repos as
|
||||
# separate images (emi/thermograph/backend, emi/thermograph/frontend),
|
||||
# separate images (jinemi/thermograph/backend, jinemi/thermograph/frontend),
|
||||
# so a deploy targets ONE service and leaves the other's running container +
|
||||
# tag untouched. Each service's live tag is persisted host-side in
|
||||
# deploy/.image-tags.env (untracked -- survives the git reset below) so a
|
||||
|
|
@ -282,7 +282,7 @@ esac
|
|||
# host's existing cred; a genuine auth problem then fails loudly at `pull`.
|
||||
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
echo "==> Logging in to the registry ($REGISTRY_HOST)"
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin
|
||||
else
|
||||
echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST"
|
||||
fi
|
||||
|
|
@ -328,7 +328,7 @@ case " ${TARGETS[*]} " in
|
|||
# then always found no daemon binary in a Postgres image and dropped
|
||||
# daemon from EVERY backend deploy, regardless of what the real backend
|
||||
# image contained -- reproduced and confirmed against beta directly.
|
||||
daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG}"
|
||||
daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG}"
|
||||
if ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then
|
||||
echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run"
|
||||
kept=()
|
||||
|
|
@ -408,8 +408,8 @@ fi
|
|||
# roll never garbage-collects the image a rollback would need; docker also
|
||||
# refuses to remove an image any container still uses.
|
||||
echo "==> Pruning old app-image tags"
|
||||
_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}"
|
||||
_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}"
|
||||
_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}"
|
||||
_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}"
|
||||
docker images --format '{{.Repository}}:{{.Tag}}' \
|
||||
| grep -E "^(${_be_repo}|${_fe_repo}):" \
|
||||
| grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
|
||||
|
|
|
|||
|
|
@ -76,6 +76,16 @@ thermograph_topology() {
|
|||
# for the reason that marker was demoted to a fallback in the first place: vps2 runs
|
||||
# prod and beta side by side, and one host-wide file cannot name two backends.
|
||||
TG_SECRETS_BACKEND=sops
|
||||
# Which AppRole credential file render-secrets-openbao.sh authenticates with. Same
|
||||
# reasoning as TG_SECRETS_BACKEND above, and the same trap: vps2 renders TWO
|
||||
# environments, so a single host-wide default cannot serve both. prod and dev each
|
||||
# get the conventional path (dev is alone on vps1, so there is no collision there);
|
||||
# beta is the one that must differ, because it shares a filesystem with prod.
|
||||
#
|
||||
# Without this, beta's render falls back to prod's credentials and tg-host-prod.hcl
|
||||
# correctly denies thermograph/data/env/beta — a 403 at deploy time on the box that
|
||||
# runs prod.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle
|
||||
|
||||
case "$env_name" in
|
||||
prod)
|
||||
|
|
@ -116,6 +126,10 @@ thermograph_topology() {
|
|||
# therefore prod's blast radius too (see .claude/hooks/prod-guard.sh).
|
||||
TG_SSH_HOST=169.58.46.181
|
||||
TG_SSH_TARGET=agent@169.58.46.181
|
||||
# The one environment that cannot use the default AppRole path: it shares a
|
||||
# filesystem with prod, so it needs its own credential file to authenticate as
|
||||
# tg-beta rather than tg-prod. bootstrap-policies.sh installs it here.
|
||||
TG_BAO_APPROLE=/etc/thermograph/openbao-approle-beta
|
||||
# A SECOND checkout on the same box. Separate from prod's so the two can
|
||||
# sit on different commits of this repo, and so `git reset --hard` in one
|
||||
# deploy can never yank the tree out from under the other's running
|
||||
|
|
@ -228,7 +242,7 @@ thermograph_topology() {
|
|||
export TG_LB_HTTP_PORT TG_LB_FE_PORT TG_DB_NAME TG_DB_USER
|
||||
export TG_TAGS_FILE TG_LOCK_FILE TG_BIND_ADDR TG_SKIP_COMMON
|
||||
export TG_SVC_PREFIX TG_DATA_NETWORK TG_DB_SERVICE TG_POST_DEPLOY
|
||||
export TG_SECRETS_BACKEND
|
||||
export TG_SECRETS_BACKEND TG_BAO_APPROLE
|
||||
export TG_SSH_HOST TG_SSH_TARGET
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,12 +7,20 @@ deploy`s that happen to run on the manager node, vps2 — see
|
|||
(`75.119.132.91`) via the `role=forge` label from
|
||||
`deploy/swarm/label-forge-node.sh`.
|
||||
|
||||
The Actions **runner** is deliberately *not* part of this stack — see
|
||||
`DEPLOY-DEV.md` and the note in `register-lan-runner.sh`'s own header for
|
||||
where it runs today. That's the canonical placement per
|
||||
`thermograph-docs/runbooks/implementation-handoff.md` Track B step 5; an
|
||||
earlier revision of this stack ran the runner as a Swarm-scheduled
|
||||
Docker-in-Docker sidecar pinned to the Forgejo node, which is gone now.
|
||||
The Actions **runner** is deliberately *not* part of this stack: a
|
||||
Swarm-scheduled runner cannot redeploy the Swarm that schedules it, so CI would
|
||||
be unavailable exactly when the cluster is what needs repairing. An earlier
|
||||
revision did run it as a Swarm-scheduled Docker-in-Docker sidecar pinned to the
|
||||
Forgejo node; that is gone.
|
||||
|
||||
Runners live in two places, and only one of them counts:
|
||||
|
||||
- **`runner-vps2/`** — the always-on runner, a plain `restart: always` container
|
||||
on vps2. This is the one CI depends on.
|
||||
- **`register-lan-runner.sh`** — the desktop's systemd runner. Extra capacity.
|
||||
**Nothing may assume it is up.** It was the only registered runner in the
|
||||
estate until 2026-08-01, and its 21-hour outage on 2026-07-31 froze every
|
||||
merge and deploy and stopped the nightly backup from firing.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
|
@ -143,14 +151,14 @@ bug hit during the frontend Go rewrite). `ci-runner` adds `docker-ce-cli` +
|
|||
`git`/`python3`/`python3-yaml` for the other jobs that need them
|
||||
(`shell-lint`, `observability-validate`).
|
||||
|
||||
Current tag: `git.thermograph.org/emi/thermograph/ci-runner:v2` (`v1` is
|
||||
Current tag: `git.thermograph.org/jinemi/thermograph/ci-runner:v2` (`v1` is
|
||||
broken — do not register any runner against it). Rebuild/push (requires a PAT
|
||||
with `write:package` scope — the embedded git-remote token lacks it, same
|
||||
requirement documented in `build-push.yml`):
|
||||
|
||||
```bash
|
||||
docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN deploy/forgejo/ci-runner
|
||||
docker push git.thermograph.org/emi/thermograph/ci-runner:vN
|
||||
docker build -t git.thermograph.org/jinemi/thermograph/ci-runner:vN deploy/forgejo/ci-runner
|
||||
docker push git.thermograph.org/jinemi/thermograph/ci-runner:vN
|
||||
```
|
||||
|
||||
`register-lan-runner.sh`'s `LABELS` default points at the current tag, so
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@
|
|||
# class, and skips the per-job install entirely.
|
||||
#
|
||||
# Build/push (manual — see ../README.md for when to rebuild):
|
||||
# docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN \
|
||||
# docker build -t git.thermograph.org/jinemi/thermograph/ci-runner:vN \
|
||||
# infra/deploy/forgejo/ci-runner
|
||||
# docker push git.thermograph.org/emi/thermograph/ci-runner:vN
|
||||
# docker push git.thermograph.org/jinemi/thermograph/ci-runner:vN
|
||||
#
|
||||
# Cutting over the live runner to a new tag means editing the `labels` array
|
||||
# in ~/forgejo-runner/.runner on the runner host directly (same runner
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@
|
|||
# from the Terraform-managed docker-compose.yml that runs the Thermograph app
|
||||
# itself: this stack's only job is Forgejo and its container registry.
|
||||
#
|
||||
# The Actions RUNNER is deliberately NOT a service in this stack. Per
|
||||
# thermograph-docs/runbooks/implementation-handoff.md (Track B step 5), the canonical
|
||||
# design registers the runner on the desktop node as a plain systemd service
|
||||
# (deploy/forgejo/register-lan-runner.sh) — the same place the pre-Forgejo
|
||||
# GitHub self-hosted runner already lived, not a Swarm-scheduled container.
|
||||
# The Actions RUNNER is deliberately NOT a service in this stack, and that is a
|
||||
# reasoned choice rather than an omission: a Swarm-scheduled runner cannot
|
||||
# redeploy the Swarm that schedules it, so CI would be gone exactly when the
|
||||
# cluster is the thing that is broken.
|
||||
#
|
||||
# The runner that CI depends on runs on vps2 as a plain restart:always container
|
||||
# — deploy/forgejo/runner-vps2/. The desktop's systemd runner
|
||||
# (deploy/forgejo/register-lan-runner.sh) stays registered as extra capacity.
|
||||
# Nothing may assume the desktop is up: it was the ONLY runner until 2026-08-01,
|
||||
# and its 21-hour outage on 2026-07-31 froze every merge, deploy and backup.
|
||||
#
|
||||
# No Traefik here. Forgejo is pinned to vps1 (node.labels.role == forge)
|
||||
# because that's what was chosen, and vps1 is ALSO the emigriffith.dev portfolio
|
||||
|
|
|
|||
|
|
@ -18,11 +18,16 @@
|
|||
# breaks by keeping it registered — no workflow requests it — but do not build
|
||||
# anything new on it.
|
||||
#
|
||||
# TODO(cutover): whether this machine keeps serving the `docker` label at all is
|
||||
# an open call. It is no longer the only runner (the Swarm-hosted one is
|
||||
# always-on), and the box's new job is AI model hosting plus flex Swarm-worker
|
||||
# capacity. Keeping it is fine — it is real CI capacity — but the estate no
|
||||
# longer depends on it, so decide deliberately rather than by inertia.
|
||||
# This machine keeps serving the `docker` label as EXTRA capacity, and that is
|
||||
# now a settled call rather than an open one. The claim it used to make here —
|
||||
# that a Swarm-hosted runner was always-on, so the estate no longer depended on
|
||||
# this box — was false: no such runner existed. This was the only registered
|
||||
# runner in the estate until 2026-08-01, and when it went offline on 2026-07-31
|
||||
# every merge, deploy and backup stopped for 21 hours.
|
||||
#
|
||||
# The runner the estate depends on is deploy/forgejo/runner-vps2/. Keeping this
|
||||
# one is worthwhile (a second runner means a queue drains instead of stalling),
|
||||
# but nothing may be built on the assumption that this box is up.
|
||||
#
|
||||
# bash deploy/forgejo/register-lan-runner.sh <forgejo_url> <registration_token>
|
||||
#
|
||||
|
|
@ -35,7 +40,7 @@ set -euo pipefail
|
|||
FORGEJO_URL="${1:?usage: $0 <forgejo_url> <registration_token>}"
|
||||
TOKEN="${2:?}"
|
||||
RUNNER_DIR="${RUNNER_DIR:-$HOME/forgejo-runner}"
|
||||
LABELS="${LABELS:-docker:docker://git.thermograph.org/emi/thermograph/ci-runner:v2,thermograph-lan}"
|
||||
LABELS="${LABELS:-docker:docker://git.thermograph.org/jinemi/thermograph/ci-runner:v2,thermograph-lan}"
|
||||
|
||||
echo "==> Stopping and disabling the old GitHub Actions runner service, if present"
|
||||
systemctl --user stop github-actions-runner 2>/dev/null || true
|
||||
|
|
|
|||
3
infra/deploy/forgejo/runner-vps2/.gitignore
vendored
Normal file
3
infra/deploy/forgejo/runner-vps2/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# The runner registration (data/.runner) holds a long-lived runner token, and
|
||||
# the daemon writes caches and job workspaces beside it. None of it belongs in git.
|
||||
data/
|
||||
140
infra/deploy/forgejo/runner-vps2/README.md
Normal file
140
infra/deploy/forgejo/runner-vps2/README.md
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
# The always-on Actions runner (vps2)
|
||||
|
||||
The runner CI actually depends on. The desktop runner
|
||||
(`../register-lan-runner.sh`) stays registered as extra capacity, but nothing in
|
||||
the estate may assume it is up.
|
||||
|
||||
Labels: `docker` only. `thermograph-lan` is deliberately **not** registered here
|
||||
— it was the host-native LAN-deploy label, dev moved to vps1 and is deployed
|
||||
over SSH like beta and prod, and no workflow requests it any more.
|
||||
|
||||
## Why this exists
|
||||
|
||||
On 2026-07-31 16:31Z the desktop — then the estate's **only** registered runner
|
||||
— went offline. For the next 21 hours nothing merged (every protected branch
|
||||
requires a check that only a runner can produce), nothing deployed, and the
|
||||
03:00Z `ops-cron` did not run. Forgejo held that scheduled run in `waiting`
|
||||
rather than dropping it, so all three jobs completed on reconnect and no backup
|
||||
was actually lost. A longer outage would not have been so kind.
|
||||
|
||||
`docker-stack.yml` and this directory's parent README both described an
|
||||
"always-on Swarm-hosted runner" during that entire window. There wasn't one; an
|
||||
early revision had it as a Docker-in-Docker sidecar and it was removed. That is
|
||||
why the single point of failure went unnoticed for weeks, and it is the reason
|
||||
the root `CLAUDE.md` treats a stale doc as a correctness bug rather than a
|
||||
documentation one.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- vps2's docker daemon is logged in to `git.thermograph.org` (it is —
|
||||
`/root/.docker/config.json`), so the daemon can pull the job image.
|
||||
- `git.thermograph.org` resolves to the **mesh** address from vps2 for registry
|
||||
traffic. Job containers get this via `--add-host` in `config.yaml`; the host
|
||||
daemon already has it.
|
||||
- The job image exists: `git.thermograph.org/jinemi/thermograph/ci-runner:v2`.
|
||||
Built from `../ci-runner/Dockerfile`; see that file for when to rebuild.
|
||||
|
||||
## Where it is deployed, and why not from the checkout
|
||||
|
||||
**`/opt/forgejo-runner/` on vps2**, holding a copy of `docker-compose.yml` and
|
||||
`config.yaml` from this directory, plus a `data/` the runner owns.
|
||||
|
||||
Deliberately *not* run in place from `/opt/thermograph/…/runner-vps2/`. That
|
||||
checkout is `git reset --hard`'d on every prod deploy; untracked files do
|
||||
survive that (it is the same property `deploy/.image-tags.env` relies on), but a
|
||||
single `git clean -fdx` during troubleshooting would delete `data/.runner` and
|
||||
de-register the runner. Putting the thing that repairs the estate inside the
|
||||
tree the estate rewrites is the same mistake as scheduling it on the Swarm it
|
||||
deploys.
|
||||
|
||||
So this directory is the source of truth; vps2 gets a copy:
|
||||
|
||||
```bash
|
||||
install -d -m 0755 /opt/forgejo-runner
|
||||
cp /opt/thermograph/infra/deploy/forgejo/runner-vps2/docker-compose.yml \
|
||||
/opt/thermograph/infra/deploy/forgejo/runner-vps2/config.yaml \
|
||||
/opt/forgejo-runner/
|
||||
```
|
||||
|
||||
Re-copy after changing either file here, then `docker compose up -d`. There is
|
||||
no automation for that step and there should not be: a workflow that redeploys
|
||||
the runner runs *on* the runner.
|
||||
|
||||
## One-time registration
|
||||
|
||||
The registration token is short-lived and single-purpose. Fetch it from the
|
||||
Forgejo UI (**Site Administration → Actions → Runners → Create new Runner**) or
|
||||
the API, and keep it out of shell history — it is a credential, however
|
||||
briefly.
|
||||
|
||||
```bash
|
||||
cd /opt/forgejo-runner
|
||||
mkdir -p data
|
||||
cp config.yaml data/config.yaml
|
||||
|
||||
read -rs REG_TOKEN && export REG_TOKEN # paste; not echoed, not in history
|
||||
|
||||
docker run --rm -it \
|
||||
-v "$PWD/data:/data" -w /data --user root \
|
||||
code.forgejo.org/forgejo/runner:6.3.1 \
|
||||
forgejo-runner register --no-interactive \
|
||||
--instance https://git.thermograph.org \
|
||||
--token "$REG_TOKEN" \
|
||||
--name thermograph-vps2 \
|
||||
--labels 'docker:docker://git.thermograph.org/jinemi/thermograph/ci-runner:v2'
|
||||
|
||||
unset REG_TOKEN
|
||||
```
|
||||
|
||||
That writes `data/.runner`, which holds the **long-lived** runner token. It is
|
||||
`0600` and must stay out of git — `data/` is gitignored for exactly this reason.
|
||||
Re-running `register` with a fresh registration token replaces it; the old
|
||||
registration then shows offline in the UI and should be deleted there.
|
||||
|
||||
## Run it
|
||||
|
||||
```bash
|
||||
docker compose up -d
|
||||
docker compose logs -f --tail=50 runner # expect "runner: ... successfully activated"
|
||||
```
|
||||
|
||||
Confirm Forgejo agrees, rather than trusting the log:
|
||||
|
||||
```bash
|
||||
# on vps1, where Forgejo's database lives
|
||||
fdb=$(docker ps -qf name=forgejo_db | head -1)
|
||||
docker exec "$fdb" sh -c 'psql -U $POSTGRES_USER -d $POSTGRES_DB -c \
|
||||
"select name, to_timestamp(last_online) as last_online, agent_labels from action_runner order by id;"'
|
||||
```
|
||||
|
||||
Two rows, both recent. `last_online` is the only honest liveness signal — a
|
||||
runner process that is up but cannot reach Forgejo looks healthy locally and is
|
||||
useless.
|
||||
|
||||
## Upgrading
|
||||
|
||||
Pin changes go in `docker-compose.yml` (`image:`) and are applied with
|
||||
`docker compose up -d`. The runner token survives an image change; it lives in
|
||||
`data/.runner`, not in the image.
|
||||
|
||||
Changing the **job** image (the `docker://…ci-runner:vN` label) is a
|
||||
re-registration or a hand-edit of `data/.runner`'s `labels` array — the same
|
||||
caveat `../ci-runner/Dockerfile` records for the desktop runner. Keep the two
|
||||
runners on the same job image or a workflow's behaviour starts depending on
|
||||
which runner happened to pick it up.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Jobs queue forever with both runners online.** The workflow asked for a label
|
||||
nobody registered. `runs-on: docker` is the only label this estate uses.
|
||||
|
||||
**`docker pull` fails inside a job, works on the host.** The `--add-host` line
|
||||
in `config.yaml` is missing or wrong. vps1's Caddy serves `/v2/*` to the mesh
|
||||
only, and the job container has no other way to learn the mesh address.
|
||||
|
||||
**Jobs fail with permission denied on the socket.** `user: root` was dropped
|
||||
from the compose file.
|
||||
|
||||
**The runner is up but `last_online` is stale.** It cannot reach
|
||||
`https://git.thermograph.org` — check egress from vps2 and that Forgejo on vps1
|
||||
is serving.
|
||||
53
infra/deploy/forgejo/runner-vps2/config.yaml
Normal file
53
infra/deploy/forgejo/runner-vps2/config.yaml
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# Forgejo Actions runner config for the vps2 runner. Mounted at /data/config.yaml
|
||||
# by runner-vps2/docker-compose.yml.
|
||||
#
|
||||
# Values that differ from `forgejo-runner generate-config` defaults are the only
|
||||
# ones written out below; everything else is left at its default deliberately, so
|
||||
# a future runner upgrade inherits new defaults instead of a frozen copy of the
|
||||
# old ones.
|
||||
|
||||
log:
|
||||
level: info
|
||||
job_level: info
|
||||
|
||||
runner:
|
||||
file: .runner
|
||||
|
||||
# ONE job at a time. The desktop runner uses capacity 8, which is right for a
|
||||
# box whose whole job is CI. This runner shares a host with prod, so the
|
||||
# binding constraint is not throughput, it is that a CI burst must never be
|
||||
# able to contend with thermograph_web for CPU. One job at a time also makes
|
||||
# the resource ceiling below exact rather than a per-job estimate multiplied
|
||||
# by an unknown.
|
||||
capacity: 1
|
||||
|
||||
timeout: 3h
|
||||
shutdown_timeout: 3h
|
||||
|
||||
container:
|
||||
# 1) --add-host: git.thermograph.org resolves publicly to vps1, but vps1's
|
||||
# Caddy rejects the /v2/* registry API from off-mesh. Job containers that
|
||||
# docker-pull or docker-push therefore need the MESH address, and DNS will
|
||||
# not give it to them. Same line the desktop runner carries; the value is
|
||||
# the same from vps2 because both boxes are on 10.10.0.0/24.
|
||||
#
|
||||
# 2) --cpus/--memory: vps2 runs prod. Without a ceiling a runaway job (a
|
||||
# `docker build` that fans out, a test that leaks) competes with the
|
||||
# production app for the same 6 cores. These bound the JOB container, which
|
||||
# is a sibling of the runner rather than its child, so the runner's own
|
||||
# compose limits do not cover it -- this is the only lever that does.
|
||||
options: --add-host=git.thermograph.org:10.10.0.2 --cpus=2 --memory=4g
|
||||
|
||||
# Jobs may mount NOTHING from the host. The workflows here do not need to:
|
||||
# they check out into the job container's own workspace and reach the estate
|
||||
# over SSH. An empty list is the difference between "a job can read
|
||||
# /etc/thermograph.env" and "a job cannot", on the box where that file is
|
||||
# prod's.
|
||||
valid_volumes: []
|
||||
|
||||
privileged: false
|
||||
|
||||
# Mounts the host docker socket into the job container. build-push.yml runs
|
||||
# `docker build`, so it needs a daemon; this points it at vps2's. See the
|
||||
# README for what that does and does not mean for trust.
|
||||
docker_host: "automount"
|
||||
83
infra/deploy/forgejo/runner-vps2/docker-compose.yml
Normal file
83
infra/deploy/forgejo/runner-vps2/docker-compose.yml
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
# The always-on Forgejo Actions runner, on vps2.
|
||||
#
|
||||
# cd /opt/forgejo-runner # a COPY of this file, not the checkout
|
||||
# docker compose up -d
|
||||
#
|
||||
# It runs from /opt/forgejo-runner rather than in place, because the checkout is
|
||||
# git reset --hard'd on every prod deploy and one `git clean -fdx` there would
|
||||
# delete data/.runner. See README.md.
|
||||
#
|
||||
# Registration is a one-off and is NOT in this file, because it takes a
|
||||
# short-lived token a human fetches. See README.md.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHY A SECOND RUNNER EXISTS AT ALL
|
||||
# ============================================================================
|
||||
# Until 2026-08-01 the estate had exactly one registered runner, on the desktop.
|
||||
# It went offline at 2026-07-31 16:31Z and everything stopped for 21 hours: no
|
||||
# merge could satisfy a required check, no deploy could run, and the 03:00Z
|
||||
# ops-cron -- THE backup for both application databases and for Forgejo -- did
|
||||
# not fire. Forgejo queued that run rather than dropping it, so it completed on
|
||||
# reconnect, but a longer outage would have meant real backup gaps.
|
||||
#
|
||||
# So this runner is not extra capacity. It is the one that has to be up.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHY A PLAIN CONTAINER AND NOT A SWARM SERVICE
|
||||
# ============================================================================
|
||||
# Same argument as infra/openbao/config/openbao.hcl makes for OpenBao: the thing
|
||||
# that repairs the estate must not depend on the estate. A Swarm-scheduled
|
||||
# runner cannot redeploy the Swarm it is scheduled by, and if the cluster is the
|
||||
# thing that is broken, CI is gone exactly when it is needed. `restart: always`
|
||||
# on the local daemon has one dependency: the local daemon.
|
||||
#
|
||||
# An earlier revision of deploy/forgejo/docker-stack.yml did run a runner as a
|
||||
# Swarm-scheduled Docker-in-Docker sidecar. It was removed, and the docs kept
|
||||
# claiming an "always-on Swarm-hosted runner" for weeks afterwards -- which is
|
||||
# how the single point of failure went unnoticed.
|
||||
#
|
||||
# ============================================================================
|
||||
# WHAT MOUNTING THE DOCKER SOCKET ON THE PROD HOST MEANS
|
||||
# ============================================================================
|
||||
# Be plain about it: /var/run/docker.sock is root-equivalent on vps2, and vps2
|
||||
# runs prod. A workflow that can start a container can mount / and read
|
||||
# /etc/thermograph.env. This runner does not create that exposure from nothing
|
||||
# -- Forgejo's own database already stores VPS2_SSH_KEY, which is root on this
|
||||
# box -- but it does move the exposure from "a secret at rest" to "a secret a
|
||||
# job can reach", and that is a real difference.
|
||||
#
|
||||
# What bounds it, in order of how much each is worth:
|
||||
# 1. Only this repo's workflows run here, and only the owner can push to it.
|
||||
# 2. config.yaml sets valid_volumes: [] -- jobs cannot bind-mount host paths.
|
||||
# 3. capacity: 1 and --cpus/--memory keep a job from contending with prod.
|
||||
# 4. This compose project joins no thermograph network, so a job container
|
||||
# reaches prod's services no more easily than any other host process.
|
||||
#
|
||||
# It is defence against accident, not against a hostile workflow author. On a
|
||||
# two-person estate where both people can already SSH to this box as root, that
|
||||
# is the honest boundary.
|
||||
|
||||
name: forgejo-runner
|
||||
|
||||
services:
|
||||
runner:
|
||||
image: code.forgejo.org/forgejo/runner:6.3.1
|
||||
restart: always
|
||||
# The socket is root:docker on the host; the image's default user is not in
|
||||
# that group. Running as root is what the socket mount requires.
|
||||
user: root
|
||||
working_dir: /data
|
||||
command: forgejo-runner daemon --config /data/config.yaml
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# Bounds the DAEMON. Job containers are siblings started through the socket,
|
||||
# not children, so they are unaffected by these -- container.options in
|
||||
# config.yaml is what bounds those.
|
||||
cpus: 1.0
|
||||
mem_limit: 1g
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
|
@ -26,7 +26,7 @@ SELF_DIR=$(cd "$(dirname "$0")" && pwd)
|
|||
. "$SELF_DIR/env-topology.sh"
|
||||
thermograph_topology dev
|
||||
|
||||
REPO_URL="${REPO_URL:-http://10.10.0.2:3080/emi/thermograph.git}"
|
||||
REPO_URL="${REPO_URL:-http://10.10.0.2:3080/Jinemi/thermograph.git}"
|
||||
APP_DIR="${APP_DIR:-$TG_APP_DIR}"
|
||||
BRANCH="${BRANCH:-$TG_BRANCH}"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
# OpenBao source backend for render-secrets.sh. Sourced, never run directly.
|
||||
#
|
||||
# Two functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
# Functions:
|
||||
# thermograph_openbao_source <env> -> merged dotenv on STDOUT
|
||||
# thermograph_render_openbao <env> <out> -> source, then write <out>
|
||||
# thermograph_render_openbao_centralis <env> [out] -> quoted /etc/centralis.env
|
||||
#
|
||||
# The first two serve the app stack (/etc/thermograph.env, raw unquoted dotenv). The
|
||||
# third serves Centralis, whose file is SOURCED by bash and therefore single-quoted —
|
||||
# a different shape for a different consumer, not an inconsistency.
|
||||
#
|
||||
# render_thermograph_secrets in render-secrets.sh dispatches to the second when
|
||||
# TG_SECRETS_BACKEND=openbao, and is otherwise completely untouched — so the SOPS path
|
||||
|
|
@ -27,7 +32,13 @@ thermograph_openbao_source() {
|
|||
local env_name="${1:?env name required}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-/etc/thermograph/openbao-approle}"
|
||||
# Explicit override first, then the per-environment value env-topology.sh derives,
|
||||
# then the conventional path. Same precedence shape as render-secrets.sh:44's
|
||||
# THERMOGRAPH_SECRETS_BACKEND / TG_SECRETS_BACKEND pair, for the same reason: the
|
||||
# THERMOGRAPH_-prefixed name is the by-hand escape hatch, the TG_ one is what the
|
||||
# deploy path sets. Falling straight through to the bare default is what made beta
|
||||
# authenticate as tg-prod and get denied its own path.
|
||||
local approle="${THERMOGRAPH_BAO_APPROLE:-${TG_BAO_APPROLE:-/etc/thermograph/openbao-approle}}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
|
|
@ -107,9 +118,16 @@ thermograph_openbao_source() {
|
|||
# /etc/thermograph.env is sourced by bash in six places (deploy.sh:133,
|
||||
# deploy-stack.sh:98, both daemon entrypoints, ops-cron.yml:85, terraform), so
|
||||
# `$(...)` or a backtick in a secret is a live command-execution path on the
|
||||
# deploy host. The SOPS path has always had this hazard and has never tripped it
|
||||
# only because every current value happens to be alphanumeric. Refusing here
|
||||
# turns a latent code-execution bug into a loud render failure.
|
||||
# deploy host. Refusing here turns a latent code-execution bug into a loud
|
||||
# render failure.
|
||||
#
|
||||
# An earlier version of this comment claimed the SOPS path had never tripped
|
||||
# this "only because every current value happens to be alphanumeric". That is
|
||||
# false: CENTRALIS_TOKENS is JSON and full of double quotes. It has never
|
||||
# tripped THESE paths because it is not in them — it lives at
|
||||
# centralis/<env> and renders to /etc/centralis.env, which is single-quoted
|
||||
# precisely because raw dotenv cannot carry it. See
|
||||
# thermograph_render_openbao_centralis at the end of this file.
|
||||
THERMOGRAPH_BAO_COMMON="$common_json" \
|
||||
THERMOGRAPH_BAO_ENV="$env_json" \
|
||||
python3 - <<'PY'
|
||||
|
|
@ -237,3 +255,239 @@ thermograph_render_openbao() {
|
|||
rm -f "$tmp"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CENTRALIS
|
||||
# ---------------------------------------------------------------------------
|
||||
# /etc/centralis.env is a different file, with a different consumer, and therefore a
|
||||
# different output shape. render-secrets.sh:206-233 carries the full argument; the
|
||||
# short version is that it is consumed by exactly one thing and that thing is a shell:
|
||||
#
|
||||
# sudo bash -c 'set -a && . /etc/centralis.env && set +a && docker compose up'
|
||||
#
|
||||
# `set -a; . file` runs every line through bash's whole expansion pipeline — quote
|
||||
# removal, parameter expansion, command substitution, field splitting. Raw dotenv is
|
||||
# therefore not a safe representation here. CENTRALIS_TOKENS is JSON: rendered
|
||||
# unquoted, bash strips its quotes, and Centralis fails CLOSED on the malformed result
|
||||
# by serving a single `shared` identity — indistinguishable from the file never having
|
||||
# been written. That is the 2026-07-24 incident. So this path emits POSIX
|
||||
# single-quoted assignments and PROVES them by sourcing the result in a clean shell
|
||||
# before anything touches $dest.
|
||||
#
|
||||
# This mirrors render_centralis_secrets rather than sharing with it, for the same
|
||||
# reason given above thermograph_render_openbao: the SOPS path stays byte-identical
|
||||
# and all new risk stays in this file. Consolidate the two when the SOPS path is being
|
||||
# deleted anyway, not before.
|
||||
|
||||
# _thermograph_openbao_login <approle_file> <addr> <ca> -> token on STDOUT
|
||||
#
|
||||
# Separate from the login inline in thermograph_openbao_source deliberately: that
|
||||
# function is now exercised by prod and beta parity and is left untouched.
|
||||
_thermograph_openbao_login() {
|
||||
local approle="$1" addr="$2" ca="$3"
|
||||
local creds role_id secret_id
|
||||
if [ -r "$approle" ]; then
|
||||
creds=$(cat "$approle")
|
||||
else
|
||||
creds=$(sudo cat "$approle" 2>/dev/null || true)
|
||||
fi
|
||||
[ -n "$creds" ] || {
|
||||
echo "!! cannot read OpenBao AppRole credentials at $approle (need sudo)" >&2
|
||||
return 1
|
||||
}
|
||||
role_id=$(printf '%s\n' "$creds" | sed -n '1p')
|
||||
secret_id=$(printf '%s\n' "$creds" | sed -n '2p')
|
||||
[ -n "$role_id" ] && [ -n "$secret_id" ] || {
|
||||
echo "!! $approle malformed: expected role_id on line 1, secret_id on line 2" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_ADDR="$addr"
|
||||
[ -f "$ca" ] && export BAO_CACERT="$ca"
|
||||
bao write -field=token auth/approle/login \
|
||||
role_id="$role_id" secret_id="$secret_id" 2>/dev/null
|
||||
}
|
||||
|
||||
# thermograph_render_openbao_centralis <env_name> [dest]
|
||||
thermograph_render_openbao_centralis() {
|
||||
local env_name="${1:?env name required}"
|
||||
local dest="${2:-${CENTRALIS_RENDER_DEST:-/etc/centralis.env}}"
|
||||
local addr="${THERMOGRAPH_BAO_ADDR:-https://10.10.0.1:8200}"
|
||||
local mount="${THERMOGRAPH_BAO_MOUNT:-thermograph}"
|
||||
local ca="${THERMOGRAPH_BAO_CACERT:-/etc/thermograph/openbao-ca.crt}"
|
||||
local approle="${THERMOGRAPH_BAO_CENTRALIS_APPROLE:-/etc/thermograph/openbao-approle-centralis}"
|
||||
|
||||
command -v bao >/dev/null 2>&1 || {
|
||||
echo "!! bao CLI not installed but this host is configured for the openbao backend" >&2
|
||||
return 1
|
||||
}
|
||||
command -v python3 >/dev/null 2>&1 || {
|
||||
echo "!! python3 not installed; needed to quote values safely for a sourced file" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "==> Rendering ${dest} from OpenBao (${mount}/centralis/${env_name})"
|
||||
|
||||
local token
|
||||
token=$(_thermograph_openbao_login "$approle" "$addr" "$ca") || return 1
|
||||
[ -n "$token" ] || {
|
||||
echo "!! OpenBao AppRole login failed for centralis" >&2
|
||||
echo "!! check that OpenBao at $addr is reachable and UNSEALED" >&2
|
||||
return 1
|
||||
}
|
||||
export BAO_TOKEN="$token"
|
||||
|
||||
# ONE 0700 temp directory holding plaintext, so cleanup is a single rm on every
|
||||
# path. Deliberately not a `trap ... RETURN`: this file is SOURCED, and such a trap
|
||||
# persists into the caller's shell and re-fires on its next `source`.
|
||||
local work rc=0
|
||||
work=$(mktemp -d) || { echo "!! mktemp -d failed" >&2; return 1; }
|
||||
chmod 700 "$work"
|
||||
|
||||
# Subshell so no failure path can skip the cleanup, and so umask cannot leak out.
|
||||
# Every step carries its own `|| exit 1`: `set -e` is DISABLED inside a compound
|
||||
# command that is the left operand of `||`, which this subshell is.
|
||||
(
|
||||
umask 077
|
||||
|
||||
bao kv get -format=json -mount="$mount" "centralis/${env_name}" \
|
||||
> "$work/kv.json" 2>/dev/null || {
|
||||
echo "!! cannot read ${mount}/centralis/${env_name}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
python3 - "$work/kv.json" "$work/plain.json" <<'UNWRAP' || exit 1
|
||||
import json, sys
|
||||
doc = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
data = doc.get("data", {}).get("data", {}) or {}
|
||||
if not data:
|
||||
sys.stderr.write("!! OpenBao returned zero keys for centralis\n")
|
||||
sys.exit(1)
|
||||
json.dump(data, open(sys.argv[2], "w", encoding="utf-8"))
|
||||
UNWRAP
|
||||
|
||||
python3 - "$work/plain.json" "$work/out.env" "$work/want.json" <<'EMIT' || exit 1
|
||||
import json, re, sys
|
||||
|
||||
src, out_env, want_json = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
|
||||
|
||||
|
||||
def shell_quote(v):
|
||||
"""POSIX single-quoting. Inside '...' every byte is literal except ' itself,
|
||||
which is closed, backslash-escaped and reopened. There is no escape sequence
|
||||
to get wrong and no character class to keep up to date: it is total, and
|
||||
total is the only property worth having for a file bash will expand."""
|
||||
return "'" + v.replace("'", "'\\''") + "'"
|
||||
|
||||
|
||||
def scalar(k, v):
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
if isinstance(v, bool): # before int: bool is an int in Python
|
||||
return "true" if v else "false"
|
||||
if isinstance(v, (int, float)):
|
||||
return json.dumps(v)
|
||||
if v is None:
|
||||
return ""
|
||||
sys.stderr.write(
|
||||
"!! %s is a %s, but an env file holds only scalars\n" % (k, type(v).__name__))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
data = json.load(open(src, encoding="utf-8"))
|
||||
want, lines = {}, []
|
||||
for k, v in data.items():
|
||||
if not NAME.match(k):
|
||||
sys.stderr.write("!! %r is not a usable shell variable name\n" % (k,))
|
||||
sys.exit(1)
|
||||
want[k] = scalar(k, v)
|
||||
lines.append("%s=%s\n" % (k, shell_quote(want[k])))
|
||||
|
||||
with open(out_env, "w", encoding="utf-8") as fh:
|
||||
fh.write("# Rendered by infra/deploy/render-secrets-openbao.sh from OpenBao.\n")
|
||||
fh.write("# DO NOT EDIT BY HAND: the next render overwrites this file, and a\n")
|
||||
fh.write("# hand-edit is how the token registry was lost on 2026-07-24.\n")
|
||||
fh.write("# Rotate with `bao kv put` against thermograph/centralis/<env> instead.\n")
|
||||
fh.writelines(lines)
|
||||
with open(want_json, "w", encoding="utf-8") as fh:
|
||||
json.dump(want, fh)
|
||||
sys.stderr.write(" %d keys quoted\n" % len(want))
|
||||
EMIT
|
||||
|
||||
# PROVE IT. Source the rendered file exactly the way the deploy does — clean
|
||||
# environment, `set -a`, nothing inherited that could mask a dropped key — and
|
||||
# read every value back. This is the check the 2026-07-24 file could not pass.
|
||||
# shellcheck disable=SC2016 # single quotes are the point: "$1" must be expanded
|
||||
# by the inner `bash -c` against the argument after `_`, not by this shell.
|
||||
env -i PATH="$PATH" bash -c \
|
||||
'set -a; . "$1"; set +a; exec python3 -c "
|
||||
import json, os, sys
|
||||
sys.stdout.write(json.dumps(dict(os.environ)))"' _ "$work/out.env" \
|
||||
> "$work/got.json" || exit 1
|
||||
|
||||
python3 - "$work/want.json" "$work/got.json" <<'VERIFY' || exit 1
|
||||
import json, sys
|
||||
|
||||
want = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
got = json.load(open(sys.argv[2], encoding="utf-8"))
|
||||
|
||||
bad = []
|
||||
for k, v in want.items():
|
||||
if k not in got:
|
||||
bad.append("%s: not set at all after sourcing" % k)
|
||||
elif got[k] != v:
|
||||
# NEVER print either value. The difference is the finding; the content is
|
||||
# not, and this runs inside a deploy log.
|
||||
bad.append("%s: survives sourcing as a DIFFERENT value (len %d -> %d)"
|
||||
% (k, len(v), len(got[k])))
|
||||
if bad:
|
||||
sys.stderr.write("!! the rendered file does not round-trip through bash:\n")
|
||||
for b in bad:
|
||||
sys.stderr.write("!! %s\n" % b)
|
||||
sys.exit(1)
|
||||
|
||||
# CENTRALIS_TOKENS is the reason for all of the above: JSON, full of double quotes,
|
||||
# in a file bash expands. Centralis fails CLOSED on malformed JSON — it drops to the
|
||||
# single `shared` identity, which looks exactly like the registry never having been
|
||||
# configured. Refuse to write a file that would do that.
|
||||
raw = got.get("CENTRALIS_TOKENS", "")
|
||||
if raw:
|
||||
try:
|
||||
reg = json.loads(raw)
|
||||
except ValueError as exc:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS is not valid JSON after sourcing: %s\n" % exc)
|
||||
sys.exit(1)
|
||||
if not isinstance(reg, dict) or not reg:
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must be a non-empty JSON object\n")
|
||||
sys.exit(1)
|
||||
if not all(isinstance(s, str) and isinstance(t, str) and t for s, t in reg.items()):
|
||||
sys.stderr.write("!! CENTRALIS_TOKENS must map subject -> non-empty token string\n")
|
||||
sys.exit(1)
|
||||
# Subjects are names on audit lines, not credentials. Printing them is the whole
|
||||
# point: it is how an operator sees at a glance that nobody was lost.
|
||||
sys.stderr.write(" CENTRALIS_TOKENS parses: %d subject(s) — %s\n"
|
||||
% (len(reg), ", ".join(sorted(reg))))
|
||||
|
||||
sys.stderr.write(" %d keys survive `set -a; . <file>` byte-for-byte\n" % len(want))
|
||||
VERIFY
|
||||
) || rc=1
|
||||
|
||||
# Only now, with a file read back through bash and matched value for value, does
|
||||
# anything touch $dest. 0600: four bearer credentials for the estate's control
|
||||
# plane, with no group that needs them.
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
if [ -f "$dest" ] && [ -w "$dest" ]; then
|
||||
cat "$work/out.env" > "$dest" || rc=1
|
||||
elif install -m 0600 "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
elif sudo install -m 0600 -o "$(id -un)" -g "$(id -gn)" "$work/out.env" "$dest" 2>/dev/null; then :
|
||||
else
|
||||
echo "!! cannot write ${dest} (need file write access or passwordless sudo)" >&2
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "!! render aborted; ${dest} left untouched" >&2
|
||||
fi
|
||||
|
||||
rm -rf "$work"
|
||||
return "$rc"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -272,6 +272,30 @@ render_centralis_secrets() {
|
|||
echo "!! no environment name at ${marker} — cannot tell which Centralis vault to render" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Same early-return dispatch as render_thermograph_secrets, for the same reason
|
||||
# given there: everything below — the age-key sudo lift, the JSON decrypt, the
|
||||
# quote-and-prove pipeline, the three-way write — stays on exactly the code path it
|
||||
# has always been on, so this cannot regress the backend still authoritative for
|
||||
# Centralis. The OpenBao path needs no age key, so it returns before that check.
|
||||
local backend="${THERMOGRAPH_SECRETS_BACKEND:-${TG_SECRETS_BACKEND:-sops}}"
|
||||
if [ "$backend" = openbao ]; then
|
||||
local bao_lib="${repo}/deploy/render-secrets-openbao.sh"
|
||||
if [ ! -f "$bao_lib" ]; then
|
||||
echo "!! TG_SECRETS_BACKEND=openbao but $bao_lib is missing" >&2
|
||||
echo "!! (a checkout predating the OpenBao backend cannot render this way)" >&2
|
||||
return 1
|
||||
fi
|
||||
# shellcheck source=/dev/null
|
||||
. "$bao_lib"
|
||||
thermograph_render_openbao_centralis "$env_name" "$dest"
|
||||
return
|
||||
fi
|
||||
if [ "$backend" != sops ]; then
|
||||
echo "!! unknown secrets backend '${backend}' (expected sops|openbao)" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$key" ]; then
|
||||
echo "!! no age key at ${key}; this host cannot decrypt the vault" >&2
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -251,17 +251,44 @@ files), so layering would only push the VAPID private key, both S3 keypairs and
|
|||
- `/etc/thermograph/age.key` (`0400`) on each VPS that renders at deploy time.
|
||||
- **Back it up** in the password manager. The public recipient is in `.sops.yaml`.
|
||||
|
||||
On **vps1** (dev) those two can end up being the same file. `render-secrets.sh`
|
||||
falls back to `sudo cat` for a root-owned key, and `deploy-dev.sh` still assumes
|
||||
the account driving a CI-triggered dev deploy has no passwordless sudo and no
|
||||
tty to answer a `sudo` prompt (a `systemd --user` Forgejo-runner service) —
|
||||
inherited from the old desktop setup, where that was true of the whole box.
|
||||
Whether it's still true of vps1's CI-runner account specifically (as opposed
|
||||
to the `agent` login, which does have passwordless sudo there per `ACCESS.md`)
|
||||
wasn't re-verified for this pass; `deploy-dev.sh` points `THERMOGRAPH_AGE_KEY`
|
||||
at the operator's keyring as a fallback either way, so the box keeps **one**
|
||||
copy of the recovery root rather than two regardless of which account ends up
|
||||
mattering.
|
||||
### The two on-host copies are a deliberate quorum
|
||||
|
||||
`/etc/thermograph/age.key` on **vps1** and **vps2** is not provisioning residue and
|
||||
must not be tidied away. Together they are the recovery quorum for the entire
|
||||
estate: five SOPS vaults, and every off-box backup, since `ops-cron.yml:142,197`
|
||||
pipe those dumps through `age -r` to this same recipient.
|
||||
|
||||
That is not theoretical. On **2026-07-30** the operator's copy was shredded from the
|
||||
desktop *before* it had been copied to its replacement machine, leaving zero
|
||||
operator-side copies. These two were the only surviving material, and the key was
|
||||
restored from vps2. The ordering rule below exists because of that hour.
|
||||
|
||||
- **Never remove the last operator-side copy** until its replacement has been
|
||||
verified against all five vaults (`sops -d … | grep -c '='` → 16 / 12 / 8 / 16 / 9).
|
||||
Deleting first and verifying second is unrecoverable if the copy is bad.
|
||||
- **Both hosts must hold byte-identical copies.** Divergence is as bad as absence:
|
||||
two different keys means one host renders secrets the other cannot read, and a
|
||||
restore silently picks the wrong one.
|
||||
- **Verify it, don't assume it** — `deploy/secrets/verify-age-quorum.sh` asserts
|
||||
presence, permissions and hash equality across both hosts. It prints SHA-256
|
||||
digests and modes only, never the key, so it is safe in a CI log and suitable as
|
||||
a cron gate. Deletion and divergence are both silent failures; nothing else in the
|
||||
estate would notice either until a restore.
|
||||
|
||||
Verified 2026-07-30: vps1 `0440 root:deploy`, vps2 `0400 root:root`, digests equal.
|
||||
Note the asymmetry — on vps1 the group `deploy` can read the key that decrypts
|
||||
`common.yaml` and `prod.yaml`, on the box that also runs Forgejo, its CI runner and
|
||||
dev's unreviewed branches. The dev render does not need that group bit: it runs as
|
||||
`agent`, which is not in group `deploy` and reaches the key through its passwordless
|
||||
`sudo` instead (`/opt/thermograph-dev` is `agent:agent`). Tightening vps1 to `0400
|
||||
root:root` is therefore expected to be safe and is the obvious follow-up; it is
|
||||
called out rather than done here because it changes a live host on the deploy path.
|
||||
|
||||
This weakens — in practice, not in intent — the rule in `infra/CLAUDE.md` that "vps1
|
||||
must never hold prod credentials". Withholding `common.yaml` from dev's *render* does
|
||||
not withhold the key that decrypts it from the *box*, because every vault is
|
||||
encrypted to a single recipient. Fixing that properly means a second age identity for
|
||||
dev, not a policy line.
|
||||
|
||||
## Prerequisites (once per machine)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@ THERMOGRAPH_BASE: ENC[AES256_GCM,data:sQ==,iv:Pop6S2qU0o2+2xMKWQD2P3Vjdh4rUafWF7
|
|||
THERMOGRAPH_COOKIE_SECURE: ENC[AES256_GCM,data:CA==,iv:TM6XiygrDQn2qps4lO6hY2ldjW8LWlO3D/R/DBPqvGQ=,tag:+thBT7fMzH+DigecgxF4CQ==,type:str]
|
||||
THERMOGRAPH_INDEXNOW_KEY: ENC[AES256_GCM,data:XZEMMJfmrSjc/sKEePaigTqEkh3ob/E25R25DvOAWwY=,iv:fqdaegzG1Na0zW+UgOh92RS4FP22pxStdD1/Jq5DRjs=,tag:/8ORtVkGUWjqgGXG/xZHmg==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:5pL+8tZbs8TZ5ezJpkPUmMss0gLcte73lU+au/476Ws=,iv:ujU3G+rbPZQx/igg50GeIPQxqgd+szIsjc3AqGhGLBU=,tag:hrkgn2DPkmBGThlsdMplqg==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:XG+6wizXMVVFixhqLhg72HoKw9kEmH+Tty8jYMycvVk=,iv:tNz/WrkMYip2QyAr6bRy6PaIee56ZZ64BELBgvidVT0=,tag:HiQ55DGbeDAOSF0vpNWpWw==,type:str]
|
||||
THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:Ui5dcmhzz84nTp1gYhJ89VOJ2eOBFyLjf0WkZ7Y3sI0=,iv:WI0QqbUW7BHo7Zz0082zxpYNIYpVIhq1NeBGbQeDc/I=,tag:urNsE8DPaldJcw9aqX1PYA==,type:str]
|
||||
THERMOGRAPH_METRICS_TOKEN: ENC[AES256_GCM,data:DmOZU3HWQTuoSvQMb7iPkClJbDH48NB3OcRIYRkqNz8=,iv:acgGh0w2mc5ZD7rB7m/GW3Awsx3RWK+02L6YkAv5Rw0=,tag:x3U9F0Dekw/r9euRMjVQAA==,type:str]
|
||||
THERMOGRAPH_S3_ACCESS_KEY: ENC[AES256_GCM,data:hTSSUIfOlY0GYI8gZ7FcOi7c89iye0Ym73l4M22zkNk=,iv:aPdbXKb4pOqigo9J3a7oM3qTip6HQcHUQDlAwg6XfoE=,tag:C4p+Ob/iSElNdF4m9E+M7A==,type:str]
|
||||
THERMOGRAPH_S3_BUCKET: ENC[AES256_GCM,data:vGJDaElN6JRUawlD55Vr5Q==,iv:4ZAQ5NXb9xuMN+qPspM17W6zm9NIDXLX9hTJ/etSL0E=,tag:Ss7xaUl+kOiVF7oDs9FiYw==,type:str]
|
||||
THERMOGRAPH_S3_ENDPOINT: ENC[AES256_GCM,data:PtZg52Uu70Ndx7L/dRTCZTmqv80NXiohOg+gfrzG,iv:hKQTy3Twd5uWwnS4UIs9oqT4NzpYUWQxWO1lUqrzpAc=,tag:kYwYp58fxkzRe5yQPSXa3g==,type:str]
|
||||
THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:1xuZRaC4p+ZM/oFAUuGDZb6AAu2PbjFrHQY3zmczRpI=,iv:xzI24qlUOvYiThBLOue0odvopuxdHCwDlwI4JwiP9EU=,tag:yRS9GjGLBEYk/kZ6miFudg==,type:str]
|
||||
THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:N9mPLx6Mcgqm5TlgqIMFhNuZsBZxVPXf3LplO9k+VA==,iv:y5+MLI0zC5Kb6pRdORTMVJ1iMMoFrVRfv99mKWMRjp8=,tag:7VvKCLU+1TJtQcHWlDwybg==,type:str]
|
||||
THERMOGRAPH_S3_SECRET_KEY: ENC[AES256_GCM,data:4u5d3RHHBeIDfRIwQVDldWUhH7PyTcNP9s3xTrL4N74=,iv:RXWu+4ZdTmWx3PGw6nWhN2587byy4wqjY74Ldu4xvuk=,tag:dwS6htP2Dv2OQyNzmjZZIA==,type:str]
|
||||
THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:hg5HqBua81dG3d4KaxLxljRkeGnt4h4=,iv:q+4tQGrlsQaRGUTJuqByqss+twR6v6NQrI/qSPjCbsQ=,tag:2D1+hv/KSDJu+4+J9S0hww==,type:str]
|
||||
THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:hTxKlgPWeW4HGBf08SxdAdK+ce6T5Fl9f+5tSGV6WpS+za5EI3zsK8BgRw==,iv:l+omFaCyL2+PHWdchadkmED2gIAoj5T0u/Ltzaw8mok=,tag:AQ3wT/wD5JAouX1M+Tjjmw==,type:str]
|
||||
THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:bgtqZFfTzYz5llh/YhAyGwGtRedK3/ze8SAWj8YdldY6s4nt2F3XeTANVO2Y9+NzvXc16PB3WWPnYEzCxnrG2KciIeYpOYZMoPpGldENPBLJtLa76Ej9,iv:HuZ7XtlXIt4wyJ79k3IYjHfWrQp7lnak4uCspyzS4BE=,tag:uPZvba8LVugDy76pKat6TA==,type:str]
|
||||
TIMESCALEDB_TAG: ENC[AES256_GCM,data:3mDU/k5fx0894+E=,iv:/iw3BZPF3mZ9M3bFNJSTzP9t15YZWCJwzoVXYL9Vj9o=,tag:Y4ez3+XfLJN1PtJyZlJj+g==,type:str]
|
||||
|
|
@ -25,7 +25,7 @@ sops:
|
|||
bncE6yn10QK5kVcF9dDvpQ6RbaG+ESpNZTnuhSL5PDqrKtjnsV0Iqw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
|
||||
lastmodified: "2026-07-25T00:38:24Z"
|
||||
mac: ENC[AES256_GCM,data:C12PnFE7MO0Ge5dCAHCcyXh650hFtRuexl5d3DL0IqTf8nLyNtRJzrPyl9whIs+S3HulsXh3xJOwwPdg6aTqdCfGSQ4Kp5qV+OHxH9lfpHs2yH+exa/eX+7Khpjk0OfvX5beC0GSPpYf9c01buaUTWcIh3pJXpDMdqr+aLC+Zd0=,iv:6VeaaEEkWU5rdTHy75rc2emb2u/HmOmLzO11D38ZXHc=,tag:iGRevzdtqZF9kyH0wxqG3w==,type:str]
|
||||
lastmodified: "2026-07-31T04:23:12Z"
|
||||
mac: ENC[AES256_GCM,data:KkItyA0iw2j4w9pf/efFcpyOP8RmO8XzfMoQhJcRXLZbrUC3/xltbxkfTIe2hRgAqKyz5d7EuAHBPwwSCubue0O2gzlG0nCnRQH7YyO4cUrP24XYb+aI+tBM2jdZIuZrGDNcq04cwKiecC6MyOOp8uQPZ41AOPH3sONGWj1VH2s=,iv:+j5uv4QbuWpXAyElvbYGKWhe9vmGTfs3Hpy5I5FetJo=,tag:wcfPuugefeQAyxdb2HV1Fg==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
|
|
|
|||
117
infra/deploy/secrets/verify-age-quorum.sh
Executable file
117
infra/deploy/secrets/verify-age-quorum.sh
Executable file
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env bash
|
||||
# Assert that the age recovery quorum is intact.
|
||||
#
|
||||
# infra/deploy/secrets/verify-age-quorum.sh
|
||||
#
|
||||
# The age private key is the single recovery root for all five SOPS vaults AND for
|
||||
# every off-box backup (ops-cron.yml:142,197 pipe dumps through `age -r` to the same
|
||||
# recipient). The copies at /etc/thermograph/age.key on vps1 and vps2 are a DELIBERATE
|
||||
# two-host quorum, not provisioning residue — see the "age key" section of
|
||||
# deploy/secrets/README.md. This script is what makes that policy checkable instead of
|
||||
# merely stated.
|
||||
#
|
||||
# It fails if the key is missing on either host, if the two copies have diverged, or
|
||||
# if either is more permissive than 0440. Divergence matters as much as absence: two
|
||||
# hosts holding different keys means one of them renders secrets nobody else can
|
||||
# decrypt, and a restore from backup silently picks the wrong one.
|
||||
#
|
||||
# OUTPUT DISCIPLINE: prints SHA-256 digests and file modes only. A digest of a
|
||||
# 32-byte random key is not reversible, so it is safe in a CI log; the key itself is
|
||||
# never read, echoed or transferred. Nothing here copies the key anywhere.
|
||||
#
|
||||
# Exit status: 0 = quorum intact; 1 = something needs a human. Suitable as a cron gate.
|
||||
set -euo pipefail
|
||||
|
||||
VPS1="${TG_VPS1_SSH:-vps1}"
|
||||
VPS2="${TG_VPS2_SSH:-vps2}"
|
||||
KEY_PATH="${TG_AGE_KEY_PATH:-/etc/thermograph/age.key}"
|
||||
SSH_OPTS="${TG_SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=10}"
|
||||
|
||||
# Reads the digest and mode of the key on one host. Uses sudo when the key is not
|
||||
# directly readable, mirroring how render-secrets.sh lifts it.
|
||||
probe() {
|
||||
local target="$1" out
|
||||
# The remote script is a QUOTED heredoc and the key path is passed as $1 to
|
||||
# `bash -s`, so nothing expands on this side. Interpolating the path into the
|
||||
# command string would work too, but it is the shape that quietly breaks the day
|
||||
# a path contains a space, and shellcheck is right to flag it (SC2029).
|
||||
# shellcheck disable=SC2086 # SSH_OPTS is a deliberate word-split option list
|
||||
out=$(ssh $SSH_OPTS "$target" bash -s -- "$KEY_PATH" 2>/dev/null <<'REMOTE'
|
||||
key="$1"
|
||||
if [ ! -e "$key" ]; then echo MISSING; exit 0; fi
|
||||
mode=$(stat -c %a "$key")
|
||||
owner=$(stat -c %U:%G "$key")
|
||||
if [ -r "$key" ]; then
|
||||
digest=$(sha256sum "$key" | cut -d' ' -f1)
|
||||
else
|
||||
digest=$(sudo sha256sum "$key" 2>/dev/null | cut -d' ' -f1)
|
||||
fi
|
||||
[ -n "$digest" ] || { echo UNREADABLE; exit 0; }
|
||||
echo "$digest $mode $owner"
|
||||
REMOTE
|
||||
) || { echo UNREACHABLE; return 0; }
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
rc=0
|
||||
declare -A DIGEST
|
||||
|
||||
for pair in "vps1:$VPS1" "vps2:$VPS2"; do
|
||||
name="${pair%%:*}"
|
||||
target="${pair#*:}"
|
||||
result="$(probe "$target")"
|
||||
case "$result" in
|
||||
MISSING)
|
||||
echo "!! ${name}: no key at ${KEY_PATH} — the quorum is DOWN TO ONE COPY" >&2
|
||||
rc=1
|
||||
;;
|
||||
UNREADABLE)
|
||||
echo "!! ${name}: key present but unreadable even via sudo" >&2
|
||||
rc=1
|
||||
;;
|
||||
UNREACHABLE)
|
||||
echo "!! ${name}: host unreachable — quorum NOT verified (this is not a pass)" >&2
|
||||
rc=1
|
||||
;;
|
||||
*)
|
||||
digest="${result%% *}"
|
||||
rest="${result#* }"
|
||||
DIGEST["$name"]="$digest"
|
||||
printf ' %-5s %s mode=%s\n' "$name" "${digest:0:16}…" "$rest"
|
||||
mode="${rest%% *}"
|
||||
# 0400 or 0440 only. Anything wider means a non-root account on that box can
|
||||
# read the key that decrypts prod — and vps1 also runs Forgejo and its CI runner.
|
||||
case "$mode" in
|
||||
400|440) ;;
|
||||
*) echo "!! ${name}: mode ${mode} is wider than 0440" >&2; rc=1 ;;
|
||||
esac
|
||||
# Advisory, not a failure: 0440 with a non-root group means that group can read
|
||||
# the key that decrypts prod. On vps1 that is doubly worth knowing, because the
|
||||
# same box runs Forgejo, its CI runner and dev's unreviewed branches. Left
|
||||
# non-fatal because it is the current provisioned state — a check that fails on
|
||||
# day one is a check that gets ignored by day three. See README.md.
|
||||
group="${rest#* }"; group="${group#*:}"
|
||||
if [ "$mode" = 440 ] && [ "$group" != root ]; then
|
||||
printf ' note: group %s can read this key\n' "$group"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -n "${DIGEST[vps1]:-}" ] && [ -n "${DIGEST[vps2]:-}" ]; then
|
||||
if [ "${DIGEST[vps1]}" = "${DIGEST[vps2]}" ]; then
|
||||
echo " quorum: 2 copies, identical"
|
||||
else
|
||||
echo "!! the two copies have DIVERGED — they are different keys" >&2
|
||||
echo "!! do not 'fix' this by overwriting one. Establish which decrypts the" >&2
|
||||
echo "!! vaults (sops -d on any deploy/secrets/*.yaml) before touching either." >&2
|
||||
rc=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
echo " OK — recovery quorum intact"
|
||||
else
|
||||
echo "!! recovery quorum degraded; see deploy/secrets/README.md" >&2
|
||||
fi
|
||||
exit "$rc"
|
||||
|
|
@ -124,8 +124,8 @@ case "$SERVICE" in
|
|||
esac
|
||||
export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
|
||||
export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
|
||||
BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:$BACKEND_IMAGE_TAG"
|
||||
FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:$FRONTEND_IMAGE_TAG"
|
||||
BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:$BACKEND_IMAGE_TAG"
|
||||
FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:$FRONTEND_IMAGE_TAG"
|
||||
|
||||
# --- timescale image pin ---------------------------------------------------------
|
||||
# Hazard #7: the db image under an existing volume must never drift. Resolve
|
||||
|
|
@ -158,7 +158,7 @@ fi
|
|||
|
||||
# --- registry ------------------------------------------------------------------
|
||||
if [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin
|
||||
fi
|
||||
echo "==> Pulling images"
|
||||
pull_ok=0
|
||||
|
|
@ -290,8 +290,8 @@ FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG
|
|||
EOF
|
||||
|
||||
# GC superseded app-image tags (keep the running pair), same as deploy.sh.
|
||||
_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}"
|
||||
_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}"
|
||||
_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}"
|
||||
_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}"
|
||||
docker images --format '{{.Repository}}:{{.Tag}}' \
|
||||
| grep -E "^(${_be_repo}|${_fe_repo}):" \
|
||||
| grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@
|
|||
|
||||
services:
|
||||
beta-web:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
# Beta's OWN role and OWN database on the shared instance. `db` resolves
|
||||
|
|
@ -108,7 +108,7 @@ services:
|
|||
failure_action: rollback
|
||||
|
||||
beta-worker:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta
|
||||
|
|
@ -144,7 +144,7 @@ services:
|
|||
condition: on-failure
|
||||
|
||||
beta-lake:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
THERMOGRAPH_ROLE: lake
|
||||
|
|
@ -176,7 +176,7 @@ services:
|
|||
failure_action: rollback
|
||||
|
||||
beta-daemon:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
# Same reasoning as prod's daemon: NOT env-entrypoint.sh, because that shim
|
||||
# execs the image's own entrypoint (Alembic + uvicorn) and migrations belong
|
||||
# to the one-shot task. Source the host-rendered env and exec the binary.
|
||||
|
|
@ -218,7 +218,7 @@ services:
|
|||
failure_action: rollback
|
||||
|
||||
beta-frontend:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
# REQUIRED: overriding `entrypoint:` with no `command:` drops the image's
|
||||
# CMD entirely, and env-entrypoint.sh's fallback (`exec uvicorn app:app`)
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ services:
|
|||
condition: on-failure
|
||||
|
||||
web:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
|
||||
|
|
@ -109,7 +109,7 @@ services:
|
|||
failure_action: rollback
|
||||
|
||||
worker:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph
|
||||
|
|
@ -153,7 +153,7 @@ services:
|
|||
# healthy and answers 503/404, and web falls through to NASA — the lake is
|
||||
# an accelerator, never a point of failure.
|
||||
lake:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
environment:
|
||||
THERMOGRAPH_ROLE: lake
|
||||
|
|
@ -186,7 +186,7 @@ services:
|
|||
# (/usr/local/bin/thermograph-daemon, built into the backend image) and the
|
||||
# app share the /internal/* API contract, so one BACKEND_IMAGE_TAG rolls
|
||||
# them together and they can never skew.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
|
||||
# Not env-entrypoint.sh: that shim's handoff execs the image's
|
||||
# deploy/entrypoint.sh (Alembic + uvicorn) whenever it exists, which is
|
||||
# exactly what this service must never run — migrations belong to the
|
||||
|
|
@ -243,7 +243,7 @@ services:
|
|||
failure_action: rollback
|
||||
|
||||
frontend:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required}
|
||||
entrypoint: ["/host/env-entrypoint.sh"]
|
||||
# REQUIRED, not cosmetic: overriding `entrypoint:` with no `command:` drops
|
||||
# the image's own CMD entirely (Docker/Swarm semantics, not merged) --
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
# build-push.yml -- backend = ${BACKEND_IMAGE_PATH}, frontend =
|
||||
# ${FRONTEND_IMAGE_PATH}, tagged independently by BACKEND_IMAGE_TAG /
|
||||
# FRONTEND_IMAGE_TAG. This replaces the earlier Stage-4 model where both
|
||||
# containers shared the single emi/thermograph/app image and
|
||||
# containers shared the single admin_emi/thermograph/app image and
|
||||
# THERMOGRAPH_SERVICE_ROLE picked the process; the split Dockerfiles now start
|
||||
# the right process directly (frontend the thermograph-frontend Go binary,
|
||||
# backend entrypoint.sh), so a backend deploy and a frontend deploy are fully
|
||||
|
|
@ -99,10 +99,10 @@ services:
|
|||
# No `build:` here -- infra holds no Dockerfile; each service's Dockerfile
|
||||
# lives in its own app repo. deploy.sh sets BACKEND_IMAGE_TAG to the SHA
|
||||
# build-push.yml pushed for the backend commit being deployed; local dev
|
||||
# builds `emi/thermograph/backend:local` from the backend repo (see
|
||||
# builds `jinemi/thermograph/backend:local` from the backend repo (see
|
||||
# infra Makefile) and this pulls/uses it. BACKEND_IMAGE_PATH/TAG are
|
||||
# independent of the frontend's, so the two services deploy separately.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
|
|
@ -180,7 +180,7 @@ services:
|
|||
# without them the service stays healthy and every read falls through to
|
||||
# NASA. Never published on a host port: only the backend talks to it.
|
||||
lake:
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
environment:
|
||||
THERMOGRAPH_ROLE: lake
|
||||
PORT: 8141
|
||||
|
|
@ -207,7 +207,7 @@ services:
|
|||
# share the /internal/* API contract, so they must roll together — a
|
||||
# separate tag could skew them. It owns the Discord gateway websocket and
|
||||
# the recurring-job timers; anything needing data calls back into backend.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
|
||||
# Bypass deploy/entrypoint.sh entirely: that script runs Alembic, and the
|
||||
# backend service's boot already owns migrations — two containers racing
|
||||
# `alembic upgrade head` against one database is a real hazard, not a
|
||||
|
|
@ -262,7 +262,7 @@ services:
|
|||
# frontend/Dockerfile (which starts the thermograph-frontend Go binary
|
||||
# directly -- no THERMOGRAPH_SERVICE_ROLE process-picking). Independent
|
||||
# FRONTEND_IMAGE_TAG so a frontend deploy never disturbs the backend's tag.
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:-local}
|
||||
image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:-local}
|
||||
# Its own register() fetches the IndexNow key from backend at boot -- must
|
||||
# wait for a real, healthy backend, not just a started container.
|
||||
depends_on:
|
||||
|
|
|
|||
|
|
@ -112,6 +112,22 @@ thermograph/data/legacy/age the age PRIVATE key — see "the age key surviv
|
|||
Inheritance lives in the render order (`common` then `env/<name>`, last-wins);
|
||||
isolation lives in the policy. Today both live in one shell variable.
|
||||
|
||||
**`centralis/<env>` renders differently from everything above it, and must.** The
|
||||
app stack's `/etc/thermograph.env` is raw unquoted `KEY=value`; `/etc/centralis.env`
|
||||
is consumed by `set -a && . /etc/centralis.env`, i.e. bash's full expansion
|
||||
pipeline, so it is emitted as POSIX single-quoted assignments and then verified by
|
||||
sourcing it in a clean shell and comparing every value back before the file is
|
||||
written. `CENTRALIS_TOKENS` is JSON and cannot survive the unquoted form: bash
|
||||
strips its quotes, Centralis fails closed on the malformed result and serves a
|
||||
single `shared` identity, which is indistinguishable from the file never having
|
||||
been written. That is the 2026-07-24 incident, and the round-trip check exists so a
|
||||
render carrying it cannot reach `/etc`.
|
||||
|
||||
Consequence for seeding: `seed-from-sops.sh` applies the raw-dotenv rules (no shell
|
||||
metacharacters, no newlines) only to `common` and `env/*`. Applying them to
|
||||
`centralis/*` would reject data that renders perfectly well — the validator matches
|
||||
the renderer that will consume the path, not a single global rule.
|
||||
|
||||
`common` is a top-level path rather than `env/common` on purpose: it makes the dev deny
|
||||
rule expressible as exactly one path, with no wildcard over `env/*` able to
|
||||
accidentally re-include it.
|
||||
|
|
@ -148,22 +164,59 @@ SOPS and retiring age are two different projects.
|
|||
### Stand it up (once, by the operator, on vps2)
|
||||
|
||||
```sh
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, unit, init
|
||||
sudo bash infra/openbao/bootstrap.sh # binary, TLS, seal key, ufw, unit, init
|
||||
# custody the recovery keys + /etc/openbao/unseal.key OFF the box, then:
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/thermograph/openbao-ca.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
sudo -E bash infra/openbao/bootstrap-policies.sh # mount, policies, approles
|
||||
bao token revoke -self
|
||||
```
|
||||
|
||||
### Seed and prove (from your own machine — it needs your age key)
|
||||
> **Do not revoke the root token here.** OpenBao 2.6.0 replaced the unauthenticated
|
||||
> `/sys/generate-root` with an authenticated `/sys/generate-root-token`
|
||||
> (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
> minting a root token requires a token you already have. **The recovery keys are not
|
||||
> a way back in**; they rekey, they do not authenticate. Unless
|
||||
> `disable_unauthed_generate_root_endpoints = false` is set in `config.hcl`, revoking
|
||||
> the last root token locks you out of an otherwise healthy, unsealed vault, and the
|
||||
> only remedy is re-initialising from scratch. Revoke only once a second admin
|
||||
> identity exists.
|
||||
|
||||
Use `/etc/thermograph/openbao-ca.crt` (mode `0444`) rather than
|
||||
`/etc/openbao/tls/bao.crt` — same certificate, but `/etc/openbao/tls/` is `0700
|
||||
openbao`, so only root can read the latter. Without `BAO_CACERT` every command fails
|
||||
with `certificate signed by unknown authority`.
|
||||
|
||||
### Seed and prove (on vps2 — the age key is already there)
|
||||
|
||||
```sh
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # key names + counts, writes nothing
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # MUST pass before any flip
|
||||
```
|
||||
|
||||
Then prove parity — **not `--all` from one box.** Each AppRole is `secret_id_bound_cidrs`
|
||||
to the host that legitimately renders it, so an environment can only be verified from
|
||||
its own host:
|
||||
|
||||
```sh
|
||||
# on vps2
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
```
|
||||
|
||||
Beta needs no special handling: `verify-parity.sh` sources `env-topology.sh` and calls
|
||||
`thermograph_topology` per environment, so `TG_BAO_APPROLE` points at
|
||||
`/etc/thermograph/openbao-approle-beta` and beta authenticates as `tg-beta` rather than
|
||||
falling back to prod's credentials and being denied its own path by `tg-host-prod.hcl`.
|
||||
|
||||
That sourcing is load-bearing and was missing until 2026-08-01: this document asserted
|
||||
the property while the verifier alone did not have it, so `--env beta` and `--all` both
|
||||
took a 403 and the gate could not check the one environment whose isolation it exists
|
||||
to prove. If you refactor `verify-parity.sh`, the `thermograph_topology` call is not
|
||||
boilerplate.
|
||||
|
||||
`seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md`
|
||||
the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this
|
||||
inherits that rule.
|
||||
|
|
@ -178,10 +231,27 @@ One PR per hop, following the estate's own promotion model.
|
|||
+ TG_SECRETS_BACKEND=openbao
|
||||
```
|
||||
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended
|
||||
gate before prod: **7 consecutive green parity runs on all three environments**, run
|
||||
nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting
|
||||
CI any vault access).
|
||||
Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. The gate before
|
||||
prod is **7 consecutive green parity runs on all three environments**. That runs
|
||||
nightly from the `secrets-parity` job in `ops-cron.yml` over SSH, which gives
|
||||
continuous evidence without granting CI any vault access — the host renders, CI only
|
||||
asks it to. Forgejo's run history for that job is the record; a night the job was
|
||||
skipped, or the runner was down, does not count as green.
|
||||
|
||||
Measured 2026-08-01, immediately after the verifier was fixed:
|
||||
|
||||
| env | keys | result |
|
||||
|---|---|---|
|
||||
| dev | 12 | **PASS** — byte-identical |
|
||||
| beta | 24 | FAIL — 3 values differ |
|
||||
| prod | 32 | FAIL — 3 values differ |
|
||||
|
||||
Identical three keys on both: `THERMOGRAPH_S3_SECRET_KEY`,
|
||||
`THERMOGRAPH_LAKE_S3_SECRET_KEY`, `THERMOGRAPH_VAPID_CONTACT`. All three live in
|
||||
`common.yaml`, which was seeded on 2026-07-31 at 02:30Z — *before* the Contabo
|
||||
rotation landed. dev passes because dev never layers `common`. One
|
||||
`seed-from-sops.sh --env common` fixes beta and prod together; the 7-day clock starts
|
||||
after that, not before.
|
||||
|
||||
`TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the
|
||||
SOPS path for a full release cycle after prod flips — it is the best mitigation
|
||||
|
|
|
|||
|
|
@ -63,15 +63,21 @@ add_role tg-centralis tg-centralis 10.10.0.1/32
|
|||
|
||||
# Install the local (vps2) credentials. prod and beta both render on this box.
|
||||
#
|
||||
# The OWNERSHIP here is a real boundary that does not exist today, and it is the one
|
||||
# concrete isolation win available on a shared host. render-secrets.sh:119-124 records
|
||||
# that beta's CI deploy user is `deploy` while prod's is `agent`. Today both reach the
|
||||
# SAME root-owned age key via `sudo cat`, so there is no deploy-user-level separation
|
||||
# at all. Giving each environment its own 0400 secret-id owned by its own deploy user
|
||||
# creates one.
|
||||
# Both files are owned by `agent`, because that is the single identity both
|
||||
# environments actually deploy as: env-topology.sh sets TG_SSH_TARGET=agent@... for
|
||||
# beta and for prod, deploy.yml uses one VPS2_SSH_USER for both, and vps2 has no
|
||||
# `deploy` user at all (only `ubuntu` and `agent`). An earlier revision chowned beta's
|
||||
# file to `deploy:deploy` on the theory that beta and prod deploy as different users;
|
||||
# they do not, and the chown simply failed.
|
||||
#
|
||||
# Be honest about the limit: root on vps2 reads both files, exactly as root today reads
|
||||
# the one age key. This defends against MISCONFIGURATION, not against intrusion.
|
||||
# So be accurate about what separate credentials buy on this box: NOT deploy-user
|
||||
# isolation — there is one deploy user, and it can read both files. What they buy is
|
||||
# per-environment ATTRIBUTION in the audit log, and a policy boundary that stops a
|
||||
# *mistake* crossing between environments: a beta render authenticating with beta's
|
||||
# secret-id is refused thermograph/data/env/prod by tg-host-beta.hcl, so a wrong
|
||||
# THERMOGRAPH_ENV fails closed instead of quietly rendering the other environment's
|
||||
# database password. Real deploy-user isolation would need a `deploy` user on vps2 and
|
||||
# a separate SSH credential for beta in deploy.yml — a larger change than this script.
|
||||
install_creds() {
|
||||
local role="$1" dest="$2" owner="$3"
|
||||
local rid sid
|
||||
|
|
@ -79,7 +85,17 @@ install_creds() {
|
|||
sid=$(bao write -f -field=secret_id "auth/approle/role/${role}/secret-id")
|
||||
# umask before creation, so the file is never briefly world-readable.
|
||||
( umask 077; printf '%s\n%s\n' "$rid" "$sid" > "$dest" )
|
||||
chown "$owner" "$dest"
|
||||
# Abort the function if chown fails, and remove the half-installed file. The call
|
||||
# site wraps this in `|| echo`, which suppresses `set -e` for everything inside —
|
||||
# so without this check a failed chown falls straight through to chmod and prints a
|
||||
# line asserting an ownership that was never applied. A root-owned credential the
|
||||
# renderer cannot read, reported as installed, is worse than no credential at all:
|
||||
# it fails at deploy time instead of here.
|
||||
if ! chown "$owner" "$dest"; then
|
||||
rm -f "$dest"
|
||||
echo " !! chown $owner failed for $dest — removed, nothing installed" >&2
|
||||
return 1
|
||||
fi
|
||||
chmod 0400 "$dest"
|
||||
echo " installed $dest (0400 $owner)"
|
||||
}
|
||||
|
|
@ -87,8 +103,7 @@ install_creds() {
|
|||
if [ "$(id -u)" = 0 ]; then
|
||||
install -d -o root -g root -m 0755 /etc/thermograph
|
||||
install_creds tg-prod /etc/thermograph/openbao-approle "agent:agent"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "deploy:deploy" \
|
||||
|| echo " (beta creds skipped — no 'deploy' user on this box?)"
|
||||
install_creds tg-beta /etc/thermograph/openbao-approle-beta "agent:agent"
|
||||
install_creds tg-centralis /etc/thermograph/openbao-approle-centralis "root:root"
|
||||
else
|
||||
echo "!! not root; skipping credential install. Re-run with sudo -E." >&2
|
||||
|
|
|
|||
|
|
@ -52,13 +52,19 @@ if ! command -v bao >/dev/null 2>&1; then
|
|||
# The checksums file is signed by the release; verify before installing.
|
||||
tmpd="$(mktemp -d)"
|
||||
base="https://github.com/openbao/openbao/releases/download/v${BAO_VERSION}"
|
||||
curl -fsSL -o "$tmpd/bao.tar.gz" "${base}/bao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/bao_${BAO_VERSION}_SHA256SUMS"
|
||||
( cd "$tmpd" && grep "linux_amd64.tar.gz" checksums | sha256sum -c - ) || {
|
||||
# Asset names are `openbao_<ver>_...`, and the checksums file is `checksums.txt`.
|
||||
# The previous `bao_<ver>_...` / `bao_<ver>_SHA256SUMS` names both 404.
|
||||
tarball="openbao_${BAO_VERSION}_linux_amd64.tar.gz"
|
||||
curl -fsSL -o "$tmpd/$tarball" "${base}/${tarball}"
|
||||
curl -fsSL -o "$tmpd/checksums" "${base}/checksums.txt"
|
||||
# Anchor to end-of-line and save under the real asset name: checksums.txt also lists
|
||||
# <tarball>.sbom.json, and `sha256sum -c` resolves each line by the filename IN it,
|
||||
# so an unanchored match fails on a file that was never fetched.
|
||||
( cd "$tmpd" && grep " ${tarball}\$" checksums | sha256sum -c - ) || {
|
||||
echo "!! checksum mismatch on the bao tarball; refusing to install" >&2
|
||||
rm -rf "$tmpd"; exit 1
|
||||
}
|
||||
tar -xzf "$tmpd/bao.tar.gz" -C "$tmpd" bao
|
||||
tar -xzf "$tmpd/$tarball" -C "$tmpd" bao
|
||||
install -m 0755 -o root -g root "$tmpd/bao" /usr/local/bin/bao
|
||||
rm -rf "$tmpd"
|
||||
fi
|
||||
|
|
@ -88,7 +94,11 @@ install -m 0444 /etc/openbao/tls/bao.crt /etc/thermograph/openbao-ca.crt
|
|||
echo "==> 3/7 static seal key"
|
||||
if [ ! -f /etc/openbao/unseal.key ]; then
|
||||
# 32 random bytes, base64. Same protection level as /etc/thermograph/age.key.
|
||||
( umask 077; openssl rand -base64 32 > /etc/openbao/unseal.key )
|
||||
# tr -d '\n' is load-bearing: `openssl rand -base64` appends a newline, and the
|
||||
# static seal reads this file RAW. That one trailing byte makes the key neither
|
||||
# 32 raw bytes nor clean base64, so bao refuses to start with
|
||||
# `Error configuring seal "static": unknown encoding for AES-256 key`.
|
||||
( umask 077; openssl rand -base64 32 | tr -d '\n' > /etc/openbao/unseal.key )
|
||||
chown openbao:openbao /etc/openbao/unseal.key
|
||||
chmod 0400 /etc/openbao/unseal.key
|
||||
echo " generated /etc/openbao/unseal.key (0400 openbao)"
|
||||
|
|
@ -125,6 +135,24 @@ systemctl is-active --quiet openbao.service || {
|
|||
exit 1
|
||||
}
|
||||
|
||||
# Mesh reachability for vps1. dev renders on vps1 and must reach this listener, but
|
||||
# ufw defaults to deny(incoming) and nothing else in this tree opens the port. Without
|
||||
# it, dev's render fails at cutover with a connection timeout that reads as an OpenBao
|
||||
# fault rather than a firewall one. Scoped to vps1's mesh address specifically: the
|
||||
# AppRole CIDR bindings are the real control, and nothing else on the mesh — the
|
||||
# desktop, the phone — has any business reaching the secret store.
|
||||
if command -v ufw >/dev/null 2>&1; then
|
||||
if ufw status | grep -q '8200.*10\.10\.0\.2'; then
|
||||
echo " ufw: 8200/tcp from 10.10.0.2 already allowed"
|
||||
else
|
||||
ufw allow in on wg0 from 10.10.0.2 to any port 8200 proto tcp \
|
||||
comment 'vps1/dev -> openbao' >/dev/null
|
||||
echo " ufw: allowed 8200/tcp on wg0 from 10.10.0.2 (vps1)"
|
||||
fi
|
||||
else
|
||||
echo " !! ufw absent -- confirm 8200/tcp is reachable from 10.10.0.2 (vps1)" >&2
|
||||
fi
|
||||
|
||||
export BAO_ADDR="https://127.0.0.1:8200"
|
||||
export BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
|
||||
|
|
@ -140,9 +168,16 @@ else
|
|||
echo " ############################################################"
|
||||
echo
|
||||
# -recovery-shares/-threshold, not -key-shares: with an auto-unseal seal, init
|
||||
# yields RECOVERY keys (which authorise rekey and generate-root) rather than unseal
|
||||
# keys. 2-of-3 here is redundancy against loss, not separation of duty — there is one
|
||||
# operator. Do NOT use -recovery-shares=0: that leaves no route to a new root token.
|
||||
# yields RECOVERY keys (which authorise rekey) rather than unseal keys. 2-of-3 here
|
||||
# is redundancy against loss, not separation of duty — there is one operator.
|
||||
#
|
||||
# These keys are NOT a route back to a root token on 2.6.x. OpenBao 2.6.0 replaced
|
||||
# the unauthenticated /sys/generate-root with an authenticated /sys/generate-root-token
|
||||
# (HCSEC-2026-08), and `bao operator generate-root` now targets the new one — so
|
||||
# minting a root token requires a token you already have. The deprecated endpoint is
|
||||
# off unless `disable_unauthed_generate_root_endpoints = false` is set in config.hcl.
|
||||
# Consequence: revoking the last root token before another admin identity exists is
|
||||
# a one-way door. See the ordering note in the handoff below.
|
||||
bao operator init -recovery-shares=3 -recovery-threshold=2
|
||||
echo
|
||||
echo " Press Enter once the recovery keys AND /etc/openbao/unseal.key are"
|
||||
|
|
@ -156,22 +191,36 @@ cat <<EOF
|
|||
|
||||
==> 6/7 and 7/7 need a root token, which is NOT stored anywhere by design.
|
||||
|
||||
Paste the initial root token (or one minted from recovery keys) and run:
|
||||
Paste the initial root token printed above and run:
|
||||
|
||||
export BAO_ADDR=https://127.0.0.1:8200 BAO_CACERT=/etc/openbao/tls/bao.crt
|
||||
export BAO_TOKEN=<root token>
|
||||
bash ${SELF_DIR}/bootstrap-policies.sh
|
||||
|
||||
KEEP THAT TOKEN until the whole sequence below is done. On 2.6.x the recovery keys
|
||||
cannot mint a replacement (see the note above ${0##*/}'s init step), so a premature
|
||||
\`bao token revoke -self\` locks you out of an otherwise healthy vault.
|
||||
|
||||
That script enables the KV mount, writes the policies, creates the AppRoles and
|
||||
installs each host's credentials. Then, from YOUR OWN machine (it needs your age
|
||||
key, and per infra/CLAUDE.md a script that reads production secrets is not for an
|
||||
agent to run):
|
||||
installs each host's credentials. Then seed — on THIS box, where the age key already
|
||||
lives at /etc/thermograph/age.key. Per infra/CLAUDE.md a script that reads production
|
||||
secrets is not for an agent to run:
|
||||
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first
|
||||
export SOPS_AGE_KEY_FILE=/etc/thermograph/age.key
|
||||
infra/openbao/seed-from-sops.sh --dry-run # check first: 16/12/8/16 keys
|
||||
infra/openbao/seed-from-sops.sh --all
|
||||
infra/openbao/verify-parity.sh --all # must PASS before any flip
|
||||
|
||||
Then revoke the root token: bao token revoke -self
|
||||
Then prove parity. NOT \`--all\` from one box: each AppRole is CIDR-bound to the host
|
||||
that legitimately renders it, so an environment can only be verified from its own host.
|
||||
|
||||
# here (vps2)
|
||||
infra/openbao/verify-parity.sh --env prod # expect 32 keys
|
||||
infra/openbao/verify-parity.sh --env beta # expect 24 keys
|
||||
# on vps1
|
||||
cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys
|
||||
|
||||
Only once all three PASS, and only after you have a second admin identity that is not
|
||||
the root token, revoke it: bao token revoke -self
|
||||
|
||||
NOTHING is cut over by any of the above. TG_SECRETS_BACKEND defaults to sops for
|
||||
every environment in infra/deploy/env-topology.sh, so SOPS stays authoritative
|
||||
|
|
|
|||
|
|
@ -25,8 +25,12 @@
|
|||
|
||||
ui = false
|
||||
cluster_name = "thermograph"
|
||||
disable_mlock = true
|
||||
log_level = "info"
|
||||
// `disable_mlock` is deliberately absent: OpenBao 2.6.1 does not recognise it and
|
||||
// logs `unknown or unsupported field disable_mlock` on every start. Carrying a
|
||||
// setting the server ignores only trains the operator to skim startup warnings,
|
||||
// which is where a real one will eventually hide. The corresponding note about
|
||||
// AmbientCapabilities=CAP_IPC_LOCK in openbao.service is moot for the same reason.
|
||||
|
||||
// STORAGE: raft, and this is now forced rather than chosen. OpenBao 2.6.0
|
||||
// DEPRECATED the `file` backend for removal in v2.7.0, so picking `file` today buys
|
||||
|
|
@ -110,9 +114,21 @@ seal "static" {
|
|||
//
|
||||
// logrotate on this path MUST signal with SIGHUP, or bao keeps writing to an
|
||||
// unlinked inode and the log silently stops growing.
|
||||
// OpenBao 2.6.1 requires `type` and `path` explicitly — the block label is the
|
||||
// device NAME, not its type — and device settings go in `options`. Written as
|
||||
// `audit "file" { file_path = ... }` the server refuses to start ("audit type must
|
||||
// be specified"); with type+path but a bare file_path it starts but silently
|
||||
// IGNORES the path and mode, which is the worse failure of the two.
|
||||
audit "file" {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
type = "file"
|
||||
path = "file/"
|
||||
options = {
|
||||
file_path = "/var/log/openbao/audit.log"
|
||||
mode = "0600"
|
||||
}
|
||||
}
|
||||
|
||||
audit "syslog" {}
|
||||
audit "syslog" {
|
||||
type = "syslog"
|
||||
path = "syslog/"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -119,7 +119,17 @@ for target in "${TARGETS[@]}"; do
|
|||
# Validate and reshape into the flat string map KV v2 wants. Refuses the same unsafe
|
||||
# values render-secrets-openbao.sh refuses, so an unrenderable value is caught at
|
||||
# SEED time — before anything depends on it — rather than at deploy time.
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" python3 - <<'PY'
|
||||
# Which renderer will consume this path decides which values are legal, so the
|
||||
# validation has to match it. The app stack's /etc/thermograph.env is raw, UNQUOTED
|
||||
# KEY=value (render-secrets-openbao.sh:14-18 freezes that shape), so a metacharacter
|
||||
# there is a live expansion hazard. /etc/centralis.env is POSIX single-quoted by
|
||||
# render_centralis_secrets and its OpenBao counterpart, where every byte inside
|
||||
# '...' is literal — so applying the dotenv rules to it rejects data that renders
|
||||
# perfectly well. CENTRALIS_TOKENS is JSON and can never satisfy them.
|
||||
quoted=0
|
||||
case "$target" in centralis.*) quoted=1 ;; esac
|
||||
|
||||
reshaped="$(THERMOGRAPH_SEED_JSON="$plain_json" THERMOGRAPH_SEED_QUOTED="$quoted" python3 - <<'PY'
|
||||
import json, os, re, sys
|
||||
|
||||
doc = json.loads(os.environ["THERMOGRAPH_SEED_JSON"])
|
||||
|
|
@ -127,6 +137,9 @@ if not isinstance(doc, dict) or not doc:
|
|||
sys.stderr.write("!! vault did not decrypt to a non-empty object\n")
|
||||
sys.exit(1)
|
||||
|
||||
# 1 => destined for a single-quoted env file; 0 => raw dotenv.
|
||||
QUOTED = os.environ.get("THERMOGRAPH_SEED_QUOTED") == "1"
|
||||
|
||||
KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
UNSAFE = set("`$\\\"'")
|
||||
|
||||
|
|
@ -141,12 +154,13 @@ for key, val in doc.items():
|
|||
val = ""
|
||||
if not isinstance(val, str):
|
||||
val = json.dumps(val, separators=(",", ":"))
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
if not QUOTED:
|
||||
if "\n" in val or "\r" in val:
|
||||
bad.append(f"{key}: contains a newline")
|
||||
continue
|
||||
if UNSAFE & set(val):
|
||||
bad.append(f"{key}: contains a shell metacharacter (` $ \\ \" ')")
|
||||
continue
|
||||
out[key] = val
|
||||
|
||||
if bad:
|
||||
|
|
|
|||
|
|
@ -36,8 +36,13 @@ usage: verify-parity.sh (--all | --env <dev|beta|prod> ...)
|
|||
Renders each environment through BOTH backends and compares the resulting
|
||||
KEY=value sets. Prints key names and counts only, never values.
|
||||
|
||||
Exit status: 0 = every requested environment is at parity; 1 = a mismatch.
|
||||
Suitable as a CI / cron gate.
|
||||
--all means "every environment that lives on THIS host", not all three: dev is
|
||||
on vps1, beta and prod are on vps2, so no single box can check them all. An
|
||||
environment with no checkout here is skipped out loud. Skipping every one of
|
||||
them is an error, not a pass.
|
||||
|
||||
Exit status: 0 = every environment checked here is at parity; 1 = a mismatch,
|
||||
or nothing was checkable. Suitable as a CI / cron gate.
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
|
@ -52,20 +57,67 @@ while [ $# -gt 0 ]; do
|
|||
done
|
||||
[ ${#TARGETS[@]} -gt 0 ] || usage
|
||||
|
||||
# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets
|
||||
# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the
|
||||
# bare default — prod's credential — for every environment. On vps2 that made
|
||||
# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on
|
||||
# thermograph/data/env/beta, so the gate could not verify the one environment whose
|
||||
# isolation it exists to prove. `--all` failed the same way, and dev with it.
|
||||
#
|
||||
# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not,
|
||||
# which is the worst place for the omission to be: the tool whose whole job is to
|
||||
# notice a divergence was itself diverging from the path it verifies.
|
||||
#
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/env-topology.sh"
|
||||
# shellcheck source=/dev/null
|
||||
. "$INFRA_DIR/deploy/render-secrets-openbao.sh"
|
||||
|
||||
overall=0
|
||||
checked=0
|
||||
for env_name in "${TARGETS[@]}"; do
|
||||
# Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a
|
||||
# filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header
|
||||
# line because --all walks three environments and may skip some of them.
|
||||
if ! thermograph_topology "$env_name"; then
|
||||
echo "!! unknown environment: $env_name" >&2
|
||||
overall=1; continue
|
||||
fi
|
||||
|
||||
# NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to
|
||||
# pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all
|
||||
# anywhere therefore guaranteed a failure on whichever environments are elsewhere,
|
||||
# and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to
|
||||
# /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so
|
||||
# prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like
|
||||
# a policy bug rather than "wrong box".
|
||||
#
|
||||
# Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its
|
||||
# checkout existing. The first version of this check tested for TG_APP_DIR and was
|
||||
# wrong: vps1 still carries a stale /opt/thermograph from before the estate split,
|
||||
# so prod looked resident there and was checked anyway. A leftover directory is
|
||||
# exactly the kind of thing that outlives the arrangement it belonged to.
|
||||
#
|
||||
# Skipping is announced, never silent. A gate that quietly narrows what it checked
|
||||
# is worse than one that fails, because the run still goes green.
|
||||
if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then
|
||||
if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
|
||||
| grep -qxF "$TG_SSH_HOST"; then
|
||||
echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "== parity check: $env_name"
|
||||
checked=$((checked + 1))
|
||||
|
||||
# dev renders WITHOUT common on both backends. On the SOPS side that is a caller
|
||||
# flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps
|
||||
# the comparison apples-to-apples — and if the flag were wrong, the OpenBao side
|
||||
# would fail closed with a 403 rather than quietly diverge, which is exactly the
|
||||
# improvement being verified.
|
||||
skip_common=0
|
||||
[ "$env_name" = dev ] && skip_common=1
|
||||
# flag; on the OpenBao side the policy also forbids it. Taking the flag from the
|
||||
# topology rather than re-deriving it here keeps the comparison apples-to-apples
|
||||
# AND keeps this script from drifting from deploy.sh:67, which reads the same
|
||||
# variable. If the flag were wrong, the OpenBao side would fail closed with a 403
|
||||
# rather than quietly diverge — which is exactly the improvement being verified.
|
||||
skip_common="${TG_SKIP_COMMON:-0}"
|
||||
|
||||
sops_out=$(mktemp); bao_out=$(mktemp)
|
||||
# Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a
|
||||
|
|
@ -146,11 +198,23 @@ for k in sorted(order):
|
|||
cleanup
|
||||
done
|
||||
|
||||
# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a
|
||||
# host that carries none of them — a new box, a renamed checkout, a typo'd --env —
|
||||
# exits 0 having verified nothing, and a green run is exactly what the 7-day gate
|
||||
# counts. "Nothing was wrong" and "nothing was examined" must not look alike.
|
||||
if [ "$checked" -eq 0 ]; then
|
||||
echo
|
||||
echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2
|
||||
echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$overall" = 0 ]; then
|
||||
echo
|
||||
echo "PARITY OK for: ${TARGETS[*]}"
|
||||
echo "PARITY OK — ${checked} environment(s) checked on this host."
|
||||
echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments."
|
||||
echo "Recommended gate before prod: 7 consecutive green runs on all three."
|
||||
echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments."
|
||||
echo "That means both hosts — a green run here says nothing about the other box."
|
||||
else
|
||||
echo
|
||||
echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ new network exposure and no interactive steps.
|
|||
**As originally written** (pre vps1/vps2 split; kept for history — see the
|
||||
note at the top of this document):
|
||||
|
||||
- Monorepo `emi/thermograph` (domain dirs: `backend/ frontend/ infra/
|
||||
- Monorepo `Jinemi/thermograph` (domain dirs: `backend/ frontend/ infra/
|
||||
observability/`). Ops tooling lives in `infra/ops/`.
|
||||
- Environments: **dev** = LAN compose stack on the dev machine; **beta** =
|
||||
`75.119.132.91`; **prod** = `169.58.46.181`. SSH as `agent` with
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ infra repo**, not the app repo — the "branch" column above is which app-repo
|
|||
tag an environment is meant to track conceptually; the actual pinned versions
|
||||
are `var.hosts[*].environments[*].backend_image_tag` / `frontend_image_tag`
|
||||
(e.g. `"sha-<12 hex>"` each — the app is two separately-published images,
|
||||
`emi/thermograph-backend/app` and `emi/thermograph-frontend/app`), since the
|
||||
`admin_emi/thermograph-backend/app` and `admin_emi/thermograph-frontend/app`), since the
|
||||
host has no app checkout to derive a tag from. `app_dir` is required with no
|
||||
default specifically so vps2's two environments can never collide on the same
|
||||
checkout path (prod: `/opt/thermograph`, beta: `/opt/thermograph-beta`). The
|
||||
|
|
|
|||
|
|
@ -239,7 +239,7 @@ resource "null_resource" "host" {
|
|||
# then bring the stack up on the EXPLICIT per-service app image tags — there's no
|
||||
# app checkout on this host to derive a tag from (see var.backend_image_tag /
|
||||
# frontend_image_tag). The compose file reads BACKEND_IMAGE_TAG / FRONTEND_IMAGE_TAG
|
||||
# (for emi/thermograph-backend/app and emi/thermograph-frontend/app), so we export
|
||||
# (for admin_emi/thermograph-backend/app and admin_emi/thermograph-frontend/app), so we export
|
||||
# those, not the old single IMAGE_PATH/IMAGE_TAG. docker runs as root (sources both
|
||||
# env files in the same shell) so it never depends on the docker group membership
|
||||
# taking effect in this session.
|
||||
|
|
@ -257,7 +257,7 @@ resource "null_resource" "host" {
|
|||
render_thermograph_secrets ${var.app_dir}
|
||||
set -a; . /etc/thermograph-topology.env; . /etc/thermograph.env; set +a
|
||||
export REGISTRY_HOST="git.thermograph.org" BACKEND_IMAGE_TAG="${var.backend_image_tag}" FRONTEND_IMAGE_TAG="${var.frontend_image_tag}"
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin
|
||||
echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin
|
||||
docker compose ${local.compose_flags} pull backend frontend
|
||||
docker compose ${local.compose_flags} up -d --remove-orphans
|
||||
'
|
||||
|
|
|
|||
|
|
@ -31,12 +31,12 @@ variable "git_branch" {
|
|||
}
|
||||
|
||||
variable "backend_image_tag" {
|
||||
description = "Backend image tag to pull (emi/thermograph-backend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the backend-repo commit) or a semver tag. The host has no app-repo checkout to derive this from, so it's always explicit."
|
||||
description = "Backend image tag to pull (admin_emi/thermograph-backend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the backend-repo commit) or a semver tag. The host has no app-repo checkout to derive this from, so it's always explicit."
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "frontend_image_tag" {
|
||||
description = "Frontend image tag to pull (emi/thermograph-frontend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the frontend-repo commit) or a semver tag. Always explicit, same as backend_image_tag."
|
||||
description = "Frontend image tag to pull (admin_emi/thermograph-frontend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the frontend-repo commit) or a semver tag. Always explicit, same as backend_image_tag."
|
||||
type = string
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -151,13 +151,13 @@ hosts = {
|
|||
# }
|
||||
|
||||
# Optional overrides (shown with their defaults):
|
||||
# repo_url = "https://git.thermograph.org/emi/thermograph-infra.git"
|
||||
# repo_url = "https://git.thermograph.org/admin_emi/thermograph-infra.git"
|
||||
# app_port = 8137
|
||||
# frontend_port = 8080
|
||||
#
|
||||
# thermograph-infra is a PRIVATE repo, so a host cloning it for the first time
|
||||
# needs read credentials embedded in repo_url, e.g. a Forgejo deploy token:
|
||||
# repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/emi/thermograph-infra.git"
|
||||
# repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/admin_emi/thermograph-infra.git"
|
||||
|
||||
# ---------------------------------------------------------------------------------
|
||||
# Self-hosted Open-Meteo archive (only used by environments with openmeteo = true)
|
||||
|
|
|
|||
|
|
@ -54,8 +54,8 @@ variable "hosts" {
|
|||
git_branch = string # this INFRA repo's branch the checkout is reset to
|
||||
# Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag
|
||||
# for the commit that app repo built) or a semver tag on a release push. The app is
|
||||
# TWO separately-published images now — emi/thermograph-backend/app and
|
||||
# emi/thermograph-frontend/app — pinned independently. Required, no default: the
|
||||
# TWO separately-published images now — admin_emi/thermograph-backend/app and
|
||||
# admin_emi/thermograph-frontend/app — pinned independently. Required, no default: the
|
||||
# host's checkout is this infra repo, not the app repos, so there's no "current
|
||||
# commit" to derive a tag from; both must be explicit. Bump these (via a normal
|
||||
# tfvars edit + apply) whenever an app repo ships a commit you want this
|
||||
|
|
@ -127,9 +127,9 @@ variable "gcp_hosts" {
|
|||
}
|
||||
|
||||
variable "repo_url" {
|
||||
description = "Git remote to clone from when a host has no checkout yet. Points at THIS repo (thermograph-infra) now, not the app repo -- the app's own source is never checked out on a host; only its published registry images are pulled (see var.hosts[*].backend_image_tag / frontend_image_tag). thermograph-infra is a private repo, so this typically needs embedded read credentials, e.g. a Forgejo deploy token: \"https://<token-name>:<token>@git.thermograph.org/emi/thermograph-infra.git\"."
|
||||
description = "Git remote to clone from when a host has no checkout yet. Points at THIS repo (thermograph-infra) now, not the app repo -- the app's own source is never checked out on a host; only its published registry images are pulled (see var.hosts[*].backend_image_tag / frontend_image_tag). thermograph-infra is a private repo, so this typically needs embedded read credentials, e.g. a Forgejo deploy token: \"https://<token-name>:<token>@git.thermograph.org/admin_emi/thermograph-infra.git\"."
|
||||
type = string
|
||||
default = "https://git.thermograph.org/emi/thermograph-infra.git"
|
||||
default = "https://git.thermograph.org/admin_emi/thermograph-infra.git"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ persistent, queryable, fleet-wide log store and UI.
|
|||
> ### ⚠️ Merging is not deploying
|
||||
>
|
||||
> `/opt/observability` on vps1 (and vps2) is a checkout of the **archived**
|
||||
> `emi/thermograph-observability` repo. It can never `git pull` again — the
|
||||
> `admin_emi/thermograph-observability` repo. It can never `git pull` again — the
|
||||
> content now lives here, in the mono repo, under `observability/`. A previous
|
||||
> observability PR merged and had **zero live effect** for exactly this reason.
|
||||
>
|
||||
|
|
|
|||
Loading…
Reference in a new issue