diff --git a/.forgejo/workflows/build-push.yml b/.forgejo/workflows/build-push.yml index e9e1fc3..034b1bc 100644 --- a/.forgejo/workflows/build-push.yml +++ b/.forgejo/workflows/build-push.yml @@ -4,8 +4,8 @@ name: Build + push images (Forgejo registry) # They were identical apart from the domain string: the paths filter, the # concurrency group, IMAGE_PATH, the tag key and the build context. # -# Images stay separate and independently deployable -- emi/thermograph/backend -# and emi/thermograph/frontend -- exactly as before. Build-once, +# Images stay separate and independently deployable -- jinemi/thermograph/backend +# and jinemi/thermograph/frontend -- exactly as before. Build-once, # deploy-everywhere; nothing about the published artefacts changes here. # # SEMVER EXCEPTION, preserved: a v*.*.* tag push carries no paths context, so a @@ -110,13 +110,49 @@ jobs: - name: Log in to the Forgejo registry if: steps.image.outputs.changed == 'true' + env: + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} run: | # NOT secrets.GITHUB_TOKEN -- Forgejo's per-job auto-injected token can # never push to the container registry (a known Forgejo limitation, # independent of any permission setting). Needs a manually provisioned # PAT with write:package scope, stored as REGISTRY_TOKEN. - echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "${{ steps.image.outputs.host }}" \ - --username emi --password-stdin + # + # WHITESPACE IS THE FAILURE THIS GUARDS AGAINST, and it is a nasty one. + # `docker login --password-stdin` strips exactly ONE trailing newline. + # `echo "$T"` adds one. So a secret pasted WITH a trailing newline -- + # which is what happens if you copy from a terminal or an editor that + # ends files with one -- arrives as "\n" and the registry answers + # + # Error response from daemon: Get "https://.../v2/": denied: + # + # with no further detail. That is indistinguishable from a revoked or + # wrong token, and it cost an afternoon on 2026-08-01 chasing a + # credential that was in fact valid. Strip first, echo never. + # + # The token also goes through the ENVIRONMENT rather than being + # interpolated into the script text. An expression is substituted before bash + # parses the line, so a value containing a quote or newline would change + # the shape of the command itself rather than just its arguments. + tok=$(printf '%s' "$REGISTRY_TOKEN" | tr -d ' \t\r\n') + if [ -z "$tok" ]; then + echo "::error::REGISTRY_TOKEN is empty or unset. Set it at" \ + "Settings -> Actions -> Secrets on this repository." + exit 1 + fi + # Shape check is a WARNING, not a failure: a hard assertion on the token + # format would block every build the day Forgejo changes it. But saying + # so up front turns the registry's opaque "denied:" into a diagnosis. + # Length and character class only -- the value is never printed. + case "$tok" in + *[!0-9a-f]*) echo "::warning::REGISTRY_TOKEN contains characters outside" \ + "[0-9a-f]; a Forgejo PAT is 40 hex characters. If the" \ + "login below fails, re-paste the secret." ;; + esac + [ "${#tok}" -eq 40 ] || echo "::warning::REGISTRY_TOKEN is ${#tok} characters," \ + "expected 40. If the login below fails, re-paste the secret." + printf '%s' "$tok" | docker login "${{ steps.image.outputs.host }}" \ + --username admin_emi --password-stdin - name: Build if: steps.image.outputs.changed == 'true' diff --git a/.forgejo/workflows/infra-sync.yml b/.forgejo/workflows/infra-sync.yml index 67cd055..1827354 100644 --- a/.forgejo/workflows/infra-sync.yml +++ b/.forgejo/workflows/infra-sync.yml @@ -86,6 +86,15 @@ jobs: exit 0 fi if [ -f infra/deploy/render-secrets.sh ]; then + # env-topology.sh FIRST: it is what sets TG_SECRETS_BACKEND, and + # render-secrets.sh:44 defaults to `sops` when it is unset. Without + # this line the cutover would be only half-applied — deploy.sh would + # read the new backend while THIS workflow, which runs on every push + # touching infra/**, kept re-rendering the same file from SOPS. That + # is silent while parity holds and a hard failure the moment the SOPS + # files are retired. + . infra/deploy/env-topology.sh + thermograph_topology dev . infra/deploy/render-secrets.sh # SKIP_COMMON is dev's standing rule, not a preference: common.yaml # is the fleet's shared production credential set, and vps1 also @@ -114,6 +123,13 @@ jobs: git fetch --prune origin main git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then + # See the note in sync-dev: env-topology.sh is what sets + # TG_SECRETS_BACKEND, and for beta it also sets TG_BAO_APPROLE to + # the -beta credential. Beta shares a filesystem with prod, so the + # bare default is prod's file and tg-host-prod.hcl denies beta's + # own path — a 403 at render time on the box that runs prod. + . infra/deploy/env-topology.sh + thermograph_topology beta . infra/deploy/render-secrets.sh render_thermograph_secrets /opt/thermograph-beta/infra beta /etc/thermograph-beta.env fi @@ -139,7 +155,113 @@ jobs: git fetch --prune origin main git reset --hard origin/main if [ -f infra/deploy/render-secrets.sh ]; then + # See the note in sync-dev — without env-topology.sh this job keeps + # rendering from SOPS regardless of what the backend selector says. + . infra/deploy/env-topology.sh + thermograph_topology prod . infra/deploy/render-secrets.sh render_thermograph_secrets /opt/thermograph/infra prod /etc/thermograph.env fi echo "synced to $(git log --oneline -1)" + + sync-centralis: + # Centralis' /etc/centralis.env has been HAND-MAINTAINED. The vault has held a + # copy since deploy/secrets/centralis.prod.yaml was seeded, and + # render_centralis_secrets has existed since the OpenBao work -- but nothing + # on any deploy path ever called it. A renderer with no caller is not a + # migration, it is a file. + # + # That gap is why enabling Google sign-in on Centralis had no safe home: the + # only way to set CENTRALIS_GOOGLE_CLIENT_ID was to hand-edit the live file, + # which is exactly what render-secrets.sh's header warns against. + # + # WHY THIS JOB RECREATES A CONTAINER WHEN THE OTHER THREE DELIBERATELY DO NOT. + # This workflow's header says it never rolls a service, because image tags are + # the app domains' axis. Centralis is different in kind: its compose file + # interpolates /etc/centralis.env at PARSE time (`set -a && . /etc/centralis.env + # && docker compose up -d`), so rendering the file without recreating the + # container changes nothing at all. Rendering and not recreating would be the + # silent no-op this estate keeps getting bitten by. It is one container with no + # replicas -- a two-second recreate, not a deploy. + if: github.ref_name == 'main' + runs-on: docker + concurrency: + group: infra-sync-centralis + cancel-in-progress: false + steps: + - name: Render /etc/centralis.env from the vault, then recreate Centralis + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + script: | + set -euo pipefail + umask 077 + cd /opt/thermograph + [ -f infra/deploy/secrets/centralis.prod.yaml ] || { echo "no Centralis vault — nothing to do"; exit 0; } + + . infra/deploy/env-topology.sh + thermograph_topology prod + . infra/deploy/render-secrets.sh + + scratch=$(mktemp -d); chmod 700 "$scratch" + trap 'rm -rf -- "$scratch"' EXIT + + # Render to scratch FIRST. CENTRALIS_RENDER_DEST is the override the + # renderer already carries for exactly this: look before writing. + CENTRALIS_RENDER_DEST="$scratch/rendered.env" \ + render_centralis_secrets /opt/thermograph/infra + + # THE GUARD. On 2026-07-24 this estate lost Centralis' token registry by + # writing a file that was missing a key the live one had. The renderer + # proves its OWN output round-trips through bash, but it cannot know + # about a key that exists only in the live file -- someone hand-editing + # /etc/centralis.env to add, say, CENTRALIS_GOOGLE_CLIENT_ID is invisible + # to it. So: the vault must be a SUPERSET of what is live, or nothing is + # written and the job fails loudly. + # + # Names only. Both files are sourced in a clean environment so the + # comparison is over what bash actually ends up with, not over text. + # The `|| true` is load-bearing under `set -o pipefail`: grep exits 1 when + # a file yields no matching keys, which would otherwise abort the job on + # the exact edge case this guard exists to handle -- an empty or + # unreadable live file, i.e. a first run. + keys() { env -i bash -c 'set -a; . "$1" >/dev/null 2>&1; set +a; compgen -v' _ "$1" \ + | { grep -E '^(CENTRALIS_|DISCORD_)' || true; } | sort -u; } + keys "$scratch/rendered.env" > "$scratch/new.keys" + if [ -r /etc/centralis.env ]; then keys /etc/centralis.env > "$scratch/live.keys"; else : > "$scratch/live.keys"; fi + + missing=$(comm -23 "$scratch/live.keys" "$scratch/new.keys" || true) + if [ -n "$missing" ]; then + echo "!! REFUSING to write /etc/centralis.env." + echo "!! The live file has keys the vault does not, so rendering would DELETE them:" + printf '!! %s\n' $missing + echo "!! Someone hand-edited the live file. Put those keys in the vault instead:" + echo "!! sops edit infra/deploy/secrets/centralis.prod.yaml" + exit 1 + fi + + added=$(comm -13 "$scratch/live.keys" "$scratch/new.keys" || true) + [ -n "$added" ] && printf ' new key from the vault: %s\n' $added + + # Same file, byte for byte? Then there is nothing to do and no reason to + # bounce a healthy container. + if [ -r /etc/centralis.env ] && cmp -s "$scratch/rendered.env" /etc/centralis.env; then + echo "==> /etc/centralis.env already matches the vault — not recreating" + exit 0 + fi + + install -m 0600 -o "$(stat -c %U /etc/centralis.env 2>/dev/null || echo "$USER")" \ + -g "$(stat -c %G /etc/centralis.env 2>/dev/null || echo "$USER")" \ + "$scratch/rendered.env" /etc/centralis.env 2>/dev/null \ + || sudo install -m 0600 -o agent -g agent "$scratch/rendered.env" /etc/centralis.env + echo "==> wrote /etc/centralis.env from the vault" + + # Recreate so compose re-reads it. `docker restart` would NOT: the values + # are interpolated when the compose file is parsed, not read at runtime. + cd /opt/centralis/deploy + set -a; . /etc/centralis.env; set +a + docker compose up -d + docker compose ps --format '{{.Name}}\t{{.State}}\t{{.Status}}' diff --git a/.forgejo/workflows/ops-cron.yml b/.forgejo/workflows/ops-cron.yml index 112d0be..6c3c948 100644 --- a/.forgejo/workflows/ops-cron.yml +++ b/.forgejo/workflows/ops-cron.yml @@ -8,8 +8,15 @@ name: Ops cron (backup + IndexNow) # # The jobs SSH into a host and run inside the already-running stack (docker # exec), the same way deploy.sh runs its own post-deploy IndexNow ping -- no new -# network exposure, no separate dependency install. Runs on the `docker` label -# (the always-on Swarm-hosted runner deploy/forgejo/ stood up). +# network exposure, no separate dependency install. Runs on the `docker` label, +# served by the vps2 runner (infra/deploy/forgejo/runner-vps2/) and, when it is +# up, the desktop. +# +# THIS WORKFLOW IS THE ESTATE'S ONLY BACKUP, so its dependency on a runner is +# load-bearing. On 2026-07-31 the desktop was the only registered runner and +# went offline for 21 hours; this schedule did not fire at 03:00Z. Forgejo held +# the run in `waiting` and it completed on reconnect, so nothing was lost that +# time. The vps2 runner exists so the next outage is not a coin flip. # # WHICH BOX EACH JOB TALKS TO, and why the secret names say so: # @@ -209,6 +216,65 @@ jobs: rclone delete --min-age 30d "archive:$base/db/" 2>/dev/null || true rclone delete --min-age 30d "archive:$base/data/" 2>/dev/null || true + secrets-parity: + name: OpenBao/SOPS parity gate + runs-on: docker + # The gate infra/openbao/README.md specifies for a cutover -- "7 consecutive + # green parity runs" -- and which, until now, nothing implemented. A gate + # that exists only in prose gets satisfied by assertion. + # + # verify-parity.sh renders each environment through BOTH backends and diffs + # the KEY=value sets after a last-wins collapse. It prints key NAMES and + # counts only, never values, so these logs are safe to read and to paste. + # + # A mismatch FAILS the job, deliberately and loudly. The failure modes on + # the other side of a bad cutover are all silent: a missing + # THERMOGRAPH_AUTH_SECRET makes every worker mint its own, so logins start + # working intermittently; a half-rendered VAPID pair makes push.py generate a + # new keypair and delete every subscription row as "gone". Red here is the + # only warning that arrives before those. + # + # Runs under sudo because the age key is 0400 root and the AppRole files are + # 0400 agent/root -- the SSH user cannot read them directly. This grants CI + # no vault access of its own: the HOST renders, CI only asks it to. + # + # The run history in Forgejo IS the record of consecutive green days. Do not + # count a day this job was skipped or the runner was down as green. + concurrency: + group: ops-secrets-parity + cancel-in-progress: false + steps: + - name: prod + beta (vps2) + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS2_SSH_HOST }} + username: ${{ secrets.VPS2_SSH_USER }} + key: ${{ secrets.VPS2_SSH_KEY }} + port: ${{ secrets.VPS2_SSH_PORT }} + script: | + set -euo pipefail + # Each environment is checked from its OWN checkout. The trees are + # identical, but running beta's from beta's directory also proves + # that checkout is current rather than assuming it. + sudo /opt/thermograph/infra/openbao/verify-parity.sh --env prod + sudo /opt/thermograph-beta/infra/openbao/verify-parity.sh --env beta + + - name: dev (vps1) + uses: https://github.com/appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.VPS1_SSH_HOST }} + username: ${{ secrets.VPS1_SSH_USER }} + key: ${{ secrets.VPS1_SSH_KEY }} + port: ${{ secrets.VPS1_SSH_PORT }} + script: | + set -euo pipefail + # dev renders dev.yaml ALONE -- verify-parity.sh takes that from + # env-topology.sh's TG_SKIP_COMMON rather than re-deriving it, so the + # comparison matches what deploy-dev.sh actually does. vps1 must + # never see common.yaml: it runs Forgejo, its CI, and dev's + # unreviewed branch. + sudo /opt/thermograph-dev/infra/openbao/verify-parity.sh --env dev + indexnow: name: IndexNow ping runs-on: docker diff --git a/.gitignore b/.gitignore index 5aee0f7..f5ea9c0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,9 @@ # Claude Code worktrees. These are other sessions' checkouts living inside this # repo; `git add -A` will otherwise commit them as embedded git repositories. .claude/worktrees/ + +# Per-operator Claude Code settings — local permission grants, not team config. +# Untracked but not ignored meant one `git add -A` would publish one operator's +# allowlist (including which secrets they may decrypt) to everyone. +# `.claude/settings.json` IS tracked and stays the shared, reviewed config. +.claude/settings.local.json diff --git a/CLAUDE.md b/CLAUDE.md index ddf58a1..e7e36f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,7 +77,7 @@ cannot describe vps2 alone, since vps2 runs beta and prod side by side. - Each service's live tag is persisted host-side, so a single-service roll never disturbs the sibling's running tag. -Images are `emi/thermograph/backend` and `emi/thermograph/frontend`, tagged +Images are `jinemi/thermograph/backend` and `jinemi/thermograph/frontend`, tagged `sha-<12hex>` on every push and by semver on `v*.*.*` tags. They build and deploy **independently** — a backend change ships without a frontend deploy and vice versa. That independence is the point of the FE/BE split and survived diff --git a/CUTOVER-NOTES.md b/CUTOVER-NOTES.md index cc6ba93..994f9a0 100644 --- a/CUTOVER-NOTES.md +++ b/CUTOVER-NOTES.md @@ -25,7 +25,7 @@ Forgejo only reads root workflows, but dead copies are a trap):** - `backend-build-push.yml` / `frontend-build-push.yml` — the old per-repo build-push, path-filtered (`paths: ['backend/**']` etc.), image paths now - **explicit**: `emi/thermograph/backend|frontend` (the old + **explicit**: `jinemi/thermograph/backend|frontend` (the old `${{ github.repository }}/app` derivation collides in a monorepo). Per-domain concurrency groups. Caveat: `v*.*.*` tag pushes carry no paths context, so a version tag builds **both** images (intended: a release names the set). @@ -65,7 +65,7 @@ Forgejo only reads root workflows, but dead copies are a trap):** name, so no project-name concern there). **Verified here:** every workflow + compose file parses (PyYAML); all three -deploy scripts pass `bash -n`; no `emi/thermograph-{backend,frontend}/app` +deploy scripts pass `bash -n`; no `admin_emi/thermograph-{backend,frontend}/app` stragglers in sh/yml. **Not verified (impossible locally):** Forgejo actually honoring `on.push.paths` and `workflow_call` inputs on our version; any live deploy. @@ -118,7 +118,7 @@ Checked every split-repo branch by dry-run merge; migrated everything real: ## Cutover runbook -1. Create `emi/thermograph` on Forgejo (a **new** repo — do not resurrect the +1. Create `Jinemi/thermograph` on Forgejo (a **new** repo — do not resurrect the archived monorepo) and push `main`. 2. Actions config, once: secrets `REGISTRY_TOKEN`, `SSH_HOST/USER/KEY/PORT` (beta), `PROD_SSH_*` (prod); variable `REGISTRY_HOST=git.thermograph.org`. @@ -132,7 +132,7 @@ Checked every split-repo branch by dry-run merge; migrated everything real: `/etc/thermograph*` files stay as-is. 5. LAN box: re-point `~/thermograph-dev` at the monorepo the same way. 6. First push builds the **new** image paths — old - `emi/thermograph-{backend,frontend}/app` packages stay in the registry for + `admin_emi/thermograph-{backend,frontend}/app` packages stay in the registry for rollback; GC after burn-in. deploy.sh's tag prune already targets the new paths only. 7. One `workflow_dispatch` of `backend-deploy.yml` after its build-push diff --git a/README.md b/README.md index b4fa580..085df17 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ for: **per-domain images, per-domain deploys, and an async FE/BE contract**. | Dir | What | CI | |---|---|---| -| `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `emi/thermograph/backend`; `deploy` | -| `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `emi/thermograph/frontend` | +| `backend/` | FastAPI graded-climate API, accounts, notifications (Discord bot, push, mail), data pipeline | `build-push` → image `jinemi/thermograph/backend`; `deploy` | +| `frontend/` | Public client: static JS/CSS + SSR pages | same `build-push` / `deploy` workflows, matrixed by domain; image `jinemi/thermograph/frontend` | | `infra/` | Compose (dev, on vps1) + two Swarm stacks co-resident on vps2 (beta, prod), deploy scripts, terraform, SOPS secrets vault, ops cron | `infra-sync` (host checkout + secrets render), `secrets-guard`, `ops-cron` | | `observability/` | Loki + Grafana + Alloy stack | `observability-validate` | @@ -27,7 +27,7 @@ which docs in this repo are currently stale. Every workflow in `.forgejo/workflows/` is **path-filtered to its domain**: a push touching only `frontend/**` builds/deploys nothing else. Images stay -separate (`emi/thermograph/backend`, `emi/thermograph/frontend`, each tagged +separate (`jinemi/thermograph/backend`, `jinemi/thermograph/frontend`, each tagged `sha-<12hex>`), deploys stay per-service (`infra/deploy/deploy.sh SERVICE=backend|frontend|all`), and the API version contract (`GET /api/version`, `PAYLOAD_VER`) still lets FE and BE ship out of lockstep. diff --git a/backend/README.md b/backend/README.md index 73c58de..f80f698 100644 --- a/backend/README.md +++ b/backend/README.md @@ -3,7 +3,7 @@ The Thermograph API service: grades recent local weather against ~45 years of climate history, and hosts the accounts, notification, and SSR-content back-end that the rest of the split Thermograph stack (`thermograph-frontend`) -talks to over HTTP. Split from the `emi/thermograph` monorepo — this repo owns +talks to over HTTP. Split from the `Jinemi/thermograph` monorepo — this repo owns the API/DB/accounts/notifications layer only; it renders no HTML/CSS/JS of its own. @@ -67,7 +67,7 @@ cities_flavor.json bundled reference data (generated by gen_cities.py / `/content/...`; it negotiates compatibility via `GET /api/version`. - **`thermograph-infra`** owns the deploy/Compose/Terraform layer — this repo only builds and publishes its own container image - (`git.thermograph.org/emi/thermograph-backend/app`) and hands infra a tag to + (`git.thermograph.org/admin_emi/thermograph-backend/app`) and hands infra a tag to roll out (`SERVICE=backend` + `BACKEND_IMAGE_TAG` into infra's `deploy/deploy.sh`). - **`thermograph-docs`** holds the cross-repo architecture/runbook docs; this diff --git a/backend/docker-compose.test.yml b/backend/docker-compose.test.yml index d5ff99e..9d8c37e 100644 --- a/backend/docker-compose.test.yml +++ b/backend/docker-compose.test.yml @@ -22,11 +22,11 @@ services: backend: # Defaults to a locally-built `:local` image (scripts/smoke.sh builds it). In CI, # set BACKEND_IMAGE_TAG=sha-<12hex> to smoke the exact published image. - # The path must match what build-push.yml publishes -- emi/thermograph/backend. - # It read emi/thermograph-backend/app (the retired split-era path) until this + # The path must match what build-push.yml publishes -- jinemi/thermograph/backend. + # It read admin_emi/thermograph-backend/app (the retired split-era path) until this # was corrected; harmless for the default `:local` build, wrong the moment # BACKEND_IMAGE_TAG names a real published tag. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} depends_on: db: condition: service_healthy diff --git a/backend/scripts/smoke.sh b/backend/scripts/smoke.sh index 7165756..230f4dc 100755 --- a/backend/scripts/smoke.sh +++ b/backend/scripts/smoke.sh @@ -14,7 +14,7 @@ export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-smoke}" # The image to smoke, pulled from the registry (no local build). CI passes the just- # pushed sha (BACKEND_IMAGE_TAG=sha-<12hex>); locally it defaults to a published tag. export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-v0.0.2-split-ci}" -IMG="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG}" +IMG="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-admin_emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG}" export SMOKE_HOST_PORT="${SMOKE_HOST_PORT:-18137}" COMPOSE=(docker compose -f docker-compose.test.yml) PORT_URL="http://127.0.0.1:${SMOKE_HOST_PORT}" diff --git a/docs/onboarding/03-repo-map.md b/docs/onboarding/03-repo-map.md index 60a3952..fd9627e 100644 --- a/docs/onboarding/03-repo-map.md +++ b/docs/onboarding/03-repo-map.md @@ -12,8 +12,8 @@ thermograph/ ├── README.md domain table + how CI stays decoupled ├── .forgejo/workflows/ ALL CI. Nine files. Never add workflows elsewhere. ├── .claude/ settings.json + enforcement hooks -├── backend/ FastAPI API + Go daemon → emi/thermograph/backend -├── frontend/ Go SSR + static assets → emi/thermograph/frontend +├── backend/ FastAPI API + Go daemon → jinemi/thermograph/backend +├── frontend/ Go SSR + static assets → jinemi/thermograph/frontend ├── infra/ compose, Swarm, deploy, secrets, terraform, ops ├── observability/ Loki + Grafana + Alloy └── docs/onboarding/ you are here diff --git a/docs/onboarding/07-ci-and-release.md b/docs/onboarding/07-ci-and-release.md index 27db954..22fd7d8 100644 --- a/docs/onboarding/07-ci-and-release.md +++ b/docs/onboarding/07-ci-and-release.md @@ -102,7 +102,7 @@ alembic + `/healthz` 200 + a real `/api/v2/place` 200. Triggers on pushes to `dev`/`main`/`release` touching `backend/**` or `frontend/**`, and on `v*.*.*` tags. Publishes -`git.thermograph.org/emi/thermograph/{backend,frontend}:sha-<12hex>`. +`git.thermograph.org/Jinemi/thermograph/{backend,frontend}:sha-<12hex>`. - **Serialised** (`max-parallel: 1`): one physical runner, and two whole image builds racing each other buys nothing. diff --git a/docs/onboarding/09-observability.md b/docs/onboarding/09-observability.md index 5ad6fab..a7eee63 100644 --- a/docs/onboarding/09-observability.md +++ b/docs/onboarding/09-observability.md @@ -148,7 +148,7 @@ pages nobody. **The only proof is a message actually arriving in > ⚠️ **Merging is not deploying — and this one has already bitten.** > `/opt/observability` on vps1 and vps2 is a checkout of the **archived** -> `emi/thermograph-observability` repo. It can never `git pull` again; the +> `admin_emi/thermograph-observability` repo. It can never `git pull` again; the > content now lives here under `observability/`. A previous observability PR > merged and had **zero live effect** for exactly this reason. diff --git a/docs/onboarding/11-traps.md b/docs/onboarding/11-traps.md index 86bff0a..2791c3f 100644 --- a/docs/onboarding/11-traps.md +++ b/docs/onboarding/11-traps.md @@ -77,7 +77,7 @@ mentally translate host names. ### `backend/README.md` is written for the split-repo era It describes `thermograph-backend` as its own repo with sibling repos, uses the -retired image path `git.thermograph.org/emi/thermograph-backend/app`, and says +retired image path `git.thermograph.org/admin_emi/thermograph-backend/app`, and says *"there is no Makefile in this repo yet"* — there is (`make test`, `make smoke`, `make clean-venv`). @@ -106,7 +106,7 @@ it — that's the standing instruction in the root `CLAUDE.md`. ### `observability/README.md`: merging is not deploying `/opt/observability` on vps1 and vps2 is a checkout of the **archived** -`emi/thermograph-observability` repo. It can never `git pull` again. A previous +`admin_emi/thermograph-observability` repo. It can never `git pull` again. A previous observability PR merged and had **zero live effect** for exactly this reason. The README documents the hand-ship procedure; until the checkouts are re-pointed at the monorepo, that's the only path. Flagged again in @@ -118,7 +118,7 @@ at the monorepo, that's the only path. Flagged again in ### ~~The `docker-compose.test.yml` files pull retired image paths~~ — fixed -**Fixed.** Both now use `emi/thermograph/backend`, the path `build-push.yml` +**Fixed.** Both now use `jinemi/thermograph/backend`, the path `build-push.yml` actually publishes. The frontend harness had the worse half of this: it also pinned diff --git a/docs/onboarding/README.md b/docs/onboarding/README.md index 9150399..2a307ac 100644 --- a/docs/onboarding/README.md +++ b/docs/onboarding/README.md @@ -1,6 +1,6 @@ # Onboarding — becoming a full contributor to Thermograph -This is the developer onboarding path for the `emi/thermograph` monorepo: what +This is the developer onboarding path for the `Jinemi/thermograph` monorepo: what the product is, how the code is shaped, how to run it, what will break silently if you get it wrong, and how a change actually travels from your editor to `thermograph.org`. @@ -35,10 +35,10 @@ readings. - **`backend/`** — Python 3.12 / FastAPI. Owns all climate data, grading, the API, accounts, notifications, plus a Go daemon (`backend/daemon/`) that owns - the Discord gateway and recurring timers. Ships as `emi/thermograph/backend`. + the Discord gateway and recurring timers. Ships as `jinemi/thermograph/backend`. - **`frontend/`** — **Go** SSR service (`frontend/server/`) plus every static asset. No climate data, no database, no compute. Ships as - `emi/thermograph/frontend`. (The Python implementation at `frontend/*.py` is + `jinemi/thermograph/frontend`. (The Python implementation at `frontend/*.py` is the superseded original — see [traps](11-traps.md).) - **`infra/`** — compose and Swarm files, deploy scripts, the SOPS secrets vault, Terraform, ops query tooling. diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 4804185..6c7219b 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -10,7 +10,7 @@ names. ## The live service is Go — `server/` `server/` (module `thermograph/frontend`, Go 1.26) is what builds, tests, ships -and runs. It is the only thing in the `emi/thermograph/frontend` image. +and runs. It is the only thing in the `jinemi/thermograph/frontend` image. The Python files at this level — `app.py`, `content.py`, `api_client.py`, `format.py`, `content_loader.py`, `paths.py`, `templates/*.html.j2` — are the diff --git a/frontend/README.md b/frontend/README.md index c22d8e4..0650bdf 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -5,7 +5,7 @@ static asset for [Thermograph](https://thermograph.org) — grades recent local weather against ~45 years of climate history. This domain holds **no climate data and does no compute**; it renders pages and serves the JS/CSS/image assets that call the backend's API from the browser. It builds its own image -(`emi/thermograph/frontend`) and deploys independently of the backend. +(`jinemi/thermograph/frontend`) and deploys independently of the backend. See [`CLAUDE.md`](CLAUDE.md) for the agent-facing detail on the deploy contract and the cross-service contracts, and [`DESIGN.md`](DESIGN.md) for the visual diff --git a/frontend/docker-compose.test.yml b/frontend/docker-compose.test.yml index 234fa2d..3a104d5 100644 --- a/frontend/docker-compose.test.yml +++ b/frontend/docker-compose.test.yml @@ -27,9 +27,9 @@ services: # domain-keyed rule build-push.yml and deploy.yml both use -- so the harness # tracks the tree instead of drifting behind a hardcoded pin. (It was pinned # to the split-era v0.0.2-split-ci, under the retired - # emi/thermograph-backend/app path, long after both were superseded.) + # admin_emi/thermograph-backend/app path, long after both were superseded.) # Override THERMOGRAPH_BACKEND_TEST_TAG to pin a specific build. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${THERMOGRAPH_BACKEND_TEST_TAG:?set it, or run via scripts/backend-for-tests.sh which derives it} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${THERMOGRAPH_BACKEND_TEST_TAG:?set it, or run via scripts/backend-for-tests.sh which derives it} depends_on: db: condition: service_healthy diff --git a/frontend/server/README.md b/frontend/server/README.md index 9eb1425..5ef58a9 100644 --- a/frontend/server/README.md +++ b/frontend/server/README.md @@ -55,5 +55,5 @@ binary; static assets and the YAML copy are read from disk. go build ./... && go vet ./... && go test ./... The deployed binary is `/usr/local/bin/thermograph-frontend` inside the -`emi/thermograph/frontend` image; the image name, `frontend-*` CI workflows and +`jinemi/thermograph/frontend` image; the image name, `frontend-*` CI workflows and deploy path are unchanged from the Python service. diff --git a/infra/.env.example b/infra/.env.example index 3da1d93..7b482c4 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -58,7 +58,7 @@ THERMOGRAPH_INTERNAL_TOKEN= # :local, which is the default here -- only set these to pull a specific # published build instead of building locally. # REGISTRY_HOST=git.thermograph.org -# BACKEND_IMAGE_PATH=emi/thermograph-backend/app +# BACKEND_IMAGE_PATH=admin_emi/thermograph-backend/app # BACKEND_IMAGE_TAG=local -# FRONTEND_IMAGE_PATH=emi/thermograph-frontend/app +# FRONTEND_IMAGE_PATH=admin_emi/thermograph-frontend/app # FRONTEND_IMAGE_TAG=local diff --git a/infra/ACCESS.md b/infra/ACCESS.md index 6713241..2116eeb 100644 --- a/infra/ACCESS.md +++ b/infra/ACCESS.md @@ -26,7 +26,7 @@ rather than which environment happens to live there: | Host | Public IP | Mesh IP | Role | |------|-----------|---------|------| -| **vps1** | `75.119.132.91` | `10.10.0.2` | "Operational programs": Forgejo (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio, and the **dev** environment (mesh-only, `10.10.0.2:8137`, no public DNS/Caddy/TLS) | +| **vps1** | `75.119.132.91` | `10.10.0.2` | "Operational programs": Forgejo (git + CI + registry; `dev.jinemi.com` canonical, `git.thermograph.org` still served and still used by the registry/mesh pins/runners), Grafana/Loki/Alloy (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio, and the **dev** environment (mesh-only, `10.10.0.2:8137`, no public DNS/Caddy/TLS) | | **vps2** | `169.58.46.181` | `10.10.0.1` | "The deployed environment": **prod** and **beta**, as two separate Docker Swarm stacks, plus Centralis, Postfix, the backups, and the one shared TimescaleDB instance both stacks use | | **desktop** | — | `10.10.0.3` | AI-model hosting (voice-to-text, an upcoming-feature LLM) plus flex Swarm-worker capacity. Hosts **no** Thermograph environment — `make dev-up` still works there, but only as a laptop-local rehearsal | diff --git a/infra/CLAUDE.md b/infra/CLAUDE.md index 390e0e4..c739633 100644 --- a/infra/CLAUDE.md +++ b/infra/CLAUDE.md @@ -12,7 +12,10 @@ Two VPS boxes plus the operator's desktop, on one WireGuard mesh, named by ROLE rather than by environment: - **vps1** — `75.119.132.91`, mesh `10.10.0.2`. "Operational programs": Forgejo - (git + CI + registry, `git.thermograph.org`), Grafana/Loki/Alloy + (git + CI + registry; `dev.jinemi.com` is canonical — Forgejo's `ROOT_URL` — + and `git.thermograph.org` is still served off the same Caddy site block and + still load-bearing for the registry, mesh pins and runner URLs, so don't + retire it; see `deploy/forgejo/README.md`), Grafana/Loki/Alloy (`dashboard.thermograph.org`), the `emigriffith.dev` portfolio site, and the **dev** environment (`/opt/thermograph-dev`, tracks `dev`), with its own Postgres container. Dev is mesh-only here — published on `10.10.0.2:8137`, diff --git a/infra/README.md b/infra/README.md index ed6b477..a0b07af 100644 --- a/infra/README.md +++ b/infra/README.md @@ -3,7 +3,7 @@ Infrastructure for [Thermograph](https://thermograph.org): Terraform host provisioning, the SOPS+age secrets vault, WireGuard/Swarm networking, Forgejo, Caddy, mail, and the deploy scripts that run the already-built app images on each -host. This is a domain of the `emi/thermograph` monorepo — a host's checkout +host. This is a domain of the `Jinemi/thermograph` monorepo — a host's checkout (`/opt/thermograph`, `/opt/thermograph-beta`, or `/opt/thermograph-dev`) is a checkout of the whole monorepo, and `infra/` never builds app source; it only runs published images. diff --git a/infra/deploy/Caddyfile.vps1 b/infra/deploy/Caddyfile.vps1 index ebd6ac2..8532948 100644 --- a/infra/deploy/Caddyfile.vps1 +++ b/infra/deploy/Caddyfile.vps1 @@ -2,7 +2,11 @@ # # vps1 serves: # emigriffith.dev -> static portfolio site (from disk) -# git.thermograph.org -> Forgejo (see deploy/forgejo/caddy-git.conf) +# dev.jinemi.com -> Forgejo, CANONICAL (Forgejo's ROOT_URL) +# git.thermograph.org -> Forgejo, same site block — still served and +# still needed: the registry host in image names, +# the mesh /etc/hosts pins and the runners' URLs +# all use it (see deploy/forgejo/caddy-git.conf) # dashboard.thermograph.org -> Grafana (see observability/caddy-grafana.conf) # # and hosts the **dev** environment at dev.thermograph.org (below). diff --git a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md index 408464b..ede3d56 100644 --- a/infra/deploy/RUNBOOK-vps1-vps2-cutover.md +++ b/infra/deploy/RUNBOOK-vps1-vps2-cutover.md @@ -79,7 +79,7 @@ or 8 is done wrong, which is what the verification lines are for. ``` ssh agent@169.58.46.181 -sudo git clone --branch main http://10.10.0.2:3080/emi/thermograph.git /opt/thermograph-beta +sudo git clone --branch main http://10.10.0.2:3080/Jinemi/thermograph.git /opt/thermograph-beta sudo chown -R agent:agent /opt/thermograph-beta ``` diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 9efeaf7..8999732 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -13,7 +13,7 @@ # ssh deploy@vps 'SERVICE=all BACKEND_IMAGE_TAG=sha- FRONTEND_IMAGE_TAG=sha- /opt/thermograph/infra/deploy/deploy.sh' # # FE/BE CI-CD split: backend and frontend are published from separate repos as -# separate images (emi/thermograph/backend, emi/thermograph/frontend), +# separate images (jinemi/thermograph/backend, jinemi/thermograph/frontend), # so a deploy targets ONE service and leaves the other's running container + # tag untouched. Each service's live tag is persisted host-side in # deploy/.image-tags.env (untracked -- survives the git reset below) so a @@ -282,7 +282,7 @@ esac # host's existing cred; a genuine auth problem then fails loudly at `pull`. if [ -n "${REGISTRY_TOKEN:-}" ]; then echo "==> Logging in to the registry ($REGISTRY_HOST)" - echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin else echo "==> No REGISTRY_TOKEN in env; relying on the host's existing docker login to $REGISTRY_HOST" fi @@ -328,7 +328,7 @@ case " ${TARGETS[*]} " in # then always found no daemon binary in a Postgres image and dropped # daemon from EVERY backend deploy, regardless of what the real backend # image contained -- reproduced and confirmed against beta directly. - daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG}" + daemon_img="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG}" if ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run" kept=() @@ -408,8 +408,8 @@ fi # roll never garbage-collects the image a rollback would need; docker also # refuses to remove an image any container still uses. echo "==> Pruning old app-image tags" -_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}" -_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}" +_be_repo="${REGISTRY_HOST}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}" +_fe_repo="${REGISTRY_HOST}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}" docker images --format '{{.Repository}}:{{.Tag}}' \ | grep -E "^(${_be_repo}|${_fe_repo}):" \ | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \ diff --git a/infra/deploy/forgejo/README.md b/infra/deploy/forgejo/README.md index f15711f..461f1ad 100644 --- a/infra/deploy/forgejo/README.md +++ b/infra/deploy/forgejo/README.md @@ -7,12 +7,20 @@ deploy`s that happen to run on the manager node, vps2 — see (`75.119.132.91`) via the `role=forge` label from `deploy/swarm/label-forge-node.sh`. -The Actions **runner** is deliberately *not* part of this stack — see -`DEPLOY-DEV.md` and the note in `register-lan-runner.sh`'s own header for -where it runs today. That's the canonical placement per -`thermograph-docs/runbooks/implementation-handoff.md` Track B step 5; an -earlier revision of this stack ran the runner as a Swarm-scheduled -Docker-in-Docker sidecar pinned to the Forgejo node, which is gone now. +The Actions **runner** is deliberately *not* part of this stack: a +Swarm-scheduled runner cannot redeploy the Swarm that schedules it, so CI would +be unavailable exactly when the cluster is what needs repairing. An earlier +revision did run it as a Swarm-scheduled Docker-in-Docker sidecar pinned to the +Forgejo node; that is gone. + +Runners live in two places, and only one of them counts: + +- **`runner-vps2/`** — the always-on runner, a plain `restart: always` container + on vps2. This is the one CI depends on. +- **`register-lan-runner.sh`** — the desktop's systemd runner. Extra capacity. + **Nothing may assume it is up.** It was the only registered runner in the + estate until 2026-08-01, and its 21-hour outage on 2026-07-31 froze every + merge and deploy and stopped the nightly backup from firing. ## Prerequisites @@ -75,6 +83,58 @@ pattern as its other site blocks, same automatic-HTTPS. reach the registry over the mesh, not the public internet — see "Registry access from mesh clients" below. +## Migrating the domain (git.thermograph.org -> dev.jinemi.com) + +**Web UI and OAuth: done.** `dev.jinemi.com` is Forgejo's `ROOT_URL` +(`FORGEJO_DOMAIN` in `docker-stack.yml`), so it is the name Forgejo generates in +clone URLs, the Google OAuth callback, webhook payload URLs and mail links. + +**Registry: deliberately not done.** Renaming the registry host is a separate +migration from renaming the web UI, and the two need not happen together. + +`caddy-git.conf` lists both names on a *single* site block. That is deliberate, +not cosmetic: the `/v2/*` mesh-only matcher is per-block, so a second block for +either name would re-expose the registry API publicly and undo hazard #15. +Whatever else changes, keep the two names in one block. + +**`git.thermograph.org` must stay served.** It is not a courtesy redirect for +old bookmarks — CI resolves it: + +- images are named by registry host, and the `git.thermograph.org/` prefix is + baked into the runner labels (`register-lan-runner.sh`, + `runner-vps2/README.md`), the CI-runner image (`ci-runner/Dockerfile`), + `REGISTRY_HOST` in `infra/.env.example`, and every already-pushed tag; +- every mesh client in "Registry access from mesh clients" below pins + `git.thermograph.org` to `10.10.0.2` in `/etc/hosts`, which is what makes + Caddy's `/v2/*` matcher see a mesh source IP instead of returning 403; +- registered runners hold the instance URL they registered with + (`https://git.thermograph.org`); they keep working while that name resolves. + +### Before changing ROOT_URL again + +Login is Google-SSO-only, so the Google OAuth client must already carry a +matching redirect URI, or nobody can reach the UI — including to undo the +change. Both of these are registered on client +`337247886376-4360kepi20f24pue62ptg0lnqhcht46l.apps.googleusercontent.com`: + + https://dev.jinemi.com/user/oauth2/google/callback + https://git.thermograph.org/user/oauth2/google/callback + +Verify rather than assume. Google answers this with no credentials: a +registered URI serves the sign-in page, an unregistered one returns +`redirect_uri_mismatch`. Substitute the host you intend to make canonical: + +```sh +CID=337247886376-4360kepi20f24pue62ptg0lnqhcht46l.apps.googleusercontent.com +curl -sL "https://accounts.google.com/o/oauth2/v2/auth?client_id=$CID&redirect_uri=https%3A%2F%2F%2Fuser%2Foauth2%2Fgoogle%2Fcallback&response_type=code&scope=openid+email+profile&state=probe" \ + | grep -q redirect_uri_mismatch && echo REJECTED || echo ACCEPTED +``` + +The stack has **no auto-deploy**: a `FORGEJO_DOMAIN` change here only takes +effect when someone re-runs `docker stack deploy` by hand on the manager (vps2) +— see "Deploy / update" above. Until then the running service keeps whatever +`ROOT_URL` it was last deployed with, regardless of what this file says. + ## Registry access from mesh clients Any node that needs `docker login`/push/pull against the registry (the CI @@ -117,14 +177,48 @@ parallel, and even capacity 3 (an earlier, undocumented hand-tune) leaves `build-backend`/`build-frontend`/`validate-observability` queued behind those three before they get a slot. -**Adding more runner capacity should mean raising this number, or adding a -second runner alongside it — not putting a runner on prod or beta (vps2).** -`container.docker_host: automount` gives job containers the *host's* Docker -socket; on vps2 that would mean any CI job has root-equivalent access to both -the prod and beta stacks running there. An earlier revision of this stack -actually ran the runner as a Swarm-hosted container on the box Forgejo was -pinned to and was deliberately reverted for this same class of reason — see -the note at the top of `docker-stack.yml`. +**This paragraph used to say a runner must never go on vps2. That was reversed +deliberately on 2026-08-01, and the reasoning is worth keeping rather than +quietly deleting.** + +The original objection stands on its merits: `container.docker_host: automount` +gives job containers the *host's* Docker socket, so a runner on vps2 means any +CI job has root-equivalent access to both the prod and beta stacks running +there. An earlier revision of this stack ran the runner as a Swarm-hosted +container on the Forgejo node and was reverted for the same class of reason. + +What changed is that the alternative turned out to be worse. The desktop was +the estate's **only** registered runner, and when it went offline on +2026-07-31 nothing merged, nothing deployed, and the nightly backup did not +fire for 21 hours — on a repo where every branch is protected and every change +is a PR, a single absent runner freezes the whole estate. Availability of CI +is not a convenience here; it is what the backup and the deploy path both +hang off. + +So the runner on vps2 (`runner-vps2/`) is a considered trade, not an +oversight, and it is bounded rather than assumed benign: + +- `capacity: 1` — one job at a time, so a CI burst cannot contend with + `thermograph_web` for the box's six cores. +- `--cpus=2 --memory=4g` on job containers, set in `container.options`. Job + containers are siblings of the runner rather than children, so the runner's + own compose limits do not reach them; that option is the only lever that + does. +- `valid_volumes: []` — a job cannot bind-mount a host path, which is the + difference between "a job can read `/etc/thermograph.env`" and "a job + cannot", on the box where that file is prod's. +- The compose project joins no `thermograph` network. + +Be honest about what that buys: it is defence against **accident**, not +against a hostile workflow author. Forgejo's own database on vps1 already +stores `VPS2_SSH_KEY`, which is root on vps2, so the material was already +reachable — what changed is that it is now reachable by a *job* rather than +only at rest. On a two-person estate where both people can already SSH to that +box as root, that is the honest boundary. + +**If you are adding capacity rather than redundancy, raise `capacity` or add a +runner on a box that hosts nothing.** The vps2 runner exists so that CI has a +second home, not because prod is a good place to run CI. ## Custom CI job image (`ci-runner/`) @@ -143,14 +237,14 @@ bug hit during the frontend Go rewrite). `ci-runner` adds `docker-ce-cli` + `git`/`python3`/`python3-yaml` for the other jobs that need them (`shell-lint`, `observability-validate`). -Current tag: `git.thermograph.org/emi/thermograph/ci-runner:v2` (`v1` is +Current tag: `git.thermograph.org/jinemi/thermograph/ci-runner:v2` (`v1` is broken — do not register any runner against it). Rebuild/push (requires a PAT with `write:package` scope — the embedded git-remote token lacks it, same requirement documented in `build-push.yml`): ```bash -docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN deploy/forgejo/ci-runner -docker push git.thermograph.org/emi/thermograph/ci-runner:vN +docker build -t git.thermograph.org/jinemi/thermograph/ci-runner:vN deploy/forgejo/ci-runner +docker push git.thermograph.org/jinemi/thermograph/ci-runner:vN ``` `register-lan-runner.sh`'s `LABELS` default points at the current tag, so diff --git a/infra/deploy/forgejo/caddy-git.conf b/infra/deploy/forgejo/caddy-git.conf index 55cb9c7..1acf0cd 100644 --- a/infra/deploy/forgejo/caddy-git.conf +++ b/infra/deploy/forgejo/caddy-git.conf @@ -1,6 +1,6 @@ -# Only the block below (from "git.thermograph.org {") goes into the live -# Caddyfile — append just that part, not this instructional header, or it -# ends up as a confusing orphaned comment in production config with no +# Only the block below (from "git.thermograph.org, dev.jinemi.com {") goes into +# the live Caddyfile — append just that part, not this instructional header, or +# it ends up as a confusing orphaned comment in production config with no # surrounding context (docker-stack.yml isn't visible from there). # # Target: /etc/caddy/Caddyfile on vps1 (the box `forgejo` is pinned to — see @@ -9,19 +9,31 @@ # Forgejo publishes its web UI to 127.0.0.1:3080 only (host-local), and this # block is the only thing that ever talks to it. # -# Registry exposure (hazard #15, see docker-stack.yml's header comment): -# git.thermograph.org/v2/* is the built-in OCI registry API. It's blocked -# from anywhere except the WireGuard mesh (10.10.0.0/24) — reachable to CI -# runners and Swarm nodes over wg0, not from the public internet. Everything -# else (web UI, git-over-HTTP, PR pages) stays public like the rest of the -# site. +# TWO hostnames, ONE block, and that is load-bearing — see "Registry exposure" +# below. Both names must share a site block so the /v2/* restriction covers +# both. Splitting dev.jinemi.com into its own block serves the same Forgejo +# with the registry API open to the public internet. # -# Update the domain below to match whatever you actually pointed at vps1's -# IP, if not git.thermograph.org. +# dev.jinemi.com is CANONICAL — it is Forgejo's ROOT_URL (docker-stack.yml, +# FORGEJO_DOMAIN), so it is the name Forgejo puts in clone URLs, the OAuth +# callback, webhook payload URLs and mail links. +# +# git.thermograph.org is kept as a served alias, NOT as dead weight: the +# registry host baked into image names, the mesh /etc/hosts pins and the +# runners' registered instance URL all still say git.thermograph.org. Removing +# this name from the block breaks CI, not just old bookmarks. Both names get +# their own Let's Encrypt cert automatically (HTTP-01); both A records point +# at vps1. See README.md, "Migrating the domain". +# +# Registry exposure (hazard #15, see docker-stack.yml's header comment): +# /v2/* is the built-in OCI registry API. It's blocked from anywhere except +# the WireGuard mesh (10.10.0.0/24) — reachable to CI runners and Swarm nodes +# over wg0, not from the public internet. Everything else (web UI, +# git-over-HTTP, PR pages) stays public like the rest of the site. # --- copy from here down into /etc/caddy/Caddyfile --- -git.thermograph.org { +git.thermograph.org, dev.jinemi.com { encode zstd gzip @registry_external { diff --git a/infra/deploy/forgejo/ci-runner/Dockerfile b/infra/deploy/forgejo/ci-runner/Dockerfile index e4259e0..798d513 100644 --- a/infra/deploy/forgejo/ci-runner/Dockerfile +++ b/infra/deploy/forgejo/ci-runner/Dockerfile @@ -19,9 +19,9 @@ # class, and skips the per-job install entirely. # # Build/push (manual — see ../README.md for when to rebuild): -# docker build -t git.thermograph.org/emi/thermograph/ci-runner:vN \ +# docker build -t git.thermograph.org/jinemi/thermograph/ci-runner:vN \ # infra/deploy/forgejo/ci-runner -# docker push git.thermograph.org/emi/thermograph/ci-runner:vN +# docker push git.thermograph.org/jinemi/thermograph/ci-runner:vN # # Cutting over the live runner to a new tag means editing the `labels` array # in ~/forgejo-runner/.runner on the runner host directly (same runner diff --git a/infra/deploy/forgejo/docker-stack.yml b/infra/deploy/forgejo/docker-stack.yml index b939025..77e5466 100644 --- a/infra/deploy/forgejo/docker-stack.yml +++ b/infra/deploy/forgejo/docker-stack.yml @@ -3,11 +3,16 @@ # from the Terraform-managed docker-compose.yml that runs the Thermograph app # itself: this stack's only job is Forgejo and its container registry. # -# The Actions RUNNER is deliberately NOT a service in this stack. Per -# thermograph-docs/runbooks/implementation-handoff.md (Track B step 5), the canonical -# design registers the runner on the desktop node as a plain systemd service -# (deploy/forgejo/register-lan-runner.sh) — the same place the pre-Forgejo -# GitHub self-hosted runner already lived, not a Swarm-scheduled container. +# The Actions RUNNER is deliberately NOT a service in this stack, and that is a +# reasoned choice rather than an omission: a Swarm-scheduled runner cannot +# redeploy the Swarm that schedules it, so CI would be gone exactly when the +# cluster is the thing that is broken. +# +# The runner that CI depends on runs on vps2 as a plain restart:always container +# — deploy/forgejo/runner-vps2/. The desktop's systemd runner +# (deploy/forgejo/register-lan-runner.sh) stays registered as extra capacity. +# Nothing may assume the desktop is up: it was the ONLY runner until 2026-08-01, +# and its 21-hour outage on 2026-07-31 froze every merge, deploy and backup. # # No Traefik here. Forgejo is pinned to vps1 (node.labels.role == forge) # because that's what was chosen, and vps1 is ALSO the emigriffith.dev portfolio @@ -75,8 +80,25 @@ services: FORGEJO__database__HOST: db:5432 FORGEJO__database__NAME: forgejo FORGEJO__database__USER: forgejo - FORGEJO__server__DOMAIN: "${FORGEJO_DOMAIN:-git.thermograph.org}" - FORGEJO__server__ROOT_URL: "https://${FORGEJO_DOMAIN:-git.thermograph.org}/" + # Canonical domain. Forgejo has exactly ONE ROOT_URL and derives every + # absolute URL from it — clone URLs, the Google OAuth callback, webhook + # payload URLs, mail links. git.thermograph.org is still served (Caddy + # answers for both names off one site block, deploy/forgejo/caddy-git.conf) + # and still works for browsing, git-over-HTTP and the API; it is simply no + # longer the name Forgejo generates. + # + # Changing this breaks login unless the matching redirect URI exists on + # the Google OAuth client FIRST — login is SSO-only, so a mismatch locks + # everyone out of the UI, including out of the ability to undo it. Both + # https://dev.jinemi.com/user/oauth2/google/callback and the + # git.thermograph.org one are registered; verify with a probe before + # changing this again (deploy/forgejo/README.md, "Migrating the domain"). + # + # NOT changed by this: the registry host baked into image names, the mesh + # /etc/hosts pins, and the runners' registered instance URL all still say + # git.thermograph.org. That is a separate migration. + FORGEJO__server__DOMAIN: "${FORGEJO_DOMAIN:-dev.jinemi.com}" + FORGEJO__server__ROOT_URL: "https://${FORGEJO_DOMAIN:-dev.jinemi.com}/" FORGEJO__server__SSH_PORT: "2222" # Without this, Forgejo starts no SSH server at all despite SSH_PORT # being set — git@ clones get "connection refused", not a slow failure. diff --git a/infra/deploy/forgejo/register-lan-runner.sh b/infra/deploy/forgejo/register-lan-runner.sh index c204798..611155d 100755 --- a/infra/deploy/forgejo/register-lan-runner.sh +++ b/infra/deploy/forgejo/register-lan-runner.sh @@ -18,11 +18,16 @@ # breaks by keeping it registered — no workflow requests it — but do not build # anything new on it. # -# TODO(cutover): whether this machine keeps serving the `docker` label at all is -# an open call. It is no longer the only runner (the Swarm-hosted one is -# always-on), and the box's new job is AI model hosting plus flex Swarm-worker -# capacity. Keeping it is fine — it is real CI capacity — but the estate no -# longer depends on it, so decide deliberately rather than by inertia. +# This machine keeps serving the `docker` label as EXTRA capacity, and that is +# now a settled call rather than an open one. The claim it used to make here — +# that a Swarm-hosted runner was always-on, so the estate no longer depended on +# this box — was false: no such runner existed. This was the only registered +# runner in the estate until 2026-08-01, and when it went offline on 2026-07-31 +# every merge, deploy and backup stopped for 21 hours. +# +# The runner the estate depends on is deploy/forgejo/runner-vps2/. Keeping this +# one is worthwhile (a second runner means a queue drains instead of stalling), +# but nothing may be built on the assumption that this box is up. # # bash deploy/forgejo/register-lan-runner.sh # @@ -35,7 +40,7 @@ set -euo pipefail FORGEJO_URL="${1:?usage: $0 }" TOKEN="${2:?}" RUNNER_DIR="${RUNNER_DIR:-$HOME/forgejo-runner}" -LABELS="${LABELS:-docker:docker://git.thermograph.org/emi/thermograph/ci-runner:v2,thermograph-lan}" +LABELS="${LABELS:-docker:docker://git.thermograph.org/jinemi/thermograph/ci-runner:v2,thermograph-lan}" echo "==> Stopping and disabling the old GitHub Actions runner service, if present" systemctl --user stop github-actions-runner 2>/dev/null || true diff --git a/infra/deploy/forgejo/runner-vps2/.gitignore b/infra/deploy/forgejo/runner-vps2/.gitignore new file mode 100644 index 0000000..ebfdccb --- /dev/null +++ b/infra/deploy/forgejo/runner-vps2/.gitignore @@ -0,0 +1,3 @@ +# The runner registration (data/.runner) holds a long-lived runner token, and +# the daemon writes caches and job workspaces beside it. None of it belongs in git. +data/ diff --git a/infra/deploy/forgejo/runner-vps2/README.md b/infra/deploy/forgejo/runner-vps2/README.md new file mode 100644 index 0000000..2b1204c --- /dev/null +++ b/infra/deploy/forgejo/runner-vps2/README.md @@ -0,0 +1,140 @@ +# The always-on Actions runner (vps2) + +The runner CI actually depends on. The desktop runner +(`../register-lan-runner.sh`) stays registered as extra capacity, but nothing in +the estate may assume it is up. + +Labels: `docker` only. `thermograph-lan` is deliberately **not** registered here +— it was the host-native LAN-deploy label, dev moved to vps1 and is deployed +over SSH like beta and prod, and no workflow requests it any more. + +## Why this exists + +On 2026-07-31 16:31Z the desktop — then the estate's **only** registered runner +— went offline. For the next 21 hours nothing merged (every protected branch +requires a check that only a runner can produce), nothing deployed, and the +03:00Z `ops-cron` did not run. Forgejo held that scheduled run in `waiting` +rather than dropping it, so all three jobs completed on reconnect and no backup +was actually lost. A longer outage would not have been so kind. + +`docker-stack.yml` and this directory's parent README both described an +"always-on Swarm-hosted runner" during that entire window. There wasn't one; an +early revision had it as a Docker-in-Docker sidecar and it was removed. That is +why the single point of failure went unnoticed for weeks, and it is the reason +the root `CLAUDE.md` treats a stale doc as a correctness bug rather than a +documentation one. + +## Prerequisites + +- vps2's docker daemon is logged in to `git.thermograph.org` (it is — + `/root/.docker/config.json`), so the daemon can pull the job image. +- `git.thermograph.org` resolves to the **mesh** address from vps2 for registry + traffic. Job containers get this via `--add-host` in `config.yaml`; the host + daemon already has it. +- The job image exists: `git.thermograph.org/jinemi/thermograph/ci-runner:v2`. + Built from `../ci-runner/Dockerfile`; see that file for when to rebuild. + +## Where it is deployed, and why not from the checkout + +**`/opt/forgejo-runner/` on vps2**, holding a copy of `docker-compose.yml` and +`config.yaml` from this directory, plus a `data/` the runner owns. + +Deliberately *not* run in place from `/opt/thermograph/…/runner-vps2/`. That +checkout is `git reset --hard`'d on every prod deploy; untracked files do +survive that (it is the same property `deploy/.image-tags.env` relies on), but a +single `git clean -fdx` during troubleshooting would delete `data/.runner` and +de-register the runner. Putting the thing that repairs the estate inside the +tree the estate rewrites is the same mistake as scheduling it on the Swarm it +deploys. + +So this directory is the source of truth; vps2 gets a copy: + +```bash +install -d -m 0755 /opt/forgejo-runner +cp /opt/thermograph/infra/deploy/forgejo/runner-vps2/docker-compose.yml \ + /opt/thermograph/infra/deploy/forgejo/runner-vps2/config.yaml \ + /opt/forgejo-runner/ +``` + +Re-copy after changing either file here, then `docker compose up -d`. There is +no automation for that step and there should not be: a workflow that redeploys +the runner runs *on* the runner. + +## One-time registration + +The registration token is short-lived and single-purpose. Fetch it from the +Forgejo UI (**Site Administration → Actions → Runners → Create new Runner**) or +the API, and keep it out of shell history — it is a credential, however +briefly. + +```bash +cd /opt/forgejo-runner +mkdir -p data +cp config.yaml data/config.yaml + +read -rs REG_TOKEN && export REG_TOKEN # paste; not echoed, not in history + +docker run --rm -it \ + -v "$PWD/data:/data" -w /data --user root \ + code.forgejo.org/forgejo/runner:6.3.1 \ + forgejo-runner register --no-interactive \ + --instance https://git.thermograph.org \ + --token "$REG_TOKEN" \ + --name thermograph-vps2 \ + --labels 'docker:docker://git.thermograph.org/jinemi/thermograph/ci-runner:v2' + +unset REG_TOKEN +``` + +That writes `data/.runner`, which holds the **long-lived** runner token. It is +`0600` and must stay out of git — `data/` is gitignored for exactly this reason. +Re-running `register` with a fresh registration token replaces it; the old +registration then shows offline in the UI and should be deleted there. + +## Run it + +```bash +docker compose up -d +docker compose logs -f --tail=50 runner # expect "runner: ... successfully activated" +``` + +Confirm Forgejo agrees, rather than trusting the log: + +```bash +# on vps1, where Forgejo's database lives +fdb=$(docker ps -qf name=forgejo_db | head -1) +docker exec "$fdb" sh -c 'psql -U $POSTGRES_USER -d $POSTGRES_DB -c \ + "select name, to_timestamp(last_online) as last_online, agent_labels from action_runner order by id;"' +``` + +Two rows, both recent. `last_online` is the only honest liveness signal — a +runner process that is up but cannot reach Forgejo looks healthy locally and is +useless. + +## Upgrading + +Pin changes go in `docker-compose.yml` (`image:`) and are applied with +`docker compose up -d`. The runner token survives an image change; it lives in +`data/.runner`, not in the image. + +Changing the **job** image (the `docker://…ci-runner:vN` label) is a +re-registration or a hand-edit of `data/.runner`'s `labels` array — the same +caveat `../ci-runner/Dockerfile` records for the desktop runner. Keep the two +runners on the same job image or a workflow's behaviour starts depending on +which runner happened to pick it up. + +## Troubleshooting + +**Jobs queue forever with both runners online.** The workflow asked for a label +nobody registered. `runs-on: docker` is the only label this estate uses. + +**`docker pull` fails inside a job, works on the host.** The `--add-host` line +in `config.yaml` is missing or wrong. vps1's Caddy serves `/v2/*` to the mesh +only, and the job container has no other way to learn the mesh address. + +**Jobs fail with permission denied on the socket.** `user: root` was dropped +from the compose file. + +**The runner is up but `last_online` is stale.** It cannot reach +`https://git.thermograph.org` — check egress from vps2 and that Forgejo on vps1 +is serving. diff --git a/infra/deploy/forgejo/runner-vps2/config.yaml b/infra/deploy/forgejo/runner-vps2/config.yaml new file mode 100644 index 0000000..9862e4c --- /dev/null +++ b/infra/deploy/forgejo/runner-vps2/config.yaml @@ -0,0 +1,53 @@ +# Forgejo Actions runner config for the vps2 runner. Mounted at /data/config.yaml +# by runner-vps2/docker-compose.yml. +# +# Values that differ from `forgejo-runner generate-config` defaults are the only +# ones written out below; everything else is left at its default deliberately, so +# a future runner upgrade inherits new defaults instead of a frozen copy of the +# old ones. + +log: + level: info + job_level: info + +runner: + file: .runner + + # ONE job at a time. The desktop runner uses capacity 8, which is right for a + # box whose whole job is CI. This runner shares a host with prod, so the + # binding constraint is not throughput, it is that a CI burst must never be + # able to contend with thermograph_web for CPU. One job at a time also makes + # the resource ceiling below exact rather than a per-job estimate multiplied + # by an unknown. + capacity: 1 + + timeout: 3h + shutdown_timeout: 3h + +container: + # 1) --add-host: git.thermograph.org resolves publicly to vps1, but vps1's + # Caddy rejects the /v2/* registry API from off-mesh. Job containers that + # docker-pull or docker-push therefore need the MESH address, and DNS will + # not give it to them. Same line the desktop runner carries; the value is + # the same from vps2 because both boxes are on 10.10.0.0/24. + # + # 2) --cpus/--memory: vps2 runs prod. Without a ceiling a runaway job (a + # `docker build` that fans out, a test that leaks) competes with the + # production app for the same 6 cores. These bound the JOB container, which + # is a sibling of the runner rather than its child, so the runner's own + # compose limits do not cover it -- this is the only lever that does. + options: --add-host=git.thermograph.org:10.10.0.2 --cpus=2 --memory=4g + + # Jobs may mount NOTHING from the host. The workflows here do not need to: + # they check out into the job container's own workspace and reach the estate + # over SSH. An empty list is the difference between "a job can read + # /etc/thermograph.env" and "a job cannot", on the box where that file is + # prod's. + valid_volumes: [] + + privileged: false + + # Mounts the host docker socket into the job container. build-push.yml runs + # `docker build`, so it needs a daemon; this points it at vps2's. See the + # README for what that does and does not mean for trust. + docker_host: "automount" diff --git a/infra/deploy/forgejo/runner-vps2/docker-compose.yml b/infra/deploy/forgejo/runner-vps2/docker-compose.yml new file mode 100644 index 0000000..a85934d --- /dev/null +++ b/infra/deploy/forgejo/runner-vps2/docker-compose.yml @@ -0,0 +1,83 @@ +# The always-on Forgejo Actions runner, on vps2. +# +# cd /opt/forgejo-runner # a COPY of this file, not the checkout +# docker compose up -d +# +# It runs from /opt/forgejo-runner rather than in place, because the checkout is +# git reset --hard'd on every prod deploy and one `git clean -fdx` there would +# delete data/.runner. See README.md. +# +# Registration is a one-off and is NOT in this file, because it takes a +# short-lived token a human fetches. See README.md. +# +# ============================================================================ +# WHY A SECOND RUNNER EXISTS AT ALL +# ============================================================================ +# Until 2026-08-01 the estate had exactly one registered runner, on the desktop. +# It went offline at 2026-07-31 16:31Z and everything stopped for 21 hours: no +# merge could satisfy a required check, no deploy could run, and the 03:00Z +# ops-cron -- THE backup for both application databases and for Forgejo -- did +# not fire. Forgejo queued that run rather than dropping it, so it completed on +# reconnect, but a longer outage would have meant real backup gaps. +# +# So this runner is not extra capacity. It is the one that has to be up. +# +# ============================================================================ +# WHY A PLAIN CONTAINER AND NOT A SWARM SERVICE +# ============================================================================ +# Same argument as infra/openbao/config/openbao.hcl makes for OpenBao: the thing +# that repairs the estate must not depend on the estate. A Swarm-scheduled +# runner cannot redeploy the Swarm it is scheduled by, and if the cluster is the +# thing that is broken, CI is gone exactly when it is needed. `restart: always` +# on the local daemon has one dependency: the local daemon. +# +# An earlier revision of deploy/forgejo/docker-stack.yml did run a runner as a +# Swarm-scheduled Docker-in-Docker sidecar. It was removed, and the docs kept +# claiming an "always-on Swarm-hosted runner" for weeks afterwards -- which is +# how the single point of failure went unnoticed. +# +# ============================================================================ +# WHAT MOUNTING THE DOCKER SOCKET ON THE PROD HOST MEANS +# ============================================================================ +# Be plain about it: /var/run/docker.sock is root-equivalent on vps2, and vps2 +# runs prod. A workflow that can start a container can mount / and read +# /etc/thermograph.env. This runner does not create that exposure from nothing +# -- Forgejo's own database already stores VPS2_SSH_KEY, which is root on this +# box -- but it does move the exposure from "a secret at rest" to "a secret a +# job can reach", and that is a real difference. +# +# What bounds it, in order of how much each is worth: +# 1. Only this repo's workflows run here, and only the owner can push to it. +# 2. config.yaml sets valid_volumes: [] -- jobs cannot bind-mount host paths. +# 3. capacity: 1 and --cpus/--memory keep a job from contending with prod. +# 4. This compose project joins no thermograph network, so a job container +# reaches prod's services no more easily than any other host process. +# +# It is defence against accident, not against a hostile workflow author. On a +# two-person estate where both people can already SSH to this box as root, that +# is the honest boundary. + +name: forgejo-runner + +services: + runner: + image: code.forgejo.org/forgejo/runner:6.3.1 + restart: always + # The socket is root:docker on the host; the image's default user is not in + # that group. Running as root is what the socket mount requires. + user: root + working_dir: /data + command: forgejo-runner daemon --config /data/config.yaml + volumes: + - ./data:/data + - /var/run/docker.sock:/var/run/docker.sock + # Bounds the DAEMON. Job containers are siblings started through the socket, + # not children, so they are unaffected by these -- container.options in + # config.yaml is what bounds those. + cpus: 1.0 + mem_limit: 1g + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/infra/deploy/provision-dev.sh b/infra/deploy/provision-dev.sh index 1a2c776..b9e6dd5 100755 --- a/infra/deploy/provision-dev.sh +++ b/infra/deploy/provision-dev.sh @@ -26,7 +26,7 @@ SELF_DIR=$(cd "$(dirname "$0")" && pwd) . "$SELF_DIR/env-topology.sh" thermograph_topology dev -REPO_URL="${REPO_URL:-http://10.10.0.2:3080/emi/thermograph.git}" +REPO_URL="${REPO_URL:-http://10.10.0.2:3080/Jinemi/thermograph.git}" APP_DIR="${APP_DIR:-$TG_APP_DIR}" BRANCH="${BRANCH:-$TG_BRANCH}" diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index f53c149..c20a850 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -124,8 +124,8 @@ case "$SERVICE" in esac export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" -BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:$BACKEND_IMAGE_TAG" -FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:$FRONTEND_IMAGE_TAG" +BACKEND_IMAGE="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:$BACKEND_IMAGE_TAG" +FRONTEND_IMAGE="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:$FRONTEND_IMAGE_TAG" # --- timescale image pin --------------------------------------------------------- # Hazard #7: the db image under an existing volume must never drift. Resolve @@ -158,7 +158,7 @@ fi # --- registry ------------------------------------------------------------------ if [ -n "${REGISTRY_TOKEN:-}" ]; then - echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin fi echo "==> Pulling images" pull_ok=0 @@ -290,8 +290,8 @@ FRONTEND_IMAGE_TAG=$FRONTEND_IMAGE_TAG EOF # GC superseded app-image tags (keep the running pair), same as deploy.sh. -_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}" -_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}" +_be_repo="$REGISTRY_HOST/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}" +_fe_repo="$REGISTRY_HOST/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}" docker images --format '{{.Repository}}:{{.Tag}}' \ | grep -E "^(${_be_repo}|${_fe_repo}):" \ | grep -v -e "^${_be_repo}:${BACKEND_IMAGE_TAG}$" -e "^${_fe_repo}:${FRONTEND_IMAGE_TAG}$" \ diff --git a/infra/deploy/stack/lb/Caddyfile b/infra/deploy/stack/lb/Caddyfile index 46402ef..8ea7ffb 100644 --- a/infra/deploy/stack/lb/Caddyfile +++ b/infra/deploy/stack/lb/Caddyfile @@ -13,6 +13,34 @@ { auto_https off admin off + + # WITHOUT THIS, EVERY OAUTH REDIRECT URI THE BACKEND BUILDS IS http://. + # + # There are two Caddy hops in front of the app: the HOST Caddy terminates TLS + # and sets X-Forwarded-Proto: https, then proxies to 127.0.0.1:8137 over plain + # HTTP; this LB is the second hop. Caddy only preserves an incoming + # X-Forwarded-* header when the immediate peer is a TRUSTED proxy — otherwise + # it overwrites the header with the scheme of the connection it just received, + # which here is http. So the app saw http:// no matter how the user arrived. + # + # accounts/oauth.py:_redirect_uri builds the callback from x-forwarded-proto, + # and that URI must match what is registered in the provider's console. + # Discord tolerates the http:// form; GOOGLE REJECTS a non-HTTPS redirect URI + # for a web client outright, so Google sign-in could not work at all until + # this was fixed — independently of whether the credentials were configured. + # + # Measured on vps2 before the fix, same request to each hop: + # direct to web, XFP=https -> https://thermograph.org/api/v2/... + # through this LB, XFP=https -> http://thermograph.org/api/v2/... + # + # private_ranges covers 127.0.0.1/8 and the Docker bridge ranges, which is + # where the host Caddy reaches this container from. Nothing outside the box + # can connect here — the LB is bound to loopback precisely so it cannot be + # reached un-fronted (hazard #6, above) — so trusting the local peer does not + # widen anything: the only party that can set these headers is the host Caddy. + servers { + trusted_proxies static private_ranges + } } :8137 { diff --git a/infra/deploy/stack/lb/Caddyfile.beta b/infra/deploy/stack/lb/Caddyfile.beta index 756d21d..834f38b 100644 --- a/infra/deploy/stack/lb/Caddyfile.beta +++ b/infra/deploy/stack/lb/Caddyfile.beta @@ -17,6 +17,17 @@ { auto_https off admin off + + # Same reason as ./Caddyfile — see the long note there. Without it this second + # Caddy hop overwrites the host Caddy's X-Forwarded-Proto: https with http, + # and every OAuth redirect URI the backend builds comes out non-HTTPS, which + # Google rejects outright for a web client. + # + # Beta needs it as much as prod does, and arguably first: beta is where an + # OAuth provider change gets tested before it reaches thermograph.org. + servers { + trusted_proxies static private_ranges + } } :8137 { diff --git a/infra/deploy/stack/thermograph-beta-stack.yml b/infra/deploy/stack/thermograph-beta-stack.yml index 1d1aca2..bc788a8 100644 --- a/infra/deploy/stack/thermograph-beta-stack.yml +++ b/infra/deploy/stack/thermograph-beta-stack.yml @@ -62,7 +62,7 @@ services: beta-web: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: # Beta's OWN role and OWN database on the shared instance. `db` resolves @@ -108,7 +108,7 @@ services: failure_action: rollback beta-worker: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph_beta:${POSTGRES_PASSWORD}@db:5432/thermograph_beta @@ -144,7 +144,7 @@ services: condition: on-failure beta-lake: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: THERMOGRAPH_ROLE: lake @@ -176,7 +176,7 @@ services: failure_action: rollback beta-daemon: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} # Same reasoning as prod's daemon: NOT env-entrypoint.sh, because that shim # execs the image's own entrypoint (Alembic + uvicorn) and migrations belong # to the one-shot task. Source the host-rendered env and exec the binary. @@ -218,7 +218,7 @@ services: failure_action: rollback beta-frontend: - image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] # REQUIRED: overriding `entrypoint:` with no `command:` drops the image's # CMD entirely, and env-entrypoint.sh's fallback (`exec uvicorn app:app`) diff --git a/infra/deploy/stack/thermograph-stack.yml b/infra/deploy/stack/thermograph-stack.yml index 91fb062..f7b02b0 100644 --- a/infra/deploy/stack/thermograph-stack.yml +++ b/infra/deploy/stack/thermograph-stack.yml @@ -64,7 +64,7 @@ services: condition: on-failure web: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph @@ -109,7 +109,7 @@ services: failure_action: rollback worker: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD}@db:5432/thermograph @@ -153,7 +153,7 @@ services: # healthy and answers 503/404, and web falls through to NASA — the lake is # an accelerator, never a point of failure. lake: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] environment: THERMOGRAPH_ROLE: lake @@ -186,7 +186,7 @@ services: # (/usr/local/bin/thermograph-daemon, built into the backend image) and the # app share the /internal/* API contract, so one BACKEND_IMAGE_TAG rolls # them together and they can never skew. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} # Not env-entrypoint.sh: that shim's handoff execs the image's # deploy/entrypoint.sh (Alembic + uvicorn) whenever it exists, which is # exactly what this service must never run — migrations belong to the @@ -243,7 +243,7 @@ services: failure_action: rollback frontend: - image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] # REQUIRED, not cosmetic: overriding `entrypoint:` with no `command:` drops # the image's own CMD entirely (Docker/Swarm semantics, not merged) -- diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 1b62a06..28a7722 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -6,7 +6,7 @@ # build-push.yml -- backend = ${BACKEND_IMAGE_PATH}, frontend = # ${FRONTEND_IMAGE_PATH}, tagged independently by BACKEND_IMAGE_TAG / # FRONTEND_IMAGE_TAG. This replaces the earlier Stage-4 model where both -# containers shared the single emi/thermograph/app image and +# containers shared the single admin_emi/thermograph/app image and # THERMOGRAPH_SERVICE_ROLE picked the process; the split Dockerfiles now start # the right process directly (frontend the thermograph-frontend Go binary, # backend entrypoint.sh), so a backend deploy and a frontend deploy are fully @@ -99,10 +99,10 @@ services: # No `build:` here -- infra holds no Dockerfile; each service's Dockerfile # lives in its own app repo. deploy.sh sets BACKEND_IMAGE_TAG to the SHA # build-push.yml pushed for the backend commit being deployed; local dev - # builds `emi/thermograph/backend:local` from the backend repo (see + # builds `jinemi/thermograph/backend:local` from the backend repo (see # infra Makefile) and this pulls/uses it. BACKEND_IMAGE_PATH/TAG are # independent of the frontend's, so the two services deploy separately. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} depends_on: db: condition: service_healthy @@ -180,7 +180,7 @@ services: # without them the service stays healthy and every read falls through to # NASA. Never published on a host port: only the backend talks to it. lake: - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} environment: THERMOGRAPH_ROLE: lake PORT: 8141 @@ -207,7 +207,7 @@ services: # share the /internal/* API contract, so they must roll together — a # separate tag could skew them. It owns the Discord gateway websocket and # the recurring-job timers; anything needing data calls back into backend. - image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-jinemi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} # Bypass deploy/entrypoint.sh entirely: that script runs Alembic, and the # backend service's boot already owns migrations — two containers racing # `alembic upgrade head` against one database is a real hazard, not a @@ -262,7 +262,7 @@ services: # frontend/Dockerfile (which starts the thermograph-frontend Go binary # directly -- no THERMOGRAPH_SERVICE_ROLE process-picking). Independent # FRONTEND_IMAGE_TAG so a frontend deploy never disturbs the backend's tag. - image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:-local} + image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-jinemi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:-local} # Its own register() fetches the IndexNow key from backend at boot -- must # wait for a real, healthy backend, not just a started container. depends_on: diff --git a/infra/openbao/README.md b/infra/openbao/README.md index 14c6ac8..6e57f2c 100644 --- a/infra/openbao/README.md +++ b/infra/openbao/README.md @@ -206,9 +206,16 @@ infra/openbao/verify-parity.sh --env beta # expect 24 k cd /opt/thermograph-dev && infra/openbao/verify-parity.sh --env dev # expect 12 keys ``` -Beta needs no special handling: `env-topology.sh` derives `TG_BAO_APPROLE` per -environment, so beta authenticates as `tg-beta` rather than falling back to prod's -credentials and being denied its own path by `tg-host-prod.hcl`. +Beta needs no special handling: `verify-parity.sh` sources `env-topology.sh` and calls +`thermograph_topology` per environment, so `TG_BAO_APPROLE` points at +`/etc/thermograph/openbao-approle-beta` and beta authenticates as `tg-beta` rather than +falling back to prod's credentials and being denied its own path by `tg-host-prod.hcl`. + +That sourcing is load-bearing and was missing until 2026-08-01: this document asserted +the property while the verifier alone did not have it, so `--env beta` and `--all` both +took a 403 and the gate could not check the one environment whose isolation it exists +to prove. If you refactor `verify-parity.sh`, the `thermograph_topology` call is not +boilerplate. `seed-from-sops.sh` reads every production secret in plaintext. Per `infra/CLAUDE.md` the equivalent `seed-from-live.sh` is explicitly *not for an agent to run*; this @@ -224,10 +231,27 @@ One PR per hop, following the estate's own promotion model. + TG_SECRETS_BACKEND=openbao ``` -Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. Recommended -gate before prod: **7 consecutive green parity runs on all three environments**, run -nightly from `ops-cron.yml` over SSH (which gives continuous evidence without granting -CI any vault access). +Order: **dev → beta → prod**, with `verify-parity.sh` green throughout. The gate before +prod is **7 consecutive green parity runs on all three environments**. That runs +nightly from the `secrets-parity` job in `ops-cron.yml` over SSH, which gives +continuous evidence without granting CI any vault access — the host renders, CI only +asks it to. Forgejo's run history for that job is the record; a night the job was +skipped, or the runner was down, does not count as green. + +Measured 2026-08-01, immediately after the verifier was fixed: + +| env | keys | result | +|---|---|---| +| dev | 12 | **PASS** — byte-identical | +| beta | 24 | FAIL — 3 values differ | +| prod | 32 | FAIL — 3 values differ | + +Identical three keys on both: `THERMOGRAPH_S3_SECRET_KEY`, +`THERMOGRAPH_LAKE_S3_SECRET_KEY`, `THERMOGRAPH_VAPID_CONTACT`. All three live in +`common.yaml`, which was seeded on 2026-07-31 at 02:30Z — *before* the Contabo +rotation landed. dev passes because dev never layers `common`. One +`seed-from-sops.sh --env common` fixes beta and prod together; the 7-day clock starts +after that, not before. `TG_SECRETS_BACKEND=sops` remains a working two-way door for the whole period. Keep the SOPS path for a full release cycle after prod flips — it is the best mitigation diff --git a/infra/openbao/verify-parity.sh b/infra/openbao/verify-parity.sh index 2e12d82..f76775e 100755 --- a/infra/openbao/verify-parity.sh +++ b/infra/openbao/verify-parity.sh @@ -36,8 +36,13 @@ usage: verify-parity.sh (--all | --env ...) Renders each environment through BOTH backends and compares the resulting KEY=value sets. Prints key names and counts only, never values. -Exit status: 0 = every requested environment is at parity; 1 = a mismatch. -Suitable as a CI / cron gate. +--all means "every environment that lives on THIS host", not all three: dev is +on vps1, beta and prod are on vps2, so no single box can check them all. An +environment with no checkout here is skipped out loud. Skipping every one of +them is an error, not a pass. + +Exit status: 0 = every environment checked here is at parity; 1 = a mismatch, +or nothing was checkable. Suitable as a CI / cron gate. EOF exit 2 } @@ -52,20 +57,67 @@ while [ $# -gt 0 ]; do done [ ${#TARGETS[@]} -gt 0 ] || usage +# env-topology.sh FIRST, and this is not cosmetic. It is the only thing that sets +# TG_BAO_APPROLE, and without it render-secrets-openbao.sh:41 falls through to the +# bare default — prod's credential — for every environment. On vps2 that made +# `--env beta` authenticate as tg-prod and take a 403 from tg-host-prod.hcl on +# thermograph/data/env/beta, so the gate could not verify the one environment whose +# isolation it exists to prove. `--all` failed the same way, and dev with it. +# +# deploy.sh and deploy-stack.sh always sourced this; only the verifier did not, +# which is the worst place for the omission to be: the tool whose whole job is to +# notice a divergence was itself diverging from the path it verifies. +# +# shellcheck source=/dev/null +. "$INFRA_DIR/deploy/env-topology.sh" # shellcheck source=/dev/null . "$INFRA_DIR/deploy/render-secrets-openbao.sh" overall=0 +checked=0 for env_name in "${TARGETS[@]}"; do + # Per-environment topology: TG_BAO_APPROLE (beta's differs, because beta shares a + # filesystem with prod), TG_SKIP_COMMON and TG_APP_DIR. Called before the header + # line because --all walks three environments and may skip some of them. + if ! thermograph_topology "$env_name"; then + echo "!! unknown environment: $env_name" >&2 + overall=1; continue + fi + + # NO ENVIRONMENT CAN BE CHECKED FROM A HOST IT DOES NOT LIVE ON, and --all used to + # pretend otherwise. dev is on vps1; beta and prod are on vps2. Running --all + # anywhere therefore guaranteed a failure on whichever environments are elsewhere, + # and the failure was actively misleading: on vps1, prod resolves TG_BAO_APPROLE to + # /etc/thermograph/openbao-approle, which on THAT box holds dev's credential, so + # prod's check authenticated as tg-dev and took a 403. Fails closed, but reads like + # a policy bug rather than "wrong box". + # + # Identity comes from the ENVIRONMENT'S OWN ADDRESS being bound here, not from its + # checkout existing. The first version of this check tested for TG_APP_DIR and was + # wrong: vps1 still carries a stale /opt/thermograph from before the estate split, + # so prod looked resident there and was checked anyway. A leftover directory is + # exactly the kind of thing that outlives the arrangement it belonged to. + # + # Skipping is announced, never silent. A gate that quietly narrows what it checked + # is worse than one that fails, because the run still goes green. + if [ -n "${TG_SSH_HOST:-}" ] && command -v ip >/dev/null 2>&1; then + if ! ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \ + | grep -qxF "$TG_SSH_HOST"; then + echo "== parity check: $env_name — SKIPPED, lives on ${TG_HOST:-another host} (${TG_SSH_HOST}), not here" + continue + fi + fi + echo "== parity check: $env_name" + checked=$((checked + 1)) # dev renders WITHOUT common on both backends. On the SOPS side that is a caller - # flag; on the OpenBao side the policy also forbids it. Setting the flag here keeps - # the comparison apples-to-apples — and if the flag were wrong, the OpenBao side - # would fail closed with a 403 rather than quietly diverge, which is exactly the - # improvement being verified. - skip_common=0 - [ "$env_name" = dev ] && skip_common=1 + # flag; on the OpenBao side the policy also forbids it. Taking the flag from the + # topology rather than re-deriving it here keeps the comparison apples-to-apples + # AND keeps this script from drifting from deploy.sh:67, which reads the same + # variable. If the flag were wrong, the OpenBao side would fail closed with a 403 + # rather than quietly diverge — which is exactly the improvement being verified. + skip_common="${TG_SKIP_COMMON:-0}" sops_out=$(mktemp); bao_out=$(mktemp) # Both temp files hold PLAINTEXT SECRETS. Removed on every exit path below; not a @@ -146,11 +198,23 @@ for k in sorted(order): cleanup done +# Zero environments checked is a FAILURE, not a pass. Otherwise a --all run on a +# host that carries none of them — a new box, a renamed checkout, a typo'd --env — +# exits 0 having verified nothing, and a green run is exactly what the 7-day gate +# counts. "Nothing was wrong" and "nothing was examined" must not look alike. +if [ "$checked" -eq 0 ]; then + echo + echo "!! nothing was checked: none of [${TARGETS[*]}] has a checkout on this host." >&2 + echo "!! dev lives on vps1; beta and prod live on vps2. Run this where they are." >&2 + exit 1 +fi + if [ "$overall" = 0 ]; then echo - echo "PARITY OK for: ${TARGETS[*]}" + echo "PARITY OK — ${checked} environment(s) checked on this host." echo "Safe to consider flipping TG_SECRETS_BACKEND for these environments." - echo "Recommended gate before prod: 7 consecutive green runs on all three." + echo "Gate before prod: 7 consecutive green runs covering ALL THREE environments." + echo "That means both hosts — a green run here says nothing about the other box." else echo echo "PARITY FAILED — do NOT flip TG_SECRETS_BACKEND." >&2 diff --git a/infra/ops/ICEBERG-HANDOFF.md b/infra/ops/ICEBERG-HANDOFF.md index 9b015cc..65249d9 100644 --- a/infra/ops/ICEBERG-HANDOFF.md +++ b/infra/ops/ICEBERG-HANDOFF.md @@ -50,7 +50,7 @@ new network exposure and no interactive steps. **As originally written** (pre vps1/vps2 split; kept for history — see the note at the top of this document): -- Monorepo `emi/thermograph` (domain dirs: `backend/ frontend/ infra/ +- Monorepo `Jinemi/thermograph` (domain dirs: `backend/ frontend/ infra/ observability/`). Ops tooling lives in `infra/ops/`. - Environments: **dev** = LAN compose stack on the dev machine; **beta** = `75.119.132.91`; **prod** = `169.58.46.181`. SSH as `agent` with diff --git a/infra/terraform/README.md b/infra/terraform/README.md index 47717e5..ea040f2 100644 --- a/infra/terraform/README.md +++ b/infra/terraform/README.md @@ -42,7 +42,7 @@ infra repo**, not the app repo — the "branch" column above is which app-repo tag an environment is meant to track conceptually; the actual pinned versions are `var.hosts[*].environments[*].backend_image_tag` / `frontend_image_tag` (e.g. `"sha-<12 hex>"` each — the app is two separately-published images, -`emi/thermograph-backend/app` and `emi/thermograph-frontend/app`), since the +`admin_emi/thermograph-backend/app` and `admin_emi/thermograph-frontend/app`), since the host has no app checkout to derive a tag from. `app_dir` is required with no default specifically so vps2's two environments can never collide on the same checkout path (prod: `/opt/thermograph`, beta: `/opt/thermograph-beta`). The diff --git a/infra/terraform/modules/thermograph-host/main.tf b/infra/terraform/modules/thermograph-host/main.tf index 6d616df..6ec6423 100644 --- a/infra/terraform/modules/thermograph-host/main.tf +++ b/infra/terraform/modules/thermograph-host/main.tf @@ -239,7 +239,7 @@ resource "null_resource" "host" { # then bring the stack up on the EXPLICIT per-service app image tags — there's no # app checkout on this host to derive a tag from (see var.backend_image_tag / # frontend_image_tag). The compose file reads BACKEND_IMAGE_TAG / FRONTEND_IMAGE_TAG - # (for emi/thermograph-backend/app and emi/thermograph-frontend/app), so we export + # (for admin_emi/thermograph-backend/app and admin_emi/thermograph-frontend/app), so we export # those, not the old single IMAGE_PATH/IMAGE_TAG. docker runs as root (sources both # env files in the same shell) so it never depends on the docker group membership # taking effect in this session. @@ -257,7 +257,7 @@ resource "null_resource" "host" { render_thermograph_secrets ${var.app_dir} set -a; . /etc/thermograph-topology.env; . /etc/thermograph.env; set +a export REGISTRY_HOST="git.thermograph.org" BACKEND_IMAGE_TAG="${var.backend_image_tag}" FRONTEND_IMAGE_TAG="${var.frontend_image_tag}" - echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username admin_emi --password-stdin docker compose ${local.compose_flags} pull backend frontend docker compose ${local.compose_flags} up -d --remove-orphans ' diff --git a/infra/terraform/modules/thermograph-host/variables.tf b/infra/terraform/modules/thermograph-host/variables.tf index ceac2a6..6dcf7f6 100644 --- a/infra/terraform/modules/thermograph-host/variables.tf +++ b/infra/terraform/modules/thermograph-host/variables.tf @@ -31,12 +31,12 @@ variable "git_branch" { } variable "backend_image_tag" { - description = "Backend image tag to pull (emi/thermograph-backend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the backend-repo commit) or a semver tag. The host has no app-repo checkout to derive this from, so it's always explicit." + description = "Backend image tag to pull (admin_emi/thermograph-backend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the backend-repo commit) or a semver tag. The host has no app-repo checkout to derive this from, so it's always explicit." type = string } variable "frontend_image_tag" { - description = "Frontend image tag to pull (emi/thermograph-frontend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the frontend-repo commit) or a semver tag. Always explicit, same as backend_image_tag." + description = "Frontend image tag to pull (admin_emi/thermograph-frontend/app), e.g. \"sha-<12 hex>\" (build-push.yml's tag for the frontend-repo commit) or a semver tag. Always explicit, same as backend_image_tag." type = string } diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example index 7b80ef5..41086b0 100644 --- a/infra/terraform/terraform.tfvars.example +++ b/infra/terraform/terraform.tfvars.example @@ -151,13 +151,13 @@ hosts = { # } # Optional overrides (shown with their defaults): -# repo_url = "https://git.thermograph.org/emi/thermograph-infra.git" +# repo_url = "https://git.thermograph.org/admin_emi/thermograph-infra.git" # app_port = 8137 # frontend_port = 8080 # # thermograph-infra is a PRIVATE repo, so a host cloning it for the first time # needs read credentials embedded in repo_url, e.g. a Forgejo deploy token: -# repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/emi/thermograph-infra.git" +# repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/admin_emi/thermograph-infra.git" # --------------------------------------------------------------------------------- # Self-hosted Open-Meteo archive (only used by environments with openmeteo = true) diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index d6c187a..d2a0992 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -54,8 +54,8 @@ variable "hosts" { git_branch = string # this INFRA repo's branch the checkout is reset to # Which app images to run, e.g. "sha-<12 hex>" (each matching build-push.yml's tag # for the commit that app repo built) or a semver tag on a release push. The app is - # TWO separately-published images now — emi/thermograph-backend/app and - # emi/thermograph-frontend/app — pinned independently. Required, no default: the + # TWO separately-published images now — admin_emi/thermograph-backend/app and + # admin_emi/thermograph-frontend/app — pinned independently. Required, no default: the # host's checkout is this infra repo, not the app repos, so there's no "current # commit" to derive a tag from; both must be explicit. Bump these (via a normal # tfvars edit + apply) whenever an app repo ships a commit you want this @@ -127,9 +127,9 @@ variable "gcp_hosts" { } variable "repo_url" { - description = "Git remote to clone from when a host has no checkout yet. Points at THIS repo (thermograph-infra) now, not the app repo -- the app's own source is never checked out on a host; only its published registry images are pulled (see var.hosts[*].backend_image_tag / frontend_image_tag). thermograph-infra is a private repo, so this typically needs embedded read credentials, e.g. a Forgejo deploy token: \"https://:@git.thermograph.org/emi/thermograph-infra.git\"." + description = "Git remote to clone from when a host has no checkout yet. Points at THIS repo (thermograph-infra) now, not the app repo -- the app's own source is never checked out on a host; only its published registry images are pulled (see var.hosts[*].backend_image_tag / frontend_image_tag). thermograph-infra is a private repo, so this typically needs embedded read credentials, e.g. a Forgejo deploy token: \"https://:@git.thermograph.org/admin_emi/thermograph-infra.git\"." type = string - default = "https://git.thermograph.org/emi/thermograph-infra.git" + default = "https://git.thermograph.org/admin_emi/thermograph-infra.git" sensitive = true } diff --git a/observability/README.md b/observability/README.md index e2ac68c..d2eb77e 100644 --- a/observability/README.md +++ b/observability/README.md @@ -62,7 +62,7 @@ persistent, queryable, fleet-wide log store and UI. > ### ⚠️ Merging is not deploying > > `/opt/observability` on vps1 (and vps2) is a checkout of the **archived** -> `emi/thermograph-observability` repo. It can never `git pull` again — the +> `admin_emi/thermograph-observability` repo. It can never `git pull` again — the > content now lives here, in the mono repo, under `observability/`. A previous > observability PR merged and had **zero live effect** for exactly this reason. >