diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a051c8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Terraform state/vars are also ignored inside terraform/.gitignore; repeated here +# in case a stray file ever lands at repo root. +*.tfstate +*.tfstate.* +.terraform/ +*.tfvars +!*.tfvars.example +crash.log +crash.*.log + +# The SOPS+age private key must never be committed (deploy/secrets/*.yaml, the +# encrypted values, are meant to be committed). +age.key +*.age.key + +.DS_Store diff --git a/INFRA.md b/ACCESS.md similarity index 92% rename from INFRA.md rename to ACCESS.md index 5d1c182..8ff1f4f 100644 --- a/INFRA.md +++ b/ACCESS.md @@ -1,15 +1,16 @@ # Agent access, Docker Swarm, and Forgejo CI/CD -This covers three additive infrastructure layers on top of the app -containerization / Postgres / Terraform work described in -**`thermograph-docs/runbooks/implementation-handoff.md`** (read that first — it's the -canonical Track A/Track B plan this whole effort implements) and -`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. Terraform provisions -**prod** (the new 48 GB / 12-core box, `thermograph.org`) and **beta** (the -old VPS, `75.119.132.91`) — see `terraform/README.md` / -`terraform.tfvars.example`. **None of this touches `backend/`, `Dockerfile`, -`docker-compose*.yml`, `terraform/`, or `deploy/db/`** — those stay owned by -the app-containerization work; this is strictly the layer above it. +This covers three additive infrastructure layers on top of the host +provisioning / Postgres / Terraform work described in `terraform/README.md` +and (for the historical Track A/Track B plan this whole effort grew from, +including the decision to split this repo out of the app monorepo) +`thermograph-docs/runbooks/implementation-handoff.md` and +`thermograph-docs/architecture/repo-topology-and-infrastructure.md`. Terraform +provisions **prod** (the new 48 GB / 12-core box, `thermograph.org`) and +**beta** (the old VPS, `75.119.132.91`) — see `terraform/README.md` / +`terraform.tfvars.example`. **None of this touches the app repo** (its +backend/frontend source, `Dockerfile`, or CI) — this repo owns only how and +where the already-built app image runs; the app repo owns building it. ``` Track 1: deploy/provision-agent-access.sh — a dedicated full-root login for me diff --git a/README.md b/README.md new file mode 100644 index 0000000..48f8e47 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# thermograph-infra + +Infrastructure for [Thermograph](https://thermograph.org): Terraform host +provisioning, the SOPS+age secrets vault, Docker Swarm/WireGuard networking, +Forgejo, Caddy, and the deploy scripts that run the already-built app image on +each host. Extracted from the app monorepo (`emi/thermograph`) — the app repo +owns building and testing the app; this repo owns running it. + +- **`terraform/`** — provisions/configures hosts (SSH-driven by default; an + optional GCP-creating module is scaffolded, no live resources yet) and + triggers each deploy. See `terraform/README.md`. +- **`deploy/secrets/`** — the git-native SOPS+age secrets vault (every app + secret, encrypted at rest, rendered at deploy time). See + `deploy/secrets/README.md`. +- **`deploy/swarm/`, `deploy/forgejo/`** — the 3-node WireGuard/Swarm cluster + that hosts Forgejo (git + CI + registry); does not run the app itself. See + `ACCESS.md` and the READMEs under each directory. +- **`deploy/deploy.sh`** — pulls the pinned app image (`IMAGE_TAG`) and rolls + the compose stack; invoked by Terraform and by the app repo's + `.forgejo/workflows/deploy.yml` over SSH. +- **`docker-compose*.yml`, `docker-stack.yml`** — how the app image runs + (compose in production today; `docker-stack.yml` is a design record for a + possible future Swarm-based app deploy, not currently live). + +The app's own source, `Dockerfile`, and build/test CI stay in the app repo — +this repo never checks out app source; hosts only pull tagged images from the +registry. See `ACCESS.md` for host access and the Swarm/Forgejo topology, and +`terraform/README.md` for the day-to-day `plan`/`apply` workflow. diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 2efa4a9..ad400a3 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash -# Pull the latest code and roll the docker-compose stack (backend + frontend + -# PostgreSQL). Run on the VPS — the Forgejo Actions workflow invokes this over -# SSH, and you can run it by hand too. +# Pull this INFRA repo's checkout up to date, then roll the docker-compose stack +# (backend + frontend + PostgreSQL) onto the app image tagged IMAGE_TAG. Run on +# the VPS — the app repo's Forgejo Actions workflow invokes this over SSH (see +# .forgejo/workflows/deploy.yml there), and you can run it by hand too. # -# ssh deploy@vps '/opt/thermograph/deploy/deploy.sh' +# ssh deploy@vps 'IMAGE_TAG=sha-<12hex> /opt/thermograph/deploy/deploy.sh' +# +# This checkout is thermograph-infra, not the app repo: BRANCH is this repo's +# branch (compose files, db init, the secrets vault, this script itself); +# IMAGE_TAG is the separately-published app image to run (see the IMAGE_TAG +# guard below) -- the two are independent and rarely change together. set -euo pipefail APP_DIR="${APP_DIR:-/opt/thermograph}" @@ -70,14 +76,16 @@ if [ -z "${DEPLOY_SH_REEXECED:-}" ]; then exec "$0" "$@" fi -# Registry-pull cutover (repo-split Stage 6): pull the tag build-push.yml -# already pushed for this exact commit instead of building in place. Mirrors -# build-push.yml's own tag computation (sha-<12 hex>) so this always resolves -# to "whatever HEAD of this branch built to" -- no separate promotion step. +# Registry-pull cutover (repo-split Stage 6): pull the app image build-push.yml +# (in the APP repo) already built and pushed, instead of building in place. This +# checkout is thermograph-infra, not the app repo, so there is no "current commit" +# to derive the tag from the way the old same-repo deploy.sh once did -- IMAGE_TAG +# must be supplied explicitly (e.g. by the caller exporting +# IMAGE_TAG="sha-<12 hex>" matching the app-repo commit to run, or a semver tag). REGISTRY_HOST="${REGISTRY_HOST:-git.thermograph.org}" IMAGE_PATH="${IMAGE_PATH:-emi/thermograph/app}" -export IMAGE_TAG="sha-$(git rev-parse --short=12 HEAD)" -export REGISTRY_HOST IMAGE_PATH +IMAGE_TAG="${IMAGE_TAG:?set IMAGE_TAG=sha-<12-hex-app-commit> (or a semver tag) -- this checkout can't derive it, it isn't the app repo}" +export IMAGE_TAG REGISTRY_HOST IMAGE_PATH echo "==> Logging in to the registry ($REGISTRY_HOST)" echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index e59e902..a015219 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -1,6 +1,26 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/hashicorp/google" { + version = "6.50.0" + constraints = "~> 6.0" + hashes = [ + "h1:faTJQOetP9/RYuHwA3r2SWnuYoyzQNm4tUWZrZggcgY=", + "zh:1f3513fcfcbf7ca53d667a168c5067a4dd91a4d4cccd19743e248ff31065503c", + "zh:3da7db8fc2c51a77dd958ea8baaa05c29cd7f829bd8941c26e2ea9cb3aadc1e5", + "zh:3e09ac3f6ca8111cbb659d38c251771829f4347ab159a12db195e211c76068bb", + "zh:7bb9e41c568df15ccf1a8946037355eefb4dfb4e35e3b190808bb7c4abae547d", + "zh:81e5d78bdec7778e6d67b5c3544777505db40a826b6eb5abe9b86d4ba396866b", + "zh:8d309d020fb321525883f5c4ea864df3d5942b6087f6656d6d8b3a1377f340fc", + "zh:93e112559655ab95a523193158f4a4ac0f2bfed7eeaa712010b85ebb551d5071", + "zh:d3efe589ffd625b300cef5917c4629513f77e3a7b111c9df65075f76a46a63c7", + "zh:d4a4d672bbef756a870d8f32b35925f8ce2ef4f6bbd5b71a3cb764f1b6c85421", + "zh:e13a86bca299ba8a118e80d5f84fbdd708fe600ecdceea1a13d4919c068379fe", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fec30c095647b583a246c39d557704947195a1b7d41f81e369ba377d997faef6", + ] +} + provider "registry.terraform.io/hashicorp/local" { version = "2.9.0" constraints = "~> 2.5" @@ -43,23 +63,23 @@ provider "registry.terraform.io/hashicorp/null" { ] } -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.6" +provider "registry.terraform.io/hashicorp/time" { + version = "0.14.0" + constraints = "~> 0.12" hashes = [ - "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", - "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", - "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", - "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", - "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", - "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", - "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", - "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "h1:4EThC3ocCFiFPMZQSUvSGSxoJqBcGWxMcFYmL67uS7Y=", + "zh:12abfd6b800e4d7fa6db7310dec8ffd440b31993861ef188c7ed5260b3073937", + "zh:23005521e800bb19e1597bf755c5f70d675d30b685d4255001ed5fa47d9df3f1", + "zh:2fea249b582ae97cd1cc10385187ea50993bb47c28cc5df0305e57ceaabf0a10", + "zh:322018d3b987b7aad08697178029a2bb667bed699e88328f0c89c52a2fd41341", + "zh:32a08e98fce2d273cb9b2c89d6c54727cc9f0a32e15bfd896be4e02cc6b48f95", + "zh:3db89aabd0e619616bd4b0f8b373a7586dfe60feffcea12a84a0bdbc445714b3", + "zh:7488f56c81d742dc020f29063626c8f07ca188aa97be61e7307e8d62397020a2", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", - "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", - "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", - "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", - "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + "zh:7cb4067f2e7559b13f7562ef722f948950901eb37834873e98360ab28f66e9d7", + "zh:9d552c8345f61e1b7db8e725144981345f18ac1014d58d6f5ddf0928a195fffb", + "zh:a8e69fb6b97fc9d86fb19a9f4d42abe33c4a68e700b15387ce2e17d2b9934bed", + "zh:aeeb900eb8dd0f790c60ea5c0e0c8d42bd6e4a54f391681d4decca15b544394b", + "zh:c239c619101a8c95e1f14061eb973c57a8d15fa0e68878ced5bbd76858ee5b79", ] } diff --git a/terraform/README.md b/terraform/README.md index e4e7ef1..ba48f0c 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,47 +1,78 @@ # Thermograph — Terraform (host provisioning) -Terraform that **provisions and configures the existing VPS hosts** and hands the app -off to `docker compose`. It does *not* create servers (no cloud provider) and does not -replace compose — it prepares each host (Docker, firewall, checkout, secrets, Caddy) -and runs `docker compose up`. +Terraform that **provisions and configures VPS hosts** and hands the app off to +`docker compose`, which pulls the published app image (built + pushed by the app +repo's CI) and runs it — no app source is ever built or checked out on a host. +By default it does *not* create servers: `modules/thermograph-host` is +SSH-provisioner-driven against a host that already exists. An optional +`modules/gcp-host` can additionally *create* the VM on GCP first (see "GCP +scaffold" below) — scaffold-only today, no live resources until you opt in. ## What it manages One reusable module (`modules/thermograph-host`) is instantiated per host via -`for_each`. This config manages **two VPS hosts**: +`for_each` (`main.tf`'s `local.all_hosts`, merging `var.hosts` — SSH-managed, +already-existing boxes — with any `var.gcp_hosts` Terraform created itself). +This config manages **two VPS hosts** today: | key | role | VPS | branch | domain | notes | |--------|--------|-------------------------|-----------|-------------------|------------------------------------| -| `prod` | prod | NEW 48 GB / 12-core box (`169.58.46.181`) | `release` | `thermograph.org` | Caddy TLS; sized up (8/8/4/16g) | -| `beta` | beta | old box `75.119.132.91` | `main` | `beta.thermograph.org` | Caddy TLS; also hosts Forgejo | +| `prod` | prod | NEW 48 GB / 12-core box (`169.58.46.181`) | `release` (of the APP repo — see `app_image_tag`) | `thermograph.org` | Caddy TLS; sized up (8/8/4/16g) | +| `beta` | beta | old box `75.119.132.91` | `main` (of the APP repo) | `beta.thermograph.org` | Caddy TLS; also hosts Forgejo | -The `dev` branch is **out of scope here** — it deploys to the LAN dev server via -`deploy/deploy-dev.sh` (a self-hosted **Forgejo** Actions runner), not Terraform. +Each host's own checkout on disk (`app_dir`, `git_branch`) is **this infra +repo**, not the app repo — the "branch" column above is which app-repo tag a +host is meant to track conceptually; the actual pinned version is +`var.hosts[*].app_image_tag` (e.g. `"sha-<12 hex>"`), since the host has no app +checkout to derive a tag from. The LAN dev server is **out of scope here** — it +builds from source via the app repo's `deploy/deploy-dev.sh` (a self-hosted +Forgejo Actions runner), not Terraform. Per host, over SSH provisioners, Terraform: - installs Docker + the compose plugin if missing; - configures a `ufw` firewall (22/80/443 always; on a host with **no** domain it also opens the app port `8137`); -- ensures the git checkout at `app_dir` exists (clones on a fresh box) and resets it to - the host's branch; -- renders `/etc/thermograph.env` from Terraform variables (secrets injected from tfvars, - pushed via provisioner `content` so they never touch local disk) and installs it - root-owned `0640`; +- ensures the git checkout at `app_dir` exists (clones **this repo** on a fresh box) + and resets it to the host's `git_branch`; +- renders `/etc/thermograph-topology.env` from Terraform variables (non-secret sizing + only — `WORKERS`/`APP_CPUS`/`DB_CPUS`/`DB_MEMORY`/etc.), then runs + `deploy/render-secrets.sh` (from that same freshly-synced checkout) to render + `/etc/thermograph.env` from the SOPS+age vault — **Terraform itself never sees or + carries an app secret** (see "Secrets" below); - for a host **with** a domain, installs a rendered Caddyfile and reloads Caddy; -- brings the stack up: `docker compose <-f each compose file> up -d --build`, running - docker as root with `/etc/thermograph.env` sourced in the same shell; +- brings the stack up on the explicit `app_image_tag`: `docker login` to the registry, + `docker compose <-f each compose file> pull backend frontend`, then + `... up -d --remove-orphans`, running docker as root with both env files sourced; - health-checks `http://127.0.0.1:8137/`. -A change to the rendered env, the compose files, the branch, or the sizing flips the -`null_resource` trigger and re-runs the provisioners on next apply. +A change to the rendered topology env, the compose files, the branch, the app image +tag, or the sizing flips the `null_resource` trigger and re-runs the provisioners on +next apply. + +## GCP scaffold (no live resources) + +`modules/gcp-host` can create a GCE VM (+ a dedicated VPC, subnet, and a firewall +opening 22/80/443) instead of assuming the host already exists — see `main.tf`'s +`module.gcp_vm`. It contributes *nothing* to provisioning: its only output +(`external_ip`) feeds into the exact same `thermograph-host` module every +SSH-managed host uses, so there's one provisioning path regardless of how a +host came to exist. `var.gcp_hosts` defaults to `{}`, so by default no +`google_*` resource is ever planned and the `google` provider is never +invoked — `terraform plan`/`validate` succeed with no GCP credentials +configured at all. To actually use it: populate an entry in +`terraform.tfvars` (see the commented example in +`terraform.tfvars.example`) and authenticate via `gcloud auth +application-default login` or `GOOGLE_APPLICATION_CREDENTIALS`. This is the +same create-then-provision composition a future Proxmox module (the +architecture doc's longer-term target — see §6/§9 there) would use. ### Container sizing is env-driven `docker-compose.yml` reads `WORKERS`, `APP_CPUS`, `DB_CPUS`, and `DB_MEMORY` from the environment (defaults `4 / 4 / 2 / 8g`, identical to before). Terraform sets them per -host through `/etc/thermograph.env`, so the big prod box can run larger caps without a -compose edit. The Postgres *internal* memory budget (`shared_buffers`, +host through `/etc/thermograph-topology.env`, so the big prod box can run larger caps +without a compose edit. The Postgres *internal* memory budget (`shared_buffers`, `effective_cache_size`, `work_mem`, `maintenance_work_mem`) is derived from the same `DB_MEMORY` by `deploy/db/init/20-tuning.sh` — so raising `db_memory` scales the container cap and the tuning together (prod 16g → shared_buffers 4 GB). The tuning @@ -51,8 +82,10 @@ header). ## Prerequisites - Terraform >= 1.6 (v1.15 is installed). -- SSH key access to **both** hosts as a **sudo-capable** user (default `deploy`). Point - `ssh_private_key_path` at that key (`~` is expanded). +- SSH key access to **both** hosts as a **sudo-capable** user (the live boxes use the + dedicated `agent` account, `~/.ssh/thermograph_agent_ed25519` — see + `deploy/provision-agent-access.sh`). Point `ssh_private_key_path` at that key (`~` + is expanded). - The hosts are Debian/Ubuntu with `apt` and outbound internet (Docker/Caddy installs pull from the network). Docker may already be present — installs are conditional. - For the `prod` host: DNS for `thermograph.org` must point at the new box before apply, @@ -70,33 +103,40 @@ terraform apply Target one host with `-target='module.host["beta"]'` if you want to apply to just one. -## Local state + secrets caveat +## Secrets -The backend is **local**: `terraform.tfstate` is written next to the config and holds -every secret in cleartext (the rendered env, VAPID keys, DB password, …). It is -gitignored (`terraform/.gitignore` and the root `.gitignore`). Keep it off shared disks -and back it up somewhere private. `terraform.tfvars` is likewise gitignored; only -`terraform.tfvars.example` (dummy values) is committed. `.terraform.lock.hcl` **is** -committed so provider versions are pinned across machines. +App secrets (`POSTGRES_PASSWORD`, `THERMOGRAPH_AUTH_SECRET`, VAPID keys, +`REGISTRY_TOKEN`, Discord/SMTP credentials, …) are **not** Terraform variables +— they live in the git-native SOPS+age vault at `../deploy/secrets/*.yaml` +(committed, encrypted) and are rendered into `/etc/thermograph.env` at deploy +time by `../deploy/render-secrets.sh`, which the provisioner's deploy step +runs from the freshly-synced checkout. Terraform only renders the non-secret +`/etc/thermograph-topology.env` (sizing/routing). See +`../deploy/secrets/README.md` to rotate a secret or add a new one — it's a +`sops edit` + commit + deploy, no Terraform apply involved. -**Terraform now originates, not just carries, some of these secrets** -(`postgres_password`/`auth_secret`/`metrics_token`/`indexnow_key` — see `secrets.tf`): -left unset, each is generated by the `random` provider and its value lives *only* in -`terraform.tfstate` — there is no other copy until it's rendered to a host's -`/etc/thermograph.env`. More generated secrets landing in state makes the local-state -posture above matter more, not less: **never commit or sync `terraform.tfstate` in -cleartext**, and before this config is used from a shared machine, CI, or a remote -backend, stand up an encrypted backend (or SOPS-wrap the state) first — see the hop-1 -cutover runbook's hazard #15 and Track B step 7. Until then, treat `terraform.tfstate` -with the same care as the secrets it contains. +`om_rclone_conf` (object-storage bucket credentials for the self-hosted ERA5 +archive) is the one exception still supplied via Terraform, in +`terraform.tfvars` — folding it into the vault too is a reasonable future +step, not done here. + +## Local state + +The backend is **local**: `terraform.tfstate` is written next to the config. +It's gitignored (`terraform/.gitignore` and the root `.gitignore`) — keep it +off shared disks and back it up somewhere private. `terraform.tfvars` is +likewise gitignored (it may carry a `repo_url` credential and always carries +`om_rclone_conf`); only `terraform.tfvars.example` (dummy values) is +committed. `.terraform.lock.hcl` **is** committed so provider versions are +pinned across machines. ## WARNING — applying against live prod `terraform apply` runs `remote-exec` **on the server**: it resets the checkout to the -branch, rewrites `/etc/thermograph.env`, and runs `docker compose up -d --build` -(rebuilding images and recreating containers — a brief app restart). Against the live -production host this is a real deploy. Review the plan, apply in a maintenance window, -and prefer `-target` to touch one host at a time. +branch, renders topology config + secrets, and runs `docker compose pull && up -d` +(pulling the pinned `app_image_tag` and recreating containers — a brief app restart). +Against the live production host this is a real deploy. Review the plan, apply in a +maintenance window, and prefer `-target` to touch one host at a time. This is **separate from the Postgres data cutover** in `deploy/POSTGRES-MIGRATION.md`. Terraform provisions the host and starts the stack; it does **not** migrate the diff --git a/terraform/main.tf b/terraform/main.tf index 57c5a7e..b8c8f98 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,3 +1,11 @@ +# No project/region default here on purpose: each var.gcp_hosts entry carries its +# own project + zone (a fleet could span projects), and with var.gcp_hosts empty +# this provider is never invoked, so there's nothing to default. When you do +# populate var.gcp_hosts, authenticate via Application Default Credentials +# (`gcloud auth application-default login`) or GOOGLE_APPLICATION_CREDENTIALS -- +# no credentials are configured in this repo. +provider "google" {} + locals { # Repo root (one level above this terraform/ dir). The module hashes the compose # files here so a compose change re-triggers the remote deploy, and this is the @@ -16,13 +24,52 @@ locals { medium = { workers = 6, app_cpus = 6, db_cpus = 3, db_memory = "12g" } large = { workers = 8, app_cpus = 8, db_cpus = 4, db_memory = "16g" } } +} - # Per-host resolved sizing: a named tier (host.size) wins when set; otherwise - # the host's own workers/app_cpus/db_cpus/db_memory fields (each individually +# GCP-created hosts (empty by default — see variables.tf and modules/gcp-host). +# Creates only the VM + minimal networking; every provisioning behavior comes from +# the SAME thermograph-host module every SSH-managed host below uses, via +# local.all_hosts merging its output IP in below. No live resources exist until +# var.gcp_hosts is populated. +module "gcp_vm" { + source = "./modules/gcp-host" + for_each = var.gcp_hosts + + name = each.key + project = each.value.project + zone = each.value.zone + machine_type = each.value.machine_type + ssh_user = each.value.ssh_user + ssh_public_key_path = each.value.ssh_public_key_path +} + +locals { + # Every host this config manages, SSH-only (var.hosts) plus GCP-created (whose + # `host` is filled in from the VM Terraform just created) — one unified map so a + # single `module.host` for_each below handles both without duplicating any + # provisioning logic. GCP hosts always use a named size tier (simpler than + # exposing the four raw sizing fields on that variable too). + all_hosts = merge( + var.hosts, + { + for name, h in var.gcp_hosts : name => merge(h, { + host = module.gcp_vm[name].external_ip + workers = null + app_cpus = null + db_cpus = null + db_memory = null + openmeteo = false + om_data_dir = "/mnt/om-archive" + }) + } + ) + + # Per-host resolved sizing: a named tier (host.size) wins when set; otherwise the + # host's own workers/app_cpus/db_cpus/db_memory fields (each individually # defaulted in variables.tf) — so existing tfvars with explicit numbers are # unaffected, and a tier is purely an opt-in shortcut. host_sizing = { - for name, h in var.hosts : name => h.size != null ? local.sizes[h.size] : { + for name, h in local.all_hosts : name => h.size != null ? local.sizes[h.size] : { workers = h.workers app_cpus = h.app_cpus db_cpus = h.db_cpus @@ -31,11 +78,12 @@ locals { } } -# One module instance per host. The module is entirely SSH-provisioner driven — it -# configures an already-existing VPS and hands the app off to docker compose. +# One module instance per host (SSH-managed or GCP-created — see local.all_hosts). +# The module is entirely SSH-provisioner driven: it configures an already-running +# VM (however it came to exist) and hands the app off to docker compose. module "host" { source = "./modules/thermograph-host" - for_each = var.hosts + for_each = local.all_hosts # Per-host config name = each.key @@ -44,6 +92,7 @@ module "host" { ssh_private_key_path = each.value.ssh_private_key_path role = each.value.role git_branch = each.value.git_branch + app_image_tag = each.value.app_image_tag domain = each.value.domain compose_files = each.value.compose_files app_dir = each.value.app_dir @@ -64,31 +113,4 @@ module "host" { om_bucket_remote = var.om_bucket_remote om_rclone_conf = var.om_rclone_conf om_vfs_cache_max = var.om_vfs_cache_max - - # Shared secrets -> /etc/thermograph.env. postgres_password/auth_secret/ - # metrics_token/indexnow_key resolve through secrets.tf's locals (explicit - # var wins if supplied, else Terraform-generated) rather than the raw vars. - postgres_password = local.postgres_password - auth_secret = local.auth_secret - metrics_token = local.metrics_token - indexnow_key = local.indexnow_key - vapid_private_key = var.vapid_private_key - vapid_public_key = var.vapid_public_key - vapid_contact = var.vapid_contact - google_verify = var.google_verify - bing_verify = var.bing_verify - mail_backend = var.mail_backend - smtp_host = var.smtp_host - smtp_port = var.smtp_port - smtp_user = var.smtp_user - smtp_password = var.smtp_password - smtp_starttls = var.smtp_starttls - mail_from = var.mail_from - mail_reply_to = var.mail_reply_to - discord_webhook = var.discord_webhook - discord_public_key = var.discord_public_key - discord_app_id = var.discord_app_id - discord_bot_token = var.discord_bot_token - discord_client_secret = var.discord_client_secret - registry_token = var.registry_token } diff --git a/terraform/modules/gcp-host/main.tf b/terraform/modules/gcp-host/main.tf new file mode 100644 index 0000000..9bb8f4a --- /dev/null +++ b/terraform/modules/gcp-host/main.tf @@ -0,0 +1,77 @@ +# Creates a GCE VM (+ the minimal networking it needs) for the thermograph-host +# module to then provision over SSH — this module contributes NOTHING to +# provisioning logic (no Docker install, no compose, no secrets rendering); it +# only stands the box up and hands its IP back. See ../../main.tf's module.gcp_vm +# / module.host composition. Scaffold only: nothing here is applied until a +# caller populates var.gcp_hosts (root variables.tf) with a real entry. + +# A dedicated VPC rather than the project's "default" network, so this doesn't +# depend on (or clutter) whatever else may already exist in the project. +resource "google_compute_network" "this" { + project = var.project + name = "thermograph-${var.name}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "this" { + project = var.project + name = "thermograph-${var.name}" + network = google_compute_network.this.id + region = join("-", slice(split("-", var.zone), 0, 2)) + ip_cidr_range = "10.20.0.0/24" +} + +# 22/80/443 only -- matches the SSH-managed hosts' own ufw rules (thermograph-host +# module's setup step). The app port itself is opened by that same module's ufw +# step when the host has no domain, same as any other host it provisions. +resource "google_compute_firewall" "allow_ingress" { + project = var.project + name = "thermograph-${var.name}-allow-ingress" + network = google_compute_network.this.id + source_ranges = ["0.0.0.0/0"] + + allow { + protocol = "tcp" + ports = ["22", "80", "443"] + } +} + +resource "google_compute_instance" "this" { + project = var.project + name = "thermograph-${var.name}" + zone = var.zone + machine_type = var.machine_type + + boot_disk { + initialize_params { + # Debian: matches the apt-based setup script thermograph-host's + # provisioner runs (Docker's get.docker.com installer, ufw, Caddy's apt repo). + image = "debian-cloud/debian-12" + size = 30 + } + } + + network_interface { + subnetwork = google_compute_subnetwork.this.id + access_config {} # ephemeral public IP + } + + metadata = { + ssh-keys = "${var.ssh_user}:${file(pathexpand(var.ssh_public_key_path))}" + } +} + +# A brief pause before the caller's thermograph-host module opens an SSH +# connection: a fresh GCE instance's sshd is not always immediately reachable +# the instant the API reports the instance RUNNING (cloud-init/sshd startup is a +# known race here, distinct from the null_resource provisioner's own timeout). +resource "time_sleep" "wait_for_ssh" { + create_duration = "30s" + depends_on = [google_compute_instance.this] +} + +output "external_ip" { + description = "Public IP of the created instance, once it (and a short settle delay) exist." + value = google_compute_instance.this.network_interface[0].access_config[0].nat_ip + depends_on = [time_sleep.wait_for_ssh] +} diff --git a/terraform/modules/gcp-host/variables.tf b/terraform/modules/gcp-host/variables.tf new file mode 100644 index 0000000..ffe275e --- /dev/null +++ b/terraform/modules/gcp-host/variables.tf @@ -0,0 +1,31 @@ +variable "name" { + description = "Short host key (e.g. \"gcp-uat\"), used to name the created resources." + type = string +} + +variable "project" { + description = "GCP project ID to create the instance in." + type = string +} + +variable "zone" { + description = "GCP zone, e.g. \"us-west1-a\"." + type = string +} + +variable "machine_type" { + description = "GCE machine type." + type = string + default = "e2-medium" +} + +variable "ssh_user" { + description = "Login user provisioned via instance metadata (must be able to sudo)." + type = string + default = "deploy" +} + +variable "ssh_public_key_path" { + description = "Path to the PUBLIC key installed into the instance's ssh-keys metadata for ssh_user." + type = string +} diff --git a/terraform/modules/gcp-host/versions.tf b/terraform/modules/gcp-host/versions.tf new file mode 100644 index 0000000..ec0dcce --- /dev/null +++ b/terraform/modules/gcp-host/versions.tf @@ -0,0 +1,12 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + time = { + source = "hashicorp/time" + version = "~> 0.12" + } + } +} diff --git a/terraform/modules/thermograph-host/main.tf b/terraform/modules/thermograph-host/main.tf index e93f827..7eaae42 100644 --- a/terraform/modules/thermograph-host/main.tf +++ b/terraform/modules/thermograph-host/main.tf @@ -1,8 +1,4 @@ locals { - # THERMOGRAPH_DATABASE_URL is built from the same password compose uses for the db - # container, so the app always matches the database it initialized. - database_url = "postgresql+asyncpg://thermograph:${var.postgres_password}@db:5432/thermograph" - # A Secure cookie is only sent over HTTPS, so enable it only where Caddy terminates # TLS (domain set). A plain-HTTP dev box (domain "") would otherwise drop its login # cookie and no one could stay signed in. @@ -20,43 +16,21 @@ locals { # Hash the local compose files so a compose edit re-triggers the remote deploy. compose_files_sha = join(",", [for f in local.effective_compose_files : filesha256("${var.repo_root}/${f}")]) - # Rendered /etc/thermograph.env (sensitive — carries every secret). - env_content = templatefile("${path.module}/templates/thermograph.env.tftpl", { - app_port = var.app_port - postgres_password = var.postgres_password - database_url = local.database_url - auth_secret = var.auth_secret - metrics_token = var.metrics_token - indexnow_key = var.indexnow_key - workers = var.workers - app_cpus = var.app_cpus - db_cpus = var.db_cpus - db_memory = var.db_memory - timescaledb_tag = var.timescaledb_tag - base = "/" - base_url = local.base_url - cookie_secure = local.cookie_secure - openmeteo = var.openmeteo - om_data_dir = var.om_data_dir - vapid_private_key = var.vapid_private_key - vapid_public_key = var.vapid_public_key - vapid_contact = var.vapid_contact - google_verify = var.google_verify - bing_verify = var.bing_verify - mail_backend = var.mail_backend - smtp_host = var.smtp_host - smtp_port = var.smtp_port - smtp_user = var.smtp_user - smtp_password = var.smtp_password - smtp_starttls = var.smtp_starttls - mail_from = var.mail_from - mail_reply_to = var.mail_reply_to - discord_webhook = var.discord_webhook - discord_public_key = var.discord_public_key - discord_app_id = var.discord_app_id - discord_bot_token = var.discord_bot_token - discord_client_secret = var.discord_client_secret - registry_token = var.registry_token + # Rendered /etc/thermograph-topology.env — non-secret sizing/routing config only. + # Every app secret is rendered separately, at deploy time, from the SOPS+age + # vault (see remote-exec step 3 below) — Terraform never sees or carries them. + topology_env_content = templatefile("${path.module}/templates/thermograph-topology.env.tftpl", { + app_port = var.app_port + workers = var.workers + app_cpus = var.app_cpus + db_cpus = var.db_cpus + db_memory = var.db_memory + timescaledb_tag = var.timescaledb_tag + base = "/" + base_url = local.base_url + cookie_secure = local.cookie_secure + openmeteo = var.openmeteo + om_data_dir = var.om_data_dir }) # Caddyfile is only meaningful on a host with a public domain. @@ -91,17 +65,17 @@ locals { } resource "null_resource" "host" { - # Re-provision when the rendered env, the compose files, the branch, sizing, or the - # Caddyfile change. sha256 of the secret-bearing env is unwrapped with nonsensitive() - # (a one-way hash leaks nothing) so plans stay readable. + # Re-provision when the rendered topology env, the compose files, the branch, + # sizing, the app image tag, or the Caddyfile change. triggers = { - env_sha = nonsensitive(sha256(local.env_content)) - compose_sha = local.compose_files_sha - compose_flags = local.compose_flags - caddy_sha = sha256(local.caddy_content) - branch = var.git_branch - app_dir = var.app_dir - sizing = "${var.workers}/${var.app_cpus}/${var.db_cpus}/${var.db_memory}/${var.timescaledb_tag}" + topology_env_sha = sha256(local.topology_env_content) + compose_sha = local.compose_files_sha + compose_flags = local.compose_flags + caddy_sha = sha256(local.caddy_content) + branch = var.git_branch + app_image_tag = var.app_image_tag + app_dir = var.app_dir + sizing = "${var.workers}/${var.app_cpus}/${var.db_cpus}/${var.db_memory}/${var.timescaledb_tag}" # Re-provision when the archive self-hosting config changes. The rclone.conf is # hashed (nonsensitive on a one-way digest) so a credential rotation redeploys. openmeteo = "${var.openmeteo}/${var.om_data_dir}/${var.om_bucket_remote}/${var.om_vfs_cache_max}" @@ -116,10 +90,11 @@ resource "null_resource" "host" { timeout = "5m" } - # Push the rendered secrets via `content` so they are never written to local disk. + # Push the rendered topology env via `content` (consistent with the other + # `file` provisioners here, though nothing in it is secret). provisioner "file" { - content = local.env_content - destination = "/tmp/thermograph.env" + content = local.topology_env_content + destination = "/tmp/thermograph-topology.env" } provisioner "file" { @@ -237,7 +212,10 @@ resource "null_resource" "host" { ] } - # 2. Sync the checkout to the host's branch (cloning first on a fresh box). + # 2. Sync the checkout to the host's branch (cloning first on a fresh box). This + # is now the INFRA repo's checkout (compose files, db init, Caddy templates, + # the secrets vault, deploy.sh itself) — the app's own source is never checked + # out on the host; only its published registry image is pulled (step 3). provisioner "remote-exec" { inline = [ <<-EOT @@ -255,21 +233,30 @@ resource "null_resource" "host" { ] } - # 3. Install /etc/thermograph.env, then bring the stack up. docker runs as root - # (sources the env file in the same shell) so it never depends on the docker - # group membership taking effect in this session. + # 3. Install /etc/thermograph-topology.env, render /etc/thermograph.env from the + # SOPS+age vault (deploy/render-secrets.sh, now part of this same checkout), + # then bring the stack up on the EXPLICIT app image tag — there's no app + # checkout on this host to derive a tag from (see var.app_image_tag). docker + # runs as root (sources both env files in the same shell) so it never depends + # on the docker group membership taking effect in this session. provisioner "remote-exec" { inline = [ <<-EOT set -eu - echo "[${var.name}] deploy: install env + docker compose pull + up" - sudo install -m 0640 -o root -g root /tmp/thermograph.env /etc/thermograph.env - rm -f /tmp/thermograph.env - sudo bash -c 'set -a; . /etc/thermograph.env; set +a; cd ${var.app_dir} && \ - export REGISTRY_HOST="git.thermograph.org" IMAGE_PATH="emi/thermograph/app" IMAGE_TAG="sha-$(git rev-parse --short=12 HEAD)" && \ - echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin && \ - docker compose ${local.compose_flags} pull backend frontend && \ - docker compose ${local.compose_flags} up -d' + echo "[${var.name}] deploy: install topology env + render secrets + docker compose pull + up" + sudo install -m 0640 -o root -g root /tmp/thermograph-topology.env /etc/thermograph-topology.env + rm -f /tmp/thermograph-topology.env + sudo bash -c ' + set -eu + cd ${var.app_dir} + . deploy/render-secrets.sh + render_thermograph_secrets ${var.app_dir} + set -a; . /etc/thermograph-topology.env; . /etc/thermograph.env; set +a + export REGISTRY_HOST="git.thermograph.org" IMAGE_PATH="emi/thermograph/app" IMAGE_TAG="${var.app_image_tag}" + echo "$REGISTRY_TOKEN" | docker login "$REGISTRY_HOST" --username emi --password-stdin + docker compose ${local.compose_flags} pull backend frontend + docker compose ${local.compose_flags} up -d --remove-orphans + ' EOT ] } diff --git a/terraform/modules/thermograph-host/templates/thermograph-topology.env.tftpl b/terraform/modules/thermograph-host/templates/thermograph-topology.env.tftpl new file mode 100644 index 0000000..0821d69 --- /dev/null +++ b/terraform/modules/thermograph-host/templates/thermograph-topology.env.tftpl @@ -0,0 +1,33 @@ +# /etc/thermograph-topology.env — RENDERED BY TERRAFORM. Do not edit on the host; +# change the tfvars and re-apply. Non-secret sizing/routing config ONLY; every app +# secret (POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, VAPID keys, REGISTRY_TOKEN, +# Discord/SMTP creds, ...) lives in /etc/thermograph.env instead, rendered at deploy +# time from the SOPS+age vault by deploy/render-secrets.sh (see deploy.sh). Both +# files are sourced before `docker compose` runs — this one first, so a same-named +# vault value (there shouldn't be one) would still win. + +# Port uvicorn binds inside the container (compose publishes it on the host). +PORT=${app_port} + +# --- Container sizing (consumed by docker-compose.yml interpolation) ------------- +# Terraform sizes the containers per host. Defaults in compose are 4 / 4 / 2 / 8g. +WORKERS=${workers} +APP_CPUS=${app_cpus} +DB_CPUS=${db_cpus} +DB_MEMORY=${db_memory} +# "latest-pg18" (the default) matches today's behavior. Pin an exact minor before +# any host of this stack could ever replicate with another (docker-compose.yml). +TIMESCALEDB_TAG=${timescaledb_tag} +%{ if openmeteo ~} + +# --- Self-hosted Open-Meteo archive (docker-compose.openmeteo.yml) --------------- +# Host rclone mount of the ERA5 object-storage bucket; the overlay bind-mounts it +# into the Open-Meteo containers. THERMOGRAPH_ARCHIVE_URL is set by the overlay. +OM_DATA_DIR=${om_data_dir} +%{ endif ~} + +# --- Serving -------------------------------------------------------------------- +THERMOGRAPH_BASE=${base} +THERMOGRAPH_BASE_URL=${base_url} +# Secure cookie is only sent over HTTPS: 1 behind Caddy TLS, 0 on a plain-HTTP host. +THERMOGRAPH_COOKIE_SECURE=${cookie_secure} diff --git a/terraform/modules/thermograph-host/templates/thermograph.env.tftpl b/terraform/modules/thermograph-host/templates/thermograph.env.tftpl deleted file mode 100644 index f99a614..0000000 --- a/terraform/modules/thermograph-host/templates/thermograph.env.tftpl +++ /dev/null @@ -1,112 +0,0 @@ -# /etc/thermograph.env — RENDERED BY TERRAFORM. Do not edit on the host; change the -# tfvars and re-apply. Read by `docker compose` (interpolation) and loaded into the -# app container (env_file in docker-compose.yml). Mirrors deploy/thermograph.env.example. - -# Port uvicorn binds inside the container (compose publishes it on the host). -PORT=${app_port} - -# --- Registry pull (repo-split Stage 6) ------------------------------------------- -# `docker login` + `docker compose pull` credential -- read:package scope is -# enough (deploy.sh only pulls, never pushes). -REGISTRY_TOKEN=${registry_token} - -# --- PostgreSQL (docker-compose stack) ------------------------------------------ -# Initializes the db container AND builds the app's THERMOGRAPH_DATABASE_URL. -POSTGRES_PASSWORD=${postgres_password} -# Provided explicitly for parity with the example; the app service also derives this -# from POSTGRES_PASSWORD in docker-compose.yml. -THERMOGRAPH_DATABASE_URL=${database_url} - -# --- Container sizing (consumed by docker-compose.yml interpolation) ------------- -# Terraform sizes the containers per host. Defaults in compose are 4 / 4 / 2 / 8g. -WORKERS=${workers} -APP_CPUS=${app_cpus} -DB_CPUS=${db_cpus} -DB_MEMORY=${db_memory} -# "latest-pg18" (the default) matches today's behavior. Pin an exact minor before -# any host of this stack could ever replicate with another (docker-compose.yml). -TIMESCALEDB_TAG=${timescaledb_tag} -%{ if openmeteo ~} - -# --- Self-hosted Open-Meteo archive (docker-compose.openmeteo.yml) --------------- -# Host rclone mount of the ERA5 object-storage bucket; the overlay bind-mounts it -# into the Open-Meteo containers. THERMOGRAPH_ARCHIVE_URL is set by the overlay. -OM_DATA_DIR=${om_data_dir} -%{ endif ~} - -# --- Serving -------------------------------------------------------------------- -THERMOGRAPH_BASE=${base} -THERMOGRAPH_BASE_URL=${base_url} -# Secure cookie is only sent over HTTPS: 1 behind Caddy TLS, 0 on a plain-HTTP host. -THERMOGRAPH_COOKIE_SECURE=${cookie_secure} - -# --- Auth ----------------------------------------------------------------------- -# Signs email-confirm / password-reset tokens; pinned so links survive restarts. -THERMOGRAPH_AUTH_SECRET=${auth_secret} - -# --- Ops metrics + IndexNow (Terraform-generated; see ../../secrets.tf) --------- -# Lets the ops dashboard authenticate remotely (over an SSH tunnel or otherwise) -# instead of requiring direct loopback access. -THERMOGRAPH_METRICS_TOKEN=${metrics_token} -# The IndexNow key, published at /{key}.txt as proof of ownership. Freely -# rotatable — nothing external depends on this specific value staying put. -THERMOGRAPH_INDEXNOW_KEY=${indexnow_key} - -# --- Web Push (VAPID) ----------------------------------------------------------- -# Pinned so existing browser subscriptions keep receiving across restarts. -THERMOGRAPH_VAPID_PRIVATE_KEY=${vapid_private_key} -THERMOGRAPH_VAPID_PUBLIC_KEY=${vapid_public_key} -%{ if vapid_contact != "" ~} -THERMOGRAPH_VAPID_CONTACT=${vapid_contact} -%{ endif ~} - -# --- SEO verification (optional) ------------------------------------------------ -%{ if google_verify != "" ~} -THERMOGRAPH_GOOGLE_VERIFY=${google_verify} -%{ endif ~} -%{ if bing_verify != "" ~} -THERMOGRAPH_BING_VERIFY=${bing_verify} -%{ endif ~} - -# --- Outbound email (optional) -------------------------------------------------- -%{ if mail_backend != "" ~} -THERMOGRAPH_MAIL_BACKEND=${mail_backend} -%{ endif ~} -%{ if smtp_host != "" ~} -THERMOGRAPH_SMTP_HOST=${smtp_host} -%{ endif ~} -%{ if smtp_port != "" ~} -THERMOGRAPH_SMTP_PORT=${smtp_port} -%{ endif ~} -%{ if smtp_user != "" ~} -THERMOGRAPH_SMTP_USER=${smtp_user} -%{ endif ~} -%{ if smtp_password != "" ~} -THERMOGRAPH_SMTP_PASSWORD=${smtp_password} -%{ endif ~} -%{ if smtp_starttls != "" ~} -THERMOGRAPH_SMTP_STARTTLS=${smtp_starttls} -%{ endif ~} -%{ if mail_from != "" ~} -THERMOGRAPH_MAIL_FROM=${mail_from} -%{ endif ~} -%{ if mail_reply_to != "" ~} -THERMOGRAPH_MAIL_REPLY_TO=${mail_reply_to} -%{ endif ~} - -# --- Discord (optional) --------------------------------------------------------- -%{ if discord_webhook != "" ~} -THERMOGRAPH_DISCORD_WEBHOOK=${discord_webhook} -%{ endif ~} -%{ if discord_public_key != "" ~} -THERMOGRAPH_DISCORD_PUBLIC_KEY=${discord_public_key} -%{ endif ~} -%{ if discord_app_id != "" ~} -THERMOGRAPH_DISCORD_APP_ID=${discord_app_id} -%{ endif ~} -%{ if discord_bot_token != "" ~} -THERMOGRAPH_DISCORD_BOT_TOKEN=${discord_bot_token} -%{ endif ~} -%{ if discord_client_secret != "" ~} -THERMOGRAPH_DISCORD_CLIENT_SECRET=${discord_client_secret} -%{ endif ~} diff --git a/terraform/modules/thermograph-host/variables.tf b/terraform/modules/thermograph-host/variables.tf index 293466a..86742cc 100644 --- a/terraform/modules/thermograph-host/variables.tf +++ b/terraform/modules/thermograph-host/variables.tf @@ -26,7 +26,12 @@ variable "role" { } variable "git_branch" { - description = "Branch the host checkout is reset to." + description = "This INFRA repo's branch the host checkout is reset to (independent of which app image is deployed — see app_image_tag)." + type = string +} + +variable "app_image_tag" { + description = "App image tag to pull, e.g. \"sha-<12 hex>\" (build-push.yml's tag for the app-repo commit) or a semver tag. The host has no app-repo checkout to derive this from, so it's always explicit." type = string } @@ -124,112 +129,7 @@ variable "timescaledb_tag" { default = "latest-pg18" } -# ---- Secrets rendered into /etc/thermograph.env ------------------------------- -variable "postgres_password" { - type = string - sensitive = true -} - -variable "auth_secret" { - type = string - sensitive = true -} - -variable "metrics_token" { - type = string - sensitive = true - default = "" -} - -variable "indexnow_key" { - type = string - sensitive = true - default = "" -} - -variable "vapid_private_key" { - type = string - sensitive = true -} - -variable "vapid_public_key" { - type = string - sensitive = true -} - -variable "vapid_contact" { - type = string - sensitive = true -} - -variable "google_verify" { - type = string - sensitive = true -} - -variable "bing_verify" { - type = string - sensitive = true -} - -variable "mail_backend" { - type = string -} - -variable "smtp_host" { - type = string -} - -variable "smtp_port" { - type = string -} - -variable "smtp_user" { - type = string - sensitive = true -} - -variable "smtp_password" { - type = string - sensitive = true -} - -variable "smtp_starttls" { - type = string -} - -variable "mail_from" { - type = string -} - -variable "mail_reply_to" { - type = string -} - -variable "discord_webhook" { - type = string - sensitive = true -} - -variable "discord_public_key" { - type = string -} - -variable "discord_app_id" { - type = string -} - -variable "discord_bot_token" { - type = string - sensitive = true -} - -variable "discord_client_secret" { - type = string - sensitive = true -} - -variable "registry_token" { - type = string - sensitive = true -} +# Secrets (POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, VAPID keys, REGISTRY_TOKEN, +# Discord/SMTP credentials, ...) are no longer Terraform variables -- they're +# rendered at deploy time from the SOPS+age vault (deploy/secrets/*.yaml) by +# deploy/render-secrets.sh, which deploy.sh calls. See main.tf's remote-exec step 3. diff --git a/terraform/secrets.tf b/terraform/secrets.tf deleted file mode 100644 index 29bf998..0000000 --- a/terraform/secrets.tf +++ /dev/null @@ -1,65 +0,0 @@ -# Terraform-generated internal secrets — values with no meaning outside this -# deployment, so Terraform owns generating them instead of requiring the operator -# to hand-generate and paste each one into terraform.tfvars. -# -# Every resource here is pinned with a static `keepers` value so `terraform -# apply` never regenerates a value already in use — regeneration would be a real -# incident (every session invalidated, the app<->DB password mismatched, a -# bookmarked dashboard token broken). Rotation is a deliberate edit to a -# resource's `keepers` value here, never a side effect of anything else -# changing. See docs/architecture/repo-topology-and-infrastructure.md §6. -# -# The corresponding variable (variables.tf) defaults to "" (Terraform -# generates); supply an explicit value only to seed an EXISTING live secret -# during a migration onto Terraform (hop-1 cutover runbook Stage 0) — -# `terraform plan` must show no change to the local below once the value -# matches what's already deployed. -# -# VAPID is deliberately NOT here: it's an EC keypair (var.vapid_private_key / -# vapid_public_key), generated once out-of-band and held as a stable REQUIRED -# input. Unlike an opaque token, there is no safe "Terraform regenerates it" -# story — regeneration breaks every existing push subscription outright. -# -# Generated values land in tfstate. tfstate must never be committed or synced in -# cleartext — stand up an encrypted backend (or keep these out of Terraform and -# supply them as pre-created Swarm secrets instead, per the hop-1 runbook's -# hazard #15) before relying on this in a shared or remote-state setup. - -resource "random_password" "postgres_password" { - length = 32 - special = false # lands raw (no URL-encoding) in THERMOGRAPH_DATABASE_URL - keepers = { - rotation = "1" - } -} - -resource "random_id" "auth_secret" { - byte_length = 32 # matches users.py's own secrets.token_urlsafe(32) fallback shape - keepers = { - rotation = "1" - } -} - -resource "random_password" "metrics_token" { - length = 32 - special = false # may travel as a query param (?token=...); stay URL-safe - keepers = { - rotation = "1" - } -} - -resource "random_id" "indexnow_key" { - byte_length = 16 # 32 hex chars, matching indexnow.py's own secrets.token_hex(16) - keepers = { - rotation = "1" - } -} - -locals { - # var.X wins when explicitly supplied (migration: seed the existing live - # value); otherwise the Terraform-generated one — see the file header. - postgres_password = var.postgres_password != "" ? var.postgres_password : random_password.postgres_password.result - auth_secret = var.auth_secret != "" ? var.auth_secret : random_id.auth_secret.b64_url - metrics_token = var.metrics_token != "" ? var.metrics_token : random_password.metrics_token.result - indexnow_key = var.indexnow_key != "" ? var.indexnow_key : random_id.indexnow_key.hex -} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index f5fc86e..cdf3673 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -1,21 +1,27 @@ -# Copy to terraform.tfvars and fill in real IPs + secrets. +# Copy to terraform.tfvars and fill in real IPs + credentials. # cp terraform.tfvars.example terraform.tfvars -# terraform.tfvars is gitignored (it holds secrets, and those land in local state). -# NEVER commit real values. +# terraform.tfvars is gitignored (repo_url may carry a credential, and om_rclone_conf +# always does — both land in local state). NEVER commit real values. +# +# App secrets (POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, VAPID keys, +# REGISTRY_TOKEN, Discord/SMTP creds, ...) are NOT here anymore -- they live in +# the SOPS+age vault (../deploy/secrets/*.yaml), rendered at deploy time. See +# deploy/secrets/README.md to rotate or add one. # --------------------------------------------------------------------------------- # Hosts # --------------------------------------------------------------------------------- -# Two VPS hosts. (The `dev` branch deploys to the LAN dev server via +# Two VPS hosts. (The `dev` branch deploys to the LAN dev server via the app repo's # deploy/deploy-dev.sh — that box is NOT managed by Terraform.) hosts = { - # Production: the NEW 48 GB / 12-core VPS serving thermograph.org (branch `release`). + # Production: the NEW 48 GB / 12-core VPS serving thermograph.org. prod = { host = "REPLACE_WITH_NEW_VPS_IP" # <-- the new prod VPS IP/hostname - ssh_user = "deploy" - ssh_private_key_path = "~/.ssh/id_ed25519" # key that can log in as deploy@ and sudo + ssh_user = "agent" + ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" role = "prod" - git_branch = "release" + git_branch = "main" # this INFRA repo's branch -- see app_image_tag for the app version + app_image_tag = "REPLACE_WITH_APP_IMAGE_TAG" # e.g. "sha-abcdef012345" -- from build-push.yml on the app repo domain = "thermograph.org" # Caddy TLS in front, app on loopback compose_files = ["docker-compose.yml"] app_dir = "/opt/thermograph" @@ -30,13 +36,14 @@ hosts = { om_data_dir = "/mnt/om-archive" } - # Beta / testing: the OLD VPS, repurposed (branch `main`). + # Beta / testing: the OLD VPS, repurposed. beta = { host = "75.119.132.91" - ssh_user = "deploy" - ssh_private_key_path = "~/.ssh/id_ed25519" + ssh_user = "agent" + ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" role = "beta" git_branch = "main" + app_image_tag = "REPLACE_WITH_APP_IMAGE_TAG" # No public domain by default: no Caddy/TLS, firewall opens the app port. NOTE: # with compose_files = ["docker-compose.yml"] the app binds 127.0.0.1 only, so # until you either set a domain (e.g. "beta.thermograph.org", which fronts it with @@ -57,10 +64,11 @@ hosts = { # not managed until then. "nano" keeps it cheap since it's destroyed when idle. # uat = { # host = "REPLACE_WITH_UAT_VM_IP" - # ssh_user = "deploy" - # ssh_private_key_path = "~/.ssh/id_ed25519" + # ssh_user = "agent" + # ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" # role = "uat" - # git_branch = "release" + # git_branch = "main" + # app_image_tag = "REPLACE_WITH_APP_IMAGE_TAG" # domain = "" # compose_files = ["docker-compose.yml"] # app_dir = "/opt/thermograph" @@ -68,10 +76,32 @@ hosts = { # } } +# GCP-created hosts — SCAFFOLD ONLY, empty by default. Populate an entry to have +# Terraform actually create a GCP Compute Engine VM (see modules/gcp-host); until +# then no google_* resource is planned and no GCP credentials are needed. Example: +# gcp_hosts = { +# gcp-uat = { +# project = "REPLACE_WITH_GCP_PROJECT_ID" +# zone = "us-west1-a" +# machine_type = "e2-medium" +# ssh_user = "agent" +# ssh_public_key_path = "~/.ssh/thermograph_agent_ed25519.pub" +# ssh_private_key_path = "~/.ssh/thermograph_agent_ed25519" +# role = "uat" +# git_branch = "main" +# app_image_tag = "REPLACE_WITH_APP_IMAGE_TAG" +# size = "nano" +# } +# } + # Optional overrides (shown with their defaults): -# repo_url = "https://github.com/griffemi/thermograph.git" +# repo_url = "https://git.thermograph.org/emi/thermograph-infra.git" # app_port = 8137 # frontend_port = 8080 +# +# thermograph-infra is a PRIVATE repo, so a host cloning it for the first time +# needs read credentials embedded in repo_url, e.g. a Forgejo deploy token: +# repo_url = "https://deploy:REPLACE_WITH_TOKEN@git.thermograph.org/emi/thermograph-infra.git" # --------------------------------------------------------------------------------- # Self-hosted Open-Meteo archive (only used by hosts with openmeteo = true) -------- @@ -89,50 +119,3 @@ om_rclone_conf = <<-RCLONE access_key_id = REPLACE_WITH_ACCESS_KEY secret_access_key = REPLACE_WITH_SECRET_KEY RCLONE - -# --------------------------------------------------------------------------------- -# Shared secrets (keep this file out of git) -# --------------------------------------------------------------------------------- -# postgres_password / auth_secret / metrics_token / indexnow_key are OPTIONAL — -# left unset (or ""), Terraform generates and owns each one (see secrets.tf), -# pinned so `apply` never regenerates a value already in use. Uncomment and set -# one only to seed an EXISTING live secret when migrating onto Terraform (hop-1 -# cutover runbook Stage 0) — `terraform plan` must then show no change to it. -# postgres_password = "REPLACE_WITH_THE_EXISTING_LIVE_DB_PASSWORD" -# auth_secret = "REPLACE_WITH_THE_EXISTING_LIVE_AUTH_SECRET" -# metrics_token = "REPLACE_WITH_THE_EXISTING_LIVE_METRICS_TOKEN" -# indexnow_key = "REPLACE_WITH_THE_EXISTING_LIVE_INDEXNOW_KEY" - -# VAPID is NOT Terraform-generated — an EC keypair, generate once out-of-band and -# hold it as a stable input; regeneration breaks every existing push subscription. -# cd backend && ../.venv/bin/python -c "import push,json;k=push._generate();print(k['private_key']);print(k['public_key'])" -vapid_private_key = "REPLACE_WITH_VAPID_PRIVATE_KEY" -vapid_public_key = "REPLACE_WITH_VAPID_PUBLIC_KEY" -vapid_contact = "mailto:you@example.com" - -# ---- Optional: search-engine verification (leave "" to omit) -------------------- -# google_verify = "" -# bing_verify = "" - -# ---- Optional: outbound email (leave "" to omit) -------------------------------- -# mail_backend = "smtp" -# smtp_host = "127.0.0.1" -# smtp_port = "25" -# smtp_user = "" -# smtp_password = "" -# smtp_starttls = "" -# mail_from = "Thermograph " -# mail_reply_to = "" - -# ---- Optional: Discord (leave "" to omit) --------------------------------------- -# discord_webhook = "" -# discord_public_key = "" -# discord_app_id = "" -# discord_bot_token = "" -# discord_client_secret = "" - -# ---- Required: registry pull credential (repo-split Stage 6) -------------------- -# A Forgejo personal access token, read:package scope, for deploy.sh's -# `docker login` + `docker compose pull` on every apply. No default -- apply -# fails loud without it rather than deploying a host that can't redeploy itself. -# registry_token = "" diff --git a/terraform/variables.tf b/terraform/variables.tf index 948482b..6959eb3 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -5,7 +5,7 @@ # for_each). This config manages TWO hosts: prod (new 48 GB / 12-core VPS, branch # `release`, thermograph.org) and beta (the old VPS 75.119.132.91, branch `main`, # no public domain by default). The `dev` branch deploys to the LAN dev server via -# deploy/deploy-dev.sh (a self-hosted runner) and is NOT managed here. +# deploy/deploy-dev.sh (a self-hosted runner, in the app repo) and is NOT managed here. # # A host with `domain = ""` gets no Caddy/TLS: the app port is opened on the firewall # and the app is reached directly (beta does this by default; set a domain to front @@ -17,20 +17,28 @@ # budget scales from db_memory automatically (deploy/db/init/20-tuning.sh), so no # separate tuning edit is needed to exploit the RAM. variable "hosts" { - description = "Map of hosts to manage, keyed by a short name (e.g. \"prod\", \"dev\")." + description = "Map of hosts to manage, keyed by a short name (e.g. \"prod\", \"beta\")." type = map(object({ host = string # IP or hostname to SSH to ssh_user = optional(string, "deploy") # SSH login user ssh_private_key_path = string # path to the private key for that user - role = string # "prod" | "dev" (informational + outputs) - git_branch = string # branch the checkout is reset to - domain = optional(string, "") # public domain; "" => no Caddy/TLS - compose_files = optional(list(string), ["docker-compose.yml"]) - app_dir = optional(string, "/opt/thermograph") # checkout path on the host - workers = optional(number, 4) # uvicorn workers (WORKERS) - app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) - db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) - db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) + role = string # "prod" | "beta" (informational + outputs) + git_branch = string # this INFRA repo's branch the checkout is reset to + # Which app image to run, e.g. "sha-<12 hex>" (matches build-push.yml's tag for + # the app-repo commit build-push.yml built) or a semver tag on a release push. + # Required, no default: since the host's checkout is this infra repo, not the app + # repo, there is no "current commit" to derive a tag from — it must be explicit. + # Bump this (via a normal tfvars edit + apply) whenever the app repo ships a + # commit you want this host running; the infra repo's own git_branch is + # independent of that and rarely needs to change. + app_image_tag = string + domain = optional(string, "") # public domain; "" => no Caddy/TLS + compose_files = optional(list(string), ["docker-compose.yml"]) + app_dir = optional(string, "/opt/thermograph") # checkout path on the host + workers = optional(number, 4) # uvicorn workers (WORKERS) + app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) + db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) + db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) # A named size tier (see locals.sizes in main.tf: nano/small/medium/large) — # when set, overrides the four fields above with the tier's preset. Leave # null (default) to keep hand-picking workers/app_cpus/db_cpus/db_memory, @@ -49,10 +57,37 @@ variable "hosts" { })) } +# GCP-created hosts, keyed the same way as `hosts`. Default {} => zero GCP resources +# planned and the google provider is never actually invoked (see versions.tf and +# modules/gcp-host). Populate an entry to have Terraform create the VM itself; its +# output IP then feeds into the SAME thermograph-host module every SSH-managed host +# uses (main.tf), so provisioning logic is never duplicated between providers. +variable "gcp_hosts" { + description = "Map of hosts for Terraform to CREATE on GCP (Compute Engine), keyed the same way as `hosts`. Empty by default -- no live GCP resources exist yet; this is a scaffold for future use. See modules/gcp-host." + type = map(object({ + project = string # GCP project ID + zone = string # e.g. "us-west1-a" + machine_type = optional(string, "e2-medium") + ssh_user = optional(string, "deploy") + ssh_public_key_path = string # path to the PUBLIC key installed on the instance + ssh_private_key_path = string # path to the matching PRIVATE key (for the module's provisioner) + role = string + git_branch = string + app_image_tag = string + domain = optional(string, "") + compose_files = optional(list(string), ["docker-compose.yml"]) + app_dir = optional(string, "/opt/thermograph") + size = optional(string, "small") + timescaledb_tag = optional(string, "latest-pg18") + })) + default = {} +} + variable "repo_url" { - description = "Git remote to clone from when a host has no checkout yet (a fresh prod box)." + description = "Git remote to clone from when a host has no checkout yet. Points at THIS repo (thermograph-infra) now, not the app repo -- the app's own source is never checked out on a host; only its published registry images are pulled (see var.hosts[*].app_image_tag). thermograph-infra is a private repo, so this typically needs embedded read credentials, e.g. a Forgejo deploy token: \"https://:@git.thermograph.org/emi/thermograph-infra.git\"." type = string - default = "https://github.com/griffemi/thermograph.git" + default = "https://git.thermograph.org/emi/thermograph-infra.git" + sensitive = true } variable "app_port" { @@ -88,165 +123,20 @@ variable "om_vfs_cache_max" { } # --------------------------------------------------------------------------------- -# Shared secrets (rendered into /etc/thermograph.env on every host) +# Secrets: owned by the SOPS+age vault now, NOT Terraform # --------------------------------------------------------------------------------- -# These live in terraform.tfvars (gitignored) and land in local state — never commit -# real values. Every host in this config shares the same secret set; split the config -# if prod and dev must diverge on a credential. +# POSTGRES_PASSWORD, THERMOGRAPH_AUTH_SECRET, THERMOGRAPH_METRICS_TOKEN, +# THERMOGRAPH_INDEXNOW_KEY, THERMOGRAPH_VAPID_*, REGISTRY_TOKEN, Discord/SMTP +# credentials, and every other app secret are no longer Terraform variables -- +# they're rendered at deploy time from deploy/secrets/*.yaml (SOPS-encrypted, +# committed) by deploy/render-secrets.sh, which deploy.sh calls before `docker +# compose up`. Terraform's job here shrank to topology/sizing only (see +# modules/thermograph-host/templates/thermograph-topology.env.tftpl) plus +# triggering the deploy. See deploy/secrets/README.md to rotate or add a secret. # -# postgres_password / auth_secret / metrics_token / indexnow_key default to "" — -# Terraform generates and owns each one (see secrets.tf), pinned so `apply` never -# regenerates a live value. Supply an explicit value only to seed an EXISTING live -# secret during a migration onto Terraform (hop-1 cutover runbook Stage 0). -variable "postgres_password" { - description = "PostgreSQL password (initializes the db container AND builds THERMOGRAPH_DATABASE_URL). \"\" (default) => Terraform-generated." - type = string - sensitive = true - default = "" -} - -variable "auth_secret" { - description = "THERMOGRAPH_AUTH_SECRET — signs email-confirm / password-reset tokens. \"\" (default) => Terraform-generated." - type = string - sensitive = true - default = "" -} - -variable "metrics_token" { - description = "THERMOGRAPH_METRICS_TOKEN — optional remote-access token for the ops metrics endpoint. \"\" (default) => Terraform-generated." - type = string - sensitive = true - default = "" -} - -variable "indexnow_key" { - description = "THERMOGRAPH_INDEXNOW_KEY — IndexNow key, published at /{key}.txt. \"\" (default) => Terraform-generated." - type = string - sensitive = true - default = "" -} - -variable "vapid_private_key" { - description = "THERMOGRAPH_VAPID_PRIVATE_KEY — signs Web Push. Pin it or existing subscriptions break." - type = string - sensitive = true -} - -variable "vapid_public_key" { - description = "THERMOGRAPH_VAPID_PUBLIC_KEY — public half of the VAPID pair." - type = string - sensitive = true -} - -variable "vapid_contact" { - description = "THERMOGRAPH_VAPID_CONTACT — mailto: or https URL sent to push services." - type = string - sensitive = true - default = "" -} - -# ---- Optional: search-engine verification ------------------------------------- -variable "google_verify" { - description = "THERMOGRAPH_GOOGLE_VERIFY — Search Console HTML-tag content value." - type = string - sensitive = true - default = "" -} - -variable "bing_verify" { - description = "THERMOGRAPH_BING_VERIFY — Bing Webmaster msvalidate.01 content value." - type = string - sensitive = true - default = "" -} - -# ---- Optional: outbound email (SMTP) ------------------------------------------ -variable "mail_backend" { - description = "THERMOGRAPH_MAIL_BACKEND — console | smtp | disabled. Empty => unset (console default)." - type = string - default = "" -} - -variable "smtp_host" { - description = "THERMOGRAPH_SMTP_HOST (usually 127.0.0.1 for the local Postfix null client)." - type = string - default = "" -} - -variable "smtp_port" { - description = "THERMOGRAPH_SMTP_PORT." - type = string - default = "" -} - -variable "smtp_user" { - description = "THERMOGRAPH_SMTP_USER (only for a remote relay)." - type = string - sensitive = true - default = "" -} - -variable "smtp_password" { - description = "THERMOGRAPH_SMTP_PASSWORD (only for a remote relay)." - type = string - sensitive = true - default = "" -} - -variable "smtp_starttls" { - description = "THERMOGRAPH_SMTP_STARTTLS — \"1\" to enable. Empty => unset." - type = string - default = "" -} - -variable "mail_from" { - description = "THERMOGRAPH_MAIL_FROM — From: header, e.g. \"Thermograph \"." - type = string - default = "" -} - -variable "mail_reply_to" { - description = "THERMOGRAPH_MAIL_REPLY_TO." - type = string - default = "" -} - -# ---- Optional: Discord -------------------------------------------------------- -variable "discord_webhook" { - description = "THERMOGRAPH_DISCORD_WEBHOOK — the URL IS the credential." - type = string - sensitive = true - default = "" -} - -variable "discord_public_key" { - description = "THERMOGRAPH_DISCORD_PUBLIC_KEY — verifies interaction requests." - type = string - default = "" -} - -variable "discord_app_id" { - description = "THERMOGRAPH_DISCORD_APP_ID." - type = string - default = "" -} - -variable "discord_bot_token" { - description = "THERMOGRAPH_DISCORD_BOT_TOKEN — a credential (command registration only)." - type = string - sensitive = true - default = "" -} - -variable "registry_token" { - description = "Forgejo registry pull credential (read:package scope) -- deploy.sh's docker login + docker compose pull (repo-split Stage 6). Required; no default -- deploys fail loud without it rather than silently falling back to a build." - type = string - sensitive = true -} - -variable "discord_client_secret" { - description = "THERMOGRAPH_DISCORD_CLIENT_SECRET — OAuth2 account linking (a credential)." - type = string - sensitive = true - default = "" -} +# The four secrets Terraform used to generate itself (postgres_password, +# auth_secret, metrics_token, indexnow_key) still exist as values -- they just +# live in the vault now, seeded once from Terraform's own generated values via +# deploy/secrets/seed-from-live.sh so the handoff changed nothing in use. Rotate +# them the same way as any other vault secret from here on (sops edit + commit + +# deploy), not via a Terraform keeper change. diff --git a/terraform/versions.tf b/terraform/versions.tf index dfd848b..c7fc49a 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,8 +2,12 @@ terraform { # Terraform v1.15 is installed; the optional() object defaults used here need >= 1.3. required_version = ">= 1.6" - # No cloud provider: the hosts already exist. Everything is driven over SSH by the - # null provider's provisioners, with local/random for rendering + generated values. + # Hosts under var.hosts already exist and are driven over SSH by the null + # provider's provisioners (local for rendering). google is scaffold-only: with + # var.gcp_hosts left at its default {}, no google_* resource is ever planned and + # the provider is never actually invoked -- `terraform plan`/`validate` work with + # no GCP credentials configured at all. Populate var.gcp_hosts (and the usual + # GOOGLE_APPLICATION_CREDENTIALS / gcloud auth) to actually create a GCP host. required_providers { null = { source = "hashicorp/null" @@ -13,9 +17,13 @@ terraform { source = "hashicorp/local" version = "~> 2.5" } - random = { - source = "hashicorp/random" - version = "~> 3.6" + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + time = { + source = "hashicorp/time" + version = "~> 0.12" } } }