From c41ade74b14618b7f07410a5731fba77ba0a039a Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Mon, 20 Jul 2026 17:39:48 -0700 Subject: [PATCH] Add the Swarm interim stack file, a pinnable TimescaleDB tag, and a Caddy health-gate (#235) * Split web/worker duties with THERMOGRAPH_ROLE Background work (the subscription notifier) is welded to the same process that serves requests, so scaling the web tier to N replicas would also scale notifier instances unless something restricts it further than leader election alone. Add THERMOGRAPH_ROLE (web|worker|all, default all - unchanged single-process behavior). Every replica runs the same image; ROLE only gates whether a process is allowed to own the notifier at all, layered on top of the existing leader election: web replicas never start it even if they'd win leader election, worker replicas start it if they win. The decision is pulled into _should_run_notifier() so it's unit-testable without booting the full app (DB init, places index, neighbor warmer). Add a minimal /healthz liveness route (no DB/upstream I/O, not under BASE) so a worker replica - which serves no real traffic - still has something Swarm can health-check. * Add the Swarm interim stack file, a pinnable TimescaleDB tag, and a Caddy health-gate Three changes toward the hop-1 interim cutover, all inert until Track B stands up the platform: docker-stack.yml: the Swarm stack file for the interim cutover, distinct from docker-compose.yml (today's plain-compose deploy, unaffected). Pulls a pre-built image (IMAGE_TAG) instead of building in place; app/worker publish no host port (127.0.0.1:8137:8137 has no Swarm equivalent - Swarm's routing mesh publishes on 0.0.0.0, which would expose the plaintext app un-fronted), reaching Caddy only over an MTU-lowered overlay network (VXLAN-over-WireGuard needs a smaller MTU or large payloads silently stall); db is placement- pinned to a labelled node; app/worker skip inline migrations (RUN_MIGRATIONS=0) so the runbook's one-shot migrate task is the only thing that ever runs Alembic; secrets are real Swarm secrets mounted at /run/secrets, read by the entrypoint shim rather than plain env vars. TIMESCALEDB_TAG: docker-compose.yml's db image now reads this (default latest-pg18, today's behavior unchanged), wired through Terraform (timescaledb_tag, default "latest-pg18") so it can actually be pinned to an exact minor without hand-editing the host - required before any host of the stack could replicate with another (a floating tag risks mismatched extension minors, which blocks a physical replica and risks compressed- chunk corruption on restore). Caddy active health-gate: both the Terraform-rendered Caddyfile and the live deploy/Caddyfile now health-check the app on the same cheap /healthz route its own Docker HEALTHCHECK uses (now /healthz instead of the SSR homepage, so it's cheap enough for a tight interval and works identically for a worker replica, which serves no public traffic at all) - Caddy won't forward into a container that's still booting or unhealthy. Verified live: built and booted the real image via docker compose - both containers report healthy via the new /healthz-based HEALTHCHECK, and GET / still renders the full SSR homepage unchanged. Both Caddyfiles validated with the real caddy binary. docker-stack.yml validated with docker compose config (required-var guards fire with clear messages; secrets correctly mount at /run/secrets/, matching the entrypoint shim's mapping). docker-compose.yml validated with and without TIMESCALEDB_TAG set, alongside the existing openmeteo overlay. terraform validate + fmt clean. --- deploy/Caddyfile | 10 +- deploy/thermograph.env.example | 10 + docker-compose.yml | 10 +- docker-stack.yml | 173 ++++++++++++++++++ terraform/main.tf | 1 + terraform/modules/thermograph-host/main.tf | 3 +- .../templates/Caddyfile.tftpl | 11 +- .../templates/thermograph.env.tftpl | 3 + .../modules/thermograph-host/variables.tf | 6 + terraform/variables.tf | 6 + 10 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 docker-stack.yml diff --git a/deploy/Caddyfile b/deploy/Caddyfile index 30b8715..389d42e 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -17,7 +17,15 @@ thermograph.org { encode zstd gzip # The app serves everything at the root now (THERMOGRAPH_BASE=/). - reverse_proxy 127.0.0.1:8137 + # Active health check on the same cheap /healthz route the container's own + # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting + # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). + reverse_proxy 127.0.0.1:8137 { + health_uri /healthz + health_interval 5s + health_timeout 3s + health_status 2xx + } log { output file /var/log/caddy/thermograph.log diff --git a/deploy/thermograph.env.example b/deploy/thermograph.env.example index 695ec06..79a1106 100644 --- a/deploy/thermograph.env.example +++ b/deploy/thermograph.env.example @@ -57,6 +57,16 @@ WORKERS=4 # deploy (today's default); the flock above is sufficient there. #THERMOGRAPH_SINGLETON_PG=1 +# Which duties this process performs. Every replica runs the same image; ROLE just +# decides whether it's allowed to own the notifier once it wins the leader election +# above — this is what lets the web tier scale to N stateless replicas under Swarm +# without also scaling notifier instances, while a single worker replica owns it. +# all -> both (default; today's single-process behavior, unchanged) +# web -> never runs the notifier, even if it would win leader election +# worker -> runs the notifier if it wins leader election +# Read once at process start; changing it needs a restart, not a live toggle. +#THERMOGRAPH_ROLE=all + # Base path the app is served under. # / -> app at the domain root (Thermograph owns the whole domain — # this is what the Caddyfile expects: thermograph.org proxies "/") diff --git a/docker-compose.yml b/docker-compose.yml index d9a3b55..a987c60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,15 @@ services: # here (see backend/data/climate_store.py), so the DB, not the filesystem, is the # source of truth. The init script CREATE EXTENSIONs timescaledb on a fresh volume # (Alembic also does, idempotently, at app boot). - image: timescale/timescaledb:latest-pg18 + # + # TIMESCALEDB_TAG defaults to the floating latest-pg18 tag (today's behavior, + # unchanged) so a plain `docker compose up` keeps working with no setup. Pin it + # to an exact minor (e.g. 2.17.2-pg18) before any host of this stack could ever + # replicate with another — a floating tag risks two hosts landing on different + # extension minors, which blocks a physical replica and risks compressed-chunk + # corruption on restore. docker-stack.yml (the Swarm interim stack) REQUIRES an + # exact pin for exactly this reason; use the SAME tag on both once you set one. + image: timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18} environment: POSTGRES_USER: thermograph POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} diff --git a/docker-stack.yml b/docker-stack.yml new file mode 100644 index 0000000..a4641ff --- /dev/null +++ b/docker-stack.yml @@ -0,0 +1,173 @@ +# Docker Swarm stack for the hop-1 interim cutover — see +# docs/runbooks/hop1-forgejo-registry-cutover.md (which this file implements) and +# docs/architecture/repo-topology-and-infrastructure.md §6/§9. +# +# Distinct from docker-compose.yml (today's plain-compose deploy, unaffected by +# this file): Swarm pulls a pre-built image from the registry (IMAGE_TAG) rather +# than building in place, and needs Swarm-specific keys (deploy:, networks:, +# secrets:, no host-published ports on the overlay-fronted services). +# +# docker stack deploy -c docker-stack.yml thermograph +# +# `docker stack deploy` only interpolates from the invoking shell's environment, +# not a .env file — export the required vars first (or `set -a; . ./stack.env; +# set +a; docker stack deploy ...`). Required shell vars: IMAGE_TAG (the tag CI +# pushed to the registry), TIMESCALEDB_TAG (an EXACT pinned minor — see the db +# service below, hazard #7). POSTGRES_PASSWORD is NOT a shell var here: unlike +# docker-compose.yml (which interpolates it into THERMOGRAPH_DATABASE_URL at +# compose time), this file reads it as the `postgres_password` Swarm secret — +# POSTGRES_PASSWORD_FILE for db (native support in the postgres/timescaledb +# image), and the chunk-3 entrypoint shim builds THERMOGRAPH_DATABASE_URL for +# app/worker from the same secret file at container start. All `postgres_password` +# / `thermograph_*` secrets below must already exist in the Swarm (`docker secret +# create -`) before the first deploy — Track B step 7 in +# docs/runbooks/implementation-handoff.md. +# +# Migrations are NOT run inline by app/worker here (RUN_MIGRATIONS=0) — the +# runbook's Stage F brings the schema to head via a one-shot task BEFORE scaling +# app up, so multiple replicas never race Alembic and nothing runs DDL against a +# still-read-only standby mid-cutover (hazard #9): +# +# docker run --rm --network thermograph_internal \ +# -e THERMOGRAPH_DATABASE_URL=... migrate + +services: + db: + # Pin the EXACT TimescaleDB minor — never latest-pg18 here. A floating tag can + # give two hosts different extension minors, which blocks a physical replica + # (a newer .so over an older catalog won't start) and risks compressed-chunk + # corruption on timescaledb_post_restore() (hazard #7). Get the exact X.Y.Z + # from the source database: SELECT extversion FROM pg_extension WHERE + # extname='timescaledb' — use the SAME tag docker-compose.yml's TIMESCALEDB_TAG + # is pinned to, on every host that could ever replicate with this one. + image: timescale/timescaledb:${TIMESCALEDB_TAG:?set TIMESCALEDB_TAG to an exact pinned minor, e.g. 2.17.2-pg18 -- not latest-pg18} + environment: + POSTGRES_USER: thermograph + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: thermograph + DB_MEMORY: ${DB_MEMORY:-8g} + volumes: + - pgdata:/var/lib/postgresql + - ./deploy/db/init:/docker-entrypoint-initdb.d + networks: + - internal + secrets: + - postgres_password + deploy: + # Pin to the labelled DB node so replication/IO never crosses the slow WG + # uplink (hazard #14): `docker node update --label-add db=true ` once, + # on whichever node holds pgdata (Track B). + placement: + constraints: ["node.labels.db == true"] + resources: + limits: + cpus: "${DB_CPUS:-2}" + memory: ${DB_MEMORY:-8g} + restart_policy: + condition: on-failure + # No published port: reachable only as db:5432 on the `internal` overlay — + # Postgres must never be public (design doc §6). + + # Stateless, freely-replicable web tier. ROLE=web means _should_run_notifier() + # is False unconditionally (web/app.py) — it never starts the notifier or the + # worker scheduler even if it would otherwise win leader election. + app: + image: ${IMAGE_TAG:?set IMAGE_TAG to the image CI pushed, e.g. forge.example/thermograph/app:sha-abc123} + environment: + THERMOGRAPH_BASE: / + PORT: 8137 + WORKERS: ${WORKERS:-4} + THERMOGRAPH_ROLE: web + RUN_MIGRATIONS: "0" + volumes: + - appdata:/app/data + - applogs:/app/logs + networks: + - internal + secrets: + - postgres_password + - thermograph_auth_secret + - thermograph_vapid_private_key + - thermograph_vapid_public_key + deploy: + replicas: 1 # single replica this hop -- homepage.json now lives in + # Postgres (chunk 4), but the notifier/scheduler split (chunk + # 2/5) is what actually lets this go multi-replica in Phase 2 + resources: + limits: + cpus: "${APP_CPUS:-4}" + restart_policy: + condition: on-failure + update_config: + order: start-first # new task must pass the image's HEALTHCHECK (now + # GET /healthz) before the old one is stopped + # No `ports:` — 127.0.0.1:8137:8137 (docker-compose.yml) has no Swarm + # equivalent: Swarm's routing mesh publishes on 0.0.0.0, which would expose + # the plaintext app un-fronted (hazard #6). Only the host's Caddy reaches + # `app`, over the `internal` overlay network — see the Caddyfile templates' + # health-gated reverse_proxy. + + # Owns the notifier + worker scheduler (chunks 1/2/5). Exactly one replica — + # the leader-election guard is belt-and-suspenders, not a substitute for it. + worker: + image: ${IMAGE_TAG:?set IMAGE_TAG to the image CI pushed, e.g. forge.example/thermograph/app:sha-abc123} + environment: + THERMOGRAPH_BASE: / + WORKERS: "1" + THERMOGRAPH_ROLE: worker + # Cluster-wide advisory lock, not the flock: the flock only arbitrates + # workers on ONE host, so under Swarm it guards nothing across replicas. + THERMOGRAPH_SINGLETON_PG: "1" + RUN_MIGRATIONS: "0" + volumes: + - appdata:/app/data + - applogs:/app/logs + networks: + - internal + secrets: + - postgres_password + - thermograph_auth_secret + - thermograph_vapid_private_key + - thermograph_vapid_public_key + - thermograph_discord_webhook + deploy: + replicas: 1 + resources: + limits: + cpus: "${WORKER_CPUS:-1}" + restart_policy: + condition: on-failure + # No `ports:` — the worker serves no public traffic; GET /healthz on its own + # container port is for the image's own HEALTHCHECK (Swarm task health) only. + +networks: + internal: + driver: overlay + driver_opts: + # VXLAN-over-WireGuard double encapsulation needs a lower MTU than the + # ~1450 overlay default, or large payloads (a /cell bundle, the homepage + # feed) silently stall while small packets (health checks) keep passing + # (hazard #13). Validate with a real large-payload transfer, not just a + # ping, once the mesh is up (Track B). + com.docker.network.driver.mtu: "1370" + +volumes: + pgdata: {} + appdata: {} + applogs: {} + +# Declared here, provisioned externally (docker secret create - < file) — +# never by this stack, and never committed. See Stage 0 of the cutover runbook +# for which values are continuity-critical (AUTH_SECRET, VAPID, POSTGRES_PASSWORD +# must be the EXISTING live values, not freshly generated ones). +secrets: + postgres_password: + external: true + thermograph_auth_secret: + external: true + thermograph_vapid_private_key: + external: true + thermograph_vapid_public_key: + external: true + thermograph_discord_webhook: + external: true diff --git a/terraform/main.tf b/terraform/main.tf index 160d02e..858dde1 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -25,6 +25,7 @@ module "host" { app_cpus = each.value.app_cpus db_cpus = each.value.db_cpus db_memory = each.value.db_memory + timescaledb_tag = each.value.timescaledb_tag openmeteo = each.value.openmeteo om_data_dir = each.value.om_data_dir diff --git a/terraform/modules/thermograph-host/main.tf b/terraform/modules/thermograph-host/main.tf index 43ba0d6..0973d68 100644 --- a/terraform/modules/thermograph-host/main.tf +++ b/terraform/modules/thermograph-host/main.tf @@ -30,6 +30,7 @@ locals { app_cpus = var.app_cpus db_cpus = var.db_cpus db_memory = var.db_memory + timescaledb_tag = var.timescaledb_tag base = "/" base_url = local.base_url cookie_secure = local.cookie_secure @@ -96,7 +97,7 @@ resource "null_resource" "host" { caddy_sha = sha256(local.caddy_content) branch = var.git_branch app_dir = var.app_dir - sizing = "${var.workers}/${var.app_cpus}/${var.db_cpus}/${var.db_memory}" + sizing = "${var.workers}/${var.app_cpus}/${var.db_cpus}/${var.db_memory}/${var.timescaledb_tag}" # Re-provision when the archive self-hosting config changes. The rclone.conf is # hashed (nonsensitive on a one-way digest) so a credential rotation redeploys. openmeteo = "${var.openmeteo}/${var.om_data_dir}/${var.om_bucket_remote}/${var.om_vfs_cache_max}" diff --git a/terraform/modules/thermograph-host/templates/Caddyfile.tftpl b/terraform/modules/thermograph-host/templates/Caddyfile.tftpl index 8e686b2..fccda31 100644 --- a/terraform/modules/thermograph-host/templates/Caddyfile.tftpl +++ b/terraform/modules/thermograph-host/templates/Caddyfile.tftpl @@ -10,7 +10,16 @@ ${domain} { encode zstd gzip - reverse_proxy 127.0.0.1:${port} + # Active health check on the same cheap /healthz route the container's own + # HEALTHCHECK uses (Dockerfile) — so a `docker compose up -d --build` deploy + # that's still restarting/booting never gets proxied into (a reload alone has + # no gate, hop-1 runbook hazard #10). health_uri is relative to the upstream. + reverse_proxy 127.0.0.1:${port} { + health_uri /healthz + health_interval 5s + health_timeout 3s + health_status 2xx + } log { output file /var/log/caddy/thermograph.log diff --git a/terraform/modules/thermograph-host/templates/thermograph.env.tftpl b/terraform/modules/thermograph-host/templates/thermograph.env.tftpl index 1b14a44..c88de7e 100644 --- a/terraform/modules/thermograph-host/templates/thermograph.env.tftpl +++ b/terraform/modules/thermograph-host/templates/thermograph.env.tftpl @@ -18,6 +18,9 @@ WORKERS=${workers} APP_CPUS=${app_cpus} DB_CPUS=${db_cpus} DB_MEMORY=${db_memory} +# "latest-pg18" (the default) matches today's behavior. Pin an exact minor before +# any host of this stack could ever replicate with another (docker-compose.yml). +TIMESCALEDB_TAG=${timescaledb_tag} %{ if openmeteo ~} # --- Self-hosted Open-Meteo archive (docker-compose.openmeteo.yml) --------------- diff --git a/terraform/modules/thermograph-host/variables.tf b/terraform/modules/thermograph-host/variables.tf index a8bdda0..15dd659 100644 --- a/terraform/modules/thermograph-host/variables.tf +++ b/terraform/modules/thermograph-host/variables.tf @@ -112,6 +112,12 @@ variable "db_memory" { type = string } +variable "timescaledb_tag" { + description = "TimescaleDB image tag (TIMESCALEDB_TAG), e.g. \"2.17.2-pg18\". \"latest-pg18\" (the default) matches today's behavior; pin an exact minor before any host could ever replicate with another." + type = string + default = "latest-pg18" +} + # ---- Secrets rendered into /etc/thermograph.env ------------------------------- variable "postgres_password" { type = string diff --git a/terraform/variables.tf b/terraform/variables.tf index a468681..e5e3182 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -31,6 +31,12 @@ variable "hosts" { app_cpus = optional(number, 4) # app container CPU cap (APP_CPUS) db_cpus = optional(number, 2) # db container CPU cap (DB_CPUS) db_memory = optional(string, "8g") # db container memory cap (DB_MEMORY) + # The floating tag matches today's behavior everywhere until you pin it. Pin to + # an exact minor (SELECT extversion FROM pg_extension WHERE extname='timescaledb' + # on the live DB) before any host of this stack could ever replicate with + # another — a floating tag risks mismatched extension minors, which blocks a + # physical replica (hop-1 cutover runbook hazard #7). Use the SAME tag everywhere. + timescaledb_tag = optional(string, "latest-pg18") # Self-host the ERA5 archive (docker-compose.openmeteo.yml + a host rclone mount # of the object-storage bucket). Only the self-hosting host (prod) sets true. openmeteo = optional(bool, false)