From 1727ebf60fadb6c4938b43b95eac241924252eaf Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Mon, 20 Jul 2026 17:39:48 -0700 Subject: [PATCH] Add the Swarm interim stack file, a pinnable TimescaleDB tag, and a Caddy health-gate (#235) * Split web/worker duties with THERMOGRAPH_ROLE Background work (the subscription notifier) is welded to the same process that serves requests, so scaling the web tier to N replicas would also scale notifier instances unless something restricts it further than leader election alone. Add THERMOGRAPH_ROLE (web|worker|all, default all - unchanged single-process behavior). Every replica runs the same image; ROLE only gates whether a process is allowed to own the notifier at all, layered on top of the existing leader election: web replicas never start it even if they'd win leader election, worker replicas start it if they win. The decision is pulled into _should_run_notifier() so it's unit-testable without booting the full app (DB init, places index, neighbor warmer). Add a minimal /healthz liveness route (no DB/upstream I/O, not under BASE) so a worker replica - which serves no real traffic - still has something Swarm can health-check. * Add the Swarm interim stack file, a pinnable TimescaleDB tag, and a Caddy health-gate Three changes toward the hop-1 interim cutover, all inert until Track B stands up the platform: docker-stack.yml: the Swarm stack file for the interim cutover, distinct from docker-compose.yml (today's plain-compose deploy, unaffected). Pulls a pre-built image (IMAGE_TAG) instead of building in place; app/worker publish no host port (127.0.0.1:8137:8137 has no Swarm equivalent - Swarm's routing mesh publishes on 0.0.0.0, which would expose the plaintext app un-fronted), reaching Caddy only over an MTU-lowered overlay network (VXLAN-over-WireGuard needs a smaller MTU or large payloads silently stall); db is placement- pinned to a labelled node; app/worker skip inline migrations (RUN_MIGRATIONS=0) so the runbook's one-shot migrate task is the only thing that ever runs Alembic; secrets are real Swarm secrets mounted at /run/secrets, read by the entrypoint shim rather than plain env vars. TIMESCALEDB_TAG: docker-compose.yml's db image now reads this (default latest-pg18, today's behavior unchanged), wired through Terraform (timescaledb_tag, default "latest-pg18") so it can actually be pinned to an exact minor without hand-editing the host - required before any host of the stack could replicate with another (a floating tag risks mismatched extension minors, which blocks a physical replica and risks compressed- chunk corruption on restore). Caddy active health-gate: both the Terraform-rendered Caddyfile and the live deploy/Caddyfile now health-check the app on the same cheap /healthz route its own Docker HEALTHCHECK uses (now /healthz instead of the SSR homepage, so it's cheap enough for a tight interval and works identically for a worker replica, which serves no public traffic at all) - Caddy won't forward into a container that's still booting or unhealthy. Verified live: built and booted the real image via docker compose - both containers report healthy via the new /healthz-based HEALTHCHECK, and GET / still renders the full SSR homepage unchanged. Both Caddyfiles validated with the real caddy binary. docker-stack.yml validated with docker compose config (required-var guards fire with clear messages; secrets correctly mount at /run/secrets/, matching the entrypoint shim's mapping). docker-compose.yml validated with and without TIMESCALEDB_TAG set, alongside the existing openmeteo overlay. terraform validate + fmt clean. --- tests/web/test_role.py | 34 ++++++++++++++++++++++++++++++++++ web/app.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 tests/web/test_role.py diff --git a/tests/web/test_role.py b/tests/web/test_role.py new file mode 100644 index 0000000..0529807 --- /dev/null +++ b/tests/web/test_role.py @@ -0,0 +1,34 @@ +"""THERMOGRAPH_ROLE: which processes are allowed to own the subscription +notifier (web/worker/all split), plus the /healthz liveness route.""" +from fastapi.testclient import TestClient + +from core import singleton +from web import app as appmod + + +def test_default_role_is_all_and_permits_the_notifier(monkeypatch): + monkeypatch.setattr(appmod, "ROLE", "all") + monkeypatch.setattr(singleton, "claim_leader", lambda: True) + assert appmod._should_run_notifier() is True + + +def test_web_role_never_runs_the_notifier_even_if_it_would_win_leader(monkeypatch): + monkeypatch.setattr(appmod, "ROLE", "web") + monkeypatch.setattr(singleton, "claim_leader", lambda: True) + assert appmod._should_run_notifier() is False + + +def test_worker_role_runs_the_notifier_only_if_it_wins_leader(monkeypatch): + monkeypatch.setattr(appmod, "ROLE", "worker") + monkeypatch.setattr(singleton, "claim_leader", lambda: False) + assert appmod._should_run_notifier() is False + monkeypatch.setattr(singleton, "claim_leader", lambda: True) + assert appmod._should_run_notifier() is True + + +def test_healthz_reports_status_and_configured_role(monkeypatch): + monkeypatch.setattr(appmod, "ROLE", "worker") + client = TestClient(appmod.app) + r = client.get("/healthz") + assert r.status_code == 200 + assert r.json() == {"status": "ok", "role": "worker"} diff --git a/web/app.py b/web/app.py index 3dee27b..3bed0ae 100644 --- a/web/app.py +++ b/web/app.py @@ -49,6 +49,13 @@ FRONTEND_DIR = paths.FRONTEND_DIR _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") BASE = f"/{_BASE}" if _BASE else "" +# Which duties this process performs: "web" (serve requests, no notifier), +# "worker" (own the notifier, still serves requests today — see _lifespan), or +# "all" (both — today's single-process default, so an unset ROLE is unchanged +# behavior). Read once at import: toggling requires a process restart, same as +# every other env-driven constant here (BASE, WORKERS, ...). +ROLE = os.environ.get("THERMOGRAPH_ROLE", "all").strip().lower() + # The PWA manifest is served as a static file; register its media type so # StaticFiles labels it correctly (not in Python's default mimetypes map). mimetypes.add_type("application/manifest+json", ".webmanifest") @@ -120,6 +127,14 @@ def _neighbor_warmer() -> None: _warm_queue.task_done() +def _should_run_notifier() -> bool: + """Whether this process should start the subscription-notifier thread: ROLE must + permit it, and it must win the leader election. Split out from _lifespan so this + decision is unit-testable without booting the full app (DB init, places index, + neighbor warmer) — see tests/web/test_role.py.""" + return ROLE in ("worker", "all") and singleton.claim_leader() + + @contextlib.asynccontextmanager async def _lifespan(app): # Warm the local place-name index (/suggest's typo tolerance) in the @@ -138,8 +153,11 @@ async def _lifespan(app): # it must run in exactly one: elect a leader (see singleton.claim_leader, which # picks a host-local flock or a cluster-wide Postgres advisory lock from env). # Unset (single worker / dev / tests) => always the leader. The neighbor warmer - # above stays per-worker: each drains its own request-fed queue. - if singleton.claim_leader(): + # above stays per-worker: each drains its own request-fed queue. ROLE further + # restricts this to processes that are allowed to own it at all — "web" replicas + # never start it even if they'd otherwise win the leader election, so a stateless + # N-replica web tier can scale without also scaling notifier instances. + if _should_run_notifier(): notify.start() yield notify.stop() @@ -152,6 +170,17 @@ app = FastAPI(title="Thermograph", version="0.2.0", lifespan=_lifespan) app.add_middleware(GZipMiddleware, minimum_size=1024) +@app.get("/healthz") +def healthz(): + """Liveness probe: the process is up and serving. Deliberately does no I/O (no + DB query, no upstream call) so it stays cheap and reliable enough for a tight + Swarm/Docker healthcheck interval — readiness (DB reachable, etc.) is a + separate concern the ingress health-gate covers by hitting a real route. Not + under BASE: a fixed, unauthenticated path a healthcheck can hit without + knowing THERMOGRAPH_BASE, on both the web and worker roles.""" + return {"status": "ok", "role": ROLE} + + def _client_ip(request) -> "str | None": """The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us (Caddy on prod sets it), otherwise the direct peer address (LAN dev)."""