Merge remote-tracking branch 'origin/dev' into feat/iceberg-ops-rebased

# Conflicts:
#	infra/ops/README.md

.forgejo/workflows/backend-deploy-dev.yml

@@ -70,4 +70,10 @@ jobs:
         # thermograph-lan is host-native (the runner job runs directly on the
         # LAN box, not over SSH like beta/prod), so plain env vars on the
         # command reach deploy-dev.sh directly -- no ssh-action needed here.
+        # The lake creds ride the Actions S3 secrets: dev renders no vault
+        # (deploy-dev.sh's no-op secrets path), and compose interpolates
+        # ${THERMOGRAPH_LAKE_S3_*} straight from the deploy environment.
+        env:
+          THERMOGRAPH_LAKE_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+          THERMOGRAPH_LAKE_S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
         run: SERVICE=backend BACKEND_IMAGE_TAG=${{ steps.tag.outputs.tag }} bash "$HOME/thermograph-dev/infra/deploy/deploy-dev.sh"

.forgejo/workflows/observability-validate.yml

@@ -12,10 +12,13 @@ name: Validate observability stack
 # path-filtered to the domain, and workflow_call lets pr-build.yml reuse this
 # as the domain's PR check under its single `gate` required status.
 #
-# Not validated here: the Alloy config (observability/alloy/config.alloy, an
+# The Alloy config (observability/alloy/config.alloy, an HCL-like format) IS
-# HCL-like format, needs the `alloy` binary) and docker-compose *schema*
+# validated below -- the static `alloy` binary is downloaded straight from its
-# (unknown-key checks, which would need the docker CLI). Add either if the
+# GitHub release (pinned to the same v1.9.1 the fleet runs; see
-# churn warrants the extra tooling.
+# observability/alloy/docker-compose.agent.yml), no docker CLI needed.
+#
+# Not validated here: docker-compose *schema* (unknown-key checks, which would
+# need the docker CLI). Add it if the churn warrants the extra tooling.
 
 on:
   workflow_call: {}
@@ -70,3 +73,21 @@ jobs:
                   print("INVALID YAML", f, "-", e); bad = True
           sys.exit(1 if bad else 0)
           PY
+
+      - name: Alloy config parses and validates (components, relabel/process wiring)
+        run: |
+          set -euo pipefail
+          apt-get update -qq && apt-get install -y -qq unzip >/dev/null
+          curl -sSL -o /tmp/alloy.zip \
+            https://github.com/grafana/alloy/releases/download/v1.9.1/alloy-linux-amd64.zip
+          unzip -q /tmp/alloy.zip -d /tmp/alloybin
+          chmod +x /tmp/alloybin/alloy-linux-amd64
+
+          # `alloy validate` builds the real component graph (catches bad field
+          # names, dangling forward_to/receiver refs, malformed relabel/process
+          # stages) but doesn't recognize the top-level `livedebugging {}` singleton
+          # block as a component -- it only knows named components, even though
+          # `alloy run` loads that block fine. Known gap in the `validate`
+          # subcommand, not a config error, so strip that one line before checking.
+          grep -v '^livedebugging ' observability/alloy/config.alloy > /tmp/config-for-validate.alloy
+          /tmp/alloybin/alloy-linux-amd64 validate /tmp/config-for-validate.alloy

.forgejo/workflows/shell-lint.yml

@@ -0,0 +1,75 @@
+name: shell-lint
+
+# Shellcheck over every *.sh in the tree. These scripts run as root over SSH
+# against production hosts (deploy, secrets provisioning, host bootstrap) with
+# no test suite in front of them -- a quoting bug or an unset-variable typo is
+# found *on the host* or not at all. The tree was driven to zero findings in
+# one pass; this guard keeps it there.
+#
+# Deliberately NOT path-filtered (same call as secrets-guard): a backstop that
+# only runs when you expect it to isn't a backstop, and shellcheck over the
+# ~26 scripts here is seconds. Scripts are discovered with `find`, not listed,
+# so a new script is covered the moment it lands -- that is the entire point.
+#
+# Shellcheck is installed from the pinned official static-binary release, NOT
+# `apt-get install shellcheck` (the observability-validate precedent): the
+# distro version drifts with the job image, and a drifted shellcheck can grow
+# NEW warnings that fail CI on an unrelated push. The pin (v0.11.0, matching
+# the koalaman/shellcheck:stable image the zero-findings pass was run against)
+# plus the sha256 makes the check reproducible; bump both together, and expect
+# to fix any new findings in the same PR as the bump.
+
+on:
+  pull_request:
+  push:
+    branches: [dev, main, release]
+
+env:
+  SHELLCHECK_VERSION: v0.11.0
+  SHELLCHECK_SHA256: 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198
+
+jobs:
+  shellcheck:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install shellcheck ${{ env.SHELLCHECK_VERSION }} (pinned static binary)
+        run: |
+          set -euo pipefail
+          # node:20-bookworm (this runner's job image) ships curl, tar and xz,
+          # so the pinned tarball needs no apt-get at all -- faster than the
+          # distro install it replaces.
+          curl -fsSL -o /tmp/shellcheck.tar.xz \
+            "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz"
+          echo "${SHELLCHECK_SHA256}  /tmp/shellcheck.tar.xz" | sha256sum -c -
+          tar -xJf /tmp/shellcheck.tar.xz -C /tmp
+          install -m 0755 "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck
+          shellcheck --version
+
+      - name: Shellcheck every *.sh in the tree
+        run: |
+          set -euo pipefail
+          mapfile -t files < <(find . -name '*.sh' -not -path './.git/*' | sort)
+          if [ ${#files[@]} -eq 0 ]; then
+            echo "no *.sh files yet — nothing to lint"
+            exit 0
+          fi
+          echo "shellcheck over ${#files[@]} scripts"
+          # -x follows sourced files: several scripts carry
+          # `# shellcheck source=` directives that only resolve with it.
+          # Default severity, no excludes: the tree is at zero findings, so
+          # anything shellcheck reports is a regression, not noise.
+          if shellcheck -x -f gcc "${files[@]}" > /tmp/findings.txt; then
+            echo "ok: all scripts clean"
+            exit 0
+          fi
+          # gcc format is file:line:col: level: message — reshape each line
+          # into a ::error annotation so findings land on the diff in the PR
+          # view. `rest` soaks up any further colons inside the message.
+          while IFS=: read -r f l _ rest; do
+            [ -n "$l" ] || continue
+            echo "::error file=${f#./},line=${l}::${rest# }"
+          done < /tmp/findings.txt
+          echo "shellcheck found problems in the files above"
+          exit 1

backend/.dockerignore

@@ -0,0 +1,10 @@
+# Local virtualenvs are disposable dev state (see CLAUDE.md), but without this
+# file `COPY . /app/` on a dev checkout ships ~400MB of site-packages the image
+# already installs via pip — and the chown -R layer then duplicates it AGAIN.
+# CI contexts never contain a .venv, so this only protects `:local` dev builds.
+# Do NOT add daemon/ here: the daemon-builder stage COPYs it from this same
+# context.
+.venv*
+__pycache__/
+*.py[co]
+.pytest_cache/

backend/Dockerfile

@@ -1,5 +1,25 @@
 # Thermograph backend: FastAPI API + accounts/notifications + the SSR content
 # JSON API frontend consumes. Split from the monorepo (repo-split Stage 7).
+
+# thermograph-daemon (daemon/): the Go process that owns the Discord gateway
+# websocket and the recurring-job timers, calling back into this app's
+# /internal/* routes for anything that needs data. It is built INTO this image
+# on purpose: daemon and backend share the internal API contract, so shipping
+# one image (compose picks the process per-service) makes version skew between
+# them impossible. CGO_ENABLED=0 gives a fully static binary that drops into
+# the python:3.12-slim final stage with no runtime deps; go.mod/go.sum are
+# copied and downloaded before the sources so the Go dep layer caches across
+# daemon code-only changes, same reasoning as the pip layer below.
+FROM golang:1.26 AS daemon-builder
+WORKDIR /src
+COPY daemon/go.mod daemon/go.sum ./
+RUN go mod download
+COPY daemon/ ./
+# -trimpath keeps the build reproducible (identical sources -> identical
+# binary), which is what lets the final stage's COPY layer cache-hit when only
+# Python code changed.
+RUN CGO_ENABLED=0 go build -trimpath -o /out/thermograph-daemon .
+
 FROM python:3.12-slim
 
 # curl is only for the container HEALTHCHECK below. Everything Python needs
@@ -14,6 +34,12 @@ RUN apt-get update \
 COPY requirements.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
+# The daemon binary lands before the app tree: it changes far less often than
+# the Python code, so this layer usually cache-hits and only the COPY below
+# rebuilds. NOT the entrypoint — deploy/entrypoint.sh stays that; compose
+# selects this binary per-service for the daemon container.
+COPY --from=daemon-builder /out/thermograph-daemon /usr/local/bin/thermograph-daemon
+
 COPY . /app/
 RUN chmod +x /app/deploy/entrypoint.sh
 
@@ -23,10 +49,16 @@ RUN chmod +x /app/deploy/entrypoint.sh
 # Docker seeds them from this image content -- including this ownership --
 # so they stay writable.
 RUN useradd --system --create-home --uid 10001 thermograph \
-    && mkdir -p /app/data /app/logs \
+    && mkdir -p /app/data /app/logs /state \
-    && chown -R thermograph:thermograph /app
+    && chown -R thermograph:thermograph /app /state
 
 USER thermograph
+# Bake DuckDB's httpfs + iceberg extensions AS THE RUNTIME USER — extensions
+# install to the invoking user's ~/.duckdb, and a root-time bake leaves uid
+# 10001 unable to LOAD them (seen live: prod lake /query 500'd on exactly
+# this). iceberg powers the era5_daily view via table metadata instead of a
+# bucket-wide glob LIST.
+RUN python -c "import duckdb; c = duckdb.connect(); c.execute('INSTALL httpfs'); c.execute('INSTALL iceberg')"
 WORKDIR /app
 
 ENV PORT=8137 \

backend/api/internal_routes.py

@@ -0,0 +1,158 @@
+"""Internal control surface for the Go daemon (backend/daemon/).
+
+The gateway bot and the recurring worker jobs moved out of this process into
+`thermograph-daemon` — Go owns only the long-lived stateful I/O (the Discord
+websocket, RESUME/heartbeat/backoff, interval timers). Anything that needs
+*data* calls back in here, because grading depends on polars + the parquet
+cache: reimplementing it out-of-process would let the bot's grades drift from
+the API's, and the slash-command path deliberately shares one grade builder so
+the two never drift. These routes hand the daemon finished answers, keeping it
+entirely out of the grading contract.
+
+Not publicly reachable: the deploy Caddyfile routes only /api/*, /digest and
+/discord/interactions to this backend, so /internal/* never resolves through
+the public domain. The shared-secret header is defence in depth on top of
+that, not the only control — and it fails closed: with no token configured
+the whole surface answers 404, never "open".
+
+Deliberately NOT under BASE (same posture as /healthz): the daemon targets
+THERMOGRAPH_API_BASE_INTERNAL directly and shouldn't need to know
+THERMOGRAPH_BASE to construct a path.
+"""
+import hashlib
+import hmac
+import os
+import threading
+
+from fastapi import APIRouter, Depends, Header, HTTPException
+from pydantic import BaseModel
+
+_TOKEN_ENV = "THERMOGRAPH_INTERNAL_TOKEN"
+_HEADER = "X-Thermograph-Internal-Token"
+
+# Domain-separation label for the derived token. MUST match the Go daemon's
+# internal/config constant byte-for-byte or every callback 401s.
+_DERIVE_LABEL = b"thermograph/internal-api/v1"
+
+
+def resolve_internal_token() -> str:
+    """The shared secret guarding /internal/*, or "" if it can't be established.
+
+    An explicit THERMOGRAPH_INTERNAL_TOKEN always wins. Otherwise it is DERIVED
+    from THERMOGRAPH_AUTH_SECRET, which every environment already provisions, so
+    standing up the daemon needs no new vault entry and no operator step -- one
+    less secret to rotate, leak, or forget on a new host.
+
+    HMAC with a distinct label rather than reusing AUTH_SECRET directly: that is
+    ordinary domain separation, so a leak of this token can't be replayed as the
+    session-signing key it was derived from, and the derivation is one-way.
+
+    Deliberately reads THERMOGRAPH_AUTH_SECRET from the environment rather than
+    users.py's effective value: users.py falls back to a random per-process
+    secret when the env var is unset, and two processes would then derive two
+    different tokens and fail every call in a way that looks like a bug rather
+    than missing config. Unset env var => "" => the surface stays closed."""
+    explicit = os.environ.get(_TOKEN_ENV, "").strip()
+    if explicit:
+        return explicit
+    auth_secret = os.environ.get("THERMOGRAPH_AUTH_SECRET", "").strip()
+    if not auth_secret:
+        return ""
+    return hmac.new(auth_secret.encode(), _DERIVE_LABEL, hashlib.sha256).hexdigest()
+
+
+def _require_internal_token(
+    x_thermograph_internal_token: str = Header(default=""),
+) -> None:
+    """Gate every internal route on the shared secret.
+
+    Read from the environment per request (not at import) so the fail-closed
+    behaviour is a property of the running config, not of import order — and so
+    tests can exercise both states without re-importing the app. An unset/empty
+    token means the surface was never provisioned: 404 (the route doesn't
+    exist), matching the metrics endpoint's don't-reveal posture. A present but
+    wrong header is a real auth failure: 401. compare_digest, not ==, so the
+    comparison doesn't leak the token's prefix through response timing."""
+    token = resolve_internal_token()
+    if not token:
+        raise HTTPException(status_code=404)
+    if not hmac.compare_digest(x_thermograph_internal_token, token):
+        raise HTTPException(status_code=401, detail="bad internal token")
+
+
+# include_in_schema=False: an internal surface has no business in the public
+# OpenAPI document, same as /api/v2/metrics.
+router = APIRouter(prefix="/internal", include_in_schema=False,
+                   dependencies=[Depends(_require_internal_token)])
+
+
+# Every handler below is a plain `def`, not `async def`, on purpose: they all do
+# blocking work (sync parquet/DB reads, grading math, upstream HTTP fetches).
+# FastAPI runs sync handlers on its threadpool, so a slow grade or a long warm
+# run never stalls the event loop the rest of the API is serving on — the same
+# reason web/app.py wraps discord_interactions.handle in run_in_threadpool.
+
+
+class _GradeBody(BaseModel):
+    query: str
+
+
+@router.post("/discord/grade")
+def discord_grade(body: _GradeBody) -> dict:
+    """The gateway-ready reply for one bot mention/DM: the same embed the /grade
+    slash command builds, via the shared discord_interactions._grade_message —
+    reusing one builder is what keeps the two paths from ever drifting. Local
+    import for the same reason the in-process gateway bot used one: don't drag
+    the interactions module's transitive deps into contexts that only mount the
+    router."""
+    from notifications import discord_interactions
+    data = discord_interactions._grade_message(body.query)
+    # Gateway messages can't be ephemeral — the flag is interactions-only, and
+    # Discord rejects it outside an interaction response.
+    data.pop("flags", None)
+    return data
+
+
+# One in-flight run per job, enforced here rather than trusted to the daemon's
+# timers: warm_cities spends the Open-Meteo quota, so a stacked second run
+# double-spends it, and a slow run must never let invocations pile up holding
+# requests open. Non-blocking acquire => the loser answers 409 immediately.
+_JOB_LOCKS: dict[str, threading.Lock] = {
+    "warm-cities": threading.Lock(),
+    "indexnow": threading.Lock(),
+}
+
+
+def _run_exclusive(job: str, fn) -> dict:
+    lock = _JOB_LOCKS[job]
+    if not lock.acquire(blocking=False):
+        raise HTTPException(status_code=409, detail=f"{job} is already running")
+    try:
+        fn()
+    finally:
+        lock.release()
+    return {"ok": True}
+
+
+@router.post("/jobs/warm-cities")
+def jobs_warm_cities() -> dict:
+    """(Re-)warm the curated city set. The daemon owns the cadence (default
+    daily — every already-cached city is a cheap skip); this endpoint just runs
+    one pass. Local import: warm_cities is a script-style top-level module, so
+    only the process that actually runs the job pays its import cost."""
+    def _run():
+        import warm_cities
+        warm_cities.main()
+    return _run_exclusive("warm-cities", _run)
+
+
+@router.post("/jobs/indexnow")
+def jobs_indexnow() -> dict:
+    """Check whether the indexable URL set changed and, if so, ping IndexNow.
+    submit_if_changed() is a cheap no-op when nothing changed, which is why the
+    daemon can tick this more often than warm-cities without spending anything
+    on most runs."""
+    def _run():
+        import indexnow
+        indexnow.submit_if_changed()
+    return _run_exclusive("indexnow", _run)

backend/core/metrics.py

@@ -96,6 +96,12 @@ def classify_inbound(path: str, base: str = "") -> str:
     the app's mount prefix (e.g. ``/thermograph`` or ``""``).
     """
     p = path or "/"
+    # The liveness probe (Dockerfile HEALTHCHECK, Caddy health_uri) is by far the
+    # highest-volume single path in prod (measured ~47% of the access log) and is
+    # never real traffic — same posture as the metrics check below. Never under
+    # BASE (see /healthz's own docstring in web/app.py), so check before stripping.
+    if p.rstrip("/") == "/healthz":
+        return "health"
     # The dashboard polls the metrics endpoint; never count it as traffic, whatever base
     # prefix it arrives under — e.g. a `/thermograph/api/v2/metrics` probe against a
     # root-served prod app would otherwise land in "other" and show as an inbound error.
@@ -118,6 +124,13 @@ def classify_inbound(path: str, base: str = "") -> str:
         # category keeps record_inbound from double-counting every interaction.
         if seg == "event":
             return "event"
+        # The SSR content API (backend/api/content_routes.py) is called only by
+        # the frontend_ssr service's own api_client.py, never by a browser — a
+        # server-to-server hop, not a page view. Its own category is what lets
+        # that (measured ~32% of the access log on prod) be excluded from the
+        # access log below without also hiding real external traffic.
+        if seg == "content":
+            return "internal"
         return f"api:{seg}" if seg else "api:other"
     if p.endswith((".js", ".css", ".html", ".webmanifest", ".png", ".svg", ".ico",
                    ".json", ".woff", ".woff2", ".map", ".txt", ".xml")):

backend/daemon/README.md

@@ -0,0 +1,50 @@
+# thermograph-daemon
+
+Single Go binary that owns the backend's long-lived stateful I/O: the Discord
+gateway connection (IDENTIFY/RESUME/heartbeat/reconnect backoff) and the
+recurring-job timers (warm-cities, IndexNow). It replaces the two in-process
+background jobs `web/app.py` used to run under leader election — one daemon
+replica makes the single-connection invariant a deployment fact instead of a
+runtime election.
+
+## The Go/Python split
+
+Go here owns **no** climate or grading logic. Grading depends on polars and the
+parquet cache, and the slash-command path deliberately shares one grade builder
+with the API so bot grades and API grades never drift. So anything data-shaped
+is a POST back into the Python app's internal-only routes:
+
+| Route | Purpose |
+| --- | --- |
+| `POST /internal/discord/grade` | `{"query": "phoenix"}` → gateway-ready Discord message JSON, relayed to Discord verbatim |
+| `POST /internal/jobs/warm-cities` | trigger the warm-cities job |
+| `POST /internal/jobs/indexnow` | trigger the IndexNow check-and-submit |
+
+Every request carries `X-Thermograph-Internal-Token`. Caddy never routes
+`/internal/*` to the backend, so the token is defence in depth, not the only
+control. If `THERMOGRAPH_INTERNAL_TOKEN` is unset, the Python side disables the
+routes and this daemon refuses to start (fail closed on both ends).
+
+## Configuration (env)
+
+| Var | Required | Meaning |
+| --- | --- | --- |
+| `THERMOGRAPH_INTERNAL_TOKEN` | yes | shared secret for the internal routes |
+| `THERMOGRAPH_API_BASE_INTERNAL` | yes | backend base URL, e.g. `http://web:8137` |
+| `THERMOGRAPH_DISCORD_BOT` | no | `1`/`true`/`yes`/`on` enables the gateway |
+| `THERMOGRAPH_DISCORD_BOT_TOKEN` | no | bot token (gateway stays off without it) |
+| `THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS` | no | default 24; malformed → default |
+| `THERMOGRAPH_INDEXNOW_INTERVAL_HOURS` | no | default 6; malformed → default |
+
+## Run locally
+
+```sh
+cd backend/daemon
+go build ./... && go vet ./... && go test ./...
+
+THERMOGRAPH_INTERNAL_TOKEN=dev-token \
+THERMOGRAPH_API_BASE_INTERNAL=http://localhost:8137 \
+go run .
+```
+
+Deployed as `/usr/local/bin/thermograph-daemon` in the backend image.

backend/daemon/go.mod

@@ -0,0 +1,5 @@
+module thermograph/daemon
+
+go 1.26
+
+require github.com/coder/websocket v1.8.15

backend/daemon/go.sum

@@ -0,0 +1,2 @@
+github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
+github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=

backend/daemon/internal/apiclient/client.go

@@ -0,0 +1,107 @@
+// Package apiclient is the daemon's ONLY channel back into the Python app.
+// The daemon owns long-lived I/O (gateway socket, timers) and nothing else:
+// grading depends on polars + the parquet cache, and the slash-command path
+// deliberately shares one grade builder with the API so the two never drift.
+// Reimplementing any of that here would reintroduce exactly that drift, so
+// everything data-shaped is a POST to the backend's /internal/* routes.
+package apiclient
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// defaultTimeout bounds every callback. Grading does synchronous parquet/DB
+// reads on the Python side (a cold city can take a while), so be generous —
+// but never infinite: a hung backend must not wedge the gateway's event loop.
+const defaultTimeout = 60 * time.Second
+
+// errBodyLimit caps how much of an error response we echo into logs; enough
+// to see a traceback's headline without dumping megabytes.
+const errBodyLimit = 512
+
+// Client calls the backend's internal-only routes. The token is defence in
+// depth — Caddy never routes /internal/* to the backend in the first place —
+// but it is still required on every request so a misrouted or LAN-local call
+// cannot hit these endpoints unauthenticated.
+type Client struct {
+	base  string
+	token string
+	http  *http.Client
+}
+
+// New returns a Client for the backend at base (e.g. http://web:8137),
+// authenticating with token.
+func New(base, token string) *Client {
+	return &Client{
+		base:  strings.TrimRight(base, "/"),
+		token: token,
+		http:  &http.Client{Timeout: defaultTimeout},
+	}
+}
+
+// post sends body to path and returns the raw response body on 2xx.
+func (c *Client) post(ctx context.Context, path string, body any) (json.RawMessage, error) {
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return nil, fmt.Errorf("apiclient: encode %s body: %w", path, err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(payload))
+	if err != nil {
+		return nil, fmt.Errorf("apiclient: build %s request: %w", path, err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Thermograph-Internal-Token", c.token)
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("apiclient: POST %s: %w", path, err)
+	}
+	defer resp.Body.Close()
+
+	// Read one byte past the limit so the truncation marker only appears when
+	// something was actually cut.
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("apiclient: POST %s: read body: %w", path, err)
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		snippet := raw
+		truncated := ""
+		if len(snippet) > errBodyLimit {
+			snippet = snippet[:errBodyLimit]
+			truncated = "... (truncated)"
+		}
+		return nil, fmt.Errorf("apiclient: POST %s: status %d: %s%s", path, resp.StatusCode, snippet, truncated)
+	}
+	return json.RawMessage(raw), nil
+}
+
+// Grade asks Python to grade a city query and returns the gateway-ready
+// Discord message JSON VERBATIM. The raw bytes are relayed to Discord without
+// parsing or reshaping — that verbatim relay is what keeps Go out of the
+// grading/embed contract: any change to the message shape lives entirely on
+// the Python side.
+func (c *Client) Grade(ctx context.Context, query string) (json.RawMessage, error) {
+	return c.post(ctx, "/internal/discord/grade", map[string]string{"query": query})
+}
+
+// WarmCities triggers the warm-cities job (the Python side owns what "warm"
+// means and which cities are curated).
+func (c *Client) WarmCities(ctx context.Context) error {
+	_, err := c.post(ctx, "/internal/jobs/warm-cities", map[string]string{})
+	return err
+}
+
+// IndexNow triggers the IndexNow check-and-submit job; a no-op on the Python
+// side when the indexable URL set has not changed.
+func (c *Client) IndexNow(ctx context.Context) error {
+	_, err := c.post(ctx, "/internal/jobs/indexnow", map[string]string{})
+	return err
+}

backend/daemon/internal/apiclient/client_test.go

@@ -0,0 +1,124 @@
+package apiclient
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestGradeHappyPathRelaysBodyVerbatim(t *testing.T) {
+	// Deliberately odd formatting/key order: Grade must hand back the exact
+	// bytes, never a re-marshalled reshaping of them.
+	const body = `{ "content":null,  "embeds": [ {"title":"Phoenix"} ] }`
+	var gotPath, gotToken, gotQuery string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		gotToken = r.Header.Get("X-Thermograph-Internal-Token")
+		var req map[string]string
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		gotQuery = req["query"]
+		w.Header().Set("Content-Type", "application/json")
+		if _, err := w.Write([]byte(body)); err != nil {
+			t.Errorf("write response: %v", err)
+		}
+	}))
+	defer srv.Close()
+
+	c := New(srv.URL, "tok123")
+	raw, err := c.Grade(context.Background(), "phoenix")
+	if err != nil {
+		t.Fatalf("Grade: %v", err)
+	}
+	if string(raw) != body {
+		t.Errorf("body not relayed verbatim:\n got %q\nwant %q", raw, body)
+	}
+	if gotPath != "/internal/discord/grade" {
+		t.Errorf("path = %q", gotPath)
+	}
+	if gotToken != "tok123" {
+		t.Errorf("token header = %q, want %q", gotToken, "tok123")
+	}
+	if gotQuery != "phoenix" {
+		t.Errorf("query = %q", gotQuery)
+	}
+}
+
+func TestJobsSendTokenAndHitRightPaths(t *testing.T) {
+	paths := map[string]bool{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			t.Errorf("method = %s, want POST", r.Method)
+		}
+		if r.Header.Get("X-Thermograph-Internal-Token") != "tok" {
+			t.Errorf("missing/wrong token header on %s", r.URL.Path)
+		}
+		paths[r.URL.Path] = true
+		if _, err := w.Write([]byte(`{"ok":true}`)); err != nil {
+			t.Errorf("write response: %v", err)
+		}
+	}))
+	defer srv.Close()
+
+	c := New(srv.URL+"/", "tok") // trailing slash must not produce "//internal"
+	if err := c.WarmCities(context.Background()); err != nil {
+		t.Fatalf("WarmCities: %v", err)
+	}
+	if err := c.IndexNow(context.Background()); err != nil {
+		t.Fatalf("IndexNow: %v", err)
+	}
+	for _, p := range []string{"/internal/jobs/warm-cities", "/internal/jobs/indexnow"} {
+		if !paths[p] {
+			t.Errorf("path %s was never hit (saw %v)", p, paths)
+		}
+	}
+}
+
+func TestNon2xxReturnsStatusAndTruncatedBody(t *testing.T) {
+	long := strings.Repeat("x", 2000)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, long, http.StatusBadGateway)
+	}))
+	defer srv.Close()
+
+	c := New(srv.URL, "tok")
+	_, err := c.Grade(context.Background(), "phoenix")
+	if err == nil {
+		t.Fatal("expected error on 502, got nil")
+	}
+	msg := err.Error()
+	if !strings.Contains(msg, "502") {
+		t.Errorf("error should include status, got: %s", msg)
+	}
+	if !strings.Contains(msg, "truncated") {
+		t.Errorf("error should mark truncation, got %d chars: %.100s...", len(msg), msg)
+	}
+	if len(msg) > 1000 {
+		t.Errorf("error message not truncated: %d chars", len(msg))
+	}
+}
+
+func TestContextCancellation(t *testing.T) {
+	block := make(chan struct{})
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-block // hold the request open until the test ends
+	}))
+	defer srv.Close()
+	defer close(block)
+
+	c := New(srv.URL, "tok")
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	_, err := c.Grade(ctx, "phoenix")
+	if err == nil {
+		t.Fatal("expected error from cancelled context, got nil")
+	}
+	if !strings.Contains(err.Error(), "deadline") && !strings.Contains(err.Error(), "cancel") {
+		t.Errorf("expected a context-shaped error, got: %v", err)
+	}
+}

backend/daemon/internal/config/config.go

@@ -0,0 +1,138 @@
+// Package config loads and validates the daemon's environment. All knobs are
+// env vars because the daemon runs as a sibling container to the web service
+// and shares its /etc/thermograph.env — a config file would be a second source
+// of truth for the same values.
+package config
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Defaults mirror the Python scheduler they replace (notifications/scheduler.py):
+// warm-cities daily — a warm cache mostly stays warm, and every already-cached
+// city is a cheap skip; IndexNow every 6h — submit_if_changed() is a no-op when
+// nothing changed, so it can tick more often without spending anything.
+const (
+	DefaultWarmCitiesIntervalHours = 24
+	DefaultIndexNowIntervalHours   = 6
+)
+
+// Config is the daemon's fully-resolved configuration. Construct it only via
+// Load so the required/optional split is enforced in one place.
+type Config struct {
+	// InternalToken is the shared secret sent as X-Thermograph-Internal-Token
+	// on every callback into Python. Required: if it is unset the web app
+	// disables its /internal/* routes (fail closed), so a daemon without it
+	// could only ever fail every call — better to refuse to start.
+	InternalToken string
+
+	// APIBase is the in-network base URL of the FastAPI app (e.g.
+	// http://web:8137). Same env var the frontend already uses for its
+	// server-side calls, so there is one name for "where is the backend".
+	APIBase string
+
+	// DiscordEnabled gates the gateway connection. Off by default: a gateway
+	// connection is a new persistent behaviour, so it takes an explicit opt-in
+	// (THERMOGRAPH_DISCORD_BOT truthy AND a token) rather than piggybacking on
+	// the token being present for DMs.
+	DiscordEnabled bool
+
+	// DiscordToken is the bot token; only meaningful when DiscordEnabled.
+	DiscordToken string
+
+	WarmCitiesInterval time.Duration
+	IndexNowInterval   time.Duration
+}
+
+// truthy matches the Python side's _TRUTHY set exactly, so the same env file
+// enables/disables the bot regardless of which process reads it.
+func truthy(s string) bool {
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "1", "true", "yes", "on":
+		return true
+	}
+	return false
+}
+
+// intervalHours parses an interval env var. A malformed or non-positive value
+// falls back to the default rather than erroring — mirrors the Python
+// `int(os.environ.get(..., "24") or 24)` behaviour where a bad knob degrades
+// to the documented default instead of taking the process down.
+func intervalHours(raw string, def int) time.Duration {
+	hours := def
+	if v := strings.TrimSpace(raw); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			hours = n
+		}
+	}
+	return time.Duration(hours) * time.Hour
+}
+
+// Load reads the environment and returns a validated Config, or an error that
+// should abort startup. The daemon must never run half-configured: a missing
+// token or API base means every callback would fail, silently, forever.
+func Load() (*Config, error) {
+	return load(os.Getenv)
+}
+
+// load takes a getenv func so tests can inject an environment without mutating
+// the real one.
+func load(getenv func(string) string) (*Config, error) {
+	cfg := &Config{
+		InternalToken: strings.TrimSpace(getenv("THERMOGRAPH_INTERNAL_TOKEN")),
+		APIBase:       strings.TrimSpace(getenv("THERMOGRAPH_API_BASE_INTERNAL")),
+		DiscordToken:  strings.TrimSpace(getenv("THERMOGRAPH_DISCORD_BOT_TOKEN")),
+	}
+	if cfg.InternalToken == "" {
+		cfg.InternalToken = deriveInternalToken(strings.TrimSpace(getenv("THERMOGRAPH_AUTH_SECRET")))
+	}
+	if cfg.InternalToken == "" {
+		return nil, errors.New("neither THERMOGRAPH_INTERNAL_TOKEN nor THERMOGRAPH_AUTH_SECRET is set; refusing to start (the web app disables /internal/* without a token, so every callback would fail)")
+	}
+	if cfg.APIBase == "" {
+		return nil, errors.New("THERMOGRAPH_API_BASE_INTERNAL is not set; refusing to start (no way to reach the backend)")
+	}
+	// The gateway needs both the opt-in flag and a token; missing either just
+	// disables it — the cron half is still worth running on its own.
+	cfg.DiscordEnabled = truthy(getenv("THERMOGRAPH_DISCORD_BOT")) && cfg.DiscordToken != ""
+
+	cfg.WarmCitiesInterval = intervalHours(getenv("THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS"), DefaultWarmCitiesIntervalHours)
+	cfg.IndexNowInterval = intervalHours(getenv("THERMOGRAPH_INDEXNOW_INTERVAL_HOURS"), DefaultIndexNowIntervalHours)
+	return cfg, nil
+}
+
+// deriveLabel is the domain-separation label for the derived internal token.
+// It MUST match backend/api/internal_routes.py's _DERIVE_LABEL byte-for-byte,
+// or the daemon and the web app compute different tokens and every callback
+// 401s. TestDeriveInternalTokenMatchesPython pins the pair to a shared vector.
+const deriveLabel = "thermograph/internal-api/v1"
+
+// deriveInternalToken computes the shared secret from THERMOGRAPH_AUTH_SECRET,
+// which every environment already provisions. This is what lets the daemon
+// stand up with no new vault entry and no operator step -- one less secret to
+// rotate, leak, or forget on a new host. An explicit THERMOGRAPH_INTERNAL_TOKEN
+// still wins when set.
+//
+// HMAC under a distinct label rather than reusing the auth secret directly:
+// ordinary domain separation, so a leak of this token cannot be replayed as the
+// session-signing key it came from, and the derivation is one-way.
+//
+// Returns "" for an empty secret, which the caller turns into a refusal to
+// start. It must never silently invent a token: the web app would have derived
+// a different one (or none), and every call would fail as an auth error rather
+// than as the missing configuration it actually is.
+func deriveInternalToken(authSecret string) string {
+	if authSecret == "" {
+		return ""
+	}
+	mac := hmac.New(sha256.New, []byte(authSecret))
+	mac.Write([]byte(deriveLabel))
+	return hex.EncodeToString(mac.Sum(nil))
+}

backend/daemon/internal/config/config_test.go

@@ -0,0 +1,96 @@
+package config
+
+import (
+	"testing"
+	"time"
+)
+
+// env builds a getenv func over a map, defaulting the two required vars so
+// each test only states what it cares about.
+func env(overrides map[string]string) func(string) string {
+	base := map[string]string{
+		"THERMOGRAPH_INTERNAL_TOKEN":    "sekrit",
+		"THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137",
+	}
+	for k, v := range overrides {
+		base[k] = v
+	}
+	return func(k string) string { return base[k] }
+}
+
+func TestRequiredVarsRefuseToStart(t *testing.T) {
+	for _, missing := range []string{"THERMOGRAPH_INTERNAL_TOKEN", "THERMOGRAPH_API_BASE_INTERNAL"} {
+		if _, err := load(env(map[string]string{missing: ""})); err == nil {
+			t.Errorf("expected error when %s is unset, got nil", missing)
+		}
+	}
+	if cfg, err := load(env(nil)); err != nil || cfg == nil {
+		t.Fatalf("both required vars set: unexpected error %v", err)
+	}
+}
+
+func TestTruthyParsing(t *testing.T) {
+	cases := []struct {
+		flag, token string
+		want        bool
+	}{
+		// Must match the Python _TRUTHY set exactly.
+		{"1", "tok", true},
+		{"true", "tok", true},
+		{"TRUE", "tok", true},
+		{"yes", "tok", true},
+		{"on", "tok", true},
+		{" on ", "tok", true}, // whitespace tolerated, like .strip() in Python
+		{"0", "tok", false},
+		{"false", "tok", false},
+		{"", "tok", false},
+		{"enabled", "tok", false}, // not in the truthy set
+		// Flag alone is not enough: no token means no gateway.
+		{"1", "", false},
+	}
+	for _, c := range cases {
+		cfg, err := load(env(map[string]string{
+			"THERMOGRAPH_DISCORD_BOT":       c.flag,
+			"THERMOGRAPH_DISCORD_BOT_TOKEN": c.token,
+		}))
+		if err != nil {
+			t.Fatalf("flag=%q token=%q: unexpected error %v", c.flag, c.token, err)
+		}
+		if cfg.DiscordEnabled != c.want {
+			t.Errorf("flag=%q token=%q: DiscordEnabled=%v, want %v", c.flag, c.token, cfg.DiscordEnabled, c.want)
+		}
+	}
+}
+
+func TestIntervalDefaultsAndFallbacks(t *testing.T) {
+	cases := []struct {
+		name           string
+		warm, indexnow string
+		wantWarm       time.Duration
+		wantIndexNow   time.Duration
+	}{
+		{"unset uses defaults", "", "", 24 * time.Hour, 6 * time.Hour},
+		{"explicit values win", "12", "3", 12 * time.Hour, 3 * time.Hour},
+		// Malformed values degrade to the default instead of erroring, like
+		// the Python int(... or 24) they replace.
+		{"garbage falls back", "soon", "1.5", 24 * time.Hour, 6 * time.Hour},
+		{"non-positive falls back", "0", "-2", 24 * time.Hour, 6 * time.Hour},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			cfg, err := load(env(map[string]string{
+				"THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS": c.warm,
+				"THERMOGRAPH_INDEXNOW_INTERVAL_HOURS":    c.indexnow,
+			}))
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if cfg.WarmCitiesInterval != c.wantWarm {
+				t.Errorf("WarmCitiesInterval=%v, want %v", cfg.WarmCitiesInterval, c.wantWarm)
+			}
+			if cfg.IndexNowInterval != c.wantIndexNow {
+				t.Errorf("IndexNowInterval=%v, want %v", cfg.IndexNowInterval, c.wantIndexNow)
+			}
+		})
+	}
+}

backend/daemon/internal/config/derive_test.go

@@ -0,0 +1,66 @@
+package config
+
+import "testing"
+
+// CROSS-LANGUAGE CONTRACT. The daemon and the web app must derive byte-identical
+// tokens from the same THERMOGRAPH_AUTH_SECRET, or every callback 401s -- and it
+// would surface as an auth error, which reads like a bug rather than like the
+// configuration drift it actually is. The same vector is asserted on the Python
+// side in backend/tests/api/test_internal_routes.py::test_derived_token_matches_go.
+// If deriveLabel or the construction changes, BOTH must change together.
+const (
+	sharedVectorSecret = "test-auth-secret"
+	sharedVectorToken  = "4c3830b2158941ff52720d198b65f7924372a3a482b8da52540a4d9be38247ea"
+)
+
+func TestDeriveInternalTokenMatchesPython(t *testing.T) {
+	if got := deriveInternalToken(sharedVectorSecret); got != sharedVectorToken {
+		t.Fatalf("deriveInternalToken(%q) = %q; want %q (must match internal_routes.py)",
+			sharedVectorSecret, got, sharedVectorToken)
+	}
+}
+
+// An empty secret must yield nothing rather than some fixed "empty HMAC" value:
+// the caller turns "" into a refusal to start, and inventing a token here would
+// let the daemon run against a web app that computed a different one.
+func TestDeriveInternalTokenEmptySecretYieldsNothing(t *testing.T) {
+	if got := deriveInternalToken(""); got != "" {
+		t.Fatalf("deriveInternalToken(%q) = %q; want empty so Load refuses to start", "", got)
+	}
+}
+
+func TestLoadPrefersExplicitTokenOverDerived(t *testing.T) {
+	env := map[string]string{
+		"THERMOGRAPH_INTERNAL_TOKEN":    "explicit-wins",
+		"THERMOGRAPH_AUTH_SECRET":       sharedVectorSecret,
+		"THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137",
+	}
+	cfg, err := load(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if cfg.InternalToken != "explicit-wins" {
+		t.Fatalf("InternalToken = %q; an explicit env var must win over derivation", cfg.InternalToken)
+	}
+}
+
+func TestLoadDerivesTokenWhenOnlyAuthSecretIsSet(t *testing.T) {
+	env := map[string]string{
+		"THERMOGRAPH_AUTH_SECRET":       sharedVectorSecret,
+		"THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137",
+	}
+	cfg, err := load(func(k string) string { return env[k] })
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if cfg.InternalToken != sharedVectorToken {
+		t.Fatalf("InternalToken = %q; want the derived %q", cfg.InternalToken, sharedVectorToken)
+	}
+}
+
+func TestLoadRefusesWhenNeitherTokenNorAuthSecretIsSet(t *testing.T) {
+	env := map[string]string{"THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137"}
+	if _, err := load(func(k string) string { return env[k] }); err == nil {
+		t.Fatal("load succeeded with no token and no auth secret; it must refuse to start")
+	}
+}

backend/daemon/internal/cron/cron.go

@@ -0,0 +1,92 @@
+// Package cron is the Go replacement for notifications/scheduler.py's
+// APScheduler pair (warm-cities + IndexNow). The Python side chose APScheduler
+// over a Postgres-native queue because exactly one process ever runs these
+// jobs — no fan-out, backpressure, or dead-letter need — and that shape
+// carries over unchanged: this daemon is the sole scheduler, so plain
+// per-job tickers are all the machinery required. procrastinate/pgqueuer
+// remain the documented upgrade path if that ever changes — see
+// docs/architecture/repo-topology-and-infrastructure.md §7.
+//
+// The jobs themselves stay in Python (called via the internal HTTP routes);
+// this package owns only the timing.
+package cron
+
+import (
+	"context"
+	"log/slog"
+	"sync"
+	"time"
+)
+
+// Job is one recurring task: a name for logs, an interval, and the callback
+// (in practice an apiclient method hitting a /internal/jobs/* route).
+type Job struct {
+	Name     string
+	Interval time.Duration
+	Run      func(ctx context.Context) error
+}
+
+// Run drives every job on its own ticker until ctx is cancelled, then returns
+// once all job goroutines have stopped. It never returns early: a failing job
+// logs and keeps ticking — one bad tick (backend briefly down, timeout) must
+// never kill the schedule, because nothing restarts it short of a container
+// restart.
+func Run(ctx context.Context, logger *slog.Logger, jobs ...Job) {
+	var wg sync.WaitGroup
+	for _, job := range jobs {
+		wg.Add(1)
+		go func(job Job) {
+			defer wg.Done()
+			runJob(ctx, logger, job)
+		}(job)
+	}
+	wg.Wait()
+}
+
+func runJob(ctx context.Context, logger *slog.Logger, job Job) {
+	// A plain ticker's first fire is one interval from now, NOT at startup —
+	// deliberately kept from the Python scheduler (its add_job had no
+	// next_run_time override for the same reason): warm-cities already runs at
+	// deploy time via the deploy hook, so an immediate run on every daemon
+	// boot/restart would just re-spend work (and, for warm-cities, archive-fetch
+	// quota) that the deploy already paid for.
+	ticker := time.NewTicker(job.Interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			start := time.Now()
+			// Run synchronously in this loop: a job can never overlap with
+			// itself by construction. That matters most for warm-cities,
+			// which spends the archive-fetch quota — two overlapping runs would
+			// double-spend it.
+			if err := job.Run(ctx); err != nil {
+				if ctx.Err() != nil {
+					// Shutdown raced the job; the cancellation is the story,
+					// not the (expected) aborted call.
+					return
+				}
+				logger.Error("cron job failed; will retry next tick",
+					"job", job.Name, "err", err, "elapsed", time.Since(start))
+			} else {
+				logger.Info("cron job ok",
+					"job", job.Name, "elapsed", time.Since(start))
+			}
+			// If the job overran its interval, a tick may have fired mid-run.
+			// Skip it rather than running again back-to-back — a late tick is
+			// the same double-spend as an overlapping one, just serialized.
+			// (Go 1.23+ tickers already drop fires with no ready receiver;
+			// this drain pins the skip semantics rather than relying on the
+			// runtime's channel buffering.)
+			select {
+			case <-ticker.C:
+				logger.Warn("cron job overran its interval; skipping the tick that fired mid-run",
+					"job", job.Name, "interval", job.Interval, "elapsed", time.Since(start))
+			default:
+			}
+		}
+	}
+}

backend/daemon/internal/cron/cron_test.go

@@ -0,0 +1,204 @@
+package cron
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// Intervals here are tens of milliseconds: long enough that scheduler jitter
+// cannot invert an assertion, short enough that the whole file runs in well
+// under a second.
+
+func discard() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, nil))
+}
+
+// The first run must be one interval after start, never at startup — the
+// deploy hook already warmed the cache, so a boot-time run is pure waste.
+func TestFirstRunIsDeferredOneInterval(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var runs atomic.Int32
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		Run(ctx, discard(), Job{
+			Name:     "test",
+			Interval: 60 * time.Millisecond,
+			Run: func(context.Context) error {
+				runs.Add(1)
+				return nil
+			},
+		})
+	}()
+
+	// Well inside the first interval: nothing may have run yet.
+	time.Sleep(20 * time.Millisecond)
+	if got := runs.Load(); got != 0 {
+		t.Fatalf("job ran %d time(s) before the first interval elapsed; first run must be deferred", got)
+	}
+
+	// Well past the first interval: it must have run by now.
+	deadline := time.After(500 * time.Millisecond)
+	for runs.Load() == 0 {
+		select {
+		case <-deadline:
+			t.Fatal("job never ran after the first interval elapsed")
+		case <-time.After(5 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	<-done
+}
+
+// A slow run must not overlap with itself, and ticks that fired mid-run must
+// be skipped, not queued — an immediate back-to-back run would double-spend
+// the archive-fetch quota just like a concurrent one.
+func TestSlowJobNeverOverlapsAndSkipsMissedTicks(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	const interval = 30 * time.Millisecond
+
+	var inFlight, maxInFlight, runs atomic.Int32
+	release := make(chan struct{})
+	started := make(chan struct{}, 16)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		Run(ctx, discard(), Job{
+			Name:     "slow",
+			Interval: interval,
+			Run: func(context.Context) error {
+				n := inFlight.Add(1)
+				for {
+					m := maxInFlight.Load()
+					if n <= m || maxInFlight.CompareAndSwap(m, n) {
+						break
+					}
+				}
+				runs.Add(1)
+				started <- struct{}{}
+				<-release // block until the test lets each run finish
+				inFlight.Add(-1)
+				return nil
+			},
+		})
+	}()
+
+	// First run starts; hold it across several intervals.
+	<-started
+	time.Sleep(4 * interval)
+	if got := maxInFlight.Load(); got != 1 {
+		t.Fatalf("job overlapped with itself: max in-flight = %d", got)
+	}
+	if got := runs.Load(); got != 1 {
+		t.Fatalf("expected exactly 1 run while the first is still blocked, got %d", got)
+	}
+	release <- struct{}{}
+
+	// After release the schedule resumes, but the ticks missed during the
+	// block must have been dropped: the next run arrives roughly one interval
+	// later, and only one more within that window (not a burst of catch-ups).
+	select {
+	case <-started:
+	case <-time.After(500 * time.Millisecond):
+		t.Fatal("job never resumed after the blocked run finished")
+	}
+	if got := runs.Load(); got != 2 {
+		t.Fatalf("missed ticks were replayed as a burst: %d runs total, want 2", got)
+	}
+	release <- struct{}{}
+
+	cancel()
+	<-done
+	if got := maxInFlight.Load(); got != 1 {
+		t.Fatalf("job overlapped with itself: max in-flight = %d", got)
+	}
+}
+
+// One failed tick must never kill the ticker — nothing restarts the schedule
+// short of a container restart.
+func TestFailedTickDoesNotStopSchedule(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var runs atomic.Int32
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		Run(ctx, discard(), Job{
+			Name:     "flaky",
+			Interval: 20 * time.Millisecond,
+			Run: func(context.Context) error {
+				if runs.Add(1) == 1 {
+					return errors.New("backend briefly down")
+				}
+				return nil
+			},
+		})
+	}()
+
+	deadline := time.After(1 * time.Second)
+	for runs.Load() < 3 {
+		select {
+		case <-deadline:
+			t.Fatalf("schedule stalled after a failure: only %d run(s)", runs.Load())
+		case <-time.After(5 * time.Millisecond):
+		}
+	}
+
+	cancel()
+	<-done
+}
+
+// Cancellation must stop Run promptly even while a job is mid-call: the job
+// callbacks are ctx-aware HTTP calls, so cancelling the context aborts the
+// in-flight request and the loop must then exit instead of ticking again.
+func TestCancelStopsRunWhileJobInFlight(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+
+	started := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		Run(ctx, discard(),
+			Job{
+				Name:     "blocking",
+				Interval: 10 * time.Millisecond,
+				Run: func(jobCtx context.Context) error {
+					close(started)
+					<-jobCtx.Done() // behaves like an HTTP call aborted by cancellation
+					return jobCtx.Err()
+				},
+			},
+			// A second, idle job proves Run waits for ALL jobs to stop.
+			Job{
+				Name:     "idle",
+				Interval: time.Hour,
+				Run:      func(context.Context) error { return nil },
+			},
+		)
+	}()
+
+	<-started
+	cancel()
+
+	select {
+	case <-done:
+	case <-time.After(1 * time.Second):
+		t.Fatal("Run did not return promptly after context cancellation")
+	}
+}

backend/daemon/internal/gateway/gateway.go

@@ -0,0 +1,529 @@
+// Package gateway holds Discord's **gateway** bot — it reacts to @mentions
+// (and DMs) in real time. The other two Discord paths need no persistent
+// connection: slash commands arrive as signed HTTP POSTs and the daily post
+// goes out over an incoming webhook, both handled on the Python side. This one
+// is different — to answer an *ordinary* message that mentions the bot, we
+// have to hold a live websocket to Discord's gateway and listen for
+// MESSAGE_CREATE events.
+//
+// Single-connection invariant. Discord permits exactly one gateway connection
+// per bot token (per shard). The Python predecessor enforced that with a
+// leader election inside web/app.py; here it holds by deployment shape — the
+// daemon runs as exactly one replica, and this package owns its only
+// connection.
+//
+// Least privilege / no privileged intent. We subscribe to GUILDS +
+// GUILD_MESSAGES + DIRECT_MESSAGES and act only on messages that @mention the
+// bot or DM it. Discord delivers message *content* for exactly those cases
+// even without the privileged MESSAGE_CONTENT intent, so the bot needs no
+// privileged-intent portal review — a real design constraint, not trivia. To
+// trigger on arbitrary keywords in channels, enable MESSAGE_CONTENT in the
+// Developer Portal and extend triage; the rest of the machinery is unchanged.
+//
+// The gateway owns NO climate/grading logic. A mention with a city query is
+// answered by calling back into the Python app (Grader / apiclient), which
+// shares one grade builder with the /grade slash command so the two never
+// drift.
+package gateway
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"math/rand"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/coder/websocket"
+
+	"thermograph/daemon/internal/config"
+)
+
+// Run is the package entrypoint: build a Bot from the resolved config and
+// drive it for ctx's lifetime. The caller has already checked
+// cfg.DiscordEnabled — an unconfigured deploy never gets this far, so an
+// unconfigured deploy pays nothing.
+func Run(ctx context.Context, cfg *config.Config, api Grader) error {
+	return New(cfg.DiscordToken, api).Run(ctx)
+}
+
+// v10 JSON gateway. On a resume we reconnect to the session's
+// resume_gateway_url (from the READY payload) instead, with the same query
+// string appended.
+const (
+	gatewayURL = "wss://gateway.discord.gg/?v=10&encoding=json"
+	gatewayQS  = "/?v=10&encoding=json"
+)
+
+// Gateway intents (bitfield). GUILDS gives guild/channel context;
+// GUILD_MESSAGES and DIRECT_MESSAGES deliver message-create events in servers
+// and DMs. We deliberately do NOT request MESSAGE_CONTENT (privileged) —
+// Discord still includes content for messages that mention us or DM us, which
+// is all we act on.
+const (
+	intentGuilds         = 1 << 0
+	intentGuildMessages  = 1 << 9
+	intentDirectMessages = 1 << 12
+	intents              = intentGuilds | intentGuildMessages | intentDirectMessages
+)
+
+// Gateway opcodes (discord.com/developers/docs/topics/gateway-events#opcodes).
+const (
+	opDispatch       = 0
+	opHeartbeat      = 1
+	opIdentify       = 2
+	opResume         = 6
+	opReconnect      = 7
+	opInvalidSession = 9
+	opHello          = 10
+	opHeartbeatACK   = 11
+)
+
+// isFatalClose reports whether a close code is unrecoverable — reconnecting
+// just loops, so we stop: 4004 auth failed (bad token), 4010 invalid shard,
+// 4011 sharding required, 4012 invalid API version, 4013 invalid intents,
+// 4014 disallowed (privileged) intents.
+func isFatalClose(code websocket.StatusCode) bool {
+	switch code {
+	case 4004, 4010, 4011, 4012, 4013, 4014:
+		return true
+	}
+	return false
+}
+
+// Reconnect backoff bounds. A server-requested reconnect (op 7) resets to the
+// start so a planned reconnect comes back promptly.
+const (
+	backoffStart = 1 * time.Second
+	backoffMax   = 60 * time.Second
+)
+
+// nextBackoff doubles the reconnect delay, capped at backoffMax.
+func nextBackoff(d time.Duration) time.Duration {
+	d *= 2
+	if d > backoffMax {
+		d = backoffMax
+	}
+	return d
+}
+
+// maxConcurrentReplies bounds in-flight MESSAGE_CREATE handling. The Python
+// version pushed each message onto a thread (asyncio.to_thread) so a slow
+// grading lookup couldn't stall heartbeats, but its queue was unbounded; the
+// semaphore here keeps the same never-block-the-read-loop property while
+// making sure a flood of mentions can't spawn unbounded goroutines against
+// the backend.
+const maxConcurrentReplies = 4
+
+// dialTimeout mirrors the Python's open_timeout=30.
+const dialTimeout = 30 * time.Second
+
+// maxFrameSize mirrors the Python's max_size=2**20 — a grade embed is tiny,
+// so anything near a megabyte is not a frame we want to buffer.
+const maxFrameSize = 1 << 20
+
+const (
+	modeIdentify = "identify"
+	modeResume   = "resume"
+)
+
+// Grader is the one slice of the internal API the gateway needs: a city query
+// in, gateway-ready Discord message JSON out. Satisfied by *apiclient.Client.
+type Grader interface {
+	Grade(ctx context.Context, query string) (json.RawMessage, error)
+}
+
+// Bot owns the single gateway connection and everything hanging off it.
+type Bot struct {
+	token string
+	api   Grader
+	sem   chan struct{} // bounds concurrent MESSAGE_CREATE handlers
+	rest  string        // Discord REST base; swapped for a test server in tests
+	http  *http.Client  // REST replies (not the gateway socket)
+}
+
+// New returns a Bot that authenticates with token and answers city queries
+// through api.
+func New(token string, api Grader) *Bot {
+	return &Bot{
+		token: token,
+		api:   api,
+		sem:   make(chan struct{}, maxConcurrentReplies),
+		rest:  discordAPIBase,
+		http:  &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+// payload is the gateway envelope: opcode, event data, sequence number,
+// dispatch type.
+type payload struct {
+	Op int             `json:"op"`
+	D  json.RawMessage `json:"d"`
+	S  *int64          `json:"s"`
+	T  string          `json:"t"`
+}
+
+// session is the per-connection state that must survive a resume: the last
+// sequence number (what RESUME replays from), the resume token/URL, and our
+// own user id (for mention detection). seq is mutex-guarded because the read
+// loop advances it while the heartbeat goroutine reads it; the other fields
+// are only ever touched from the read/run loop.
+type session struct {
+	mu        sync.Mutex
+	seq       int64
+	haveSeq   bool
+	sessionID string
+	resumeURL string
+	userID    string
+}
+
+func (s *session) setSeq(v int64) {
+	s.mu.Lock()
+	s.seq, s.haveSeq = v, true
+	s.mu.Unlock()
+}
+
+func (s *session) lastSeq() (int64, bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.seq, s.haveSeq
+}
+
+// wsConn is a thin JSON send/receive wrapper. coder/websocket allows
+// concurrent calls to everything except Read, and only the read loop reads,
+// so no extra locking is needed for the heartbeat goroutine's writes.
+type wsConn struct {
+	c *websocket.Conn
+}
+
+func (w *wsConn) send(ctx context.Context, v any) error {
+	data, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return w.c.Write(ctx, websocket.MessageText, data)
+}
+
+func (w *wsConn) read(ctx context.Context) ([]byte, error) {
+	_, data, err := w.c.Read(ctx)
+	return data, err
+}
+
+func (w *wsConn) close(code websocket.StatusCode, reason string) {
+	// Best-effort: the connection may already be gone, and a failed close on
+	// a dead socket changes nothing about what happens next.
+	_ = w.c.Close(code, reason)
+}
+
+// jitter is the 0..1s randomness added to every reconnect sleep so a fleet of
+// clients dropped by the same incident doesn't reconnect in lockstep.
+func jitter() time.Duration {
+	return time.Duration(rand.Float64() * float64(time.Second))
+}
+
+// sleepCtx sleeps for d, returning false if ctx was cancelled first.
+func sleepCtx(ctx context.Context, d time.Duration) bool {
+	select {
+	case <-ctx.Done():
+		return false
+	case <-time.After(d):
+		return true
+	}
+}
+
+// Run maintains the gateway connection for the daemon's lifetime: connect,
+// IDENTIFY (or RESUME), dispatch events, and reconnect with capped
+// exponential backoff. It returns nil on context cancellation and an error
+// only on a fatal close (bad token / disallowed intents) — reconnecting after
+// those loops forever, so the caller must not restart us.
+func (b *Bot) Run(ctx context.Context) error {
+	sess := &session{}
+	mode := modeIdentify
+	backoff := backoffStart
+	for {
+		useResume := mode == modeResume && sess.resumeURL != ""
+		url := gatewayURL
+		if useResume {
+			url = sess.resumeURL
+		}
+		dialCtx, cancel := context.WithTimeout(ctx, dialTimeout)
+		conn, _, err := websocket.Dial(dialCtx, url, nil)
+		cancel()
+		if err != nil {
+			if ctx.Err() != nil {
+				log.Print("gateway: shutting down")
+				return nil
+			}
+			// A failed dial carries no gateway close code, so it is never
+			// fatal — back off and retry.
+			mode = modeIdentify
+			if sess.sessionID != "" {
+				mode = modeResume
+			}
+			log.Printf("gateway: connect failed (%v); reconnecting in %s", err, backoff)
+			if !sleepCtx(ctx, backoff+jitter()) {
+				return nil
+			}
+			backoff = nextBackoff(backoff)
+			continue
+		}
+		conn.SetReadLimit(maxFrameSize)
+		ws := &wsConn{c: conn}
+
+		next, err := b.connection(ctx, ws, sess, useResume)
+		if ctx.Err() != nil {
+			// Shutdown: close clean (1000) — we are gone for good, so the
+			// session should not be held open for a RESUME that never comes.
+			ws.close(websocket.StatusNormalClosure, "shutting down")
+			log.Print("gateway: shutting down")
+			return nil
+		}
+		if err != nil {
+			if code := websocket.CloseStatus(err); isFatalClose(code) {
+				log.Printf("gateway: fatal gateway close %d; stopping", code)
+				return fmt.Errorf("gateway: fatal close code %d", code)
+			}
+			mode = modeIdentify
+			if sess.sessionID != "" {
+				mode = modeResume
+			}
+			log.Printf("gateway: connection dropped (%v); reconnecting in %s", err, backoff)
+			if !sleepCtx(ctx, backoff+jitter()) {
+				return nil
+			}
+			backoff = nextBackoff(backoff)
+			continue
+		}
+
+		// Deliberate reconnect (op 7 / op 9): close it ourselves. Non-1000
+		// when we intend to RESUME — Discord invalidates the session on a
+		// 1000/1001 close, which would force a fresh IDENTIFY.
+		if next == modeResume {
+			ws.close(websocket.StatusCode(4000), "resuming")
+		} else {
+			ws.close(websocket.StatusNormalClosure, "reidentifying")
+		}
+		mode = next
+		backoff = backoffStart // planned reconnect: come back promptly
+	}
+}
+
+// parseHello validates the op-10 HELLO that opens every connection and returns
+// its heartbeat interval.
+//
+// Every failure here is an ERROR rather than a clean "reidentify". That
+// distinction is load-bearing: a clean return is Run's PLANNED-reconnect path,
+// which resets the backoff to 1s and redials with no sleep at all. That is
+// correct for a server-requested reconnect (op 7), but a gateway persistently
+// answering with a malformed HELLO would turn it into a tight reconnect loop
+// against Discord -- precisely what the capped backoff exists to prevent. The
+// error path applies backoff and still picks the next mode from surviving
+// session state.
+func parseHello(raw []byte) (time.Duration, error) {
+	var hello payload
+	if err := json.Unmarshal(raw, &hello); err != nil {
+		return 0, fmt.Errorf("gateway: unparseable HELLO: %w", err)
+	}
+	if hello.Op != opHello {
+		return 0, fmt.Errorf("gateway: expected HELLO (op %d), got op %d", opHello, hello.Op)
+	}
+	var hd struct {
+		HeartbeatInterval float64 `json:"heartbeat_interval"`
+	}
+	if err := json.Unmarshal(hello.D, &hd); err != nil {
+		return 0, fmt.Errorf("gateway: unparseable HELLO payload: %w", err)
+	}
+	// A non-positive interval would busy-loop the heartbeat goroutine.
+	if hd.HeartbeatInterval <= 0 {
+		return 0, fmt.Errorf("gateway: invalid heartbeat_interval %v", hd.HeartbeatInterval)
+	}
+	return time.Duration(hd.HeartbeatInterval * float64(time.Millisecond)), nil
+}
+
+// connection drives one live connection until it ends. A clean return carries
+// the mode for the next connect ("resume" to reconnect and RESUME, "identify"
+// to start a fresh session); an error return means the socket died and the
+// caller picks the next mode from the surviving session state.
+func (b *Bot) connection(ctx context.Context, ws *wsConn, sess *session, resume bool) (string, error) {
+	raw, err := ws.read(ctx)
+	if err != nil {
+		return "", err
+	}
+	interval, err := parseHello(raw)
+	if err != nil {
+		return modeIdentify, err
+	}
+
+	// The heartbeat lives exactly as long as this connection.
+	hbCtx, stopHB := context.WithCancel(ctx)
+	defer stopHB()
+	acked := &atomic.Bool{}
+	acked.Store(true)
+	beatNow := make(chan struct{}, 1)
+	go b.heartbeat(hbCtx, ws, interval, sess, acked, beatNow)
+
+	seq, haveSeq := sess.lastSeq()
+	if resume && sess.sessionID != "" && haveSeq {
+		err = ws.send(ctx, map[string]any{"op": opResume, "d": map[string]any{
+			"token":      b.token,
+			"session_id": sess.sessionID,
+			"seq":        seq,
+		}})
+	} else {
+		err = ws.send(ctx, map[string]any{"op": opIdentify, "d": map[string]any{
+			"token":      b.token,
+			"intents":    intents,
+			"properties": map[string]string{"os": "linux", "browser": "thermograph", "device": "thermograph"},
+		}})
+	}
+	if err != nil {
+		return "", err
+	}
+
+	for {
+		data, err := ws.read(ctx)
+		if err != nil {
+			return "", err
+		}
+		var p payload
+		if err := json.Unmarshal(data, &p); err != nil {
+			log.Printf("gateway: dropping unparseable frame: %v", err)
+			continue
+		}
+		if p.S != nil {
+			// Every payload that carries a sequence number advances it — it
+			// is what RESUME replays from.
+			sess.setSeq(*p.S)
+		}
+		switch p.Op {
+		case opDispatch:
+			b.dispatch(ctx, p, sess)
+		case opHeartbeat:
+			// Server asked for an immediate beat; route it through the
+			// heartbeat goroutine, which owns beat sending.
+			select {
+			case beatNow <- struct{}{}:
+			default:
+			}
+		case opHeartbeatACK:
+			acked.Store(true)
+		case opReconnect:
+			return modeResume, nil // planned reconnect — resume
+		case opInvalidSession:
+			// d == true => the session can still be resumed; d == false =>
+			// it is dead and the next connect must IDENTIFY fresh.
+			var resumable bool
+			_ = json.Unmarshal(p.D, &resumable)
+			if !resumable {
+				sess.sessionID = ""
+				sess.resumeURL = ""
+			}
+			// Discord wants a short randomized wait before re-authing.
+			if !sleepCtx(ctx, time.Second+jitter()) {
+				return "", ctx.Err()
+			}
+			if sess.sessionID != "" {
+				return modeResume, nil
+			}
+			return modeIdentify, nil
+		}
+	}
+}
+
+// heartbeat sends op-1 heartbeats every interval, the first one jittered by
+// interval*rand per Discord's guidance. If the previous beat was never ACKed
+// the link is a zombie — a half-open TCP connection where our writes
+// "succeed" but nothing comes back — so close with a non-1000 code (1000
+// would invalidate the session) and stop; the next connect can then RESUME.
+func (b *Bot) heartbeat(ctx context.Context, ws *wsConn, interval time.Duration, sess *session, acked *atomic.Bool, beatNow <-chan struct{}) {
+	timer := time.NewTimer(time.Duration(rand.Float64() * float64(interval)))
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-beatNow:
+			// Server-requested beat (op 1): send immediately, outside the
+			// zombie accounting — it is not the reply to one of ours.
+			if err := b.sendHeartbeat(ctx, ws, sess); err != nil {
+				return // socket gone; the read loop owns reconnecting
+			}
+		case <-timer.C:
+			if !acked.Load() {
+				ws.close(websocket.StatusCode(4000), "heartbeat ack missed")
+				return
+			}
+			acked.Store(false)
+			if err := b.sendHeartbeat(ctx, ws, sess); err != nil {
+				return
+			}
+			timer.Reset(interval)
+		}
+	}
+}
+
+func (b *Bot) sendHeartbeat(ctx context.Context, ws *wsConn, sess *session) error {
+	var d any // null until the first sequence number arrives
+	if seq, ok := sess.lastSeq(); ok {
+		d = seq
+	}
+	return ws.send(ctx, map[string]any{"op": opHeartbeat, "d": d})
+}
+
+// dispatch handles an op-0 DISPATCH event we care about.
+func (b *Bot) dispatch(ctx context.Context, p payload, sess *session) {
+	switch p.T {
+	case "READY":
+		var d struct {
+			SessionID        string `json:"session_id"`
+			ResumeGatewayURL string `json:"resume_gateway_url"`
+			User             struct {
+				ID snowflake `json:"id"`
+			} `json:"user"`
+		}
+		if err := json.Unmarshal(p.D, &d); err != nil {
+			log.Printf("gateway: unparseable READY: %v", err)
+			return
+		}
+		sess.sessionID = d.SessionID
+		sess.resumeURL = ""
+		if r := strings.TrimRight(d.ResumeGatewayURL, "/"); r != "" {
+			sess.resumeURL = r + gatewayQS
+		}
+		sess.userID = string(d.User.ID)
+		log.Printf("gateway: ready (user %s)", sess.userID)
+	case "MESSAGE_CREATE":
+		// Until READY tells us our own user id we can't detect mentions (or
+		// guard against answering ourselves), so stay silent.
+		if sess.userID == "" {
+			return
+		}
+		var m gwMessage
+		if err := json.Unmarshal(p.D, &m); err != nil {
+			log.Printf("gateway: unparseable MESSAGE_CREATE: %v", err)
+			return
+		}
+		// Grading is a synchronous parquet/DB round trip on the Python side,
+		// so it must never run on the read loop — a slow lookup would stall
+		// heartbeats (the Python used asyncio.to_thread for the same
+		// reason). Each message gets its own goroutine, with the semaphore
+		// bounding in-flight replies so a flood of mentions can't spawn
+		// unbounded goroutines against the backend — an improvement over
+		// to_thread's unbounded queue. Over the bound we drop and log rather
+		// than block the read loop.
+		botID := sess.userID
+		select {
+		case b.sem <- struct{}{}:
+			go func() {
+				defer func() { <-b.sem }()
+				b.handleMessage(ctx, &m, botID)
+			}()
+		default:
+			log.Printf("gateway: dropping message %s: too many replies in flight", m.ID)
+		}
+	}
+}

backend/daemon/internal/gateway/gateway_test.go

@@ -0,0 +1,182 @@
+// Connection-machinery tests, restricted to the pure parts (backoff, fatal
+// close codes, READY/session bookkeeping, sequence tracking). Deliberately no
+// live-gateway test — that path is exercised in staging, not CI.
+
+package gateway
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/coder/websocket"
+)
+
+func TestBackoffDoublesAndCaps(t *testing.T) {
+	want := []time.Duration{
+		2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second,
+		32 * time.Second, 60 * time.Second, 60 * time.Second, 60 * time.Second,
+	}
+	d := backoffStart
+	for i, w := range want {
+		d = nextBackoff(d)
+		if d != w {
+			t.Fatalf("step %d: want %s, got %s", i, w, d)
+		}
+	}
+}
+
+func TestFatalCloseCodes(t *testing.T) {
+	for _, c := range []websocket.StatusCode{4004, 4010, 4011, 4012, 4013, 4014} {
+		if !isFatalClose(c) {
+			t.Errorf("close code %d must be fatal (reconnecting loops forever)", c)
+		}
+	}
+	// 4008 (rate limited) and 4009 (session timed out) are resumable; 4000 is
+	// our own zombie close; -1 is CloseStatus's not-a-close-error sentinel.
+	for _, c := range []websocket.StatusCode{-1, 1000, 1001, 4000, 4001, 4008, 4009} {
+		if isFatalClose(c) {
+			t.Errorf("close code %d must not be fatal", c)
+		}
+	}
+}
+
+func TestReadyCapturesResumeState(t *testing.T) {
+	b := New("tok", &fakeGrader{})
+	sess := &session{}
+	d := []byte(`{"session_id":"abc","resume_gateway_url":"wss://resume.example/","user":{"id":"999"}}`)
+
+	b.dispatch(context.Background(), payload{Op: opDispatch, T: "READY", D: d}, sess)
+
+	if sess.sessionID != "abc" {
+		t.Fatalf("want session id abc, got %q", sess.sessionID)
+	}
+	// Trailing slash trimmed, gateway query string appended — RESUME must go
+	// to the session's own URL with the same v10/json parameters.
+	if sess.resumeURL != "wss://resume.example/?v=10&encoding=json" {
+		t.Fatalf("bad resume url: %q", sess.resumeURL)
+	}
+	if sess.userID != botID {
+		t.Fatalf("want user id %s, got %q", botID, sess.userID)
+	}
+}
+
+func TestReadyWithoutResumeURLLeavesItEmpty(t *testing.T) {
+	b := New("tok", &fakeGrader{})
+	sess := &session{resumeURL: "wss://stale.example/?v=10&encoding=json"}
+	d := []byte(`{"session_id":"abc","user":{"id":"999"}}`)
+
+	b.dispatch(context.Background(), payload{Op: opDispatch, T: "READY", D: d}, sess)
+
+	if sess.resumeURL != "" {
+		t.Fatalf("a READY without resume_gateway_url must clear the stale url, got %q", sess.resumeURL)
+	}
+}
+
+func TestMessageCreateBeforeReadyIsIgnored(t *testing.T) {
+	// Until READY supplies our user id we cannot detect mentions (or guard
+	// against answering ourselves), so nothing may be handled.
+	g := &fakeGrader{resp: json.RawMessage(`{}`)}
+	b := New("tok", g)
+	sess := &session{} // no userID yet
+	d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID)))
+	if err != nil {
+		t.Fatalf("marshal: %v", err)
+	}
+
+	b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess)
+
+	time.Sleep(20 * time.Millisecond)
+	if g.callCount() != 0 {
+		t.Fatal("pre-READY message must be ignored")
+	}
+}
+
+func TestPayloadSequenceIsNullable(t *testing.T) {
+	var withSeq, withoutSeq payload
+	if err := json.Unmarshal([]byte(`{"op":0,"t":"X","s":7,"d":{}}`), &withSeq); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if withSeq.S == nil || *withSeq.S != 7 {
+		t.Fatalf("want seq 7, got %v", withSeq.S)
+	}
+	if err := json.Unmarshal([]byte(`{"op":11,"s":null,"d":null}`), &withoutSeq); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if withoutSeq.S != nil {
+		t.Fatalf("null seq must stay nil, got %v", *withoutSeq.S)
+	}
+}
+
+func TestSessionSeqTracking(t *testing.T) {
+	sess := &session{}
+	if _, ok := sess.lastSeq(); ok {
+		t.Fatal("fresh session must have no sequence number")
+	}
+	sess.setSeq(41)
+	sess.setSeq(42)
+	if seq, ok := sess.lastSeq(); !ok || seq != 42 {
+		t.Fatalf("want seq 42, got %d (ok=%v)", seq, ok)
+	}
+}
+
+// A malformed HELLO must be an ERROR, not a clean "reidentify". Run treats a
+// clean return as a PLANNED reconnect: backoff resets to 1s and it redials with
+// no sleep at all. So if these ever regress to returning nil, a gateway stuck
+// sending a bad HELLO becomes a tight reconnect loop against Discord.
+func TestParseHelloRejectsMalformedSoBackoffApplies(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		raw  string
+	}{
+		{"not json", `{`},
+		{"wrong op", `{"op":0,"d":{"heartbeat_interval":41250}}`},
+		{"missing interval", `{"op":10,"d":{}}`},
+		{"zero interval", `{"op":10,"d":{"heartbeat_interval":0}}`},
+		{"negative interval", `{"op":10,"d":{"heartbeat_interval":-1}}`},
+		{"interval wrong type", `{"op":10,"d":{"heartbeat_interval":"soon"}}`},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := parseHello([]byte(tc.raw))
+			if err == nil {
+				t.Fatalf("parseHello(%s) = %v, nil; want an error so Run applies backoff", tc.raw, got)
+			}
+			if got != 0 {
+				t.Fatalf("parseHello(%s) interval = %v; want 0 on error", tc.raw, got)
+			}
+		})
+	}
+}
+
+func TestParseHelloAcceptsValid(t *testing.T) {
+	got, err := parseHello([]byte(`{"op":10,"d":{"heartbeat_interval":41250}}`))
+	if err != nil {
+		t.Fatalf("parseHello: unexpected error %v", err)
+	}
+	if want := 41250 * time.Millisecond; got != want {
+		t.Fatalf("interval = %v; want %v", got, want)
+	}
+}
+
+// Discord's advertised retry_after is clamped to 5s, matching the Python REST
+// helper's _MAX_BACKOFF_S. Replies run on a small fixed worker pool, so a bogus
+// server value must not park a slot for minutes.
+func TestRetryAfterClamp(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		body string
+		want time.Duration
+	}{
+		{"honours a sane value", `{"retry_after":2}`, 2 * time.Second},
+		{"clamps a hostile value", `{"retry_after":600}`, 5 * time.Second},
+		{"defaults when absent", `{}`, 1 * time.Second},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := retryAfter(http.Header{}, []byte(tc.body)); got != tc.want {
+				t.Fatalf("retryAfter(%s) = %v; want %v", tc.body, got, tc.want)
+			}
+		})
+	}
+}

backend/daemon/internal/gateway/message.go

@@ -0,0 +1,118 @@
+// Message triage: the pure decision of whether — and how — to answer one
+// MESSAGE_CREATE. Ported from the Python bot's _response_for_message /
+// _mentions_bot / _is_dm / _strip_mentions; it is plain string/JSON logic
+// with no data dependency, so it lives in Go. Anything needing climate data
+// (the actual grade) goes back to Python via Grader.
+
+package gateway
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+// helpText answers an empty query. Go owns this constant because it needs no
+// data; the wording matches the Python original exactly.
+const helpText = "Mention me with a city name — e.g. `@Thermograph Phoenix` — and I'll grade " +
+	"today's weather against ~45 years of local history."
+
+// snowflake decodes a Discord id that may arrive as a JSON string (what v10
+// sends) or a bare number. The Python compared ids through str() coercion;
+// this keeps that tolerance instead of betting the payload shape never
+// wobbles.
+type snowflake string
+
+func (s *snowflake) UnmarshalJSON(b []byte) error {
+	if string(b) == "null" {
+		*s = ""
+		return nil
+	}
+	var str string
+	if err := json.Unmarshal(b, &str); err == nil {
+		*s = snowflake(str)
+		return nil
+	}
+	var n json.Number
+	if err := json.Unmarshal(b, &n); err == nil {
+		*s = snowflake(n.String())
+		return nil
+	}
+	return fmt.Errorf("snowflake: cannot decode %s", b)
+}
+
+// gwMessage is the slice of a MESSAGE_CREATE payload the triage needs.
+type gwMessage struct {
+	ID        snowflake  `json:"id"`
+	ChannelID snowflake  `json:"channel_id"`
+	GuildID   snowflake  `json:"guild_id"`
+	Content   string     `json:"content"`
+	Author    gwAuthor   `json:"author"`
+	Mentions  []gwAuthor `json:"mentions"`
+}
+
+type gwAuthor struct {
+	ID  snowflake `json:"id"`
+	Bot bool      `json:"bot"`
+}
+
+// action is triage's verdict for one message.
+type action int
+
+const (
+	actSilent action = iota // no reply
+	actHelp                 // reply with helpText
+	actGrade                // call back into Python with the query
+)
+
+// mentionsBot reports whether the message @mentions our own user id.
+func mentionsBot(m *gwMessage, botID string) bool {
+	for _, u := range m.Mentions {
+		if string(u.ID) == botID {
+			return true
+		}
+	}
+	return false
+}
+
+// isDM: a DM has no guild_id; a guild message always carries one.
+func isDM(m *gwMessage) bool {
+	return m.GuildID == ""
+}
+
+// stripMentions drops every form of the bot's own mention (<@id> and the
+// legacy <@!id>) and collapses the surrounding whitespace, leaving just the
+// user's query text.
+func stripMentions(content, botID string) string {
+	out := strings.ReplaceAll(content, "<@"+botID+">", " ")
+	out = strings.ReplaceAll(out, "<@!"+botID+">", " ")
+	return strings.Join(strings.Fields(out), " ")
+}
+
+// triage decides the reply for one MESSAGE_CREATE: silence, the help line, or
+// a grade lookup for the returned query.
+//
+// Silent unless the message DMs the bot or @mentions it, and never for a bot
+// author — which includes ourselves, the guard against a mention-loop (our
+// reply quoting the mention must not trigger another reply). The text after
+// the mention is treated as a city query; DMs are the query as-is (trimmed
+// but not collapsed, matching the Python's .strip()).
+func triage(m *gwMessage, botID string) (action, string) {
+	if m.Author.Bot || string(m.Author.ID) == botID {
+		return actSilent, ""
+	}
+	dm := isDM(m)
+	if !dm && !mentionsBot(m, botID) {
+		return actSilent, ""
+	}
+	var query string
+	if dm {
+		query = strings.TrimSpace(m.Content)
+	} else {
+		query = stripMentions(m.Content, botID)
+	}
+	if query == "" {
+		return actHelp, ""
+	}
+	return actGrade, query
+}

backend/daemon/internal/gateway/message_test.go

@@ -0,0 +1,162 @@
+// Triage tests: every behaviour asserted by the Python suite
+// (backend/tests/notifications/test_discord_bot.py) has an equivalent here,
+// minus the ephemeral-flag drop — the /internal/discord/grade route now
+// returns gateway-ready JSON, so that concern moved to the Python side.
+
+package gateway
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+const botID = "999"
+
+// testMsg mirrors the Python suite's _msg fixture: author "1" (not a bot),
+// message id 42 in channel 77, guild 5 unless overridden.
+func testMsg(content string, opts ...func(*gwMessage)) *gwMessage {
+	m := &gwMessage{
+		ID:        "42",
+		ChannelID: "77",
+		GuildID:   "5",
+		Content:   content,
+		Author:    gwAuthor{ID: "1"},
+	}
+	for _, opt := range opts {
+		opt(m)
+	}
+	return m
+}
+
+func fromBot() func(*gwMessage) {
+	return func(m *gwMessage) { m.Author.Bot = true }
+}
+
+func fromID(id string) func(*gwMessage) {
+	return func(m *gwMessage) { m.Author.ID = snowflake(id) }
+}
+
+func mentioning(ids ...string) func(*gwMessage) {
+	return func(m *gwMessage) {
+		for _, id := range ids {
+			m.Mentions = append(m.Mentions, gwAuthor{ID: snowflake(id)})
+		}
+	}
+}
+
+func asDM() func(*gwMessage) {
+	return func(m *gwMessage) { m.GuildID = "" }
+}
+
+// --- when the bot stays silent ----------------------------------------------
+
+func TestIgnoresOtherBots(t *testing.T) {
+	act, _ := triage(testMsg("<@999> Phoenix", fromID("8"), fromBot(), mentioning(botID)), botID)
+	if act != actSilent {
+		t.Fatalf("bot author should be ignored, got action %d", act)
+	}
+}
+
+func TestIgnoresItsOwnMessages(t *testing.T) {
+	act, _ := triage(testMsg("hello", fromID(botID), mentioning(botID)), botID)
+	if act != actSilent {
+		t.Fatalf("own message should be ignored, got action %d", act)
+	}
+}
+
+func TestIgnoresGuildMessageWithoutAMention(t *testing.T) {
+	act, _ := triage(testMsg("just chatting"), botID)
+	if act != actSilent {
+		t.Fatalf("unmentioned guild message should be ignored, got action %d", act)
+	}
+}
+
+// --- when the bot replies ----------------------------------------------------
+
+func TestMentionWithCityGrades(t *testing.T) {
+	act, query := triage(testMsg("<@999> Phoenix", mentioning(botID)), botID)
+	if act != actGrade || query != "Phoenix" {
+		t.Fatalf("want grade %q, got action %d query %q", "Phoenix", act, query)
+	}
+}
+
+func TestLegacyNicknameMentionIsStripped(t *testing.T) {
+	act, query := triage(testMsg("<@!999>   Tokyo", mentioning(botID)), botID)
+	if act != actGrade || query != "Tokyo" {
+		t.Fatalf("want grade %q, got action %d query %q", "Tokyo", act, query)
+	}
+}
+
+func TestDMNeedsNoMention(t *testing.T) {
+	act, query := triage(testMsg("Berlin", asDM()), botID)
+	if act != actGrade || query != "Berlin" {
+		t.Fatalf("want grade %q, got action %d query %q", "Berlin", act, query)
+	}
+}
+
+func TestDMTrimsButKeepsInnerWhitespace(t *testing.T) {
+	// Parity with the Python DM path (.strip(), not a collapse): only the
+	// edges are trimmed.
+	act, query := triage(testMsg("  New   York  ", asDM()), botID)
+	if act != actGrade || query != "New   York" {
+		t.Fatalf("want grade %q, got action %d query %q", "New   York", act, query)
+	}
+}
+
+func TestMentionWithNoQueryGivesHelp(t *testing.T) {
+	act, query := triage(testMsg("<@999>", mentioning(botID)), botID)
+	if act != actHelp || query != "" {
+		t.Fatalf("want help, got action %d query %q", act, query)
+	}
+	if !strings.Contains(strings.ToLower(helpText), "mention me") {
+		t.Fatalf("help text lost its instruction: %q", helpText)
+	}
+}
+
+// --- helpers -----------------------------------------------------------------
+
+func TestStripMentionsHandlesBothForms(t *testing.T) {
+	if got := stripMentions("<@999> <@!999>  Sao Paulo", botID); got != "Sao Paulo" {
+		t.Fatalf("want %q, got %q", "Sao Paulo", got)
+	}
+}
+
+func TestIsDMIsTrueWithoutGuildID(t *testing.T) {
+	if !isDM(testMsg("x", asDM())) {
+		t.Fatal("message without guild_id should be a DM")
+	}
+	if isDM(testMsg("x")) {
+		t.Fatal("message with guild_id should not be a DM")
+	}
+}
+
+func TestSnowflakeAcceptsStringOrNumber(t *testing.T) {
+	// The Python compared ids through str() coercion; a numeric id in the
+	// payload must still count as a mention.
+	var m gwMessage
+	raw := `{"id":42,"channel_id":"77","guild_id":"5","content":"<@999> Lima",
+	         "author":{"id":1,"bot":false},"mentions":[{"id":999}]}`
+	if err := json.Unmarshal([]byte(raw), &m); err != nil {
+		t.Fatalf("unmarshal: %v", err)
+	}
+	if m.ID != "42" || string(m.Author.ID) != "1" {
+		t.Fatalf("numeric ids mis-decoded: id=%q author=%q", m.ID, m.Author.ID)
+	}
+	act, query := triage(&m, botID)
+	if act != actGrade || query != "Lima" {
+		t.Fatalf("want grade %q, got action %d query %q", "Lima", act, query)
+	}
+}
+
+func TestIntentsExcludeMessageContent(t *testing.T) {
+	// The no-privileged-intent constraint is load-bearing: adding
+	// MESSAGE_CONTENT (1<<15) silently would make Discord close the gateway
+	// with 4014 until the portal review happens.
+	if intents != intentGuilds|intentGuildMessages|intentDirectMessages {
+		t.Fatalf("unexpected intents bitfield: %d", intents)
+	}
+	if intents&(1<<15) != 0 {
+		t.Fatal("intents must not request MESSAGE_CONTENT")
+	}
+}

backend/daemon/internal/gateway/reply.go

@@ -0,0 +1,138 @@
+// The REST half of answering a message: replies go out over plain HTTP with
+// the bot token, not the gateway socket (the gateway is receive-only for us).
+
+package gateway
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+const discordAPIBase = "https://discord.com/api/v10"
+
+// replyBodyLimit caps how much of a failed reply's response body we echo into
+// logs.
+const replyBodyLimit = 4096
+
+// injectReplyFields adds message_reference + allowed_mentions to an otherwise
+// verbatim message body. Only the top level is decoded — every value that
+// came from Python (embeds and all) passes through as raw bytes, unparsed, so
+// the grading/embed contract stays entirely on the Python side.
+//
+// allowed_mentions is a SECURITY control, not decoration: the graded reply
+// echoes the user's query text, so without {"parse":[]} a crafted query could
+// turn our reply into an @everyone/role ping. Only the person who asked is
+// pinged, via the reply reference.
+func injectReplyFields(body []byte, messageID string) ([]byte, error) {
+	var top map[string]json.RawMessage
+	if err := json.Unmarshal(body, &top); err != nil {
+		return nil, err
+	}
+	if messageID != "" {
+		ref, err := json.Marshal(map[string]string{"message_id": messageID})
+		if err != nil {
+			return nil, err
+		}
+		top["message_reference"] = ref
+		top["allowed_mentions"] = json.RawMessage(`{"parse":[],"replied_user":true}`)
+	}
+	return json.Marshal(top)
+}
+
+// reply posts body to the triggering message's channel as a proper reply. A
+// failed reply is logged and swallowed — it must never kill the read loop.
+func (b *Bot) reply(ctx context.Context, channelID, messageID string, body json.RawMessage) {
+	if channelID == "" {
+		return
+	}
+	payload, err := injectReplyFields(body, messageID)
+	if err != nil {
+		log.Printf("gateway: reply body is not a JSON object: %v", err)
+		return
+	}
+	url := b.rest + "/channels/" + channelID + "/messages"
+	for attempt := 0; ; attempt++ {
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload))
+		if err != nil {
+			log.Printf("gateway: reply failed: %v", err)
+			return
+		}
+		req.Header.Set("Authorization", "Bot "+b.token)
+		req.Header.Set("Content-Type", "application/json")
+		resp, err := b.http.Do(req)
+		if err != nil {
+			log.Printf("gateway: reply failed: %v", err)
+			return
+		}
+		raw, _ := io.ReadAll(io.LimitReader(resp.Body, replyBodyLimit))
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusTooManyRequests && attempt == 0 {
+			// One retry honouring the advertised wait — parity with the
+			// Python REST helper's 429 handling.
+			if !sleepCtx(ctx, retryAfter(resp.Header, raw)) {
+				return
+			}
+			continue
+		}
+		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+			log.Printf("gateway: reply failed: status %d: %s", resp.StatusCode, raw)
+		}
+		return
+	}
+}
+
+// retryAfter extracts Discord's requested wait from a 429 (JSON retry_after
+// in seconds, falling back to the Retry-After header), clamped so a bogus
+// server value can't park the handler for minutes.
+func retryAfter(h http.Header, body []byte) time.Duration {
+	seconds := 1.0
+	var d struct {
+		RetryAfter float64 `json:"retry_after"`
+	}
+	if err := json.Unmarshal(body, &d); err == nil && d.RetryAfter > 0 {
+		seconds = d.RetryAfter
+	} else if v, err := strconv.ParseFloat(h.Get("Retry-After"), 64); err == nil && v > 0 {
+		seconds = v
+	}
+	// 5s matches the Python REST helper's _MAX_BACKOFF_S. It also bounds the cost
+	// of a bogus/hostile retry_after: replies run on a small fixed worker pool, so
+	// a parked handler holds one of very few slots and mentions start being
+	// dropped that much sooner.
+	if seconds > 5 {
+		seconds = 5
+	}
+	return time.Duration(seconds * float64(time.Second))
+}
+
+// handleMessage triages one MESSAGE_CREATE and sends whatever reply it calls
+// for. Runs in its own goroutine (see dispatch) so a slow grade lookup can
+// never stall heartbeats or the read loop.
+func (b *Bot) handleMessage(ctx context.Context, m *gwMessage, botID string) {
+	act, query := triage(m, botID)
+	switch act {
+	case actSilent:
+	case actHelp:
+		body, err := json.Marshal(map[string]string{"content": helpText})
+		if err != nil {
+			return
+		}
+		b.reply(ctx, string(m.ChannelID), string(m.ID), body)
+	case actGrade:
+		// The callback owns all grading; its JSON is relayed VERBATIM (no
+		// parsing, no reshaping) so the bot's grades can never drift from
+		// the API's. A failed callback stays silent — better no reply than a
+		// made-up one.
+		raw, err := b.api.Grade(ctx, query)
+		if err != nil {
+			log.Printf("gateway: grade callback failed for %q: %v", query, err)
+			return
+		}
+		b.reply(ctx, string(m.ChannelID), string(m.ID), raw)
+	}
+}

backend/daemon/internal/gateway/reply_test.go

@@ -0,0 +1,278 @@
+// Reply-path tests: a fake Grader plus an httptest server standing in for
+// Discord's REST API. No live gateway is involved anywhere.
+
+package gateway
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+type fakeGrader struct {
+	mu    sync.Mutex
+	resp  json.RawMessage
+	err   error
+	calls []string
+}
+
+func (f *fakeGrader) Grade(_ context.Context, query string) (json.RawMessage, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls = append(f.calls, query)
+	return f.resp, f.err
+}
+
+func (f *fakeGrader) callCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.calls)
+}
+
+type captured struct {
+	method, path, auth string
+	body               []byte
+}
+
+// newTestBot wires a Bot to a Grader and an httptest stand-in for Discord.
+func newTestBot(t *testing.T, g Grader, handler http.HandlerFunc) *Bot {
+	t.Helper()
+	srv := httptest.NewServer(handler)
+	t.Cleanup(srv.Close)
+	b := New("tok-123", g)
+	b.rest = srv.URL
+	return b
+}
+
+// capture records each request and replies 200 {}.
+func capture(got chan<- captured) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		got <- captured{r.Method, r.URL.Path, r.Header.Get("Authorization"), body}
+		_, _ = w.Write([]byte(`{}`))
+	}
+}
+
+func TestGradeReplyIsRelayedVerbatimAsAReply(t *testing.T) {
+	grade := json.RawMessage(`{"embeds":[{"title":"grade:Phoenix"}]}`)
+	got := make(chan captured, 1)
+	b := newTestBot(t, &fakeGrader{resp: grade}, capture(got))
+
+	b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID)
+
+	c := <-got
+	if c.method != http.MethodPost || c.path != "/channels/77/messages" {
+		t.Fatalf("want POST /channels/77/messages, got %s %s", c.method, c.path)
+	}
+	if c.auth != "Bot tok-123" {
+		t.Fatalf("want bot-token auth, got %q", c.auth)
+	}
+	var top map[string]json.RawMessage
+	if err := json.Unmarshal(c.body, &top); err != nil {
+		t.Fatalf("reply body not JSON: %v", err)
+	}
+	// The embed must be byte-for-byte what Python returned — the verbatim
+	// relay is the no-drift guarantee.
+	if string(top["embeds"]) != `[{"title":"grade:Phoenix"}]` {
+		t.Fatalf("embeds not relayed verbatim: %s", top["embeds"])
+	}
+	if string(top["message_reference"]) != `{"message_id":"42"}` {
+		t.Fatalf("bad message_reference: %s", top["message_reference"])
+	}
+	// The locked-down allowed_mentions is the anti-@everyone control; it must
+	// be present exactly.
+	if string(top["allowed_mentions"]) != `{"parse":[],"replied_user":true}` {
+		t.Fatalf("bad allowed_mentions: %s", top["allowed_mentions"])
+	}
+}
+
+func TestEmptyQueryRepliesWithHelpWithoutGrading(t *testing.T) {
+	got := make(chan captured, 1)
+	g := &fakeGrader{}
+	b := newTestBot(t, g, capture(got))
+
+	b.handleMessage(context.Background(), testMsg("<@999>", mentioning(botID)), botID)
+
+	c := <-got
+	var body struct {
+		Content string `json:"content"`
+	}
+	if err := json.Unmarshal(c.body, &body); err != nil {
+		t.Fatalf("help body not JSON: %v", err)
+	}
+	if body.Content != helpText {
+		t.Fatalf("want help text, got %q", body.Content)
+	}
+	if g.callCount() != 0 {
+		t.Fatal("help reply must not hit the grade callback")
+	}
+}
+
+func TestSilentMessagesMakeNoRequests(t *testing.T) {
+	var hits atomic.Int32
+	g := &fakeGrader{resp: json.RawMessage(`{}`)}
+	b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+	})
+
+	b.handleMessage(context.Background(), testMsg("just chatting"), botID)
+
+	if g.callCount() != 0 || hits.Load() != 0 {
+		t.Fatalf("silent message leaked: grades=%d posts=%d", g.callCount(), hits.Load())
+	}
+}
+
+func TestGradeFailureStaysSilent(t *testing.T) {
+	var hits atomic.Int32
+	g := &fakeGrader{err: context.DeadlineExceeded}
+	b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+	})
+
+	// Must log-and-return, not panic or post a broken reply.
+	b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID)
+
+	if hits.Load() != 0 {
+		t.Fatal("failed grade must not produce a reply")
+	}
+}
+
+func TestFailedReplyIsSwallowed(t *testing.T) {
+	g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)}
+	b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "boom", http.StatusInternalServerError)
+	})
+
+	// A failed reply must never kill the caller (in production: the read
+	// loop) — returning normally is the assertion.
+	b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID)
+}
+
+func TestReplyRetriesOnceOn429(t *testing.T) {
+	var hits atomic.Int32
+	bodies := make(chan []byte, 2)
+	g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)}
+	b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		bodies <- body
+		if hits.Add(1) == 1 {
+			w.WriteHeader(http.StatusTooManyRequests)
+			_, _ = w.Write([]byte(`{"retry_after":0.01}`))
+			return
+		}
+		_, _ = w.Write([]byte(`{}`))
+	})
+
+	b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID)
+
+	if hits.Load() != 2 {
+		t.Fatalf("want 1 retry after 429, got %d requests", hits.Load())
+	}
+	first, second := <-bodies, <-bodies
+	if string(first) != string(second) {
+		t.Fatal("retry must resend the identical payload")
+	}
+}
+
+func TestNoChannelMeansNoPost(t *testing.T) {
+	var hits atomic.Int32
+	g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)}
+	b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+	})
+
+	b.handleMessage(context.Background(),
+		testMsg("<@999> Phoenix", mentioning(botID), func(m *gwMessage) { m.ChannelID = "" }), botID)
+
+	if hits.Load() != 0 {
+		t.Fatal("a message without a channel id must not be answered")
+	}
+}
+
+func TestInjectReplyFieldsWithoutMessageIDLeavesBodyAlone(t *testing.T) {
+	out, err := injectReplyFields([]byte(`{"content":"x"}`), "")
+	if err != nil {
+		t.Fatalf("inject: %v", err)
+	}
+	var top map[string]json.RawMessage
+	if err := json.Unmarshal(out, &top); err != nil {
+		t.Fatalf("out not JSON: %v", err)
+	}
+	if _, ok := top["message_reference"]; ok {
+		t.Fatal("no triggering id => no message_reference")
+	}
+	if _, ok := top["allowed_mentions"]; ok {
+		t.Fatal("no triggering id => no allowed_mentions")
+	}
+}
+
+// --- dispatch: same reply, off the read loop ---------------------------------
+
+func TestDispatchDeliversTheSameReplyViaAGoroutine(t *testing.T) {
+	// Mirrors the Python test_dispatch_delivers_the_same_reply_via_a_thread:
+	// the concurrency offload (goroutine + semaphore here, asyncio.to_thread
+	// there) must not change what gets sent.
+	grade := json.RawMessage(`{"embeds":[{"title":"grade:Phoenix"}]}`)
+	got := make(chan captured, 1)
+	b := newTestBot(t, &fakeGrader{resp: grade}, capture(got))
+	sess := &session{userID: botID}
+	d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID)))
+	if err != nil {
+		t.Fatalf("marshal: %v", err)
+	}
+
+	b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess)
+
+	select {
+	case c := <-got:
+		if c.path != "/channels/77/messages" {
+			t.Fatalf("want /channels/77/messages, got %s", c.path)
+		}
+		var top map[string]json.RawMessage
+		if err := json.Unmarshal(c.body, &top); err != nil {
+			t.Fatalf("reply body not JSON: %v", err)
+		}
+		if string(top["embeds"]) != `[{"title":"grade:Phoenix"}]` {
+			t.Fatalf("embeds not relayed verbatim: %s", top["embeds"])
+		}
+		if string(top["message_reference"]) != `{"message_id":"42"}` {
+			t.Fatalf("bad message_reference: %s", top["message_reference"])
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("dispatched message never produced a reply")
+	}
+}
+
+func TestMessageFloodIsBoundedWithoutBlockingTheReadLoop(t *testing.T) {
+	g := &fakeGrader{resp: json.RawMessage(`{}`)}
+	b := New("tok", g)
+	b.sem = make(chan struct{}, 1)
+	b.sem <- struct{}{} // occupy the only slot: simulate a reply in flight
+	sess := &session{userID: botID}
+	d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID)))
+	if err != nil {
+		t.Fatalf("marshal: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess)
+		close(done)
+	}()
+	select {
+	case <-done:
+		// dispatch returned promptly — the read loop would not have stalled
+	case <-time.After(time.Second):
+		t.Fatal("dispatch blocked on a full semaphore")
+	}
+	time.Sleep(20 * time.Millisecond)
+	if g.callCount() != 0 {
+		t.Fatal("over-the-bound message must be dropped, not queued")
+	}
+}

backend/daemon/main.go

@@ -0,0 +1,130 @@
+// thermograph-daemon owns the backend's long-lived stateful I/O — the Discord
+// gateway websocket (IDENTIFY/RESUME/heartbeat/backoff) and the recurring-job
+// timers — and nothing else. Anything that needs data calls back into the
+// Python app over its internal-only HTTP routes (see internal/apiclient).
+//
+// This replaces the two in-process background jobs web/app.py used to start
+// under leader election. One daemon process replaces the election entirely:
+// there is exactly one replica of this container, so the single-connection
+// invariant (Discord tolerates only one gateway session per token; duplicated
+// APScheduler replicas would repeat every job) holds by deployment shape
+// rather than by runtime coordination.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+	"time"
+
+	"thermograph/daemon/internal/apiclient"
+	"thermograph/daemon/internal/config"
+	"thermograph/daemon/internal/cron"
+	"thermograph/daemon/internal/gateway"
+)
+
+// shutdownGrace bounds how long we wait for the gateway and cron halves to
+// stop after a signal. Long enough for the gateway to close its session
+// cleanly (a dirty exit leaves Discord holding a stale session), short enough
+// to stay inside a container runtime's stop timeout so we exit on our own
+// terms rather than being SIGKILLed mid-close.
+const shutdownGrace = 10 * time.Second
+
+func main() {
+	os.Exit(run())
+}
+
+// run is main minus os.Exit, so deferred cleanup actually runs.
+func run() int {
+	// Everything to stdout: the container collects stdout, nothing else.
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+
+	// Refuse to start on bad config rather than running half-configured: a
+	// daemon that cannot authenticate to the backend can only fail silently.
+	cfg, err := config.Load()
+	if err != nil {
+		logger.Error("invalid configuration; refusing to start", "err", err)
+		return 1
+	}
+
+	// One context for everything long-lived; SIGINT/SIGTERM cancels it so the
+	// gateway can close its session cleanly (Discord treats an abrupt drop
+	// worse than a clean close for RESUME purposes).
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	api := apiclient.New(cfg.APIBase, cfg.InternalToken)
+
+	logger.Info("thermograph-daemon starting",
+		"api", cfg.APIBase,
+		"discord", cfg.DiscordEnabled,
+		"warm_cities_interval", cfg.WarmCitiesInterval,
+		"indexnow_interval", cfg.IndexNowInterval)
+
+	var wg sync.WaitGroup
+
+	// The cron half runs unconditionally — it is the floor of what this
+	// daemon does, gateway or not.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		cron.Run(ctx, logger,
+			cron.Job{
+				Name:     "warm-cities",
+				Interval: cfg.WarmCitiesInterval,
+				Run:      func(ctx context.Context) error { return api.WarmCities(ctx) },
+			},
+			cron.Job{
+				Name:     "indexnow",
+				Interval: cfg.IndexNowInterval,
+				Run:      func(ctx context.Context) error { return api.IndexNow(ctx) },
+			},
+		)
+	}()
+
+	if cfg.DiscordEnabled {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			// A fatal gateway error (bad token, disallowed intents) must not
+			// take the cron half down: the schedule is independently useful,
+			// and exiting here would turn a Discord misconfiguration into a
+			// crash-loop that also stops warm-cities/IndexNow. Log loudly and
+			// keep running instead.
+			if err := gateway.Run(ctx, cfg, api); err != nil && ctx.Err() == nil {
+				logger.Error("discord gateway exited with a fatal error; cron jobs keep running without it",
+					"err", err)
+			}
+		}()
+	} else {
+		// An unconfigured deploy must pay nothing, same as the Python side:
+		// no connection, no retries — just say so once and run cron-only.
+		logger.Info("discord gateway disabled (THERMOGRAPH_DISCORD_BOT not truthy or no bot token); running cron-only")
+	}
+
+	<-ctx.Done()
+	// Restore default signal handling immediately: a second SIGINT/SIGTERM
+	// during the grace period kills us the ordinary way instead of being
+	// swallowed by the (already-cancelled) NotifyContext.
+	stop()
+	logger.Info("shutting down", "grace", shutdownGrace)
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+		logger.Info("shutdown complete")
+	case <-time.After(shutdownGrace):
+		// This process is the sole holder of the gateway connection, so a
+		// clean close matters — but a wedged goroutine must not hold the
+		// container's stop hostage forever.
+		logger.Warn("shutdown grace period elapsed before all loops stopped; exiting anyway")
+	}
+	return 0
+}

backend/data/climate.py

@@ -21,6 +21,7 @@ from core import audit
 from core import metrics
 import paths
 from data import climate_store
+from data import era5lake
 from data import meteostat
 from data import store
 
@@ -789,21 +790,32 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]:
         def _serve_stale():
             return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None}
 
-        # Fetch. NASA POWER is the primary source (keyless, independent of Open-Meteo);
+        # Fetch. The ERA5 lake is first when configured (true ERA5 with measured
-        # the gusts it lacks are filled from Meteostat inside _fetch_history_nasa.
+        # gusts, served from our own bucket — no third-party API in the path);
-        # Open-Meteo is the fallback when NASA is unavailable — still guarded by its
+        # unconfigured or missing-point cells fall through at zero cost. NASA
-        # rate-limit cooldown. Both sources must return a plausibly-full span before
+        # POWER is next (keyless, independent of Open-Meteo; its missing gusts
-        # being accepted and cached as a complete record (a short/partial response is
+        # are filled from Meteostat inside _fetch_history_nasa), then Open-Meteo
-        # rejected rather than held indefinitely).
+        # — still guarded by its rate-limit cooldown. Every source must return a
+        # plausibly-full span before being accepted and cached as a complete
+        # record (a short/partial response is rejected rather than held
+        # indefinitely).
         df = None
         source = None
         try:
-            fetched = _fetch_history_nasa(cell)
+            fetched = era5lake.fetch_history_lake(cell, start=START_DATE)
             if fetched.height >= MIN_ARCHIVE_DAYS:
                 df = fetched
-                source = "nasa-power"
+                source = "era5-lake"
-        except Exception:  # noqa: BLE001 - primary unavailable; fall through to Open-Meteo
+        except Exception:  # noqa: BLE001 - lake unconfigured/miss; fall through to NASA
             pass
+        if df is None:
+            try:
+                fetched = _fetch_history_nasa(cell)
+                if fetched.height >= MIN_ARCHIVE_DAYS:
+                    df = fetched
+                    source = "nasa-power"
+            except Exception:  # noqa: BLE001 - primary unavailable; fall through to Open-Meteo
+                pass
         if df is None and time.time() >= _archive_cooldown_until:
             try:
                 fetched = _fetch_history(cell)

backend/data/era5lake.py

@@ -0,0 +1,173 @@
+"""The ERA5 lake: layout, grid math, and read clients for the bucket-hosted archive.
+
+The lake is a set of parquet files in an S3-compatible bucket (Contabo Object
+Storage, ``era5-thermograph``), extracted once from the public Earthmover ERA5
+Icechunk archive by ``gen_era5_lake.py`` and read three ways:
+
+- the **lake service** (``lake_app.py``, prod-only Swarm service) serves point
+  histories and SQL-style queries with a local disk cache in front;
+- the **web/worker backend** asks the lake service first
+  (``THERMOGRAPH_LAKE_URL``), or reads the bucket directly when only S3
+  credentials are configured (beta/LAN, no lake service);
+- ad-hoc analysis hits the hive table straight from the bucket.
+
+Two layouts, one manifest (all under ``era5/``):
+
+- ``daily/tile=<ti>_<tj>/year=<Y>/month=<M>/part.parquet`` — the analytical
+  table, hive-partitioned by location/year/month. The location grain is a
+  12x12-point tile (~3°x3°), aligned to the Earthmover store's chunk grid; a
+  per-point x per-month grain would be ~170M objects, which no object store
+  enjoys.
+- ``points/lat=<i>/lon=<j>.parquet`` — the serving projection: one file per
+  ERA5 grid point holding its full daily history, so the hot path (one cell's
+  history) is a single GET.
+- ``manifest.parquet`` — Iceberg-manifest-style index of every point file:
+  grid indices, coordinates, row count, date span. Readers prune with it and
+  the extractor resumes from it.
+
+Columns everywhere: the finalized history-store schema (date, tmax, tmin,
+precip, wind, gust, humid, fmax, fmin, feels, doy — °F/in/mph/%, as
+``climate._finalize_approximated`` emits it), so a lake frame drops straight
+into ``climate._write_history_backed`` exactly like a NASA fetch does.
+"""
+import io
+import os
+
+import polars as pl
+
+# --- ERA5 native grid (0.25°, 721 x 1440, lat 90..-90, lon 0..359.75) ---------
+LAT_N = 721
+LON_N = 1440
+STEP = 0.25
+TILE = 12                      # matches the Earthmover chunk grid (8736h x 12 x 12)
+
+PREFIX = "era5"
+MANIFEST_KEY = f"{PREFIX}/manifest.parquet"
+
+MIN_LAKE_DAYS = 10_000         # a plausibly-full 1980+ record, same spirit as
+                               # climate.MIN_ARCHIVE_DAYS (rejects partial files)
+
+
+def to_idx(lat: float, lon: float) -> tuple[int, int]:
+    """Nearest ERA5 grid indices for a lat/lon. lat_idx runs north->south."""
+    i = round((90.0 - lat) / STEP)
+    j = round((lon % 360.0) / STEP) % LON_N
+    return min(max(i, 0), LAT_N - 1), j
+
+
+def to_coords(i: int, j: int) -> tuple[float, float]:
+    """Center lat/lon ([-180, 180) longitude) of grid indices."""
+    lon = j * STEP
+    return 90.0 - i * STEP, lon - 360.0 if lon >= 180.0 else lon
+
+
+def tile_of(i: int, j: int) -> tuple[int, int]:
+    return i // TILE, j // TILE
+
+
+def point_key(i: int, j: int) -> str:
+    return f"{PREFIX}/points/lat={i}/lon={j}.parquet"
+
+
+def daily_part_key(ti: int, tj: int, year: int, month: int) -> str:
+    return f"{PREFIX}/daily/tile={ti}_{tj}/year={year}/month={month:02d}/part.parquet"
+
+
+# --- Config (all optional: unset means "no lake configured") ------------------
+
+def lake_url() -> str:
+    """Lake service base URL (prod: the Swarm VIP), '' when not deployed."""
+    return os.environ.get("THERMOGRAPH_LAKE_URL", "").rstrip("/")
+
+
+def s3_config() -> "dict | None":
+    """Bucket access for direct reads/writes, None unless fully configured.
+    Contabo is path-style; region is a signing formality there ('default')."""
+    key = os.environ.get("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "")
+    secret = os.environ.get("THERMOGRAPH_LAKE_S3_SECRET_KEY", "")
+    if not (key and secret):
+        return None
+    return {
+        "endpoint": os.environ.get("THERMOGRAPH_LAKE_S3_ENDPOINT",
+                                   "https://eu2.contabostorage.com"),
+        "bucket": os.environ.get("THERMOGRAPH_LAKE_S3_BUCKET", "era5-thermograph"),
+        "region": os.environ.get("THERMOGRAPH_LAKE_S3_REGION", "default"),
+        "access_key": key,
+        "secret_key": secret,
+    }
+
+
+def local_dir() -> str:
+    """Filesystem mirror of the lake layout — tests and offline dev reads."""
+    return os.environ.get("THERMOGRAPH_LAKE_LOCAL_DIR", "")
+
+
+def storage_options(cfg: dict) -> dict:
+    """polars/object_store options for scan_parquet against the bucket."""
+    return {
+        "aws_access_key_id": cfg["access_key"],
+        "aws_secret_access_key": cfg["secret_key"],
+        "aws_region": cfg["region"],
+        "endpoint_url": cfg["endpoint"],
+        # Contabo serves buckets on the path, not as a subdomain.
+        "aws_virtual_hosted_style_request": "false",
+    }
+
+
+# --- Point-history reads (the app's hot path) ----------------------------------
+
+class LakeUnavailable(Exception):
+    """No lake configured, point not in the lake, or the read failed."""
+
+
+def _read_local(i: int, j: int) -> pl.DataFrame:
+    path = os.path.join(local_dir(), point_key(i, j))
+    if not os.path.exists(path):
+        raise LakeUnavailable(f"no local lake file for point ({i}, {j})")
+    return pl.read_parquet(path)
+
+
+def _read_service(i: int, j: int) -> pl.DataFrame:
+    import httpx  # noqa: PLC0415 - keep module import light for the extractor venv
+
+    lat, lon = to_coords(i, j)
+    r = httpx.get(f"{lake_url()}/history", params={"lat": lat, "lon": lon},
+                  timeout=httpx.Timeout(20.0, connect=3.0))
+    if r.status_code == 404:
+        raise LakeUnavailable(f"point ({i}, {j}) not in the lake")
+    r.raise_for_status()
+    return pl.read_parquet(io.BytesIO(r.content))
+
+
+def _read_bucket(i: int, j: int, cfg: dict) -> pl.DataFrame:
+    try:
+        return pl.read_parquet(f"s3://{cfg['bucket']}/{point_key(i, j)}",
+                               storage_options=storage_options(cfg))
+    except Exception as e:  # noqa: BLE001 - object_store errors are not one type
+        raise LakeUnavailable(f"bucket read failed for point ({i}, {j}): {e}") from e
+
+
+def fetch_history_lake(cell: dict, start: "str | None" = None) -> pl.DataFrame:
+    """Finalized daily history for the ERA5 point nearest a grid cell, from the
+    first configured source: local dir -> lake service -> bucket. The lake
+    stores each point's whole 1940+ record; pass ``start`` (the caller's
+    grading window, e.g. climate.START_DATE) to slice — grading semantics stay
+    the caller's, the deep record stays in the lake. Raises LakeUnavailable
+    when unconfigured, missing, or implausibly short."""
+    import datetime  # noqa: PLC0415
+
+    i, j = to_idx(cell["center_lat"], cell["center_lon"])
+    if local_dir():
+        df = _read_local(i, j)
+    elif lake_url():
+        df = _read_service(i, j)
+    else:
+        cfg = s3_config()
+        if cfg is None:
+            raise LakeUnavailable("no lake configured")
+        df = _read_bucket(i, j, cfg)
+    if start is not None:
+        df = df.filter(pl.col("date") >= datetime.date.fromisoformat(start))
+    if df.height < MIN_LAKE_DAYS:
+        raise LakeUnavailable(f"lake point ({i}, {j}) is short: {df.height} rows")
+    return df.drop([c for c in ("lat_idx", "lon_idx") if c in df.columns])

backend/deploy/entrypoint.sh

@@ -62,6 +62,13 @@ run_migrations() {
   done
 }
 
+# The lake role serves the ERA5 bucket only — no database, so no migrations: a
+# lake replica must come up (and stay up) even when Postgres is down.
+if [ "${THERMOGRAPH_ROLE:-}" = "lake" ]; then
+  echo "==> Starting lake service on 0.0.0.0:${PORT:-8141} with ${WORKERS:-1} worker(s)"
+  exec uvicorn lake_app:app --host 0.0.0.0 --port "${PORT:-8141}" --workers "${WORKERS:-1}"
+fi
+
 # One-shot migrate mode (`entrypoint.sh migrate`, or THERMOGRAPH_MIGRATE_ONLY=1): run
 # the migration and exit — a one-shot task, so it runs once against the DB rather
 # than racing per web replica.
@@ -78,4 +85,7 @@ if [ "${RUN_MIGRATIONS:-1}" != "0" ]; then
 fi
 
 echo "==> Starting uvicorn on 0.0.0.0:${PORT:-8137} with ${WORKERS:-4} worker(s)"
-exec uvicorn app:app --host 0.0.0.0 --port "${PORT:-8137}" --workers "${WORKERS:-4}"
+# --no-access-log: the app's own request-logging middleware (audit.log_access,
+# web/app.py) already writes a structured line per request; uvicorn's own access
+# log just duplicated every one of them for no benefit.
+exec uvicorn app:app --host 0.0.0.0 --port "${PORT:-8137}" --workers "${WORKERS:-4}" --no-access-log

backend/gen_era5_lake.py

@@ -0,0 +1,237 @@
+"""Extract the ERA5 lake: Earthmover Icechunk archive -> the Thermograph bucket.
+
+Builds the lake that ``data/era5lake.py`` reads (layout documented there):
+per-point serving files, the tile/year/month hive table, and the manifest.
+Pulls are tile-aligned to the source's chunk grid (8736h x 12 x 12) — one
+46-year pull (~19s measured) decodes all 144 points of a tile, so per-point
+pulls would waste 144x the wire. Roughly: ~600 tiles cover the curated
+cities (hours), ~3000 cover land (a day or two); both resume from the
+manifest, so ctrl-C and re-run is always safe.
+
+    pip install -r requirements-seed.txt 'numcodecs[pcodec]' boto3
+    python gen_era5_lake.py --cities --dry-run      # one tile, no writes
+    python gen_era5_lake.py --cities                # tiles covering cities.json
+    python gen_era5_lake.py --tiles 20:30,40:60     # explicit tile-index window
+    python gen_era5_lake.py --land                  # every land-containing tile
+
+Destination: the bucket from THERMOGRAPH_LAKE_S3_* (see era5lake.s3_config),
+or a local mirror when THERMOGRAPH_LAKE_LOCAL_DIR is set (tests/rehearsal).
+Seed-only deps (icechunk/xarray/zarr) stay out of the app image, same policy
+as seed_era5.py.
+"""
+import argparse
+import concurrent.futures
+import datetime
+import io
+import os
+import sys
+import time
+
+import polars as pl
+
+from data import cities
+from data import climate
+from data import era5lake
+from seed_era5 import ERA5_BUCKET, ERA5_GROUP, ERA5_PREFIX, ERA5_REGION, ERA5_VARS, \
+    hourly_to_daily
+
+# The lake keeps each location's WHOLE available record (the store reaches
+# back to 1940); the app slices to its own grading window (climate.START_DATE)
+# at read time, so the extra decades are analytical surface, not a grading
+# change.
+START_DATE = "1940-01-01"
+
+
+def _open_store():
+    import icechunk           # noqa: PLC0415 - seed-only dep
+    import xarray as xr       # noqa: PLC0415
+
+    storage = icechunk.s3_storage(
+        bucket=ERA5_BUCKET, prefix=ERA5_PREFIX, region=ERA5_REGION, anonymous=True)
+    repo = icechunk.Repository.open(storage)
+    return xr.open_zarr(repo.readonly_session("main").store,
+                        group=ERA5_GROUP, consolidated=False)
+
+
+class _Sink:
+    """Writes lake objects to the bucket (boto3) or a local mirror dir."""
+
+    def __init__(self):
+        self.local = era5lake.local_dir()
+        self.cfg = era5lake.s3_config()
+        if not self.local and self.cfg is None:
+            sys.exit("no destination: set THERMOGRAPH_LAKE_S3_* or "
+                     "THERMOGRAPH_LAKE_LOCAL_DIR")
+        self._s3 = None
+        if not self.local:
+            import boto3  # noqa: PLC0415 - extractor-only dep
+            from botocore.config import Config  # noqa: PLC0415
+            # Contabo throttles bursts with 429s (seen live at 16 concurrent
+            # PUTs); adaptive mode rate-limits client-side and retries with
+            # backoff instead of dying after botocore's default 4 attempts.
+            self._s3 = boto3.client(
+                "s3", endpoint_url=self.cfg["endpoint"],
+                region_name=self.cfg["region"],
+                aws_access_key_id=self.cfg["access_key"],
+                aws_secret_access_key=self.cfg["secret_key"],
+                config=Config(retries={"max_attempts": 10, "mode": "adaptive"}))
+
+    def put(self, key: str, df: pl.DataFrame) -> None:
+        if self.local:
+            path = os.path.join(self.local, key)
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            df.write_parquet(path)
+            return
+        buf = io.BytesIO()
+        df.write_parquet(buf)
+        self._s3.put_object(Bucket=self.cfg["bucket"], Key=key,
+                            Body=buf.getvalue())
+
+    def read_manifest(self) -> "pl.DataFrame | None":
+        try:
+            if self.local:
+                return pl.read_parquet(
+                    os.path.join(self.local, era5lake.MANIFEST_KEY))
+            obj = self._s3.get_object(Bucket=self.cfg["bucket"],
+                                      Key=era5lake.MANIFEST_KEY)
+            return pl.read_parquet(io.BytesIO(obj["Body"].read()))
+        except Exception:  # noqa: BLE001 - a fresh lake has no manifest yet
+            return None
+
+
+def _tile_frame(ds, ti: int, tj: int, end: str) -> "pl.DataFrame | None":
+    """Pull one 12x12 tile's hourly series and return the finalized daily frame
+    for all its points (lat_idx, lon_idx, date, ...). None for all-sea tiles."""
+    t = era5lake.TILE
+    i0, j0 = ti * t, tj * t
+    lat_sl = slice(i0, min(i0 + t, era5lake.LAT_N))
+    lon_sl = slice(j0, j0 + t)
+    if "lsm" in ds:            # land-sea mask: skip all-sea tiles outright
+        lsm = ds["lsm"].isel(latitude=lat_sl, longitude=lon_sl)
+        if "valid_time" in lsm.dims:   # static field, stored with a time axis
+            lsm = lsm.isel(valid_time=-1)
+        if float(lsm.max()) < 0.05:
+            return None
+    sub = (ds[list(ERA5_VARS)]
+           .isel(latitude=lat_sl, longitude=lon_sl)
+           .sel(valid_time=slice(START_DATE, end))
+           .load())
+
+    frames = []
+    for a, lat in enumerate(sub.latitude.values):
+        for b, lon in enumerate(sub.longitude.values):
+            cols = {"time": sub.valid_time.values}
+            for src, dst in ERA5_VARS.items():
+                cols[dst] = sub[src].isel(latitude=a, longitude=b).values
+            daily = climate._finalize_approximated(hourly_to_daily(pl.DataFrame(cols)))
+            i, j = era5lake.to_idx(float(lat), float(lon))
+            frames.append(daily.with_columns(
+                pl.lit(i, dtype=pl.Int32).alias("lat_idx"),
+                pl.lit(j, dtype=pl.Int32).alias("lon_idx")))
+    return pl.concat(frames)
+
+
+def _write_tile(sink: _Sink, ti: int, tj: int, tile_df: pl.DataFrame) -> pl.DataFrame:
+    """Write one tile's point files + hive partitions; return its manifest rows.
+    Uploads run on a thread pool: a tile is ~1.2k small objects and sequential
+    PUTs are pure round-trip latency (measured 2.5 obj/s -> 8 min/tile; the
+    pool brings that to well under a minute). boto3 clients are thread-safe."""
+    puts = []
+    rows = []
+    for (i, j), pdf in tile_df.group_by(["lat_idx", "lon_idx"]):
+        pdf = pdf.sort("date")
+        puts.append((era5lake.point_key(i, j), pdf.drop(["lat_idx", "lon_idx"])))
+        lat, lon = era5lake.to_coords(i, j)
+        rows.append({"lat_idx": i, "lon_idx": j, "lat": lat, "lon": lon,
+                     "tile": f"{ti}_{tj}", "rows": pdf.height,
+                     "date_min": str(pdf["date"].min()),
+                     "date_max": str(pdf["date"].max())})
+    parts = tile_df.with_columns(
+        pl.col("date").dt.year().alias("year"),
+        pl.col("date").dt.month().alias("month"))
+    for (year, month), mdf in parts.group_by(["year", "month"]):
+        puts.append((era5lake.daily_part_key(ti, tj, year, month),
+                     mdf.drop(["year", "month"]).sort(["lat_idx", "lon_idx", "date"])))
+    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
+        for f in [pool.submit(sink.put, k, df) for k, df in puts]:
+            f.result()   # surface the first failure instead of hiding it
+    return pl.DataFrame(rows)
+
+
+def _city_tiles() -> list:
+    from data import grid  # noqa: PLC0415
+
+    tiles = set()
+    for c in cities.all_cities():
+        cell = grid.snap(c["lat"], c["lon"])
+        tiles.add(era5lake.tile_of(
+            *era5lake.to_idx(cell["center_lat"], cell["center_lon"])))
+    return sorted(tiles)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--cities", action="store_true")
+    ap.add_argument("--land", action="store_true")
+    ap.add_argument("--tiles", help="ti0:ti1,tj0:tj1 tile-index window")
+    ap.add_argument("--limit", type=int)
+    ap.add_argument("--overwrite", action="store_true")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    if args.cities:
+        todo = _city_tiles()
+    elif args.tiles:
+        (a, b), (c, d) = (p.split(":") for p in args.tiles.split(","))
+        todo = [(ti, tj) for ti in range(int(a), int(b))
+                for tj in range(int(c), int(d))]
+    elif args.land:
+        todo = [(ti, tj) for ti in range(era5lake.LAT_N // era5lake.TILE + 1)
+                for tj in range(era5lake.LON_N // era5lake.TILE)]
+    else:
+        ap.error("pick one of --cities / --tiles / --land")
+    if args.limit:
+        todo = todo[:args.limit]
+
+    sink = _Sink()
+    ds = _open_store()
+    end = (datetime.date.today()
+           - datetime.timedelta(days=climate.ARCHIVE_LATENCY_DAYS)).isoformat()
+
+    manifest = sink.read_manifest()
+    done = set() if manifest is None or args.overwrite else \
+        set(manifest["tile"].unique().to_list())
+
+    written = skipped = empty = failed = 0
+    for n, (ti, tj) in enumerate(todo, 1):
+        if f"{ti}_{tj}" in done:
+            skipped += 1
+            continue
+        t0 = time.time()
+        try:
+            tile_df = _tile_frame(ds, ti, tj, end)
+            if tile_df is None:
+                empty += 1
+                continue
+            if args.dry_run:
+                print(f"[dry-run] tile {ti}_{tj}: {tile_df.height} rows "
+                      f"({time.time() - t0:.1f}s); nothing written")
+                print(tile_df.head())
+                return
+            rows = _write_tile(sink, ti, tj, tile_df)
+        except Exception as e:  # noqa: BLE001 - keep going; the tile isn't in the
+            failed += 1         # manifest, so the next run retries it
+            print(f"[{n}/{len(todo)}] FAILED tile {ti}_{tj}: {e}")
+            continue
+        manifest = rows if manifest is None else pl.concat(
+            [manifest.filter(pl.col("tile") != f"{ti}_{tj}"), rows])
+        sink.put(era5lake.MANIFEST_KEY, manifest)   # after every tile: resumable
+        written += 1
+        print(f"[{n}/{len(todo)}] tile {ti}_{tj}: {rows.height} points "
+              f"in {time.time() - t0:.1f}s")
+    print(f"done: tiles written={written} skipped(done)={skipped} sea={empty} "
+          f"failed={failed}")
+
+
+if __name__ == "__main__":
+    main()

backend/indexnow.py

@@ -179,7 +179,8 @@ def _write_state(sig: str) -> None:
 def submit_if_changed(base_url: str | None = None) -> dict | None:
     """Submit every indexable page, but only when the URL set changed since the
     last successful submission — the shared logic behind the CLI's --if-changed
-    flag and the worker scheduler's periodic ping (notifications/scheduler.py), so
+    flag and the daemon's periodic ping (thermograph-daemon's indexnow timer,
+    via api/internal_routes.py), so
     a code-only deploy or an unremarkable scheduled tick never resubmits. Returns
     the submit_all() result, or None when skipped (unchanged, or a partial/failed
     submission that leaves the prior state in place so the next attempt retries)."""

backend/lake_app.py

@@ -0,0 +1,204 @@
+"""The lake service: low-latency SQL-ish reads over the ERA5 bucket (prod-only).
+
+A small FastAPI app run as the Swarm ``lake`` service (same backend image,
+``THERMOGRAPH_ROLE=lake`` — the entrypoint launches this module instead of the
+web app; no database, no migrations). Web replicas call it instead of touching
+the bucket themselves, so a cell's history costs one LAN round-trip against a
+warm disk cache instead of an S3 fetch, and analytical queries get
+partition-pruned scans without shipping S3 credentials to every web task.
+
+Endpoints:
+- ``GET /healthz``            liveness (also reports manifest row count)
+- ``GET /history?lat=&lon=``  the hot path: nearest ERA5 point's full daily
+                              history as parquet bytes, disk-cached per point
+                              (a plain single-file read — no engine needed)
+- ``POST /query {"sql": ..}`` SELECT-only SQL over two views: ``era5_daily``
+                              (the hive table, partition-pruned on
+                              tile/year/month predicates) and ``manifest``.
+                              Runs on DuckDB — a real lake engine (full SQL,
+                              parallel scans, hive + row-group pushdown); the
+                              httpfs extension is baked into the image so S3
+                              needs no runtime extension fetch.
+
+Scale-out safe: replicas share nothing — each keeps its own cache under
+THERMOGRAPH_LAKE_CACHE (a per-task local volume), so 1..2 replicas behind the
+Swarm VIP need no coordination.
+"""
+import io
+import os
+import re
+import threading
+import time
+
+import polars as pl
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import FileResponse, Response
+from pydantic import BaseModel
+
+from data import era5lake
+
+CACHE_DIR = os.environ.get("THERMOGRAPH_LAKE_CACHE", "/state/lake-cache")
+MANIFEST_TTL = 6 * 3600        # re-pull the manifest at most this often
+QUERY_ROW_CAP = 10_000         # /query responses are JSON; keep them bounded
+
+app = FastAPI(title="thermograph-lake")
+
+_manifest_lock = threading.Lock()
+_manifest: "pl.DataFrame | None" = None
+_manifest_at = 0.0
+
+
+def _source():
+    """(mode, cfg) for this process: a local lake dir (tests/dev) or the bucket."""
+    if era5lake.local_dir():
+        return "local", None
+    cfg = era5lake.s3_config()
+    return ("bucket", cfg) if cfg else ("none", None)
+
+
+def _manifest_frame() -> "pl.DataFrame | None":
+    global _manifest, _manifest_at
+    with _manifest_lock:
+        if _manifest is not None and time.time() - _manifest_at < MANIFEST_TTL:
+            return _manifest
+        mode, cfg = _source()
+        try:
+            if mode == "local":
+                df = pl.read_parquet(
+                    os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY))
+            elif mode == "bucket":
+                df = pl.read_parquet(
+                    f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}",
+                    storage_options=era5lake.storage_options(cfg))
+            else:
+                return None
+        except Exception:  # noqa: BLE001 - empty/unreachable lake: serve health, 404 reads
+            return None
+        _manifest, _manifest_at = df, time.time()
+        return df
+
+
+@app.get("/healthz")
+def healthz():
+    mode, _ = _source()
+    m = _manifest_frame()
+    return {"status": "ok", "source": mode,
+            "points": None if m is None else m.height}
+
+
+@app.get("/history")
+def history(lat: float, lon: float):
+    """One ERA5 point's full daily history, as parquet bytes (cached on disk —
+    the cache file IS the response body, so a warm hit is a sendfile)."""
+    i, j = era5lake.to_idx(lat, lon)
+    path = os.path.join(CACHE_DIR, f"p{i}_{j}.parquet")
+    if not os.path.exists(path):
+        mode, cfg = _source()
+        if mode == "none":
+            raise HTTPException(status_code=503, detail="lake not configured")
+        m = _manifest_frame()
+        if m is not None and m.filter(
+                (pl.col("lat_idx") == i) & (pl.col("lon_idx") == j)).is_empty():
+            raise HTTPException(status_code=404, detail="point not in the lake")
+        try:
+            if mode == "local":
+                df = era5lake._read_local(i, j)
+            else:
+                df = era5lake._read_bucket(i, j, cfg)
+        except era5lake.LakeUnavailable as e:
+            raise HTTPException(status_code=404, detail=str(e)) from e
+        try:
+            os.makedirs(CACHE_DIR, exist_ok=True)
+            tmp = f"{path}.{os.getpid()}.partial"
+            df.write_parquet(tmp)
+            os.replace(tmp, path)   # atomic: concurrent readers see whole files only
+        except OSError:
+            # An unwritable cache (bad volume ownership, full disk) must never
+            # fail the hot path — serve from memory; the cache is an
+            # accelerator, not a dependency.
+            buf = io.BytesIO()
+            df.write_parquet(buf)
+            return Response(buf.getvalue(),
+                            media_type="application/vnd.apache.parquet",
+                            headers={"X-Lake-Point": f"{i},{j}"})
+    return FileResponse(path, media_type="application/vnd.apache.parquet",
+                        headers={"X-Lake-Point": f"{i},{j}"})
+
+
+class Query(BaseModel):
+    sql: str
+
+
+_FORBIDDEN = re.compile(
+    r"\b(insert|update|delete|drop|create|alter|attach|copy|truncate|install|load"
+    r"|export|import|pragma|set|call)\b", re.I)
+
+
+def _connect():
+    """A fresh in-memory DuckDB with the lake views registered. Returns None
+    when no lake is configured.
+
+    Bucket mode reads era5_daily through the ICEBERG table, not a hive glob:
+    a glob over era5/daily/ must LIST every object under it (~1k per tile —
+    minutes at a few hundred tiles, seen live timing out /query at 390),
+    while Iceberg gets the file list from table metadata in a handful of
+    reads. Local mode (tests, small trees) keeps the plain hive glob — no
+    Iceberg table exists in the fixtures and listing a local dir is free."""
+    import duckdb  # noqa: PLC0415 - only the lake role pays the import
+
+    mode, cfg = _source()
+    if mode == "none":
+        return None
+    con = duckdb.connect()
+    if mode == "local":
+        daily_glob = os.path.join(era5lake.local_dir(), era5lake.PREFIX,
+                                  "daily/*/*/*/*.parquet")
+        con.execute(f"""
+            CREATE VIEW era5_daily AS
+            SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""")
+        manifest_src = os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY)
+        con.execute(
+            f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')")
+        return con
+
+    con.execute("LOAD httpfs")   # both baked into the image at build time,
+    con.execute("LOAD iceberg")  # as the runtime user (see Dockerfile)
+    host = cfg["endpoint"].split("://", 1)[-1]
+    con.execute(f"SET s3_endpoint='{host}'")
+    con.execute("SET s3_url_style='path'")
+    con.execute(f"SET s3_region='{cfg['region']}'")
+    con.execute("SET s3_access_key_id=?", [cfg["access_key"]])
+    con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]])
+    # Newest metadata JSON wins (one single-page LIST of metadata/ — cheap);
+    # pyiceberg names them NNNNN-uuid.metadata.json, so lexicographic max is
+    # the numeric max.
+    meta = con.execute(
+        f"SELECT max(file) FROM glob('s3://{cfg['bucket']}/iceberg/era5_daily/"
+        f"metadata/*.metadata.json')").fetchone()[0]
+    if meta is None:
+        return None   # lake bucket exists but the Iceberg table doesn't yet
+    con.execute(f"CREATE VIEW era5_daily AS SELECT * FROM iceberg_scan('{meta}')")
+    con.execute(f"""CREATE VIEW manifest AS SELECT * FROM
+        read_parquet('s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}')""")
+    return con
+
+
+@app.post("/query")
+def query(q: Query):
+    sql = q.sql.strip().rstrip(";").strip()
+    if not sql.lower().startswith(("select", "with")) or ";" in sql \
+            or _FORBIDDEN.search(sql):
+        raise HTTPException(status_code=400,
+                            detail="only a single SELECT statement is allowed")
+    con = _connect()
+    if con is None:
+        raise HTTPException(status_code=503, detail="lake not configured")
+    try:
+        out = con.execute(
+            f"SELECT * FROM ({sql}) LIMIT {QUERY_ROW_CAP}").fetchall()
+        cols = [d[0] for d in con.description]
+    except Exception as e:  # noqa: BLE001 - surface the engine's message to the caller
+        raise HTTPException(status_code=400, detail=f"query failed: {e}") from e
+    finally:
+        con.close()
+    return {"columns": cols, "rows": out, "row_cap": QUERY_ROW_CAP}

backend/notifications/discord_bot.py

@@ -1,295 +0,0 @@
-"""Discord **gateway** bot — reacts to @mentions (and DMs) in real time.
-
-The other two Discord paths need no persistent connection: slash commands arrive as
-signed HTTP POSTs (discord_interactions.py) and the daily post goes out over an
-incoming webhook (discord.py). This one is different — to answer an *ordinary*
-message that mentions the bot, we have to hold a live websocket to Discord's
-gateway and listen for MESSAGE_CREATE events.
-
-Single-connection invariant. Discord permits exactly one gateway connection per bot
-token (per shard), so — like the subscription notifier and the daily post — this
-runs in exactly one process: web/app.py starts it only when the process wins the
-leader election (see core/singleton.claim_leader). No new container or daemon; it
-rides the app's asyncio event loop as a background task.
-
-Least privilege / no privileged intent. We subscribe to GUILD_MESSAGES +
-DIRECT_MESSAGES and act only on messages that @mention the bot or DM it. Discord
-delivers message *content* for exactly those cases even without the privileged
-MESSAGE_CONTENT intent, so the bot works with no portal review. To trigger on
-arbitrary keywords in channels, enable the MESSAGE_CONTENT intent in the Developer
-Portal and extend `_response_for_message` — the rest of the machinery is unchanged.
-
-Enable with THERMOGRAPH_DISCORD_BOT=1 (plus THERMOGRAPH_DISCORD_BOT_TOKEN, the same
-bot token discord.py uses for DMs). Unset => no gateway connection; everything here
-no-ops, so an unconfigured deploy pays nothing.
-
-Behaviour: mention the bot with a city name ("@Thermograph Phoenix") and it replies
-with the same grade embed the /grade slash command returns, reusing
-discord_interactions._grade_message so the two never drift.
-"""
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-import os
-import random
-
-from notifications import discord
-
-log = logging.getLogger("thermograph.discord_bot")
-
-# v10 JSON gateway. On a resume we reconnect to the session's resume_gateway_url
-# (from the READY payload) instead, with the same query string appended.
-_GATEWAY_URL = "wss://gateway.discord.gg/?v=10&encoding=json"
-_GATEWAY_QS = "/?v=10&encoding=json"
-
-# Gateway intents (bitfield). GUILDS gives guild/channel context; GUILD_MESSAGES and
-# DIRECT_MESSAGES deliver message-create events in servers and DMs. We deliberately
-# do NOT request MESSAGE_CONTENT (privileged) — Discord still includes content for
-# messages that mention us or DM us, which is all we act on.
-_INTENT_GUILDS = 1 << 0
-_INTENT_GUILD_MESSAGES = 1 << 9
-_INTENT_DIRECT_MESSAGES = 1 << 12
-INTENTS = _INTENT_GUILDS | _INTENT_GUILD_MESSAGES | _INTENT_DIRECT_MESSAGES
-
-# Gateway opcodes (discord.com/developers/docs/topics/gateway-events#opcodes).
-_OP_DISPATCH = 0
-_OP_HEARTBEAT = 1
-_OP_IDENTIFY = 2
-_OP_RESUME = 6
-_OP_RECONNECT = 7
-_OP_INVALID_SESSION = 9
-_OP_HELLO = 10
-_OP_HEARTBEAT_ACK = 11
-
-# Close codes for unrecoverable conditions — reconnecting just loops, so we stop:
-# 4004 auth failed (bad token), 4010 invalid shard, 4011 sharding required, 4012
-# invalid API version, 4013 invalid intents, 4014 disallowed (privileged) intents.
-_FATAL_CLOSE = {4004, 4010, 4011, 4012, 4013, 4014}
-
-_TRUTHY = {"1", "true", "yes", "on"}
-
-_HELP = ("Mention me with a city name — e.g. `@Thermograph Phoenix` — and I'll grade "
-         "today's weather against ~45 years of local history.")
-
-# Reconnect backoff bounds (seconds).
-_BACKOFF_START = 1.0
-_BACKOFF_MAX = 60.0
-
-
-def enabled() -> bool:
-    """True iff the gateway bot is switched on AND a bot token is configured. Off by
-    default: a gateway connection is a new persistent behaviour, so it takes an
-    explicit opt-in rather than piggybacking on the token being present for DMs."""
-    flag = os.environ.get("THERMOGRAPH_DISCORD_BOT", "").strip().lower() in _TRUTHY
-    return flag and bool(discord.BOT_TOKEN)
-
-
-# --- message handling (pure; unit-tested without a live gateway) --------------
-def _mentions_bot(msg: dict, bot_user_id: str) -> bool:
-    return any(str(u.get("id")) == str(bot_user_id) for u in (msg.get("mentions") or []))
-
-
-def _is_dm(msg: dict) -> bool:
-    # A DM has no guild_id; a guild message always carries one.
-    return not msg.get("guild_id")
-
-
-def _strip_mentions(content: str, bot_user_id: str) -> str:
-    """Drop every form of the bot's own mention (<@id> and the legacy <@!id>) and
-    collapse the surrounding whitespace, leaving just the user's query text."""
-    out = content.replace(f"<@{bot_user_id}>", " ").replace(f"<@!{bot_user_id}>", " ")
-    return " ".join(out.split())
-
-
-def _response_for_message(msg: dict, bot_user_id: str) -> dict | None:
-    """The reply payload for one MESSAGE_CREATE, or None to stay silent.
-
-    Silent unless the message DMs the bot or @mentions it, and never for a bot
-    author (that includes ourselves — the guard against a mention-loop). The text
-    after the mention is treated as a city query and answered with the same embed
-    the /grade slash command builds; an empty query gets the help line."""
-    author = msg.get("author") or {}
-    if author.get("bot") or str(author.get("id")) == str(bot_user_id):
-        return None
-    dm = _is_dm(msg)
-    if not dm and not _mentions_bot(msg, bot_user_id):
-        return None
-    content = msg.get("content") or ""
-    query = content.strip() if dm else _strip_mentions(content, bot_user_id)
-    if not query:
-        return {"content": _HELP}
-    # Import here (not at module top) to avoid importing the FastAPI web layer's
-    # transitive deps for a module that also loads in the notifier-only context.
-    from notifications import discord_interactions
-    data = discord_interactions._grade_message(query)
-    # Gateway messages can't be ephemeral — the flag is interactions-only.
-    data.pop("flags", None)
-    return data
-
-
-# --- REST reply ---------------------------------------------------------------
-async def _send_reply(channel_id, message_id, data: dict) -> None:
-    """Post the reply in-channel as a proper reply to the triggering message.
-
-    Reuses discord._bot_post (sync httpx with a 429 retry) off the event loop via a
-    thread so the gateway read-loop never blocks. allowed_mentions is locked down so
-    a crafted query can't turn the reply into an @everyone/role ping; only the person
-    who asked is pinged, via the reply reference."""
-    if not channel_id:
-        return
-    payload = dict(data)
-    if message_id:
-        payload["message_reference"] = {"message_id": str(message_id)}
-        payload["allowed_mentions"] = {"parse": [], "replied_user": True}
-    try:
-        await asyncio.to_thread(
-            discord._bot_post, f"/channels/{channel_id}/messages", payload)
-    except Exception:  # noqa: BLE001 - a failed reply must never kill the read loop
-        log.warning("discord_bot: reply failed", exc_info=True)
-
-
-# --- gateway session ----------------------------------------------------------
-class _Session:
-    """Mutable per-connection state that survives a resume: the last sequence
-    number, the resume token/URL, and our own user id (for mention detection)."""
-
-    __slots__ = ("seq", "session_id", "resume_url", "user_id")
-
-    def __init__(self) -> None:
-        self.seq: int | None = None
-        self.session_id: str | None = None
-        self.resume_url: str | None = None
-        self.user_id: str = ""
-
-
-async def _heartbeat(ws, interval: float, sess: _Session, acked: dict) -> None:
-    """Send op-1 heartbeats every `interval` seconds (first beat jittered per
-    Discord's guidance). If the previous beat was never ACKed the link is a zombie —
-    close with a non-1000 code so the next connect can resume, and stop."""
-    await asyncio.sleep(interval * random.random())
-    while True:
-        if not acked["ok"]:
-            await ws.close(code=4000)
-            return
-        acked["ok"] = False
-        try:
-            await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq}))
-        except Exception:  # noqa: BLE001 - closed socket; the read loop handles reconnect
-            return
-        await asyncio.sleep(interval)
-
-
-async def _dispatch(msg: dict, sess: _Session) -> None:
-    """Handle an op-0 DISPATCH event we care about."""
-    t = msg.get("t")
-    d = msg.get("d") or {}
-    if t == "READY":
-        sess.session_id = d.get("session_id")
-        resume = (d.get("resume_gateway_url") or "").rstrip("/")
-        sess.resume_url = f"{resume}{_GATEWAY_QS}" if resume else None
-        sess.user_id = str((d.get("user") or {}).get("id") or "")
-        log.info("discord_bot: gateway ready (user %s)", sess.user_id)
-    elif t == "MESSAGE_CREATE" and sess.user_id:
-        # _response_for_message reaches into discord_interactions._grade_message for
-        # anything but the help line -- sync parquet/DB reads + grading math, same as
-        # web/app.py's discord_interactions route -- keep it off the (single, shared)
-        # gateway event loop so one grading lookup can't stall heartbeats/reads.
-        reply = await asyncio.to_thread(_response_for_message, d, sess.user_id)
-        if reply:
-            await _send_reply(d.get("channel_id"), d.get("id"), reply)
-
-
-async def _connection(ws, sess: _Session, resume: bool) -> str:
-    """Drive one live connection until it closes. Returns the mode for the next
-    connect: "resume" to reconnect and RESUME, "identify" to start a fresh session."""
-    hello = json.loads(await ws.recv())
-    if hello.get("op") != _OP_HELLO:
-        return "identify"  # protocol out of step — start clean
-    interval = float(hello["d"]["heartbeat_interval"]) / 1000.0
-    acked = {"ok": True}
-    hb = asyncio.create_task(_heartbeat(ws, interval, sess, acked))
-    try:
-        if resume and sess.session_id and sess.seq is not None:
-            await ws.send(json.dumps({"op": _OP_RESUME, "d": {
-                "token": discord.BOT_TOKEN,
-                "session_id": sess.session_id,
-                "seq": sess.seq,
-            }}))
-        else:
-            await ws.send(json.dumps({"op": _OP_IDENTIFY, "d": {
-                "token": discord.BOT_TOKEN,
-                "intents": INTENTS,
-                "properties": {"os": "linux", "browser": "thermograph", "device": "thermograph"},
-            }}))
-        async for raw in ws:
-            msg = json.loads(raw)
-            if msg.get("s") is not None:
-                sess.seq = msg["s"]
-            op = msg.get("op")
-            if op == _OP_DISPATCH:
-                await _dispatch(msg, sess)
-            elif op == _OP_HEARTBEAT:            # server asked for an immediate beat
-                await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq}))
-            elif op == _OP_HEARTBEAT_ACK:
-                acked["ok"] = True
-            elif op == _OP_RECONNECT:            # planned reconnect — resume
-                return "resume"
-            elif op == _OP_INVALID_SESSION:
-                # d == True => the session can still be resumed; else it's dead and
-                # the next connect must IDENTIFY fresh.
-                if not msg.get("d"):
-                    sess.session_id = None
-                    sess.resume_url = None
-                await asyncio.sleep(1 + random.random())
-                return "resume" if sess.session_id else "identify"
-    finally:
-        hb.cancel()
-    return "resume" if sess.session_id else "identify"
-
-
-def _close_code(exc: Exception):
-    """Best-effort websocket close code across websockets library versions (older
-    exposes .code; newer nests it under .rcvd)."""
-    code = getattr(exc, "code", None)
-    if code:
-        return code
-    return getattr(getattr(exc, "rcvd", None), "code", None)
-
-
-async def run() -> None:
-    """Maintain the gateway connection for the app's lifetime: connect, IDENTIFY (or
-    RESUME), dispatch events, and reconnect with capped exponential backoff. Returns
-    only on a fatal close (bad token / disallowed intents) or task cancellation, and
-    never raises out — a misbehaving gateway must not take the app down."""
-    if not enabled():
-        return
-    try:
-        import websockets
-    except Exception:  # noqa: BLE001 - shipped via uvicorn[standard]; guard anyway
-        log.warning("discord_bot: 'websockets' unavailable; gateway bot disabled")
-        return
-    sess = _Session()
-    mode = "identify"
-    backoff = _BACKOFF_START
-    while True:
-        use_resume = mode == "resume" and bool(sess.resume_url)
-        url = sess.resume_url if use_resume else _GATEWAY_URL
-        try:
-            async with websockets.connect(url, max_size=2 ** 20, open_timeout=30) as ws:
-                mode = await _connection(ws, sess, resume=use_resume)
-            backoff = _BACKOFF_START   # server-requested reconnect: come back promptly
-            continue
-        except asyncio.CancelledError:
-            log.info("discord_bot: shutting down")
-            raise
-        except Exception as exc:  # noqa: BLE001 - any drop -> back off and reconnect
-            if _close_code(exc) in _FATAL_CLOSE:
-                log.error("discord_bot: fatal gateway close %s; stopping", _close_code(exc))
-                return
-            mode = "resume" if sess.session_id else "identify"
-            log.warning("discord_bot: gateway dropped (%s); reconnecting in %.0fs",
-                        type(exc).__name__, backoff)
-        await asyncio.sleep(backoff + random.random())
-        backoff = min(backoff * 2, _BACKOFF_MAX)

backend/notifications/scheduler.py

@@ -1,68 +0,0 @@
-"""Recurring worker-tier jobs: keep the curated city set warm and ping IndexNow
-on a schedule, so neither needs a human to run it after every deploy anymore.
-
-Only the process that already won leader election for the notifier runs this
-too (gated by the same check in web/app.py's _lifespan) — APScheduler has no
-cross-host coordination of its own, so running it unguarded on every worker
-replica would repeat every job once per replica, same reason the notifier
-itself needs a leader.
-
-APScheduler, not a Postgres-native queue: there is exactly one process that
-ever runs this, no fan-out/backpressure/dead-letter need, and it needs no new
-infrastructure. procrastinate/pgqueuer are the documented upgrade path if that
-changes — see docs/architecture/repo-topology-and-infrastructure.md §7.
-"""
-import os
-
-from apscheduler.schedulers.background import BackgroundScheduler
-
-# How often the curated city set is (re-)warmed. warm_cities.py's own docstring
-# frames it as a deploy-time script ("Run at/after deploy") because a warm cache
-# mostly just stays warm; scheduling it periodically instead catches a newly
-# added city, or one that fell out of cache, without waiting for the next
-# deploy. Daily is plenty — every already-cached city is a cheap skip.
-WARM_CITIES_INTERVAL_HOURS = int(os.environ.get("THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS", "24") or 24)
-
-# How often to check whether the indexable URL set changed and, if so, ping
-# IndexNow. submit_if_changed() is a cheap no-op when nothing changed, so this
-# can run more often than warm_cities without spending anything on most ticks.
-INDEXNOW_INTERVAL_HOURS = int(os.environ.get("THERMOGRAPH_INDEXNOW_INTERVAL_HOURS", "6") or 6)
-
-_scheduler: "BackgroundScheduler | None" = None
-
-
-def _warm_cities_job() -> None:
-    import warm_cities   # local import: a script-style top-level module — only
-                          # pay its import cost on the process that runs the job
-    warm_cities.main()
-
-
-def _indexnow_job() -> None:
-    import indexnow
-    indexnow.submit_if_changed()
-
-
-def start() -> None:
-    """Start the scheduler (idempotent, mirroring notify.start()). The caller is
-    responsible for the leader-election gate — see web/app.py's _lifespan."""
-    global _scheduler
-    if _scheduler is not None:
-        return
-    sched = BackgroundScheduler(daemon=True)
-    # No next_run_time override: the default first run is one interval from now,
-    # not immediately — warm_cities already runs at deploy time (warm_cities.py /
-    # the deploy hook), so an immediate duplicate run on every worker boot/restart
-    # would just be wasted work.
-    sched.add_job(_warm_cities_job, "interval", hours=WARM_CITIES_INTERVAL_HOURS,
-                 id="warm_cities")
-    sched.add_job(_indexnow_job, "interval", hours=INDEXNOW_INTERVAL_HOURS,
-                 id="indexnow")
-    sched.start()
-    _scheduler = sched
-
-
-def stop() -> None:
-    global _scheduler
-    if _scheduler is not None:
-        _scheduler.shutdown(wait=False)
-        _scheduler = None

backend/requirements-seed.txt

@@ -7,3 +7,10 @@
 icechunk
 xarray
 zarr
+# The live store compresses with PCodec; without this extra every read fails
+# with "codec not available: 'pcodec'".
+numcodecs[pcodec]
+# gen_era5_lake.py's bucket writer.
+boto3
+numcodecs[pcodec]
+boto3

backend/requirements.txt

@@ -19,9 +19,11 @@ alembic==1.14.0
 pywebpush==2.0.0
 # Ed25519 verification of Discord interaction webhooks (see notifications/discord_interactions.py).
 PyNaCl==1.5.0
-# Discord gateway bot's websocket client (notifications/discord_bot.py). Already
+# The Discord gateway bot and the recurring worker jobs moved to the
-# pulled in transitively by uvicorn[standard]; pinned here since we import it directly.
+# thermograph-daemon Go binary (backend/daemon/), which calls back over
-websockets==16.0
+# api/internal_routes.py — so the direct `websockets` pin (only discord_bot.py
-# Recurring worker-tier jobs — city warming, IndexNow pings (see
+# imported it; uvicorn[standard] still pulls it transitively for its own ws
-# notifications/scheduler.py). In-process; no broker/queue needed at this scale.
+# protocol support) and `apscheduler` are gone from this list.
-apscheduler==3.11.3
+# The lake role's SQL engine (lake_app.py /query): embedded, parallel
+# partition-pruned scans over the ERA5 hive table. Only lake_app imports it.
+duckdb==1.4.2

backend/scripts/smoke.sh

@@ -29,7 +29,7 @@ echo "==> starting backend + db ($IMG)"
 "${COMPOSE[@]}" up -d
 
 echo "==> waiting for /healthz (up to 90s)"
-for i in $(seq 1 45); do
+for _ in $(seq 1 45); do
   if curl -fsS -o /dev/null "$PORT_URL/healthz"; then ok=1; break; fi
   sleep 2
 done

backend/seed_era5.py

@@ -16,10 +16,10 @@ Idempotent by default: a cell that already has a cached history is skipped (pass
 --overwrite to reseed). Not run in CI or the app image — the icechunk/xarray/zarr
 stack lives only in requirements-seed.txt.
 
-IMPORTANT — the S3/Icechunk *access* layer (_open_store / fetch_point) targets a
+The access layer was VERIFIED against the live store 2026-07-23 (prefix
-young, fast-moving API and the store's exact variable/coordinate names; it is NOT
+`icechunkV2`, group `single/temporal`, coord `valid_time`, ECMWF short variable
-exercised by the test suite. VERIFY it with `--dry-run` on the seed box before a
+names, and the pcodec codec — install `numcodecs[pcodec]`). Re-verify with
-full run. The hourly→daily transform (hourly_to_daily) IS unit-tested and is the
+`--dry-run` if the store revs. The hourly→daily transform (hourly_to_daily) IS unit-tested and is the
 part that must be numerically correct.
 
 Source: https://registry.opendata.aws/earthmover-era5/  (group="temporal", anonymous)
@@ -38,19 +38,19 @@ START_DATE = climate.START_DATE            # "1980-01-01"
 
 # Anonymous AWS Open Data Icechunk store; overridable in case the registry moves.
 ERA5_BUCKET = os.environ.get("THERMOGRAPH_ERA5_BUCKET", "earthmover-icechunk-era5")
-ERA5_PREFIX = os.environ.get("THERMOGRAPH_ERA5_PREFIX", "")   # verify on the seed box
+ERA5_PREFIX = os.environ.get("THERMOGRAPH_ERA5_PREFIX", "icechunkV2")
 ERA5_REGION = os.environ.get("THERMOGRAPH_ERA5_REGION", "us-east-1")
-ERA5_GROUP = os.environ.get("THERMOGRAPH_ERA5_GROUP", "temporal")  # time-contiguous layout
+ERA5_GROUP = os.environ.get("THERMOGRAPH_ERA5_GROUP", "single/temporal")  # time-contiguous layout
 
 # ERA5 store variable name -> our short name. ERA5 gusts (i10fg) are real here, so the
 # seed keeps genuine measured gusts rather than the Meteostat/estimate fallback.
 ERA5_VARS = {
-    "2m_temperature": "t2m",                 # K
+    "t2m": "t2m",      # 2m temperature, K
-    "2m_dewpoint_temperature": "d2m",        # K (→ RH with t2m)
+    "d2m": "d2m",      # 2m dewpoint, K (→ RH with t2m)
-    "total_precipitation": "tp",             # m, hourly accumulation
+    "tp": "tp",        # total precipitation, m, hourly accumulation
-    "10m_u_component_of_wind": "u10",        # m/s
+    "u10": "u10",      # 10m wind u, m/s
-    "10m_v_component_of_wind": "v10",        # m/s
+    "v10": "v10",      # 10m wind v, m/s
-    "instantaneous_10m_wind_gust": "i10fg",  # m/s
+    "fg10": "i10fg",   # 10m wind gust, m/s (store short name; transform expects i10fg)
 }
 
 _M_TO_IN = 39.37007874
@@ -117,9 +117,9 @@ def fetch_point(ds, lat: float, lon: float, start: str, end: str) -> pl.DataFram
     may differ per store revision — verify with --dry-run."""
     sub = (ds[list(ERA5_VARS)]
            .sel(latitude=lat, longitude=lon % 360.0, method="nearest")
-           .sel(time=slice(start, end))
+           .sel(valid_time=slice(start, end))
            .load())
-    data = {"time": sub["time"].values}
+    data = {"time": sub["valid_time"].values}
     for src, dst in ERA5_VARS.items():
         data[dst] = sub[src].values
     return pl.DataFrame(data)

backend/tests/api/test_internal_routes.py

@@ -0,0 +1,134 @@
+"""The internal control surface the thermograph-daemon Go binary calls back
+into (api/internal_routes.py): shared-secret gating (fail closed when the token
+was never provisioned), the gateway-ready /grade reply with the interactions-only
+ephemeral flag dropped, the two job triggers, and the one-in-flight-run guard.
+The underlying job/grade functions are stubbed — hermetic, no real network."""
+import pytest
+from fastapi.testclient import TestClient
+
+from api import internal_routes
+from notifications import discord_interactions as di
+from web import app as appmod
+
+TOKEN = "test-internal-token"
+HDR = {"X-Thermograph-Internal-Token": TOKEN}
+
+
+@pytest.fixture
+def client(monkeypatch):
+    monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", TOKEN)
+    return TestClient(appmod.app)
+
+
+@pytest.fixture
+def grade_stub(monkeypatch):
+    """Reply payload for the shared grade builder, mirroring the real shape
+    (embeds + an ephemeral flag the gateway path must drop)."""
+    calls = []
+    monkeypatch.setattr(
+        di, "_grade_message",
+        lambda query: calls.append(query) or {"embeds": [{"title": f"grade:{query}"}],
+                                              "flags": 64})
+    return calls
+
+
+# --- auth: fail closed, then reject before accept ----------------------------
+
+def test_router_is_disabled_when_no_token_is_provisioned(monkeypatch):
+    """No THERMOGRAPH_INTERNAL_TOKEN => the surface doesn't exist (404), even
+    for a caller presenting a header — never fall open to no-auth."""
+    monkeypatch.delenv("THERMOGRAPH_INTERNAL_TOKEN", raising=False)
+    client = TestClient(appmod.app)
+    assert client.post("/internal/discord/grade", json={"query": "x"},
+                       headers=HDR).status_code == 404
+    assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 404
+    assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 404
+
+
+def test_missing_header_is_rejected(client):
+    assert client.post("/internal/jobs/indexnow", json={}).status_code == 401
+
+
+def test_wrong_token_is_rejected(client):
+    r = client.post("/internal/jobs/indexnow", json={},
+                    headers={"X-Thermograph-Internal-Token": "not-the-token"})
+    assert r.status_code == 401
+
+
+def test_correct_token_is_accepted(client, monkeypatch):
+    import indexnow
+    monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None)
+    assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200
+
+
+# --- /internal/discord/grade -------------------------------------------------
+
+def test_grade_returns_the_shared_builder_message_without_ephemeral_flag(client, grade_stub):
+    r = client.post("/internal/discord/grade", json={"query": "Phoenix"}, headers=HDR)
+    assert r.status_code == 200
+    # Gateway messages can't be ephemeral — the flag must be gone, everything
+    # else exactly as _grade_message built it.
+    assert r.json() == {"embeds": [{"title": "grade:Phoenix"}]}
+    assert grade_stub == ["Phoenix"]
+
+
+def test_grade_requires_a_query(client, grade_stub):
+    assert client.post("/internal/discord/grade", json={}, headers=HDR).status_code == 422
+    assert grade_stub == []
+
+
+# --- job triggers ------------------------------------------------------------
+
+def test_warm_cities_invokes_warm_cities_main(client, monkeypatch):
+    import warm_cities
+    calls = []
+    monkeypatch.setattr(warm_cities, "main", lambda: calls.append(1))
+    r = client.post("/internal/jobs/warm-cities", json={}, headers=HDR)
+    assert r.status_code == 200
+    assert r.json() == {"ok": True}
+    assert calls == [1]
+
+
+def test_indexnow_invokes_submit_if_changed(client, monkeypatch):
+    import indexnow
+    calls = []
+    monkeypatch.setattr(indexnow, "submit_if_changed", lambda: calls.append(1))
+    r = client.post("/internal/jobs/indexnow", json={}, headers=HDR)
+    assert r.status_code == 200
+    assert r.json() == {"ok": True}
+    assert calls == [1]
+
+
+# --- one-in-flight-run guard -------------------------------------------------
+
+def test_concurrent_job_invocation_answers_409(client, monkeypatch):
+    """A second trigger while a run is in flight must not stack — warm_cities
+    spends the Open-Meteo quota, so a stacked run double-spends it. Holding the
+    job's lock stands in for an in-flight run."""
+    import indexnow
+    monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None)
+    lock = internal_routes._JOB_LOCKS["warm-cities"]
+    assert lock.acquire(blocking=False)
+    try:
+        assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 409
+        # The guard is per job: an in-flight warm run doesn't block indexnow.
+        assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200
+    finally:
+        lock.release()
+    # Released => the next trigger runs again.
+    import warm_cities
+    monkeypatch.setattr(warm_cities, "main", lambda: None)
+    assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 200
+
+
+def test_job_lock_is_released_after_a_failing_run(monkeypatch):
+    """A crashed run must not wedge the job forever: the 500 surfaces, the lock
+    is released, and the next trigger runs."""
+    monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", TOKEN)
+    client = TestClient(appmod.app, raise_server_exceptions=False)
+    import indexnow
+    monkeypatch.setattr(indexnow, "submit_if_changed",
+                        lambda: (_ for _ in ()).throw(RuntimeError("boom")))
+    assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 500
+    monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None)
+    assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200

backend/tests/api/test_internal_token_derivation.py

@@ -0,0 +1,70 @@
+"""The internal token's derivation, and its cross-language contract with the Go daemon.
+
+The daemon (backend/daemon/) and this app must compute byte-identical tokens from
+the same THERMOGRAPH_AUTH_SECRET. If they drift, every callback fails with 401 --
+which reads like an auth bug rather than like the configuration drift it is, so
+the shared vector below is pinned on both sides.
+"""
+import hashlib
+import hmac
+
+import pytest
+
+from api import internal_routes
+
+# Must equal backend/daemon/internal/config/derive_test.go's sharedVector*.
+SHARED_VECTOR_SECRET = "test-auth-secret"
+SHARED_VECTOR_TOKEN = "4c3830b2158941ff52720d198b65f7924372a3a482b8da52540a4d9be38247ea"
+
+
+@pytest.fixture(autouse=True)
+def _clear_token_env(monkeypatch):
+    """Both knobs start unset so each test states exactly the config it exercises."""
+    monkeypatch.delenv("THERMOGRAPH_INTERNAL_TOKEN", raising=False)
+    monkeypatch.delenv("THERMOGRAPH_AUTH_SECRET", raising=False)
+
+
+def test_derived_token_matches_go(monkeypatch):
+    """The pinned cross-language vector. Changing the label or construction here
+    requires the identical change in the Go daemon's config package."""
+    monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET)
+    assert internal_routes.resolve_internal_token() == SHARED_VECTOR_TOKEN
+
+
+def test_vector_is_actually_hmac_of_the_label():
+    """Guards against the vector and the implementation drifting together if
+    someone regenerates the constant from the code it is supposed to check."""
+    expected = hmac.new(
+        SHARED_VECTOR_SECRET.encode(),
+        internal_routes._DERIVE_LABEL,
+        hashlib.sha256,
+    ).hexdigest()
+    assert expected == SHARED_VECTOR_TOKEN
+
+
+def test_explicit_token_wins_over_derivation(monkeypatch):
+    monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET)
+    monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", "explicit-wins")
+    assert internal_routes.resolve_internal_token() == "explicit-wins"
+
+
+def test_no_secret_at_all_leaves_the_surface_closed():
+    """Neither knob set => no token => the router 404s. Fail closed, never open."""
+    assert internal_routes.resolve_internal_token() == ""
+
+
+def test_derivation_is_not_the_auth_secret_itself(monkeypatch):
+    """Domain separation: a leak of the internal token must not hand over the
+    session-signing key it was derived from."""
+    monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET)
+    token = internal_routes.resolve_internal_token()
+    assert token != SHARED_VECTOR_SECRET
+    assert SHARED_VECTOR_SECRET not in token
+
+
+def test_different_secrets_derive_different_tokens(monkeypatch):
+    monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", "secret-a")
+    a = internal_routes.resolve_internal_token()
+    monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", "secret-b")
+    b = internal_routes.resolve_internal_token()
+    assert a != b

backend/tests/core/test_metrics.py

@@ -40,6 +40,16 @@ def test_classify_inbound_categories():
     # dashboard probing /thermograph/... against a root-served (base="") prod app.
     assert c("/thermograph/api/v2/metrics", "") == "metrics"
     assert c("/api/v2/metrics", "") == "metrics"
+    # The SSR content API (backend/api/content_routes.py) is a server-to-server
+    # hop from frontend_ssr's own api_client.py, never a browser -- its own
+    # category is what lets it be excluded from the access log without also
+    # hiding real external traffic under the generic "api:*" bucket.
+    assert c("/thermograph/api/v2/content/hub", "/thermograph") == "internal"
+    assert c("/api/v2/content/city/tokyo", "") == "internal"
+    # The liveness probe: never under BASE, whatever base the app happens to be
+    # mounted at (it's registered at a fixed path -- see web/app.py's healthz).
+    assert c("/healthz") == "health"
+    assert c("/healthz", "/thermograph") == "health"
     # pages, static, seo
     assert c("/thermograph/", "/thermograph") == "page"
     assert c("/thermograph/calendar", "/thermograph") == "page"

backend/tests/data/test_climate.py

@@ -311,6 +311,44 @@ def _full_history_frame(n=4000):
     }))
 
 
+def test_lake_is_first_history_source(monkeypatch, tmp_path):
+    """A configured ERA5 lake short-circuits the whole third-party chain: neither
+    NASA POWER nor Open-Meteo is consulted."""
+    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
+    monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
+    cell = {"id": "lake_primary", "center_lat": 47.6, "center_lon": -122.3}
+
+    full = _full_history_frame()
+    monkeypatch.setattr(climate.era5lake, "fetch_history_lake",
+                        lambda c, start=None: full)
+    def _nasa_should_not_run(c): raise AssertionError("NASA should not be called")
+    def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called")
+    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run)
+    monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run)
+
+    df, meta = climate._load_history(cell)
+    assert meta["source"] == "era5-lake"
+    assert df.height == full.height
+
+
+def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path):
+    """An unconfigured lake (or a point outside it) costs nothing: NASA serves
+    exactly as before the lake existed."""
+    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
+    monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
+    cell = {"id": "lake_miss", "center_lat": 47.6, "center_lon": -122.3}
+
+    def _no_lake(c, start=None):
+        raise climate.era5lake.LakeUnavailable("no lake configured")
+    monkeypatch.setattr(climate.era5lake, "fetch_history_lake", _no_lake)
+    full = _full_history_frame()
+    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full)
+
+    df, meta = climate._load_history(cell)
+    assert meta["source"] == "nasa-power"
+    assert df.height == full.height
+
+
 def test_nasa_is_primary_history_source(monkeypatch, tmp_path):
     """NASA POWER is the primary history source: a full NASA frame is accepted and
     Open-Meteo is never consulted."""

backend/tests/data/test_era5lake.py

@@ -0,0 +1,90 @@
+"""The ERA5 lake: grid math, layout keys, and the read/slice client (no network
+— the local-dir source stands in for the bucket)."""
+import datetime
+
+import polars as pl
+import pytest
+
+from data import era5lake
+
+
+def _daily_frame(start="1940-01-01", days=32000):
+    d0 = datetime.date.fromisoformat(start)
+    dates = [d0 + datetime.timedelta(days=k) for k in range(days)]
+    n = len(dates)
+    return pl.DataFrame({
+        "date": dates,
+        "tmax": [70.0] * n, "tmin": [50.0] * n, "precip": [0.1] * n,
+        "wind": [8.0] * n, "gust": [14.0] * n, "humid": [55.0] * n,
+        "fmax": [71.0] * n, "fmin": [48.0] * n,
+    })
+
+
+def test_grid_roundtrip_and_bounds():
+    # Nearest-index snap, poles clamped, longitude wrapped to [0, 360).
+    assert era5lake.to_idx(90.0, 0.0) == (0, 0)
+    assert era5lake.to_idx(-90.0, 0.0) == (720, 0)
+    assert era5lake.to_idx(51.5074, -0.1278) == (154, 1439)
+    i, j = era5lake.to_idx(35.6762, 139.6503)
+    lat, lon = era5lake.to_coords(i, j)
+    assert abs(lat - 35.6762) <= era5lake.STEP / 2 + 1e-9
+    assert abs(lon - 139.6503) <= era5lake.STEP / 2 + 1e-9
+
+
+def test_tile_and_keys_align_with_hive_layout():
+    assert era5lake.tile_of(154, 1439) == (12, 119)
+    assert era5lake.point_key(154, 1439) == "era5/points/lat=154/lon=1439.parquet"
+    assert era5lake.daily_part_key(12, 119, 1994, 3) == \
+        "era5/daily/tile=12_119/year=1994/month=03/part.parquet"
+
+
+def test_s3_config_requires_both_keys(monkeypatch):
+    monkeypatch.delenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", raising=False)
+    monkeypatch.delenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", raising=False)
+    assert era5lake.s3_config() is None
+    monkeypatch.setenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "k")
+    assert era5lake.s3_config() is None
+    monkeypatch.setenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", "s")
+    cfg = era5lake.s3_config()
+    assert cfg["bucket"] == "era5-thermograph"
+    opts = era5lake.storage_options(cfg)
+    assert opts["aws_virtual_hosted_style_request"] == "false"
+
+
+def _write_point(tmp_path, cell, df):
+    i, j = era5lake.to_idx(cell["center_lat"], cell["center_lon"])
+    path = tmp_path / era5lake.point_key(i, j)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    df.write_parquet(path)
+
+
+CELL = {"center_lat": 51.5, "center_lon": -0.125}
+
+
+def test_fetch_slices_to_start_and_keeps_schema(monkeypatch, tmp_path):
+    monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path))
+    _write_point(tmp_path, CELL, _daily_frame())
+    df = era5lake.fetch_history_lake(CELL, start="1980-01-01")
+    assert df["date"].min() == datetime.date(1980, 1, 1)
+    assert set(df.columns) == {"date", "tmax", "tmin", "precip", "wind",
+                               "gust", "humid", "fmax", "fmin"}
+    # Without a start, the whole 1940+ record comes back.
+    assert era5lake.fetch_history_lake(CELL).height == 32000
+
+
+def test_fetch_rejects_short_and_missing(monkeypatch, tmp_path):
+    monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path))
+    with pytest.raises(era5lake.LakeUnavailable):
+        era5lake.fetch_history_lake(CELL)          # nothing written
+    _write_point(tmp_path, CELL, _daily_frame(days=500))
+    with pytest.raises(era5lake.LakeUnavailable):  # implausibly short record
+        era5lake.fetch_history_lake(CELL)
+
+
+def test_fetch_unconfigured_raises(monkeypatch):
+    monkeypatch.delenv("THERMOGRAPH_LAKE_LOCAL_DIR", raising=False)
+    monkeypatch.delenv("THERMOGRAPH_LAKE_URL", raising=False)
+    monkeypatch.delenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", raising=False)
+    monkeypatch.delenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", raising=False)
+    with pytest.raises(era5lake.LakeUnavailable):
+        era5lake.fetch_history_lake(CELL)

backend/tests/notifications/test_discord_bot.py

@@ -1,125 +0,0 @@
-"""Discord gateway bot: the message-handling logic that decides when to reply and
-with what. No live gateway — we feed MESSAGE_CREATE payloads straight to the pure
-handler and stub the shared /grade lookup so these stay data-independent."""
-import asyncio
-
-import pytest
-
-from notifications import discord
-from notifications import discord_bot as bot
-from notifications import discord_interactions as di
-
-BOT_ID = "999"
-
-
-@pytest.fixture(autouse=True)
-def _stub_grade(monkeypatch):
-    """Reply payload for /grade, mirroring the real shape (embeds + an ephemeral
-    flag the gateway path must drop)."""
-    monkeypatch.setattr(
-        di, "_grade_message",
-        lambda query: {"embeds": [{"title": f"grade:{query}"}], "flags": 64})
-
-
-def _msg(content="", *, author_id="1", bot_author=False, mentions=(), guild=True):
-    m = {
-        "content": content,
-        "id": "42",
-        "channel_id": "77",
-        "author": {"id": author_id, "bot": bot_author},
-        "mentions": [{"id": mid} for mid in mentions],
-    }
-    if guild:
-        m["guild_id"] = "5"
-    return m
-
-
-# --- when the bot stays silent ----------------------------------------------
-
-def test_ignores_other_bots():
-    assert bot._response_for_message(
-        _msg("<@999> Phoenix", author_id="8", bot_author=True, mentions=[BOT_ID]), BOT_ID) is None
-
-
-def test_ignores_its_own_messages():
-    assert bot._response_for_message(
-        _msg("hello", author_id=BOT_ID, mentions=[BOT_ID]), BOT_ID) is None
-
-
-def test_ignores_guild_message_without_a_mention():
-    assert bot._response_for_message(_msg("just chatting", mentions=[]), BOT_ID) is None
-
-
-# --- when the bot replies ----------------------------------------------------
-
-def test_mention_with_city_returns_grade_without_ephemeral_flag():
-    out = bot._response_for_message(_msg("<@999> Phoenix", mentions=[BOT_ID]), BOT_ID)
-    assert out == {"embeds": [{"title": "grade:Phoenix"}]}   # flags dropped
-    assert "flags" not in out
-
-
-def test_legacy_nickname_mention_is_stripped():
-    out = bot._response_for_message(_msg("<@!999>   Tokyo", mentions=[BOT_ID]), BOT_ID)
-    assert out["embeds"][0]["title"] == "grade:Tokyo"
-
-
-def test_dm_needs_no_mention():
-    out = bot._response_for_message(_msg("Berlin", guild=False, mentions=[]), BOT_ID)
-    assert out["embeds"][0]["title"] == "grade:Berlin"
-
-
-def test_mention_with_no_query_gives_help():
-    out = bot._response_for_message(_msg("<@999>", mentions=[BOT_ID]), BOT_ID)
-    assert "content" in out and "mention me" in out["content"].lower()
-
-
-# --- helpers -----------------------------------------------------------------
-
-def test_strip_mentions_handles_both_forms():
-    assert bot._strip_mentions("<@999> <@!999>  Sao Paulo", BOT_ID) == "Sao Paulo"
-
-
-def test_is_dm_is_true_without_guild_id():
-    assert bot._is_dm(_msg("x", guild=False)) is True
-    assert bot._is_dm(_msg("x", guild=True)) is False
-
-
-# --- _dispatch: same reply, off the event loop -------------------------------
-
-def test_dispatch_delivers_the_same_reply_via_a_thread(monkeypatch):
-    """_dispatch now runs _response_for_message through asyncio.to_thread (the
-    event-loop fix) -- confirm that still produces the exact same reply and gets
-    it to _send_reply, i.e. the offload didn't change behaviour."""
-    sent = {}
-
-    async def _fake_send_reply(channel_id, message_id, data):
-        sent["channel_id"] = channel_id
-        sent["message_id"] = message_id
-        sent["data"] = data
-
-    monkeypatch.setattr(bot, "_send_reply", _fake_send_reply)
-    sess = bot._Session()
-    sess.user_id = BOT_ID
-    msg = {"t": "MESSAGE_CREATE", "d": _msg("<@999> Phoenix", mentions=[BOT_ID])}
-
-    asyncio.run(bot._dispatch(msg, sess))
-
-    assert sent["channel_id"] == "77"
-    assert sent["message_id"] == "42"
-    assert sent["data"] == {"embeds": [{"title": "grade:Phoenix"}]}
-
-
-# --- enabled() gating --------------------------------------------------------
-
-def test_enabled_is_off_by_default(monkeypatch):
-    monkeypatch.delenv("THERMOGRAPH_DISCORD_BOT", raising=False)
-    monkeypatch.setattr(discord, "BOT_TOKEN", "a-token")
-    assert bot.enabled() is False
-
-
-def test_enabled_needs_both_flag_and_token(monkeypatch):
-    monkeypatch.setenv("THERMOGRAPH_DISCORD_BOT", "1")
-    monkeypatch.setattr(discord, "BOT_TOKEN", "")
-    assert bot.enabled() is False
-    monkeypatch.setattr(discord, "BOT_TOKEN", "a-token")
-    assert bot.enabled() is True

backend/tests/notifications/test_scheduler.py

@@ -1,79 +0,0 @@
-"""Tests for the recurring worker-tier jobs (scheduler.py): job registration,
-cadence from env, idempotent start/stop, and that each job calls the right
-underlying function. No real hours-long wait — inspects APScheduler's own job
-registry rather than letting anything actually fire."""
-import datetime
-
-import pytest
-
-from notifications import scheduler
-
-
-@pytest.fixture(autouse=True)
-def _clean_scheduler():
-    """Every test starts from "not running" and leaves nothing behind, even on
-    failure — scheduler._scheduler is module-global state shared across tests."""
-    scheduler.stop()
-    yield
-    scheduler.stop()
-
-
-def test_start_registers_both_jobs_with_configured_intervals(monkeypatch):
-    monkeypatch.setattr(scheduler, "WARM_CITIES_INTERVAL_HOURS", 24)
-    monkeypatch.setattr(scheduler, "INDEXNOW_INTERVAL_HOURS", 6)
-    scheduler.start()
-    jobs = {j.id: j for j in scheduler._scheduler.get_jobs()}
-    assert set(jobs) == {"warm_cities", "indexnow"}
-    assert jobs["warm_cities"].trigger.interval == datetime.timedelta(hours=24)
-    assert jobs["indexnow"].trigger.interval == datetime.timedelta(hours=6)
-    assert jobs["warm_cities"].func is scheduler._warm_cities_job
-    assert jobs["indexnow"].func is scheduler._indexnow_job
-
-
-def test_start_does_not_run_a_job_immediately():
-    """warm_cities already runs at deploy time; an immediate duplicate run on
-    every worker boot/restart would just be wasted work — the first scheduled
-    run must be one interval away, not "now"."""
-    scheduler.start()
-    now = datetime.datetime.now(datetime.timezone.utc)
-    for job in scheduler._scheduler.get_jobs():
-        assert job.next_run_time > now
-
-
-def test_start_is_idempotent(monkeypatch):
-    scheduler.start()
-    first = scheduler._scheduler
-    calls = []
-    monkeypatch.setattr(scheduler.BackgroundScheduler, "__init__",
-                        lambda self, **kw: calls.append(1) or None)
-    scheduler.start()
-    assert scheduler._scheduler is first
-    assert calls == []   # a second BackgroundScheduler was never constructed
-
-
-def test_stop_shuts_down_and_clears_state():
-    scheduler.start()
-    assert scheduler._scheduler is not None
-    scheduler.stop()
-    assert scheduler._scheduler is None
-
-
-def test_stop_without_start_is_a_safe_no_op():
-    scheduler.stop()   # must not raise
-    assert scheduler._scheduler is None
-
-
-def test_warm_cities_job_calls_warm_cities_main(monkeypatch):
-    import warm_cities
-    calls = []
-    monkeypatch.setattr(warm_cities, "main", lambda: calls.append(1))
-    scheduler._warm_cities_job()
-    assert calls == [1]
-
-
-def test_indexnow_job_calls_submit_if_changed(monkeypatch):
-    import indexnow
-    calls = []
-    monkeypatch.setattr(indexnow, "submit_if_changed", lambda: calls.append(1))
-    scheduler._indexnow_job()
-    assert calls == [1]

backend/tests/web/test_lake_app.py

@@ -0,0 +1,114 @@
+"""The lake service over a local-dir lake fixture: health, the point hot path
+(+ its disk cache), and the SELECT-only SQL surface."""
+import datetime
+import io
+
+import polars as pl
+import pytest
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture
+def lake(tmp_path, monkeypatch):
+    """A tiny two-point lake (points, hive partitions, manifest) on disk."""
+    from data import era5lake
+
+    points = [(154, 1439), (154, 0)]           # around London
+    d0 = datetime.date(1940, 1, 1)
+    dates = [d0 + datetime.timedelta(days=k) for k in range(12000)]
+    n = len(dates)
+    rows = []
+    for i, j in points:
+        df = pl.DataFrame({
+            "date": dates,
+            "tmax": [70.0] * n, "tmin": [50.0] * n, "precip": [0.1] * n,
+            "wind": [8.0] * n, "gust": [14.0] * n, "humid": [55.0] * n,
+            "fmax": [71.0] * n, "fmin": [48.0] * n,
+        })
+        p = tmp_path / era5lake.point_key(i, j)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        df.write_parquet(p)
+        ti, tj = era5lake.tile_of(i, j)
+        part = df.with_columns(pl.lit(i, dtype=pl.Int32).alias("lat_idx"),
+                               pl.lit(j, dtype=pl.Int32).alias("lon_idx"))
+        hp = tmp_path / era5lake.daily_part_key(ti, tj, 1940, 1)
+        hp.parent.mkdir(parents=True, exist_ok=True)
+        part.filter((pl.col("date").dt.year() == 1940)
+                    & (pl.col("date").dt.month() == 1)).write_parquet(hp)
+        lat, lon = era5lake.to_coords(i, j)
+        rows.append({"lat_idx": i, "lon_idx": j, "lat": lat, "lon": lon,
+                     "tile": f"{ti}_{tj}", "rows": n,
+                     "date_min": "1940-01-01", "date_max": str(dates[-1])})
+    (tmp_path / "era5").mkdir(exist_ok=True)
+    pl.DataFrame(rows).write_parquet(tmp_path / era5lake.MANIFEST_KEY)
+
+    monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path))
+    monkeypatch.setenv("THERMOGRAPH_LAKE_CACHE", str(tmp_path / "cache"))
+    import lake_app
+    monkeypatch.setattr(lake_app, "CACHE_DIR", str(tmp_path / "cache"))
+    monkeypatch.setattr(lake_app, "_manifest", None)
+    return TestClient(lake_app.app)
+
+
+def test_healthz_reports_points(lake):
+    body = lake.get("/healthz").json()
+    assert body["status"] == "ok"
+    assert body["source"] == "local"
+    assert body["points"] == 2
+
+
+def test_history_serves_parquet_and_caches(lake, tmp_path):
+    r = lake.get("/history", params={"lat": 51.5074, "lon": -0.1278})
+    assert r.status_code == 200
+    assert r.headers["x-lake-point"] == "154,1439"
+    df = pl.read_parquet(io.BytesIO(r.content))
+    assert df.height == 12000
+    assert (tmp_path / "cache" / "p154_1439.parquet").exists()
+    # Second hit comes off the cache file and is byte-identical.
+    assert lake.get("/history",
+                    params={"lat": 51.5074, "lon": -0.1278}).content == r.content
+
+
+def test_history_serves_from_memory_when_cache_unwritable(lake, tmp_path, monkeypatch):
+    """An unwritable cache dir (bad volume ownership — seen live) degrades to
+    serving from memory, never a 500."""
+    import lake_app
+    ro = tmp_path / "ro-cache" / "nested"   # parent made read-only below
+    (tmp_path / "ro-cache").mkdir()
+    (tmp_path / "ro-cache").chmod(0o555)
+    monkeypatch.setattr(lake_app, "CACHE_DIR", str(ro))
+    r = lake.get("/history", params={"lat": 51.5074, "lon": -0.1278})
+    (tmp_path / "ro-cache").chmod(0o755)
+    assert r.status_code == 200
+    assert r.headers["x-lake-point"] == "154,1439"
+
+
+def test_history_404_for_point_outside_lake(lake):
+    r = lake.get("/history", params={"lat": -45.0, "lon": 170.0})
+    assert r.status_code == 404
+
+
+def test_query_selects_with_partition_predicates(lake):
+    r = lake.post("/query", json={"sql":
+        "SELECT year, month, count(*) AS n FROM era5_daily "
+        "WHERE year = 1940 AND month = 1 GROUP BY year, month"})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["columns"] == ["year", "month", "n"]
+    assert body["rows"][0][2] == 2 * 31          # two points x January 1940
+
+
+def test_query_manifest_view(lake):
+    r = lake.post("/query", json={"sql": "SELECT count(*) FROM manifest"})
+    assert r.status_code == 200
+    assert r.json()["rows"][0][0] == 2
+
+
+@pytest.mark.parametrize("sql", [
+    "DROP TABLE era5_daily",
+    "SELECT 1; SELECT 2",
+    "INSERT INTO era5_daily VALUES (1)",
+    "COPY era5_daily TO 'x'",
+])
+def test_query_rejects_non_select(lake, sql):
+    assert lake.post("/query", json={"sql": sql}).status_code == 400

backend/tests/web/test_request_logging.py

@@ -0,0 +1,87 @@
+"""The request-logging middleware (web/app.py's revalidate_static): what gets
+counted, what gets written to the access log, and what gets truncated before
+it's persisted. Companion to core/test_metrics.py's classify_inbound tests --
+these exercise the actual ASGI request path, not just the classifier function.
+"""
+import pytest
+from fastapi.testclient import TestClient
+
+from web import app as appmod
+
+metrics = appmod.metrics   # the exact module object app.py's own calls use --
+audit = appmod.audit       # see the note on module-reload isolation below.
+
+
+@pytest.fixture
+def client():
+    return TestClient(appmod.app)
+
+
+def _patch_log_access(monkeypatch):
+    calls = []
+    monkeypatch.setattr(audit, "log_access", lambda record: calls.append(record))
+    return calls
+
+
+def test_healthz_is_never_audited(monkeypatch, client):
+    """/healthz is ~47% of prod's daily access log; it must never even reach
+    audit.log_access (not just be cheap once there -- see classify_inbound's
+    "health" category and the exclusion in revalidate_static)."""
+    calls = _patch_log_access(monkeypatch)
+    r = client.get("/healthz")
+    assert r.status_code == 200
+    assert calls == []
+
+
+def test_internal_category_is_never_audited(monkeypatch, client):
+    """The SSR content API's own category ("internal") is excluded from the
+    access log the same way -- it's a server-to-server hop (frontend_ssr's
+    api_client.py calling backend), not a page view."""
+    calls = _patch_log_access(monkeypatch)
+    monkeypatch.setattr(metrics, "classify_inbound", lambda path, base: "internal")
+    r = client.get("/thermograph/api/version")
+    assert r.status_code == 200
+    assert calls == []
+
+
+def test_access_log_ip_is_truncated_not_raw(monkeypatch, client):
+    calls = _patch_log_access(monkeypatch)
+    r = client.get("/thermograph/api/version",
+                    headers={"X-Forwarded-For": "203.0.113.77, 10.0.0.1"})
+    assert r.status_code == 200
+    assert len(calls) == 1
+    assert calls[0]["ip"] == "203.0.113.0"      # /24, not the exact address
+    assert calls[0]["ip"] != "203.0.113.77"
+
+
+def test_loggable_ip_truncation():
+    assert appmod._loggable_ip("203.0.113.77") == "203.0.113.0"
+    assert appmod._loggable_ip("2001:db8:1234:5678::1") == "2001:db8:1234::"
+    assert appmod._loggable_ip(None) is None
+    assert appmod._loggable_ip("") == ""
+    assert appmod._loggable_ip("not-an-ip") == "not-an-ip"   # best-effort, never raises
+
+
+def test_rate_limiter_still_sees_the_full_precision_ip(monkeypatch, client):
+    """_client_ip (which feeds both the access log and the event rate limiter)
+    is never itself truncated -- only what audit.log_access persists is. A
+    /24-truncated rate-limit key would let one abuser exhaust a whole NAT'd
+    office's quota, which is exactly what must NOT happen here."""
+    seen = {}
+    monkeypatch.setattr(metrics, "record_event", lambda name, **kw: seen.update(kw))
+    r = client.post("/thermograph/api/v2/event", json={"event": "home.locate"},
+                    headers={"X-Forwarded-For": "203.0.113.77"})
+    assert r.status_code == 204
+    assert seen["ip"] == "203.0.113.77"
+
+
+def test_event_route_records_end_to_end(client):
+    """Full HTTP round trip through the real ASGI route -- confirms the write
+    path backend/api/event -> metrics.record_event -> the counters store works
+    end to end (the existing suite only ever called record_event directly)."""
+    before = metrics.snapshot()["events_total"]
+    r = client.post("/thermograph/api/v2/event", json={"event": "home.locate"})
+    assert r.status_code == 204
+    after = metrics.snapshot()
+    assert after["events_total"] == before + 1
+    assert after["events"]["home.locate"]["direct"]   # no Referer sent -> "direct"

backend/web/app.py

@@ -1,9 +1,9 @@
 """Thermograph API — grade recent local weather against ~45 years of climatology."""
-import asyncio
 import contextlib
 import datetime
 import hashlib
 import hmac
+import ipaddress
 import json
 import os
 import queue
@@ -19,17 +19,16 @@ from fastapi.responses import JSONResponse, RedirectResponse
 
 from accounts import api_accounts
 from api import content_routes
+from api import internal_routes
 from core import audit
 from data import climate
 from notifications import digest
-from notifications import discord_bot
 from notifications import discord_interactions as discord_interactions_mod
 from notifications import discord_link
 from accounts import db
 from data import grid
 from core import metrics
 from notifications import notify
-from notifications import scheduler
 import paths
 from data import places
 from core import singleton
@@ -163,33 +162,19 @@ async def _lifespan(app):
     # restricts this to processes that are allowed to own it at all — "web" replicas
     # never start it even if they'd otherwise win the leader election, so a stateless
     # N-replica web tier can scale without also scaling notifier instances.
-    bot_task = None
+    # The Discord gateway bot and the recurring worker jobs (city warming,
+    # IndexNow) used to ride this same leader gate in-process; both now live in
+    # the thermograph-daemon Go binary, which owns the connection/timers and
+    # calls back over api/internal_routes.py for anything needing data. The
+    # notifier is the one singleton still running here.
     if _should_run_notifier():
         notify.start()
-        # Same leader gate as the notifier above — the recurring worker jobs
-        # (city warming, IndexNow) must likewise run in exactly one process.
-        scheduler.start()
-        # Discord gateway bot (opt-in). Discord allows one gateway connection per
-        # bot token, so it belongs on the same single leader; it rides this event
-        # loop as a background task rather than a new process. No-op unless
-        # THERMOGRAPH_DISCORD_BOT is set with a bot token (discord_bot.enabled()).
-        if discord_bot.enabled():
-            bot_task = asyncio.create_task(discord_bot.run(), name="discord-bot")
     # Deploy/restart marker: one line correlates a metric shift with a boot.
     audit.log_activity("app.start", {"version": app.version, "role": ROLE, "base": BASE,
                                      "notifier": _should_run_notifier(), "pid": os.getpid()})
     yield
     audit.log_activity("app.stop", {"role": ROLE, "pid": os.getpid()})
-    if bot_task is not None:
-        bot_task.cancel()
-        # Wait for the cancellation to actually land before tearing anything else
-        # down -- otherwise shutdown races the bot coroutine's own teardown (closing
-        # the websocket, cancelling its heartbeat task) and the CancelledError it
-        # raises on its way out surfaces as unhandled-task noise in the logs.
-        with contextlib.suppress(asyncio.CancelledError):
-            await bot_task
     notify.stop()
-    scheduler.stop()
     await _frontend_client.aclose()
 
 
@@ -279,13 +264,34 @@ def api_version():
 
 def _client_ip(request) -> "str | None":
     """The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us
-    (Caddy on prod sets it), otherwise the direct peer address (LAN dev)."""
+    (Caddy on prod sets it), otherwise the direct peer address (LAN dev). Full
+    precision — the rate limiter (metrics._rate_ok) needs the exact address, since
+    a truncated key would let one abuser exhaust a whole NAT'd office's quota.
+    Truncate at the point of persistence instead (see _loggable_ip)."""
     xff = request.headers.get("x-forwarded-for")
     if xff:
         return xff.split(",")[0].strip()
     return request.client.host if request.client else None
 
 
+def _loggable_ip(ip: "str | None") -> "str | None":
+    """Coarsen a client IP before it's written to the access log: /24 for IPv4,
+    /48 for IPv6. Keeps rough geo/abuse signal without keeping a full, joinable
+    address sitting in structured logs for the log's 30-day retention -- the
+    public privacy page promises IPs aren't logged beyond normal request handling,
+    and a raw address shipped to Loki was a live gap against that. Never the
+    input to the rate limiter (_client_ip's callers pass the untruncated value
+    there) -- this is only what gets persisted."""
+    if not ip:
+        return ip
+    try:
+        addr = ipaddress.ip_address(ip)
+    except ValueError:
+        return ip  # not a parseable address (e.g. a test/placeholder value) -- pass through
+    prefix = 24 if addr.version == 4 else 48
+    return str(ipaddress.ip_network(f"{addr}/{prefix}", strict=False).network_address)
+
+
 @app.middleware("http")
 async def revalidate_static(request, call_next):
     """Serve the frontend with no-cache (NOT no-store): browsers may keep a copy
@@ -304,11 +310,15 @@ async def revalidate_static(request, call_next):
         # loop so one request's instrumentation can't stall every other request
         # this worker is serving concurrently.
         await run_in_threadpool(metrics.record_inbound, cat, response.status_code)
-        # Retain per-request client IPs for later analysis; skip static assets and the
+        # Retain per-request client IPs for later analysis; skip static assets, the
-        # dashboard's own metrics polling to keep the log to real, meaningful traffic.
+        # dashboard's own metrics polling, the liveness probe (health -- by far the
-        if cat not in ("static", "metrics", "event"):
+        # highest-volume single path, and never real traffic), and the internal
+        # SSR->API hop (internal -- frontend_ssr's own server-to-server calls, not
+        # a page view) to keep the log to real, meaningful traffic. The IP itself is
+        # truncated (see _loggable_ip) -- coarse geo/abuse signal, not a full address.
+        if cat not in ("static", "metrics", "event", "health", "internal"):
             await run_in_threadpool(audit.log_access, {
-                "ip": _client_ip(request), "method": request.method,
+                "ip": _loggable_ip(_client_ip(request)), "method": request.method,
                 "path": path, "status": response.status_code, "cat": cat})
         # Threshold-gated slow-request line: RunAudit only times the 7 graded
         # endpoints, so this is the only latency signal for auth/account/content
@@ -838,6 +848,13 @@ app.include_router(v2, prefix=f"{BASE}/api/v2")
 # service (Stage 3) instead of anything in this process. See
 # backend/api/content_routes.py.
 app.include_router(content_routes.router, prefix=f"{BASE}/api/v2")
+# Internal control surface for the thermograph-daemon Go binary (the gateway
+# bot + recurring jobs that used to run in _lifespan). Deliberately NOT under
+# BASE — the daemon hits THERMOGRAPH_API_BASE_INTERNAL directly, same posture
+# as /healthz — and never publicly routed (the Caddyfile only forwards
+# /api/*, /digest and /discord/interactions here). Registered before the
+# catch-all frontend proxy below so /internal/* can't fall through to it.
+app.include_router(internal_routes.router)
 
 
 # --- proxy to the frontend SSR service (repo-split Stage 4) ------------------

frontend/scripts/capture-fixtures.sh

@@ -18,7 +18,7 @@ trap 'scripts/backend-for-tests.sh down >/dev/null 2>&1 || true' EXIT
 
 fetch() {  # <endpoint-path> <fixture-name>
   echo "==> $2.json  <-  $1"
-  curl -fsS "$BASE/$1" --max-time 40 | { command -v jq >/dev/null && jq . || cat; } > "$OUT/$2.json"
+  curl -fsS "$BASE/$1" --max-time 40 | { if command -v jq >/dev/null; then jq .; else cat; fi; } > "$OUT/$2.json"
 }
 
 fetch "api/v2/content/home"                          home
@@ -28,4 +28,4 @@ fetch "api/v2/content/city/$SLUG"                    city
 fetch "api/v2/content/city/$SLUG/month/$MONTH"       month
 fetch "api/v2/content/city/$SLUG/records"            records
 
-echo "==> captured $(ls "$OUT"/*.json | wc -l) fixtures for $SLUG into $OUT/"
+echo "==> captured $(find "$OUT" -maxdepth 1 -name '*.json' | wc -l) fixtures for $SLUG into $OUT/"

infra/.env.example

@@ -9,6 +9,21 @@
 # build the app's THERMOGRAPH_DATABASE_URL. Change it before first `up`.
 POSTGRES_PASSWORD=change-me
 
+# OPTIONAL. Shared secret between the daemon service (Discord gateway + job
+# timers) and the backend's /internal/* routes.
+#
+# Leave it empty and both ends DERIVE the same token from THERMOGRAPH_AUTH_SECRET
+# (HMAC-SHA256 under a fixed label), which every environment already provisions —
+# so the daemon needs no new vault entry and no operator step. Set it only to
+# override that, e.g. to rotate this surface independently of the auth secret:
+#   openssl rand -hex 32
+#
+# With neither this nor THERMOGRAPH_AUTH_SECRET set, both ends fail closed: the
+# backend disables the internal routes and the daemon refuses to start. Never
+# routed publicly — Caddy only forwards /api/*, /digest and
+# /discord/interactions to the backend.
+THERMOGRAPH_INTERNAL_TOKEN=
+
 # --- Everything below is optional -- docker-compose.yml already defaults each
 # --- of these, so a plain `make up` works with none of it set. Uncomment to
 # --- override.

infra/.gitignore

@@ -15,6 +15,10 @@ age.key
 
 .DS_Store
 
+# lake-iceberg's pytest suite runs in place.
+__pycache__/
+*.pyc
+
 # Host-side deploy state (deploy.sh): the live per-service image tags and the
 # cross-repo deploy lock. Untracked on purpose -- they must survive the
 # `git reset --hard` at the top of every deploy (deploy.sh's comments already

infra/deploy/Caddyfile

@@ -31,10 +31,12 @@ thermograph.org {
 	# Active health check on the same cheap /healthz route each container's own
 	# HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting
 	# never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10).
+	# 15s (was 5s): plenty responsive for a process that only restarts on a deploy,
+	# and a quarter of the polling load.
 	handle @backend_paths {
 		reverse_proxy 127.0.0.1:8137 {
 			health_uri /healthz
-			health_interval 5s
+			health_interval 15s
 			health_timeout 3s
 			health_status 2xx
 		}
@@ -43,14 +45,35 @@ thermograph.org {
 	handle {
 		reverse_proxy 127.0.0.1:8080 {
 			health_uri /healthz
-			health_interval 5s
+			health_interval 15s
 			health_timeout 3s
 			health_status 2xx
 		}
 	}
 
+	# Access-log hygiene: the default JSON encoder serializes full request headers,
+	# the TLS block, and response headers on every line (measured ~1,133B/line) --
+	# strip those with the `filter` format encoder. Also strip the query string from
+	# the logged URI: Caddy's default logger records request.uri *including* the
+	# query string, so every `?q=<search text>` a visitor typed sat in Loki next to
+	# their client IP for the full 30-day retention -- a real privacy leak, not just
+	# noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7
+	# and an upgrade is out of scope, so that's handled downstream in Alloy's
+	# loki.process "caddy" stage instead (see observability/alloy/config.alloy).
 	log {
-		output file /var/log/caddy/thermograph.log
+		output file /var/log/caddy/thermograph.log {
+			roll_size 20MiB
+			roll_keep 5
+		}
+		format filter {
+			wrap json
+			fields {
+				request>headers delete
+				request>tls delete
+				resp_headers delete
+				request>uri regexp \?.* ""
+			}
+		}
 	}
 }
 

infra/deploy/deploy.sh

@@ -70,11 +70,16 @@ fi
 # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the
 # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/.
 if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then
-  # shellcheck source=deploy/render-secrets.sh
+  # shellcheck source=infra/deploy/render-secrets.sh
   . "$INFRA_DIR/deploy/render-secrets.sh"
   render_thermograph_secrets "$INFRA_DIR"
 fi
-set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
+# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it
+# cannot exist at lint time, so don't ask shellcheck to follow it.
+set -a
+# shellcheck source=/dev/null
+. /etc/thermograph.env 2>/dev/null || true
+set +a
 
 # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a
 # search-engine crawl never bursts the archive API quota. Detached inside the
@@ -156,7 +161,10 @@ TAGS_FILE="$INFRA_DIR/deploy/.image-tags.env"
 _incoming_backend="${BACKEND_IMAGE_TAG:-}"
 _incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
 if [ -f "$TAGS_FILE" ]; then
-  set -a; . "$TAGS_FILE"; set +a
+  set -a
+  # shellcheck source=/dev/null # untracked runtime artifact this script writes below
+  . "$TAGS_FILE"
+  set +a
 fi
 [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
 [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
@@ -178,10 +186,20 @@ export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}"
 export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}"
 
 # Which compose services this run pulls/rolls.
+#
+# `daemon` and `lake` ride with `backend` and are never targets on their own.
+# Both run the SAME image at the SAME tag (a second binary and a second role
+# inside the backend image): the daemon talks to the web process over the
+# /internal/* contract, the lake serves it history — rolling one without the
+# other is exactly the version skew those contracts have no negotiation for.
+# Pairing them here is also what makes the services exist at all: a
+# single-service deploy runs `up -d --no-deps <targets>`, so a service listed
+# only in compose but absent from TARGETS would never be created on a
+# backend-only deploy.
 case "$SERVICE" in
-  backend) TARGETS=(backend) ;;
+  backend) TARGETS=(backend lake daemon) ;;
   frontend) TARGETS=(frontend) ;;
-  all) TARGETS=(backend frontend) ;;
+  all) TARGETS=(backend lake frontend daemon) ;;
 esac
 
 # Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml,
@@ -218,6 +236,30 @@ if [ "$pull_ok" != 1 ]; then
   exit 1
 fi
 
+# The daemon binary ships INSIDE the backend image, but infra and app images
+# advance on different axes by design: this checkout tracks `main`, while image
+# tags are env-staged (prod's come from `release`). So a host can legitimately
+# be asked to roll a backend image OLDER than this compose file -- one built
+# before the daemon existed and with no /usr/local/bin/thermograph-daemon in it.
+# Creating the service anyway would leave a container crash-looping on a missing
+# binary, on a deploy that otherwise succeeded. Probe the image we are actually
+# about to roll and drop the daemon from this run if it can't support it; the
+# next deploy of a tag that has it picks it up with no further action.
+case " ${TARGETS[*]} " in
+  *" daemon "*)
+    daemon_img="$(docker compose config --images daemon 2>/dev/null | head -1 || true)"
+    if [ -n "$daemon_img" ] \
+       && ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then
+      echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run"
+      kept=()
+      for t in "${TARGETS[@]}"; do
+        [ "$t" = daemon ] || kept+=("$t")
+      done
+      TARGETS=("${kept[@]}")
+    fi
+    ;;
+esac
+
 # Roll only the target service(s). Backend schema migrations run inside its own
 # entrypoint (alembic upgrade head) before uvicorn, so there's no separate
 # migrate step; frontend is stateless.

infra/deploy/render-secrets.sh

@@ -48,15 +48,26 @@ render_thermograph_secrets() {
     key_env=("SOPS_AGE_KEY=$keymat")
   fi
   local tmp; tmp=$(mktemp)
-  : > "$tmp"
+  # The tmp file holds DECRYPTED secrets, so every exit path below removes it.
-  # set -e in the caller makes a decrypt failure fatal here (no partial env). common
+  #
-  # first, host second, so a host value overrides a shared one (last-wins).
+  # Deliberately NOT a `trap ... RETURN`, which looks like the tidier way to make
+  # that unconditional: a RETURN trap set inside a SOURCED function persists in
+  # the CALLER's shell after the function returns, and a RETURN trap also fires
+  # when a `.`/source completes. deploy.sh sources /etc/thermograph.env a few
+  # lines after calling us, which would re-fire the trap at top level where `tmp`
+  # (function-local) is unset -- fatal under deploy.sh's `set -u`, and silent,
+  # because that line already redirects stderr to /dev/null. Explicit removal on
+  # each path is duller and correct.
+  : > "$tmp" || { rm -f "$tmp"; return 1; }
+  # Decrypt failures return 1 explicitly, so a partial env is never written and
+  # correctness doesn't depend on the caller's shell options. common first, host
+  # second, so a host value overrides a shared one (last-wins).
   if [ -f "$repo/deploy/secrets/common.yaml" ]; then
     env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \
-      "$repo/deploy/secrets/common.yaml" >> "$tmp"
+      "$repo/deploy/secrets/common.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; }
   fi
   env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \
-    "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp"
+    "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; }
 
   # Write /etc/thermograph.env. Prefer an in-place write when the existing file is
   # writable by us (e.g. a group-writable 0660 root:<deploygroup> on a box whose CI
@@ -67,14 +78,19 @@ render_thermograph_secrets() {
   # next line of deploy.sh (`. /etc/thermograph.env` as that non-root user) can't read
   # it, so POSTGRES_PASSWORD never enters the env and `docker compose` dies on
   # interpolation. Fail loudly rather than deploy against stale secrets.
+  # rc + a single cleanup point: the plaintext tmp must go whichever branch runs,
+  # but the old trailing `rm` was also the function's last command, so it masked a
+  # failed in-place `cat` write to status 0. Capture the status instead, so a
+  # half-written /etc/thermograph.env fails loudly rather than deploying stale.
+  local rc=0
   if [ -f /etc/thermograph.env ] && [ -w /etc/thermograph.env ]; then
-    cat "$tmp" > /etc/thermograph.env
+    cat "$tmp" > /etc/thermograph.env || rc=1
   elif install -m 0640 "$tmp" /etc/thermograph.env 2>/dev/null; then :
   elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" /etc/thermograph.env 2>/dev/null; then :
   else
-    rm -f "$tmp"
     echo "!! cannot write /etc/thermograph.env (need file write access or passwordless sudo)" >&2
-    return 1
+    rc=1
   fi
   rm -f "$tmp"
+  return "$rc"
 }

infra/deploy/secrets/README.md

@@ -15,7 +15,7 @@ no per-host duplication.
 | File | Holds | Encrypted? |
 |------|-------|------------|
 | `common.yaml` | Secrets shared by every host — Discord tokens/secret, VAPID keypair, `THERMOGRAPH_AUTH_SECRET`, metrics token, indexnow key, SMTP creds | ✅ |
-| `prod.yaml` | Prod-only overrides — `POSTGRES_PASSWORD`, `REGISTRY_TOKEN` | ✅ |
+| `prod.yaml` | Prod-only overrides — `POSTGRES_PASSWORD`, `REGISTRY_TOKEN`, ERA5 lake bucket creds (`THERMOGRAPH_LAKE_S3_ACCESS_KEY` / `THERMOGRAPH_LAKE_S3_SECRET_KEY`, consumed by the `lake` Swarm service and `gen_era5_lake.py`) | ✅ |
 | `beta.yaml` | Beta-only overrides | ✅ |
 | `example.yaml` | Format reference / CI fixture (fake values) | ✅ |
 | `../../.sops.yaml` | Which age recipient files are encrypted to (plaintext config) | — |

infra/deploy/secrets/beta.yaml

@@ -16,6 +16,8 @@ THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:sTtkAbMVUtwvRssCZJa2PWUjlfnYS
 THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:SnFw+pwNgZlxMEhMUH/1daMZlRB2QFNJHtL+4C3S1eFcJJdT2uNGkY7Ym9FPfx1V7cwt/3urHg1oVgdDzYaIal2wye4l5HhKBP5PjtQkDW+Pck24kxul,iv:Ku4MhsrAfyA2Y3CqhGIsYKrjhKQ4cqAYjCqjFeEvBWE=,tag:hw42iS+79D7gNf+BPTYM0g==,type:str]
 THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:ox+hHiFMTPZqYwICDSwPwcT13ZDSKiGcAVHfVVMNYQ==,iv:DssV/olPXERpdmXNMmzjVYvhn/0zmuSp2XlTs42pIi0=,tag:4YHXI1BFbGZ6JsdXqJuf3w==,type:str]
 REGISTRY_TOKEN: ENC[AES256_GCM,data:pC1Zzs56la8qjzWcbkuKyeVHwS5WfIe+mrLm5Sdi/LKYziAvRU5cOA==,iv:57PlJEBdgwNJsfFdNOGqAnKk8l6zpmMSWYFEKS6pn+4=,tag:pPftNHUVU6+CY2nWMaXO6w==,type:str]
+THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:ZEQ278P7azMKlYMjUraP6hIqnjymqL90zEbvTNC9iXU=,iv:5nXXZwx7RfqwCMqx5LWtxQy2nQcq+6Yr79ulBi6BOnE=,tag:jXcCCCcz4NsDoIZwa/ji4A==,type:str]
+THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:p5qCrHWoH1i3smDZZBIO8zMSWWxDARYsTY12AA0E8ao=,iv:PVOEQ0kBFeYaE/FT2UswtSTY7S84U/YOPSbcsXR51f4=,tag:JFwaOwAYInPs199li66U/g==,type:str]
 sops:
     age:
         - enc: |
@@ -27,7 +29,7 @@ sops:
             0+PQdtDxom72BqrAoj5lhepxW8YWPqQIRXsVS/XYQEQfPJuuuiRMKw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-22T03:17:28Z"
+    lastmodified: "2026-07-23T22:30:10Z"
-    mac: ENC[AES256_GCM,data:npF+ngF4Khj3NJ2PNFcQjQYIrZVpdPM0drk43pAI6HfFaynX8JOqRFZRsH0sCfMoPkXB/ffCJxla1Tt3Mu3zgKTNSWcn+mjzl+dYvlruY1VylHLS+GnTO9zKeQcwbrYzTfZNWzQhm6/mw8KCL3791U/xNHwJc6g5VxRUWl6TBdo=,iv:JLzY8rm33NLG8CqyAlU6vBhlODUiRipKu9daoRvgyzg=,tag:yIsxVZAnw8FD7Bt/x55wMQ==,type:str]
+    mac: ENC[AES256_GCM,data:JWfkQk4Z1ohPha9LID/hvZOycmDy0sUyjiqDdCidfCQEsPx+OwhPatcM31x1ZKFgh3TyYTxrauoSTlIeuwfy7+R9jATDdBBZe9zMqgs6Szu9WUdFiaZzIFpG7SEVw+8IqIyAuqZVSW3k5yHmchcmkNyDekV+1RIF8wxUrOeSCVQ=,iv:6PXUiCnYBiFyNPnCRAs6f7a+RuvWI4jSIMXwuMn7WjA=,tag:y+Ix2m65hQRkSt8FylfKSg==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

infra/deploy/secrets/dry-run-render.sh

@@ -19,6 +19,10 @@ tmp="$(mktemp)"; live="$(mktemp)"; trap 'rm -f "$tmp" "$live"' EXIT
 [ -f "$common" ] && SOPS_AGE_KEY="$keymat" sops -d --input-type yaml --output-type dotenv "$common" >> "$tmp"
 SOPS_AGE_KEY="$keymat" sops -d --input-type yaml --output-type dotenv "$host" >> "$tmp"
 
+# sudo is only for the READ of the root-owned live env; the redirect target is our
+# own mktemp file, so the redirect correctly runs as the invoking user (sudo tee
+# would wrongly write it as root).
+# shellcheck disable=SC2024
 sudo cat /etc/thermograph.env > "$live"
 python3 - "$live" "$tmp" <<'PY'
 import sys

infra/deploy/secrets/prod.yaml

@@ -24,6 +24,8 @@ THERMOGRAPH_DISCORD_BOT_TOKEN: ENC[AES256_GCM,data:zfVJLH8ddct2Jvstl/olZLQ9SM15c
 REGISTRY_TOKEN: ENC[AES256_GCM,data:3VETZR026YXvWA7DLly4/mz+hqEK7vkEMwKlDKU6CQZeZiFpbQXWig==,iv:ixZJi2VWAZak6dFPvVyErVRanp9bzqXyV3MVa4QsBuU=,tag:Pc2Mdw58S0tSdkgJ5yuVNw==,type:str]
 THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Kg==,iv:CPYN8aCu79PHaz3Zj6OgqyJhXdHLt0tLodgML2NuwQM=,tag:1B3IhlfjCwSlPq7FnKGeRg==,type:str]
 THERMOGRAPH_DISCORD_WEATHER_CHANNEL: ENC[AES256_GCM,data:WIxYcoOnBDdDcNj8uJ5eUqXjVQ==,iv:l6kU5PGVcW2Nv7EqBUGCRxBzaJOJ3AW1nEEH33aAbmI=,tag:/KgdCKmrlQioIWdz21PcTg==,type:str]
+THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:goKTqruUdKUpSSrhu1Ivbh+yS7/DhGdCBeW/xb7AIkI=,iv:TmuHDgUi7dJZYv6uuUgqpnl5kG0tFNLgn11yizhHAV8=,tag:kkH+qBWA5rJwySIILk1cPg==,type:str]
+THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:7q07fpPgjgJN1NivbhAHn5JRyhI/+f0nV7Oll9bxW3w=,iv:hKZNH4PLmyyhA/crJpy1RA6peL9HzosyiNi+Y4yMKwo=,tag:sb5ailD6K44d4m8TmzZD9g==,type:str]
 sops:
     age:
         - enc: |
@@ -35,7 +37,7 @@ sops:
             PDXCB8EqKwdC3BnKlpD9ptYcuPHVUSgve1wIZYJ7iHIEhW9Kc6RXOw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2
-    lastmodified: "2026-07-23T04:33:06Z"
+    lastmodified: "2026-07-23T21:26:01Z"
-    mac: ENC[AES256_GCM,data:BjNMEudsS9ZViR55OjSXu5qrOJ9yu2kCGx61qu4vE1ZZJIZtnnyZ3CfpsPygvLLhiFRj76xcqFnuSqTdr17BCEHz3UzsHFv0vBi17lZ5ksN5Oz+9wr+3Pg3TvxC0VF2VHJVP84VH61CmB5+G4WOms8WqLtmIGSLYcTVtcIFoUhk=,iv:jTEB8Mxg8qIyNfhpSgJtUvULVx4+XFrPLyQKhNcbkTg=,tag:SIyvNltdGCMfDGvi4kD3MA==,type:str]
+    mac: ENC[AES256_GCM,data:RHLp+0waxaKQXfAzipchk/vZnE3NtMOHAJwqddTlJYpxl9Sl47IqZcdV05j8kveySNoedZbpmu6+unso9HPaaJ9lmunZVmOPc9wqn+5JJNw8kkPIosrXL8UzKW9gO4OIbTNkEC4xSQP4qyQeUO/RaDP2aH0d+GCa7sKTePrd6Gg=,iv:9027dfABH2VuEJKWzY1+eTns+IdCUXW2KQTAdlj4iP4=,tag:JLLkj0g4gLMseLdUy896/g==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

infra/deploy/stack/autoscale.sh

@@ -18,10 +18,16 @@
 #   - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence
 #     (--detach=false), so a stuck rollout blocks further changes rather than
 #     stacking them.
-set -eu
+# pipefail: without it a `docker stats` failure on the left of the avg_cpu
+# pipe is silently swallowed by awk's exit 0 and the loop trusts the output.
+# (Runs under busybox ash in docker:*-cli, which supports pipefail.)
+# shellcheck disable=SC3040 # pipefail is not POSIX, but the busybox ash this runs under has it
+set -euo pipefail
 
 STACK_NAME="${STACK_NAME:-thermograph}"
-SERVICE="${STACK_NAME}_web"
+# Which stack service to scale (the stack runs one autoscaler instance per
+# scaled service — web and lake today; same loop, different bounds).
+SERVICE="${STACK_NAME}_${TARGET_SERVICE:-web}"
 MIN="${MIN_REPLICAS:-1}"
 MAX="${MAX_REPLICAS:-3}"
 UP_AT="${SCALE_UP_CPU:-220}"
@@ -64,7 +70,10 @@ while :; do
 
   cur=$(replicas)
   [ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; }
-  cpu=$(avg_cpu)
+  # With pipefail, a transient `docker stats` failure makes avg_cpu return
+  # non-zero; treat that as a missed sample (don't trust partial output, and
+  # don't let set -e kill the autoscaler over a daemon hiccup).
+  cpu=$(avg_cpu) || cpu=""
   [ -n "$cpu" ] || continue   # no running tasks visible this sample
 
   if [ "$cpu" -gt "$UP_AT" ]; then

infra/deploy/stack/deploy-stack.sh

@@ -49,11 +49,16 @@ export STACK_NAME
 # install the uid-10001-readable copy the tasks' env-entrypoint shim sources.
 # 10001 = the app images' `thermograph` user; the file is 0400 to that uid.
 if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then
-  # shellcheck source=deploy/render-secrets.sh
+  # shellcheck source=infra/deploy/render-secrets.sh
   . "$APP_DIR/infra/deploy/render-secrets.sh"
   render_thermograph_secrets "$APP_DIR/infra"
 fi
-set -a; . /etc/thermograph.env 2>/dev/null || true; set +a
+# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it
+# cannot exist at lint time, so don't ask shellcheck to follow it.
+set -a
+# shellcheck source=/dev/null
+. /etc/thermograph.env 2>/dev/null || true
+set +a
 sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \
   || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env
 
@@ -65,6 +70,7 @@ export REGISTRY_HOST
 TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env"
 _incoming_backend="${BACKEND_IMAGE_TAG:-}"
 _incoming_frontend="${FRONTEND_IMAGE_TAG:-}"
+# shellcheck source=/dev/null # untracked runtime artifact this script writes below
 if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi
 [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend"
 [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend"
@@ -157,9 +163,22 @@ if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/
 else
   case "$SERVICE" in
     backend)
-      echo "==> Rolling web + worker to $BACKEND_IMAGE"
+      echo "==> Rolling web + worker + lake to $BACKEND_IMAGE"
       docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web"
       docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker"
+      # lake and daemon ship in the same image; a stack file predating either
+      # has no service yet — the next SERVICE=all stack deploy creates it, so
+      # don't fail here. The daemon especially must roll with web: they share
+      # the /internal/* contract, and a version skew between them is exactly
+      # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the
+      # first post-creation backend roll left the daemon a release behind).
+      for extra in lake daemon; do
+        if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then
+          docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}"
+        else
+          echo "   (no ${STACK_NAME}_${extra} service yet; created on the next full stack deploy)"
+        fi
+      done
       ;;
     frontend)
       echo "==> Rolling frontend to $FRONTEND_IMAGE"

infra/deploy/stack/thermograph-stack.yml

@@ -82,6 +82,10 @@ services:
       # not the compose bridge's 172.19.0.1 (an overlay has no host gateway).
       # provision-mail.sh's DOCKER_MAIL_GATEWAY/SUBNET cover this listener.
       THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
+      # History reads ask the lake service (Swarm VIP over 1..2 replicas)
+      # before any third-party source; unset on beta/LAN, where compose has no
+      # lake service and climate.py falls straight through to NASA.
+      THERMOGRAPH_LAKE_URL: http://lake:8141
     volumes:
       - appdata:/state
       - applogs:/app/logs
@@ -121,6 +125,7 @@ services:
       THERMOGRAPH_SINGLETON_PG: "1"
       RUN_MIGRATIONS: "0"
       THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1}
+      THERMOGRAPH_LAKE_URL: http://lake:8141
     volumes:
       - appdata:/state
       - applogs:/app/logs
@@ -138,6 +143,105 @@ services:
       restart_policy:
         condition: on-failure
 
+  # The ERA5 lake indexer: SQL-style, partition-pruned reads over the
+  # era5-thermograph bucket with a node-local parquet cache, so a cell's
+  # history is one LAN hop for web instead of an S3 fetch (or a NASA call).
+  # Same backend image; THERMOGRAPH_ROLE=lake makes the entrypoint start
+  # lake_app (no database, no migrations). Its S3 credentials
+  # (THERMOGRAPH_LAKE_S3_ACCESS_KEY/_SECRET_KEY) arrive via the same rendered
+  # /host/thermograph.env as every other secret. Without them it stays
+  # healthy and answers 503/404, and web falls through to NASA — the lake is
+  # an accelerator, never a point of failure.
+  lake:
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
+    entrypoint: ["/host/env-entrypoint.sh"]
+    environment:
+      THERMOGRAPH_ROLE: lake
+      PORT: 8141
+      THERMOGRAPH_SERVICE_ROLE: backend
+      WORKERS: "1"
+      THERMOGRAPH_LAKE_CACHE: /state/lake-cache
+    volumes:
+      # Replicas share the cache; lake_app's tmp+rename writes make that safe.
+      - lakecache:/state
+      - /opt/thermograph/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro
+      - /etc/thermograph/stack.env:/host/thermograph.env:ro
+    networks:
+      - internal
+    deploy:
+      replicas: ${LAKE_MIN_REPLICAS:-1}
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "${LAKE_CPUS:-2}"
+      restart_policy:
+        condition: on-failure
+      update_config:
+        order: start-first
+        failure_action: rollback
+
+  daemon:
+    # SAME image and tag as web/worker on purpose: the daemon binary
+    # (/usr/local/bin/thermograph-daemon, built into the backend image) and the
+    # app share the /internal/* API contract, so one BACKEND_IMAGE_TAG rolls
+    # them together and they can never skew.
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required}
+    # Not env-entrypoint.sh: that shim's handoff execs the image's
+    # deploy/entrypoint.sh (Alembic + uvicorn) whenever it exists, which is
+    # exactly what this service must never run — migrations belong to the
+    # one-shot task, and a second migrator racing it is a real hazard. Instead
+    # source the same host-rendered env directly (it is shell-sourceable by
+    # contract — deploy.sh sources it too) and exec the binary. stack.env holds
+    # vault secrets, never service-topology URLs, so it cannot collide with the
+    # `environment:` block below.
+    entrypoint:
+      - /bin/bash
+      - -c
+      - 'set -a; [ -f /host/thermograph.env ] && . /host/thermograph.env; set +a; exec /usr/local/bin/thermograph-daemon'
+    environment:
+      # The service VIP: the daemon's /internal/* calls load-balance across web
+      # replicas like any other client of the app.
+      THERMOGRAPH_API_BASE_INTERNAL: http://web:8137
+    # Only the env mount — no appdata/applogs: warm-cities/IndexNow work
+    # executes inside the web process (the daemon just fires the internal
+    # endpoints on a timer), so the daemon holds no state and logs to stdout.
+    volumes:
+      - /etc/thermograph/stack.env:/host/thermograph.env:ro
+    # The image HEALTHCHECK curls /healthz on ${PORT} — right for the app,
+    # meaningless for the daemon (it serves nothing). Without this override
+    # Swarm marks the task unhealthy and restarts it forever (seen live on the
+    # first prod stack deploy: gateway up, killed ~100s later, 0/1 loop). The
+    # compose file already disables it; the stack file must too.
+    healthcheck:
+      disable: true
+    networks:
+      - internal
+    deploy:
+      # EXACTLY 1, load-bearing — do not scale this service, ever. Discord
+      # permits ONE gateway connection per bot token; the Python bot enforced
+      # that with a leader election (core/singleton.claim_leader), and this
+      # pinned replica count is what REPLACES that election. A second replica
+      # means two gateway sessions fighting over the token (Discord kicks
+      # both into a reconnect storm) and every timer job firing twice. The
+      # autoscaler cannot touch this — it scales ${STACK_NAME}_web only.
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "0.5"
+          memory: 128m
+      restart_policy:
+        condition: on-failure
+      update_config:
+        # stop-first (the default), NOT start-first: a start-first roll would
+        # briefly run two daemons — two gateway sessions on one token, the
+        # exact thing replicas:1 exists to prevent. A few seconds of gateway
+        # downtime per deploy is the correct trade.
+        order: stop-first
+        failure_action: rollback
+
   frontend:
     image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required}
     entrypoint: ["/host/env-entrypoint.sh"]
@@ -200,6 +304,39 @@ services:
       restart_policy:
         condition: any
 
+  # Second autoscaler instance, watching `lake` (1..2 replicas). The lake is
+  # read-only and cache-heavy, so it saturates on decode CPU well below web's
+  # thresholds — scale up sooner, still down slowly.
+  autoscaler-lake:
+    image: docker:27-cli
+    entrypoint: ["/bin/sh", "/host/autoscale.sh"]
+    environment:
+      STACK_NAME: ${STACK_NAME:-thermograph}
+      TARGET_SERVICE: lake
+      MIN_REPLICAS: ${LAKE_MIN_REPLICAS:-1}
+      MAX_REPLICAS: ${LAKE_MAX_REPLICAS:-2}
+      SCALE_UP_CPU: ${LAKE_SCALE_UP_CPU:-120}
+      SCALE_DOWN_CPU: ${LAKE_SCALE_DOWN_CPU:-30}
+      POLL_SECONDS: ${POLL_SECONDS:-15}
+      UP_SAMPLES: ${UP_SAMPLES:-3}
+      DOWN_SAMPLES: ${DOWN_SAMPLES:-20}
+      COOLDOWN_SECONDS: ${COOLDOWN_SECONDS:-180}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /opt/thermograph/infra/deploy/stack/autoscale.sh:/host/autoscale.sh:ro
+    networks:
+      - internal
+    deploy:
+      replicas: 1
+      placement:
+        constraints: ["node.role == manager"]
+      resources:
+        limits:
+          cpus: "0.2"
+          memory: 64m
+      restart_policy:
+        condition: any
+
 networks:
   internal:
     driver: overlay
@@ -221,3 +358,6 @@ volumes:
   applogs:
     external: true
     name: ${APPLOGS_VOLUME:-thermograph_applogs}
+  # Lake parquet cache: node-local and disposable (a cold cache just refills
+  # from the bucket), so NOT external — the stack owns its lifecycle.
+  lakecache: {}

infra/deploy/thermograph.env.example

@@ -83,12 +83,35 @@ WORKERS=4
 # Read once at process start; changing it needs a restart, not a live toggle.
 #THERMOGRAPH_ROLE=all
 
-# The worker's own recurring jobs (city warming, IndexNow), gated by the same
+# The recurring jobs (city warming, IndexNow). Hours between runs; both are
-# leader election as the notifier above. Hours between runs; both are cheap
+# cheap no-op skips when there's nothing to do, so the defaults rarely need
-# no-op skips when there's nothing to do, so the defaults rarely need changing.
+# changing. The daemon service (below) owns these timers and fires the work
+# through the backend's internal routes.
 #THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS=24
 #THERMOGRAPH_INDEXNOW_INTERVAL_HOURS=6
 
+# --- Internal daemon<->backend API ------------------------------------------------
+# Shared secret authenticating the daemon service's calls to the backend's
+# /internal/* routes (X-Thermograph-Internal-Token header). A credential:
+# OPTIONAL — leave unset unless you specifically want to override the derivation.
+#
+# When unset, the backend and the daemon each DERIVE this token from
+# THERMOGRAPH_AUTH_SECRET (HMAC-SHA256 under a fixed domain-separation label).
+# That secret is already required and already in every vault, so the daemon
+# stands up with NO new key to add, rotate, or forget on a new host. HMAC under a
+# distinct label rather than reusing the auth secret directly, so a leak of this
+# token can't be replayed as the session-signing key.
+#
+# Set it explicitly to rotate this surface independently of the auth secret
+# (e.g. openssl rand -hex 32). On a SOPS host that means `sops edit` on
+# deploy/secrets/*.yaml, never hand-editing /etc/thermograph.env — it is a
+# rendered artifact. Both ends must agree, so set it on both or neither.
+#
+# With neither this nor THERMOGRAPH_AUTH_SECRET set, both ends fail closed: the
+# backend disables the internal routes and the daemon refuses to start. Defence
+# in depth: Caddy never routes /internal/* to the backend anyway.
+#THERMOGRAPH_INTERNAL_TOKEN=
+
 # Base path the app is served under.
 #   /            -> app at the domain root (Thermograph owns the whole domain —
 #                   this is what the Caddyfile expects: thermograph.org proxies "/")
@@ -188,12 +211,13 @@ THERMOGRAPH_BASE_URL=https://thermograph.org
 # alone). Leave unset to disable. In the Thermograph.org server:
 #   weather-events = 1529274516746932307
 #THERMOGRAPH_DISCORD_WEATHER_CHANNEL=
-# Discord gateway bot (@mention/DM grading): runs INSIDE the backend leader
+# Discord gateway bot (@mention/DM grading): runs in the daemon service (the
-# process (same singleton election as the notifier — see web/app.py's lifespan),
+# Go thermograph-daemon container — no longer inside the backend leader
-# riding its event loop; no separate bot process. Opt-in: any truthy value here
+# process). Opt-in: any truthy value here starts it, provided BOT_TOKEN above
-# starts it, provided BOT_TOKEN above is also set (flag alone is a silent no-op).
+# is also set (flag alone is a silent no-op). Discord allows ONE gateway
-# Discord allows ONE gateway connection per bot token, so enable this in exactly
+# connection per bot token, so enable this in exactly one environment — prod,
-# one environment — prod, the only vault with the token.
+# the only vault with the token (the stack pins the daemon to 1 replica for
+# the same reason).
 #THERMOGRAPH_DISCORD_BOT=
 # Account linking (OAuth2 "identify"): lets a signed-in user connect their Discord
 # account, storing their Discord user id for DM alerts. CLIENT_ID is the same App
@@ -202,8 +226,9 @@ THERMOGRAPH_BASE_URL=https://thermograph.org
 #THERMOGRAPH_DISCORD_CLIENT_SECRET=
 # Gateway bot (opt-in): holds a live websocket so the bot replies to messages that
 # @mention it (or DM it) with a city grade — e.g. "@Thermograph Phoenix". Reuses
-# THERMOGRAPH_DISCORD_BOT_TOKEN above. Runs on the single notifier leader only, so
+# THERMOGRAPH_DISCORD_BOT_TOKEN above. Runs in the single-replica daemon service
-# it needs THERMOGRAPH_ENABLE_NOTIFIER on (the default). No privileged intent
+# (which also needs THERMOGRAPH_INTERNAL_TOKEN below — it grades via the
-# required — Discord delivers content for mentions/DMs. Unset/0 => no gateway
+# backend's internal API). No privileged intent required — Discord delivers
-# connection. (Code: thermograph-backend notifications/discord_bot.py.)
+# content for mentions/DMs. Unset/0 => no gateway connection.
+# (Code: backend/daemon/, calling backend's /internal/discord/grade.)
 #THERMOGRAPH_DISCORD_BOT=1

infra/docker-compose.dev.yml

@@ -37,9 +37,17 @@ services:
     ports: !reset null
     cpus: !reset null
     deploy: !reset null
+  # Same uncapped-on-dev treatment for the daemon; it has no ports to touch
+  # (outbound-only in every environment).
+  daemon:
+    cpus: !reset null
+    deploy: !reset null
   db:
     cpus: !reset null
     deploy: !reset null
+  lake:
+    cpus: !reset null
+    deploy: !reset null
     # Uncapped memory on dev too (prod ceilings it at 8g). The Postgres/DuckDB
     # memory *budget* is still the ~8 GB derived by deploy/db/init/20-tuning.sh from
     # the default DB_MEMORY (raise DB_MEMORY to give dev more); shm_size stays (it's

infra/docker-compose.yml

@@ -124,6 +124,18 @@ services:
       # (NOT /app/data) so it never shadows the Python `data/` package -- see the
       # volumes: note below and thermograph-backend paths.py.
       THERMOGRAPH_SINGLETON_LOCK: /state/notifier.lock
+      # History reads ask the lake service before any third-party source; if
+      # the lake has no bucket creds (LAN without them exported) it answers
+      # 503 and climate.py falls straight through to NASA — an accelerator,
+      # never a dependency.
+      THERMOGRAPH_LAKE_URL: http://lake:8141
+      # Shared secret for the /internal/* routes the daemon service calls.
+      # Interpolated (like POSTGRES_PASSWORD) so local runs get it from the
+      # repo-root .env; prod/beta have it in /etc/thermograph.env, which
+      # deploy.sh sources before compose so this resolves there too. Empty =>
+      # the backend disables the routes and the daemon refuses to start -- fail
+      # closed on both ends.
+      THERMOGRAPH_INTERNAL_TOKEN: ${THERMOGRAPH_INTERNAL_TOKEN:-}
     # Prod secrets live in /etc/thermograph.env: POSTGRES_PASSWORD,
     # THERMOGRAPH_AUTH_SECRET, THERMOGRAPH_VAPID_PRIVATE_KEY/_PUBLIC_KEY,
     # THERMOGRAPH_COOKIE_SECURE=1, mail/Discord keys, ... (see
@@ -157,6 +169,92 @@ services:
       - "127.0.0.1:8137:8137"
     restart: unless-stopped
 
+  # The ERA5 lake service: partition-pruned reads over the era5-thermograph
+  # bucket with a local parquet cache, same backend image with
+  # THERMOGRAPH_ROLE=lake (the entrypoint starts lake_app — no database, no
+  # migrations, so no depends_on: db). Bucket creds come from
+  # /etc/thermograph.env where the vault renders them (beta/prod) or from the
+  # deploy environment (LAN — deploy-dev.yml injects the Actions S3 secrets);
+  # without them the service stays healthy and every read falls through to
+  # NASA. Never published on a host port: only the backend talks to it.
+  lake:
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
+    environment:
+      THERMOGRAPH_ROLE: lake
+      PORT: 8141
+      THERMOGRAPH_SERVICE_ROLE: backend
+      WORKERS: "1"
+      THERMOGRAPH_LAKE_CACHE: /state/lake-cache
+      THERMOGRAPH_LAKE_S3_ACCESS_KEY: ${THERMOGRAPH_LAKE_S3_ACCESS_KEY:-}
+      THERMOGRAPH_LAKE_S3_SECRET_KEY: ${THERMOGRAPH_LAKE_S3_SECRET_KEY:-}
+    env_file:
+      - path: /etc/thermograph.env
+        required: false
+    volumes:
+      - lakecache:/state
+    cpus: ${LAKE_CPUS:-2}
+    deploy:
+      resources:
+        limits:
+          cpus: "${LAKE_CPUS:-2}"
+    restart: unless-stopped
+
+  daemon:
+    # The SAME image and tag as backend, on purpose: the daemon (Go, built into
+    # the backend image as /usr/local/bin/thermograph-daemon) and the backend
+    # share the /internal/* API contract, so they must roll together — a
+    # separate tag could skew them. It owns the Discord gateway websocket and
+    # the recurring-job timers; anything needing data calls back into backend.
+    image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local}
+    # Bypass deploy/entrypoint.sh entirely: that script runs Alembic, and the
+    # backend service's boot already owns migrations — two containers racing
+    # `alembic upgrade head` against one database is a real hazard, not a
+    # nicety. The daemon binary never touches the DB.
+    entrypoint: ["/usr/local/bin/thermograph-daemon"]
+    # Its first acts are calls to backend's /internal/* routes, so wait for a
+    # genuinely healthy backend, not just a started container.
+    depends_on:
+      backend:
+        condition: service_healthy
+    environment:
+      # Same var the frontend uses for the same purpose: the backend's URL on
+      # the compose-internal network.
+      THERMOGRAPH_API_BASE_INTERNAL: http://backend:8137
+      # See the backend service's note on this var — same interpolation, same
+      # fail-closed-when-empty behavior on both ends. Normally EMPTY: with no
+      # explicit value both ends derive the same token from
+      # THERMOGRAPH_AUTH_SECRET, which arrives via the env_file below. That is
+      # why this service must keep reading that file even though it serves
+      # nothing — without the auth secret it has nothing to derive from and
+      # refuses to start.
+      THERMOGRAPH_INTERNAL_TOKEN: ${THERMOGRAPH_INTERNAL_TOKEN:-}
+    # THERMOGRAPH_AUTH_SECRET (derivation input, see above), the bot token/flag
+    # and the job intervals (THERMOGRAPH_DISCORD_BOT[_TOKEN],
+    # THERMOGRAPH_*_INTERVAL_HOURS) all arrive via the host env file, same as
+    # the backend's secrets do — so both processes read one source of truth and
+    # cannot disagree about the derived token.
+    env_file:
+      - path: /etc/thermograph.env
+        required: false
+    # The image's HEALTHCHECK curls /healthz on ${PORT} — right for the backend
+    # process, meaningless for the daemon, which serves nothing. Without this
+    # the container would sit permanently "unhealthy".
+    healthcheck:
+      disable: true
+    # No volumes: warm-cities/IndexNow work executes inside the BACKEND process
+    # (the daemon only fires the internal endpoints on a timer), the parquet
+    # cache and locks stay on backend's appdata volume, and the daemon logs to
+    # stdout. It holds no state a volume would protect.
+    #
+    # No ports: outbound-only (Discord gateway + calls to backend). Publishing
+    # anything here would only widen the surface for no benefit.
+    cpus: 0.5
+    deploy:
+      resources:
+        limits:
+          cpus: "0.5"
+    restart: unless-stopped
+
   frontend:
     # Frontend's OWN image, published by thermograph-frontend's build-push.yml
     # from its own Dockerfile (which starts `uvicorn app:app` directly -- no
@@ -187,6 +285,7 @@ volumes:
   pgdata: {}
   appdata: {}
   applogs: {}
+  lakecache: {}
 
 networks:
   # Pin the default network's subnet + gateway so the host Postfix can rely on a

infra/lake-iceberg/Dockerfile

@@ -0,0 +1,20 @@
+# One-shot Iceberg sync for the ERA5 lake: registers new hive tiles from the
+# era5-thermograph bucket into the iceberg/era5_daily table (see
+# sync_iceberg.py). Everything ships as manylinux wheels — no toolchain.
+FROM python:3.12-slim
+
+# The pyiceberg extras pull sqlalchemy (sqlite catalog), pyarrow (FileIO +
+# parquet footers) and s3fs (bucket listing/manifest reads).
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+WORKDIR /app
+COPY sync_iceberg.py .
+
+# The sqlite catalog lives on a volume so runs share it; losing it is fine
+# (the next run re-registers the table from its latest metadata JSON).
+ENV THERMOGRAPH_ICEBERG_CATALOG_DB=/state/iceberg-catalog.db \
+    PYTHONUNBUFFERED=1
+VOLUME /state
+
+ENTRYPOINT ["python", "sync_iceberg.py"]

infra/lake-iceberg/README.md

@@ -0,0 +1,123 @@
+# lake-iceberg — Iceberg conversion for the ERA5 lake
+
+Maintains an Apache Iceberg v2 table `era5_daily` at
+`s3://era5-thermograph/iceberg/era5_daily` over the lake's existing hive
+parquet (`era5/daily/tile=<ti>_<tj>/year=<Y>/month=<M>/part.parquet`).
+
+Key properties:
+
+- **No data rewrite.** `sync_iceberg.py` uses pyiceberg `add_files`: the
+  Iceberg metadata references the existing part files in place. Storage is
+  not doubled and nothing under `era5/` is ever written, moved, or deleted —
+  the tool only writes under the `iceberg/` prefix.
+- **Incremental + idempotent.** Each run diffs the tiles in
+  `era5/manifest.parquet` (the source of truth for *complete* tiles — the
+  extractor uploads all of a tile's objects before its manifest rows appear)
+  against the `thermograph.synced-tiles` table property, and registers only
+  the new ones. Files and the property update land in the same commit, so an
+  interrupted run never half-syncs a tile; re-running is always safe, even
+  while an extraction job is writing new tiles.
+- **Self-contained catalog.** A pyiceberg SqlCatalog on local sqlite
+  (`THERMOGRAPH_ICEBERG_CATALOG_DB`, default `/state/iceberg-catalog.db`).
+  The catalog is only a pointer: if the db is lost, the next run finds the
+  newest `*.metadata.json` under the table location and re-registers it —
+  the synced-tiles set lives in the table metadata itself, so nothing is
+  re-added.
+- **Partitioning `(tile, year, month)`** via order-preserving transforms of
+  real columns: `truncate[12](lat_idx)` / `truncate[12](lon_idx)` (partition
+  values are `ti*12` / `tj*12` — the part files carry no tile column, the 12
+  matches the lake's 12x12-point tile grid) plus `year(date)` and
+  `month(date)`. Predicates on `lat_idx`/`lon_idx`/`date` prune partitions
+  directly — no synthetic tile/year/month columns needed in queries.
+
+## Build
+
+```sh
+docker build -t thermograph-lake-iceberg infra/lake-iceberg/
+```
+
+## Run (one-shot)
+
+Config is the backend's lake variables (same names and defaults as
+`backend/data/era5lake.py`): `THERMOGRAPH_LAKE_S3_ACCESS_KEY`,
+`THERMOGRAPH_LAKE_S3_SECRET_KEY`, and optionally
+`THERMOGRAPH_LAKE_S3_ENDPOINT` / `_BUCKET` / `_REGION`.
+
+```sh
+docker run --rm --env-file /path/to/lake.env \
+    -v lake-iceberg-state:/state \
+    thermograph-lake-iceberg
+```
+
+Flags: `--dry-run` (list pending tiles, write nothing), `--limit N` (cap
+tiles per run), `--batch N` (tiles per Iceberg commit, default 5),
+`--catalog-db PATH`. `PYICEBERG_MAX_WORKERS` controls the parquet-footer
+read parallelism inside `add_files` (32 is a good value for the ~1000 small
+footers per tile).
+
+Each run prints the row-count reconciliation (records added vs the
+manifest's per-tile row counts) and ends with the table's current metadata
+JSON path. It also refreshes `metadata/version-hint.text` (Hadoop-catalog
+convention) so readers can scan the bare table root without knowing that
+path. Transient S3 errors (Contabo intermittently returns SLOW_DOWN /
+connection timeouts under load) are retried per batch with backoff
+(`--retries`, default 3); a run that still dies resumes at the next batch
+boundary on re-run.
+
+## Scheduling
+
+Meant to run as a one-shot on prod after extraction batches finish — e.g. a
+cron or an ops-cron workflow step invoking the `docker run` above, with the
+secrets rendered from the SOPS vault like every other consumer of the
+`THERMOGRAPH_LAKE_S3_*` pair. Deliberately **not** wired into the Swarm
+stack or CI: the sync is an offline maintenance job, not a service, and each
+run costs one footer read per new part file (~1000 per tile), so it belongs
+after extraction batches, not on a tight loop.
+
+## Querying from DuckDB
+
+No catalog server needed — the version-hint pointer lets DuckDB scan the
+table root directly:
+
+```sql
+INSTALL iceberg; LOAD iceberg;
+INSTALL httpfs;  LOAD httpfs;
+CREATE SECRET lake (
+    TYPE s3,
+    KEY_ID '<THERMOGRAPH_LAKE_S3_ACCESS_KEY>',
+    SECRET '<THERMOGRAPH_LAKE_S3_SECRET_KEY>',
+    ENDPOINT 'eu2.contabostorage.com',
+    REGION 'default',
+    URL_STYLE 'path'
+);
+SELECT count(*)
+FROM iceberg_scan('s3://era5-thermograph/iceberg/era5_daily');
+```
+
+To pin an exact table version, scan the metadata JSON path the sync run
+printed (`iceberg_scan('s3://…/metadata/<NNNNN-uuid>.metadata.json')`).
+
+Partition pruning works on the raw columns, e.g. one tile-month:
+
+```sql
+SELECT lat_idx, lon_idx, avg(tmax)
+FROM iceberg_scan('s3://era5-thermograph/iceberg/era5_daily')
+WHERE lat_idx BETWEEN 120 AND 131 AND lon_idx BETWEEN 120 AND 131
+  AND date >= DATE '2020-07-01' AND date < DATE '2020-08-01'
+GROUP BY 1, 2;
+```
+
+The part files carry no Iceberg field ids; the table sets
+`schema.name-mapping.default` so engines resolve columns by name (validated
+with pyiceberg 0.11.1 and DuckDB 1.5.5, values byte-identical to the hive
+side).
+
+## Tests
+
+```sh
+pip install -r infra/lake-iceberg/requirements.txt pytest
+pytest infra/lake-iceberg/test_sync.py
+```
+
+Local-filesystem fixtures only (tiny parquet files, `file://` warehouse,
+sqlite catalog in tmp) — no network, no credentials.

infra/lake-iceberg/requirements.txt

@@ -0,0 +1,5 @@
+# Pinned to the versions the sync was validated with (add_files partition
+# inference from footer stats, sqlite catalog, path-style S3).
+pyiceberg[sql-sqlite,pyarrow,s3fs]==0.11.1
+pyarrow==25.0.0
+s3fs==2026.6.0

infra/lake-iceberg/sync_iceberg.py

@@ -0,0 +1,451 @@
+"""Maintain an Apache Iceberg table over the ERA5 lake's hive parquet.
+
+The lake extractor (backend/gen_era5_lake.py) writes plain hive-partitioned
+parquet to the era5-thermograph bucket:
+
+    era5/daily/tile=<ti>_<tj>/year=<Y>/month=<M>/part.parquet
+
+This tool registers those files, in place, into an Iceberg v2 table
+``era5_daily`` located at ``s3://<bucket>/iceberg/era5_daily`` via pyiceberg
+``add_files`` — no data is rewritten or copied, the Iceberg metadata simply
+points at the existing objects. Everything this tool writes lives under the
+``iceberg/`` prefix; the ``era5/`` prefixes are read-only to it by design.
+
+Partitioning: the part files carry no tile/year/month columns (those live in
+the object path), so the table partitions by order-preserving transforms of
+real columns instead — ``truncate[12](lat_idx)`` / ``truncate[12](lon_idx)``
+(a 12x12-point tile, so the partition value is ``ti*12``/``tj*12``) and
+``year(date)`` / ``month(date)``. Each part file holds exactly one tile and
+one calendar month, so every partition value is single-valued per file and
+``add_files`` can derive it from the parquet footer statistics alone.
+
+Incremental and safe under concurrent extraction: ``era5/manifest.parquet``
+is the source of truth for complete tiles (the extractor uploads all of a
+tile's objects before adding it to the manifest), so each run diffs the
+manifest's tiles against the ``thermograph.synced-tiles`` table property and
+adds only the difference. The property is updated in the same commit as the
+files, so a run can die at any point and the next one resumes cleanly.
+
+Catalog: a local pyiceberg SqlCatalog (sqlite, path from
+THERMOGRAPH_ICEBERG_CATALOG_DB, default /state/iceberg-catalog.db). The
+catalog is a convenience pointer only — if it is lost, the next run finds the
+latest ``*.metadata.json`` under the table location and re-registers it. Each
+run ends by printing the current metadata JSON path, which DuckDB can read
+directly: ``iceberg_scan('s3://.../metadata/NNNNN-....metadata.json')``.
+
+S3 config comes from the same THERMOGRAPH_LAKE_S3_* variables (and defaults)
+as backend/data/era5lake.py; THERMOGRAPH_LAKE_LOCAL_DIR switches to a local
+mirror of the lake layout (tests, rehearsal).
+"""
+import argparse
+import os
+import re
+import sys
+import time
+
+import pyarrow.parquet as pq
+from pyiceberg.catalog.sql import SqlCatalog
+from pyiceberg.exceptions import NoSuchTableError
+from pyiceberg.partitioning import PartitionField, PartitionSpec
+from pyiceberg.schema import Schema
+from pyiceberg.table.name_mapping import create_mapping_from_schema
+from pyiceberg.transforms import MonthTransform, TruncateTransform, YearTransform
+from pyiceberg.types import (
+    DateType,
+    DoubleType,
+    FloatType,
+    IntegerType,
+    NestedField,
+)
+
+PREFIX = "era5"
+MANIFEST_KEY = f"{PREFIX}/manifest.parquet"
+ICEBERG_PREFIX = "iceberg"
+
+NAMESPACE = "lake"
+TABLE_NAME = "era5_daily"
+IDENTIFIER = f"{NAMESPACE}.{TABLE_NAME}"
+SYNCED_PROP = "thermograph.synced-tiles"
+
+TILE = 12                      # grid points per tile side, as in era5lake.TILE
+DEFAULT_BATCH = 5              # tiles per Iceberg commit
+DEFAULT_RETRIES = 3            # per-batch retries on transient S3 errors
+BACKOFF_SECONDS = 15           # first retry delay, doubled per attempt
+DEFAULT_CATALOG_DB = "/state/iceberg-catalog.db"
+
+
+class LakeInvariantError(RuntimeError):
+    """The lake violated an extractor invariant; never retried."""
+
+# The part-file schema as the extractor writes it (verified against the
+# bucket): float32 for the heat-index-derived and raw met columns, float64
+# where polars promoted (fmin/sun/feels), int16 doy, int32 grid indices.
+# Iceberg has no 16-bit type, so doy maps to int; all fields optional.
+SCHEMA = Schema(
+    NestedField(1, "date", DateType(), required=False),
+    NestedField(2, "tmax", FloatType(), required=False),
+    NestedField(3, "tmin", FloatType(), required=False),
+    NestedField(4, "precip", FloatType(), required=False),
+    NestedField(5, "wind", FloatType(), required=False),
+    NestedField(6, "gust", FloatType(), required=False),
+    NestedField(7, "humid", FloatType(), required=False),
+    NestedField(8, "fmax", FloatType(), required=False),
+    NestedField(9, "fmin", DoubleType(), required=False),
+    NestedField(10, "sun", DoubleType(), required=False),
+    NestedField(11, "feels", DoubleType(), required=False),
+    NestedField(12, "doy", IntegerType(), required=False),
+    NestedField(13, "lat_idx", IntegerType(), required=False),
+    NestedField(14, "lon_idx", IntegerType(), required=False),
+)
+
+# (tile, year, month): truncate[12] of the grid indices is the tile (values
+# are ti*12 / tj*12), year/month come from the date column. All four
+# transforms preserve order, which is what lets add_files infer the partition
+# value from footer min/max stats (single value per file by construction).
+PARTITION_SPEC = PartitionSpec(
+    PartitionField(source_id=13, field_id=1000,
+                   transform=TruncateTransform(TILE), name="tile_lat"),
+    PartitionField(source_id=14, field_id=1001,
+                   transform=TruncateTransform(TILE), name="tile_lon"),
+    PartitionField(source_id=1, field_id=1002,
+                   transform=YearTransform(), name="year"),
+    PartitionField(source_id=1, field_id=1003,
+                   transform=MonthTransform(), name="month"),
+)
+
+
+def s3_config() -> "dict | None":
+    """Bucket access, or None when unconfigured. Same variable names and
+    defaults as backend/data/era5lake.py's s3_config (Contabo: path-style
+    addressing, region is a signing formality)."""
+    key = os.environ.get("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "")
+    secret = os.environ.get("THERMOGRAPH_LAKE_S3_SECRET_KEY", "")
+    if not (key and secret):
+        return None
+    return {
+        "endpoint": os.environ.get("THERMOGRAPH_LAKE_S3_ENDPOINT",
+                                   "https://eu2.contabostorage.com"),
+        "bucket": os.environ.get("THERMOGRAPH_LAKE_S3_BUCKET", "era5-thermograph"),
+        "region": os.environ.get("THERMOGRAPH_LAKE_S3_REGION", "default"),
+        "access_key": key,
+        "secret_key": secret,
+    }
+
+
+class HiveLake:
+    """Read-only view of the extracted lake (bucket or local mirror) plus the
+    iceberg/ prefix the table lives under. ``root`` is an fsspec path prefix
+    ("<bucket>" or "/abs/dir"), ``uri`` the equivalent URI prefix pyiceberg
+    sees ("s3://<bucket>" or "file:///abs/dir")."""
+
+    def __init__(self, fs, root: str, uri: str, cfg: "dict | None"):
+        self.fs = fs
+        self.root = root.rstrip("/")
+        self.uri = uri.rstrip("/")
+        self.cfg = cfg
+
+    @property
+    def warehouse(self) -> str:
+        return f"{self.uri}/{ICEBERG_PREFIX}"
+
+    def to_uri(self, path: str) -> str:
+        rel = path.removeprefix(self.root).lstrip("/")
+        return f"{self.uri}/{rel}"
+
+    def tile_rows(self) -> "dict[str, int]":
+        """tile -> row count, from the manifest (the source of truth for
+        complete tiles: the extractor uploads every object of a tile before
+        its manifest rows appear, so listing incomplete tiles is impossible
+        here even while extraction is running)."""
+        with self.fs.open(f"{self.root}/{MANIFEST_KEY}", "rb") as f:
+            m = pq.read_table(f, columns=["tile", "rows"])
+        out: dict[str, int] = {}
+        for tile, rows in zip(m.column("tile").to_pylist(),
+                              m.column("rows").to_pylist()):
+            out[tile] = out.get(tile, 0) + rows
+        return out
+
+    def tile_files(self, tile: str) -> "list[str]":
+        """URIs of one complete tile's hive part files. A flat find() on the
+        tile prefix, not a glob: fsspec expands year=*/month=* by walking
+        every directory level (~1000 LIST calls per tile on S3), while find()
+        is one paginated listing."""
+        prefix = f"{self.root}/{PREFIX}/daily/tile={tile}"
+        part = re.compile(r"/year=\d+/month=\d+/part\.parquet$")
+        try:
+            paths = self.fs.find(prefix)
+        except FileNotFoundError:
+            return []
+        return sorted(self.to_uri(p) for p in paths if part.search(p))
+
+    def latest_metadata_uri(self) -> "str | None":
+        """Newest *.metadata.json under the table location, for re-registering
+        the table into a fresh catalog db (pyiceberg names them with a
+        zero-padded, monotonically increasing version prefix)."""
+        prefix = f"{self.root}/{ICEBERG_PREFIX}/{TABLE_NAME}/metadata"
+        try:
+            paths = [p for p in self.fs.find(prefix)
+                     if p.endswith(".metadata.json")]
+        except FileNotFoundError:
+            return None
+        if not paths:
+            return None
+
+        def version(p: str) -> int:
+            m = re.match(r"(\d+)", os.path.basename(p))
+            return int(m.group(1)) if m else -1
+
+        return self.to_uri(max(paths, key=version))
+
+
+def lake_from_env() -> HiveLake:
+    local = os.environ.get("THERMOGRAPH_LAKE_LOCAL_DIR", "")
+    if local:
+        from fsspec.implementations.local import LocalFileSystem
+        root = os.path.abspath(local)
+        return HiveLake(LocalFileSystem(), root, f"file://{root}", None)
+    cfg = s3_config()
+    if cfg is None:
+        sys.exit("no lake configured: set THERMOGRAPH_LAKE_S3_ACCESS_KEY/"
+                 "_SECRET_KEY (or THERMOGRAPH_LAKE_LOCAL_DIR for a local mirror)")
+    import s3fs
+    fs = s3fs.S3FileSystem(
+        key=cfg["access_key"], secret=cfg["secret_key"],
+        endpoint_url=cfg["endpoint"],
+        client_kwargs={"region_name": cfg["region"]},
+        config_kwargs={"s3": {"addressing_style": "path"}})
+    return HiveLake(fs, cfg["bucket"], f"s3://{cfg['bucket']}", cfg)
+
+
+def open_catalog(catalog_db: str, lake: HiveLake) -> SqlCatalog:
+    parent = os.path.dirname(os.path.abspath(catalog_db))
+    os.makedirs(parent, exist_ok=True)
+    props = {"uri": f"sqlite:///{os.path.abspath(catalog_db)}",
+             "warehouse": lake.warehouse}
+    if lake.cfg:
+        props |= {
+            "py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO",
+            "s3.endpoint": lake.cfg["endpoint"],
+            "s3.region": lake.cfg["region"],
+            "s3.access-key-id": lake.cfg["access_key"],
+            "s3.secret-access-key": lake.cfg["secret_key"],
+            # Contabo serves buckets on the path, not as a subdomain.
+            "s3.force-virtual-addressing": "false",
+        }
+    return SqlCatalog("thermograph", **props)
+
+
+def ensure_table(catalog: SqlCatalog, lake: HiveLake, create: bool = True):
+    """Load the table; adopt existing S3 metadata into a fresh catalog db;
+    otherwise create it. ``create=False`` (dry runs) returns None instead of
+    creating anything."""
+    try:
+        return catalog.load_table(IDENTIFIER)
+    except NoSuchTableError:
+        pass
+    existing = lake.latest_metadata_uri()
+    if existing:
+        catalog.create_namespace_if_not_exists(NAMESPACE)
+        print(f"registering existing table metadata: {existing}")
+        return catalog.register_table(IDENTIFIER, existing)
+    if not create:
+        return None
+    catalog.create_namespace_if_not_exists(NAMESPACE)
+    return catalog.create_table(
+        IDENTIFIER,
+        schema=SCHEMA,
+        location=f"{lake.warehouse}/{TABLE_NAME}",
+        partition_spec=PARTITION_SPEC,
+        properties={
+            "format-version": "2",
+            # The part files carry no Iceberg field ids; every reader needs
+            # this name mapping to resolve columns (DuckDB included).
+            "schema.name-mapping.default":
+                create_mapping_from_schema(SCHEMA).model_dump_json(),
+        })
+
+
+def synced_tiles(table) -> "set[str]":
+    if table is None:
+        return set()
+    raw = table.properties.get(SYNCED_PROP, "")
+    return {t for t in raw.split(",") if t}
+
+
+def tile_key(tile: str) -> "tuple[int, int]":
+    ti, tj = tile.split("_")
+    return int(ti), int(tj)
+
+
+def pending_tiles(manifest_tiles, synced: "set[str]") -> "list[str]":
+    return sorted(set(manifest_tiles) - synced, key=tile_key)
+
+
+def sync_batch(table, lake: HiveLake, chunk: "list[str]",
+               retries: int = DEFAULT_RETRIES) -> "list[str]":
+    """Commit one batch of tiles (files + synced-tiles property, atomically),
+    retrying transient S3 failures with backoff. Each attempt re-reads the
+    synced set from the (refreshed) table, so an attempt that committed but
+    errored afterwards is not re-added. Returns the tiles actually added."""
+    for attempt in range(retries + 1):
+        try:
+            table.refresh()
+            left = [t for t in chunk if t not in synced_tiles(table)]
+            if not left:
+                return []
+            files = []
+            for t in left:
+                tf = lake.tile_files(t)
+                if not tf:
+                    raise LakeInvariantError(
+                        f"tile {t} is in the manifest but has no daily part "
+                        f"files — refusing to mark it synced")
+                files.extend(tf)
+            new_prop = ",".join(
+                sorted(synced_tiles(table) | set(left), key=tile_key))
+            with table.transaction() as tx:
+                tx.add_files(
+                    file_paths=files,
+                    snapshot_properties={"thermograph.tiles-added":
+                                         ",".join(left)})
+                tx.set_properties({SYNCED_PROP: new_prop})
+            return left
+        except LakeInvariantError:
+            raise
+        except Exception as e:  # noqa: BLE001 - S3 throttling has many faces
+            if attempt >= retries:
+                raise
+            wait = BACKOFF_SECONDS * 2 ** attempt
+            print(f"batch {chunk[0]}..{chunk[-1]} attempt {attempt + 1} "
+                  f"failed ({e}); retrying in {wait}s", file=sys.stderr)
+            time.sleep(wait)
+    return []  # unreachable
+
+
+def write_version_hint(lake: HiveLake, table) -> "str | None":
+    """Hadoop-catalog-style pointer next to the metadata files so engines can
+    scan the bare table root — DuckDB: iceberg_scan('s3://…/era5_daily') —
+    without knowing the exact metadata path. Content is the current metadata
+    file's basename minus '.metadata.json' (DuckDB's default
+    version_name_format tries '%s.metadata.json', which resolves it). A
+    convenience pointer only: refreshed every run, so a failed write just
+    lags one version."""
+    hint = os.path.basename(table.metadata_location) \
+        .removesuffix(".metadata.json")
+    path = f"{lake.root}/{ICEBERG_PREFIX}/{TABLE_NAME}/metadata/version-hint.text"
+    for attempt in range(3):
+        try:
+            with lake.fs.open(path, "wb") as f:
+                f.write(hint.encode())
+            return hint
+        except Exception as e:  # noqa: BLE001
+            if attempt == 2:
+                print(f"WARNING: version-hint.text write failed ({e}); "
+                      f"table-root scans will lag one version", file=sys.stderr)
+                return None
+            time.sleep(BACKOFF_SECONDS)
+    return None
+
+
+def run(lake: HiveLake, catalog_db: str, *, limit: "int | None" = None,
+        batch: int = DEFAULT_BATCH, retries: int = DEFAULT_RETRIES,
+        dry_run: bool = False) -> dict:
+    tile_rows = lake.tile_rows()
+    catalog = open_catalog(catalog_db, lake)
+    table = ensure_table(catalog, lake, create=not dry_run)
+    done = synced_tiles(table)
+    todo = pending_tiles(tile_rows, done)
+    if limit:
+        todo = todo[:limit]
+    print(f"manifest tiles={len(tile_rows)} synced={len(done)} "
+          f"pending this run={len(todo)}")
+
+    added_files = added_records = 0
+    mismatches = []
+    if dry_run:
+        for t in todo:
+            files = lake.tile_files(t)
+            print(f"[dry-run] tile {t}: {len(files)} part files, "
+                  f"{tile_rows[t]} manifest rows")
+    else:
+        for k in range(0, len(todo), batch):
+            chunk = todo[k:k + batch]
+            # One commit per batch: the files and the synced-tiles marker land
+            # atomically, so an interrupted run never half-syncs a tile.
+            added = sync_batch(table, lake, chunk, retries)
+            done |= set(chunk)
+            if not added:
+                print(f"[{min(k + batch, len(todo))}/{len(todo)}] tiles "
+                      f"{chunk[0]}..{chunk[-1]}: already synced")
+                continue
+            summary = table.current_snapshot().summary
+            got = int(summary.get("added-records", "0"))
+            expected = sum(tile_rows[t] for t in added)
+            n_files = int(summary.get("added-data-files", "0"))
+            added_files += n_files
+            added_records += got
+            status = "ok" if got == expected else "MISMATCH"
+            if got != expected:
+                mismatches.append((added, expected, got))
+            print(f"[{min(k + batch, len(todo))}/{len(todo)}] tiles "
+                  f"{added[0]}..{added[-1]}: +{n_files} files, "
+                  f"+{got} records (manifest says {expected}: {status})")
+
+    result = {"pending": todo, "added_files": added_files,
+              "added_records": added_records, "mismatches": mismatches,
+              "metadata_location": None, "total_records": 0,
+              "total_files": 0}
+    if table is not None:
+        snap = table.current_snapshot()
+        if snap is not None:
+            result["total_records"] = int(snap.summary.get("total-records", "0"))
+            result["total_files"] = int(snap.summary.get("total-data-files", "0"))
+        result["metadata_location"] = table.metadata_location
+        if not dry_run:
+            result["version_hint"] = write_version_hint(lake, table)
+        expected_total = sum(tile_rows[t] for t in done if t in tile_rows)
+        print(f"table: {result['total_files']} data files, "
+              f"{result['total_records']} records "
+              f"(manifest rows for synced tiles: {expected_total})")
+        print(f"metadata: {table.metadata_location}")
+        if lake.cfg:
+            print("read it from DuckDB (no catalog needed):\n"
+                  "  LOAD iceberg; LOAD httpfs;  -- plus an S3 secret for the "
+                  "endpoint\n"
+                  f"  SELECT count(*) FROM iceberg_scan("
+                  f"'{lake.warehouse}/{TABLE_NAME}');\n"
+                  f"  -- or pin the exact version: iceberg_scan("
+                  f"'{table.metadata_location}')")
+    if mismatches:
+        print(f"WARNING: {len(mismatches)} batch(es) added a record count "
+              f"that differs from the manifest — investigate before trusting "
+              f"the table", file=sys.stderr)
+    return result
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(
+        description="Register new ERA5 lake tiles into the Iceberg table")
+    ap.add_argument("--catalog-db",
+                    default=os.environ.get("THERMOGRAPH_ICEBERG_CATALOG_DB",
+                                           DEFAULT_CATALOG_DB),
+                    help="sqlite catalog path (default: %(default)s)")
+    ap.add_argument("--batch", type=int, default=DEFAULT_BATCH,
+                    help="tiles per Iceberg commit (default: %(default)s)")
+    ap.add_argument("--limit", type=int,
+                    help="sync at most this many new tiles this run")
+    ap.add_argument("--retries", type=int, default=DEFAULT_RETRIES,
+                    help="per-batch retries on transient S3 errors "
+                         "(default: %(default)s)")
+    ap.add_argument("--dry-run", action="store_true",
+                    help="list pending tiles; write nothing")
+    args = ap.parse_args()
+
+    lake = lake_from_env()
+    result = run(lake, args.catalog_db, limit=args.limit, batch=args.batch,
+                 retries=args.retries, dry_run=args.dry_run)
+    return 1 if result["mismatches"] else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

infra/lake-iceberg/test_sync.py

@@ -0,0 +1,247 @@
+"""Unit tests for the Iceberg lake sync: incremental tile diffing and
+add_files planning against a local filesystem fixture (a miniature lake in a
+tmp dir, a file:// warehouse, a sqlite catalog). No network."""
+import datetime
+import os
+import sys
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from fsspec.implementations.local import LocalFileSystem
+
+sys.path.insert(0, os.path.dirname(__file__))
+import sync_iceberg as si  # noqa: E402
+
+# Mirrors the real part-file schema (float32/float64 split, int16 doy,
+# all-null float64 sun) so the compatibility check is exercised for real.
+PART_SCHEMA = pa.schema([
+    ("date", pa.date32()),
+    ("tmax", pa.float32()),
+    ("tmin", pa.float32()),
+    ("precip", pa.float32()),
+    ("wind", pa.float32()),
+    ("gust", pa.float32()),
+    ("humid", pa.float32()),
+    ("fmax", pa.float32()),
+    ("fmin", pa.float64()),
+    ("sun", pa.float64()),
+    ("feels", pa.float64()),
+    ("doy", pa.int16()),
+    ("lat_idx", pa.int32()),
+    ("lon_idx", pa.int32()),
+])
+
+POINTS = 2      # points per fixture tile
+DAYS = 3        # rows per point per month
+
+
+def write_tile(root, ti, tj, months):
+    """Write one tile's hive part files; return its total row count."""
+    total = 0
+    for year, month in months:
+        rows = {name: [] for name in PART_SCHEMA.names}
+        for p in range(POINTS):
+            i, j = ti * si.TILE + p, tj * si.TILE + p
+            for d in range(DAYS):
+                day = datetime.date(year, month, d + 1)
+                rows["date"].append(day)
+                for col in ("tmax", "tmin", "precip", "wind", "gust",
+                            "humid", "fmax", "fmin", "feels"):
+                    rows[col].append(float(10 * p + d))
+                rows["sun"].append(None)
+                rows["doy"].append(day.timetuple().tm_yday)
+                rows["lat_idx"].append(i)
+                rows["lon_idx"].append(j)
+        part = pa.table(rows, schema=PART_SCHEMA)
+        path = os.path.join(root, si.PREFIX, "daily", f"tile={ti}_{tj}",
+                            f"year={year}", f"month={month:02d}",
+                            "part.parquet")
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        pq.write_table(part, path)
+        total += part.num_rows
+    return total
+
+
+def write_manifest(root, tile_rows):
+    """Manifest with one row per (tile, point) like the extractor writes;
+    the sync only reads tile+rows."""
+    tiles, rows = [], []
+    for tile, total in tile_rows.items():
+        per_point = total // POINTS
+        for _ in range(POINTS):
+            tiles.append(tile)
+            rows.append(per_point)
+    m = pa.table({
+        "lat_idx": pa.array([0] * len(tiles), pa.int64()),
+        "lon_idx": pa.array([0] * len(tiles), pa.int64()),
+        "lat": pa.array([0.0] * len(tiles)),
+        "lon": pa.array([0.0] * len(tiles)),
+        "tile": pa.array(tiles, pa.large_string()),
+        "rows": pa.array(rows, pa.int64()),
+        "date_min": pa.array(["1940-01-01"] * len(tiles), pa.large_string()),
+        "date_max": pa.array(["2026-01-01"] * len(tiles), pa.large_string()),
+    })
+    path = os.path.join(root, si.MANIFEST_KEY)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    pq.write_table(m, path)
+
+
+@pytest.fixture
+def lake(tmp_path):
+    root = str(tmp_path / "lake")
+    counts = {
+        "10_10": write_tile(root, 10, 10, [(2000, 1), (2000, 2)]),
+        "10_11": write_tile(root, 10, 11, [(2000, 1)]),
+    }
+    write_manifest(root, counts)
+    hl = si.HiveLake(LocalFileSystem(), root, f"file://{root}", None)
+    hl.fixture_counts = counts
+    hl.fixture_root = root
+    return hl
+
+
+@pytest.fixture
+def catalog_db(tmp_path):
+    return str(tmp_path / "catalog.db")
+
+
+def test_pending_tiles_diff():
+    manifest = {"10_10": 1, "10_11": 1, "2_3": 1}
+    assert si.pending_tiles(manifest, set()) == ["2_3", "10_10", "10_11"]
+    assert si.pending_tiles(manifest, {"10_10"}) == ["2_3", "10_11"]
+    assert si.pending_tiles(manifest, set(manifest)) == []
+    # numeric, not lexicographic, tile ordering
+    assert si.pending_tiles({"10_2": 1, "9_11": 1}, set()) == ["9_11", "10_2"]
+
+
+def test_initial_sync_registers_all_files_in_place(lake, catalog_db):
+    result = si.run(lake, catalog_db, batch=1)
+    expected = sum(lake.fixture_counts.values())
+    assert result["added_records"] == expected
+    assert result["total_records"] == expected
+    assert result["added_files"] == 3          # 2 months + 1 month
+    assert not result["mismatches"]
+
+    table = si.open_catalog(catalog_db, lake).load_table(si.IDENTIFIER)
+    assert si.synced_tiles(table) == set(lake.fixture_counts)
+    # add_files, not a rewrite: every data file is an original hive part file
+    files = table.inspect.files().column("file_path").to_pylist()
+    assert len(files) == 3
+    assert all(f"/{si.PREFIX}/daily/tile=" in f for f in files)
+    # nothing was written outside iceberg/ (the hive tree is untouched, the
+    # table tree holds only metadata)
+    fs = LocalFileSystem()
+    written = fs.find(os.path.join(lake.fixture_root, si.ICEBERG_PREFIX))
+    assert written and all("/metadata/" in p for p in written)
+
+
+def test_rerun_is_noop(lake, catalog_db):
+    first = si.run(lake, catalog_db)
+    again = si.run(lake, catalog_db)
+    assert again["pending"] == []
+    assert again["added_records"] == 0
+    assert again["total_records"] == first["total_records"]
+    assert again["metadata_location"] == first["metadata_location"]
+
+
+def test_incremental_new_tile_only(lake, catalog_db):
+    si.run(lake, catalog_db)
+    counts = dict(lake.fixture_counts)
+    counts["11_10"] = write_tile(lake.fixture_root, 11, 10, [(2001, 6)])
+    write_manifest(lake.fixture_root, counts)
+
+    result = si.run(lake, catalog_db)
+    assert result["pending"] == ["11_10"]
+    assert result["added_records"] == counts["11_10"]
+    assert result["total_records"] == sum(counts.values())
+
+
+def test_version_hint_tracks_current_metadata(lake, catalog_db):
+    first = si.run(lake, catalog_db)
+    hint_path = os.path.join(lake.fixture_root, si.ICEBERG_PREFIX,
+                             si.TABLE_NAME, "metadata", "version-hint.text")
+    with open(hint_path) as f:
+        assert f.read() == os.path.basename(
+            first["metadata_location"]).removesuffix(".metadata.json")
+
+    counts = dict(lake.fixture_counts)
+    counts["11_10"] = write_tile(lake.fixture_root, 11, 10, [(2001, 6)])
+    write_manifest(lake.fixture_root, counts)
+    second = si.run(lake, catalog_db)
+    assert second["metadata_location"] != first["metadata_location"]
+    with open(hint_path) as f:
+        assert f.read() == os.path.basename(
+            second["metadata_location"]).removesuffix(".metadata.json")
+
+
+def test_partition_spec_and_pruned_scan(lake, catalog_db):
+    si.run(lake, catalog_db)
+    table = si.open_catalog(catalog_db, lake).load_table(si.IDENTIFIER)
+    assert [f.name for f in table.spec().fields] == \
+        ["tile_lat", "tile_lon", "year", "month"]
+
+    parts = table.inspect.partitions()
+    keys = {(r["partition"]["tile_lat"], r["partition"]["tile_lon"])
+            for r in parts.to_pylist()}
+    assert keys == {(120, 120), (120, 132)}    # ti*12, tj*12
+
+    # a tile-shaped predicate resolves to that tile's rows only
+    scan = table.scan(row_filter="lat_idx >= 120 and lat_idx < 132 "
+                                 "and lon_idx >= 132 and lon_idx < 144")
+    assert scan.to_arrow().num_rows == lake.fixture_counts["10_11"]
+    # and file planning pruned to that tile's single part file
+    assert len(list(scan.plan_files())) == 1
+
+
+def test_lost_catalog_db_reregisters_without_readding(lake, catalog_db):
+    first = si.run(lake, catalog_db)
+    os.remove(catalog_db)                       # simulate losing local state
+    again = si.run(lake, catalog_db)
+    # synced-tiles lives in table metadata, so nothing is re-added
+    assert again["pending"] == []
+    assert again["added_records"] == 0
+    assert again["total_records"] == first["total_records"]
+
+
+def test_dry_run_writes_nothing(lake, catalog_db):
+    result = si.run(lake, catalog_db, dry_run=True)
+    assert sorted(result["pending"]) == sorted(lake.fixture_counts)
+    assert result["metadata_location"] is None
+    assert not os.path.exists(
+        os.path.join(lake.fixture_root, si.ICEBERG_PREFIX))
+
+
+def test_manifest_tile_without_files_refuses(lake, catalog_db):
+    counts = dict(lake.fixture_counts)
+    counts["50_50"] = 6                         # in the manifest, no files
+    write_manifest(lake.fixture_root, counts)
+    # an invariant break aborts immediately: no retries, nothing marked synced
+    with pytest.raises(si.LakeInvariantError, match="50_50"):
+        si.run(lake, catalog_db)
+
+
+def test_transient_failure_retries_and_succeeds(lake, catalog_db, monkeypatch):
+    monkeypatch.setattr(si, "BACKOFF_SECONDS", 0)
+    fails = {"n": 2}
+    original = si.HiveLake.tile_files
+
+    def flaky(self, tile):
+        if fails["n"]:
+            fails["n"] -= 1
+            raise OSError("AWS Error SLOW_DOWN during CompleteMultipartUpload")
+        return original(self, tile)
+
+    monkeypatch.setattr(si.HiveLake, "tile_files", flaky)
+    result = si.run(lake, catalog_db, retries=2)
+    assert result["added_records"] == sum(lake.fixture_counts.values())
+    assert not result["mismatches"]
+
+
+def test_transient_failure_exhausts_retries(lake, catalog_db, monkeypatch):
+    monkeypatch.setattr(si, "BACKOFF_SECONDS", 0)
+    monkeypatch.setattr(
+        si.HiveLake, "tile_files",
+        lambda self, tile: (_ for _ in ()).throw(OSError("SLOW_DOWN")))
+    with pytest.raises(OSError):
+        si.run(lake, catalog_db, retries=1)

infra/terraform/modules/thermograph-host/templates/Caddyfile.tftpl

@@ -12,6 +12,16 @@
 #
 # NOTE: the repo's deploy/Caddyfile additionally serves the emigriffith.dev portfolio
 # and legacy redirects; those are host-specific and intentionally not templated here.
+#
+# Access-log hygiene: the default JSON encoder serializes full request headers, the
+# TLS block, and response headers on every line (measured ~1,133B/line on prod,
+# ~922B on beta) -- strip those with the `filter` format encoder. Also strip the
+# query string from the logged URI: Caddy's default logger records request.uri
+# *including* the query string, so every `?q=<search text>` a visitor typed sat in
+# Loki next to their client IP for the full 30-day retention -- a real privacy
+# leak, not just noise. Bot/crawler skipping stays out of here: `log_skip` needs
+# Caddy >= 2.7 and an upgrade is out of scope, so that's handled downstream in
+# Alloy's loki.process "caddy" stage instead (see observability/alloy/config.alloy).
 ${domain} {
 	encode zstd gzip
 
@@ -21,10 +31,12 @@ ${domain} {
 	# HEALTHCHECK uses (Dockerfile) — so a `docker compose up -d --build` deploy
 	# that's still restarting/booting never gets proxied into (a reload alone has
 	# no gate, hop-1 runbook hazard #10). health_uri is relative to the upstream.
+	# 15s (was 5s): plenty responsive for an active health check against a process
+	# that only restarts on a deploy, and a quarter of the polling load.
 	handle @backend_paths {
 		reverse_proxy 127.0.0.1:${port} {
 			health_uri /healthz
-			health_interval 5s
+			health_interval 15s
 			health_timeout 3s
 			health_status 2xx
 		}
@@ -33,13 +45,25 @@ ${domain} {
 	handle {
 		reverse_proxy 127.0.0.1:${frontend_port} {
 			health_uri /healthz
-			health_interval 5s
+			health_interval 15s
 			health_timeout 3s
 			health_status 2xx
 		}
 	}
 
 	log {
-		output file /var/log/caddy/thermograph.log
+		output file /var/log/caddy/thermograph.log {
+			roll_size 20MiB
+			roll_keep 5
+		}
+		format filter {
+			wrap json
+			fields {
+				request>headers delete
+				request>tls delete
+				resp_headers delete
+				request>uri regexp \?.* ""
+			}
+		}
 	}
 }

observability/alloy/config.alloy

@@ -15,12 +15,22 @@
 livedebugging { enabled = false }
 
 // --- 1. All Docker container logs ------------------------------------------------
+// refresh_interval defaults to 60s; Swarm task churn reshuffles the target set on
+// roughly that cadence, which restarts tailers ~every 90s and was costing ~11% of
+// Alloy's own CPU in tailer restarts alone. 5m is still fast enough to pick up a
+// real deploy without paying that churn cost.
 discovery.docker "containers" {
-	host = "unix:///var/run/docker.sock"
+	host             = "unix:///var/run/docker.sock"
+	refresh_interval = "5m"
 }
 
 // Turn Docker metadata into tidy labels: `container` (short name) and `service`
-// (the compose service, e.g. app/db). Drop Alloy's own container to avoid a loop.
+// (the compose service, e.g. app/db). Drop noise/duplicate containers so Loki never
+// ingests them: Alloy itself (loop), the autoscaler, throwaway `thermograph-test_*`
+// stacks, the loopback LB bridge (`thermograph-lb`, pure plumbing, nothing to debug
+// from its logs), and the app's own worker (`thermograph_worker`'s stdout is 100%
+// `/healthz` poll noise — the app's `access/*.jsonl` under source #3 is a strict
+// superset of anything useful it logs).
 discovery.relabel "containers" {
 	targets = discovery.docker.containers.targets
 
@@ -35,27 +45,55 @@ discovery.relabel "containers" {
 	}
 	rule {
 		source_labels = ["container"]
-		regex         = ".*alloy.*"
+		regex         = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*"
 		action        = "drop"
 	}
 }
 
+// Pass RAW targets here (not discovery.relabel.containers.output) alongside
+// relabel_rules: loki.source.docker applies relabel_rules itself, once, using the
+// __meta_docker_* metadata it still holds at collection time. Passing the
+// already-relabelled output *and* relabel_rules ran the same rules twice per log
+// entry, and made the drop rules above a no-op on the second pass since the
+// __meta_docker_* labels are already gone from the pre-relabelled output.
 loki.source.docker "containers" {
 	host          = "unix:///var/run/docker.sock"
-	targets       = discovery.relabel.containers.output
+	targets       = discovery.docker.containers.targets
 	forward_to    = [loki.write.central.receiver]
 	relabel_rules = discovery.relabel.containers.rules
 	labels        = { job = "docker" }
 }
 
 // --- 2. Caddy host access logs ---------------------------------------------------
+// sync_period (glob rescan) defaults to 10s; 1m is plenty for a log file that only
+// appears/rotates on the order of hours.
 local.file_match "caddy" {
 	path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }]
+	sync_period  = "1m"
 }
 
 loki.source.file "caddy" {
 	targets    = local.file_match.caddy.targets
+	forward_to = [loki.process.caddy.receiver]
+
+	// PollingFileWatcher defaults (250ms/250ms) stat every tailed file 4x/second
+	// forever. Caddy's access log doesn't need sub-second latency into Loki.
+	file_watch {
+		min_poll_frequency = "2s"
+		max_poll_frequency = "10s"
+	}
+}
+
+// Drop well-known crawler/bot traffic before it hits Loki. Caddy itself can't do
+// this cheaply (log_skip needs Caddy >= 2.7; both hosts run older Caddy, and an
+// upgrade is out of scope here), so filter it at the shipper instead.
+loki.process "caddy" {
 	forward_to = [loki.write.central.receiver]
+
+	stage.drop {
+		expression          = "(?i)(semrushbot|claudebot|ahrefsbot|yandexbot|bytespider|mj12bot|petalbot)"
+		drop_counter_reason = "crawler"
+	}
 }
 
 // --- 3. App structured JSON logs (errors / access / audit) -----------------------
@@ -63,11 +101,17 @@ loki.source.file "caddy" {
 // compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable.
 local.file_match "app_jsonl" {
 	path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }]
+	sync_period  = "1m"
 }
 
 loki.source.file "app_jsonl" {
 	targets    = local.file_match.app_jsonl.targets
 	forward_to = [loki.process.app_jsonl.receiver]
+
+	file_watch {
+		min_poll_frequency = "2s"
+		max_poll_frequency = "10s"
+	}
 }
 
 loki.process "app_jsonl" {

observability/loki/config.yml

@@ -32,6 +32,17 @@ schema_config:
         prefix: index_
         period: 24h
 
+# 30 low-rate streams almost always hit chunk_idle_period (30m default) long before
+# they'd ever fill chunk_target_size, so chunks were flushed 1.98% full on average
+# (972 near-empty chunks for 30MB of actual log data). Give idle streams far longer
+# to accumulate before an idle flush, and cap age/size so a chunk still can't grow
+# unbounded.
+ingester:
+  chunk_idle_period: 2h
+  max_chunk_age: 12h
+  chunk_target_size: 1572864
+  chunk_encoding: snappy
+
 limits_config:
   # A hobby fleet's volume is tiny; keep 30 days and cap ingestion generously.
   retention_period: 720h
@@ -40,6 +51,15 @@ limits_config:
   max_query_series: 5000
   allow_structured_metadata: true
   volume_enabled: true
+  # No explicit ingestion/stream limits meant Loki fell back to its (much stricter)
+  # built-in defaults, which is why 25 pushes came back HTTP 429 with nothing in
+  # this file explaining why. These are sized for a three-node hobby fleet, not the
+  # multi-tenant defaults.
+  ingestion_rate_mb: 8
+  ingestion_burst_size_mb: 16
+  per_stream_rate_limit: 3MB
+  per_stream_rate_limit_burst: 10MB
+  max_global_streams_per_user: 1000
 
 compactor:
   working_directory: /loki/compactor