From 46c90914b34f7b3120fd6dafb9dda53516084555 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 11 Jul 2026 13:47:31 -0700 Subject: [PATCH] Warm neighbor cells server-side (/cell?neighbors=1) (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cache.js contained a JavaScript clone of backend/grid.py's snapping math (its own comment said so) to compute the 8 surrounding cells and fire 8 staggered prefetch requests — grid geometry had two homes, one per language, plus a client-side guess at Nominatim pacing. The server now owns it: grid.neighbors(cell) steps one cell width from the center and re-snaps (adjacent rows have different longitude steps; poles and the antimeridian handled by snap), and /api/v2/cell grew a neighbors=1 flag that enqueues those cells for a single background worker. The warm-only guarantee matches prefetch=1 — a cell with no cached archive is skipped, so no weather-API quota is ever spent — and reverse_geocode's own lock paces the at-most-one Nominatim call per never-labeled cell. Re-enqueues are TTL-deduped; the worker starts from the lifespan hook, so tests and offline importers never spawn it. The client now sends its one conditional bundle request with neighbors=1 (a warm spot costs an empty 304) instead of skipping the bundle and firing 8 extra requests; the grid-math clone and the now-unused hasFreshCache are deleted. Tests (114): grid.neighbors mid-latitude/pole/antimeridian, the neighbors=1 enqueue + TTL dedupe, _warm_cell materializing the history-derived store rows, and the never-fetch-upstream guarantee. Verified with the headless-Chromium smoke across all five pages. --- app.py | 66 +++++++++++++++++++++++++++++++++++++++++++++- grid.py | 20 ++++++++++++++ tests/test_api.py | 41 ++++++++++++++++++++++++++++ tests/test_grid.py | 25 ++++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 68112ba..8f7e61a 100644 --- a/app.py +++ b/app.py @@ -5,6 +5,9 @@ import functools import hashlib import json import os +import queue +import threading +import time import pandas as pd from fastapi import APIRouter, FastAPI, HTTPException, Query, Request, Response @@ -41,14 +44,69 @@ def _weather_fetch_error(e) -> HTTPException: return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}") +# --- background neighbor warming --------------------------------------------- +# /cell?neighbors=1 asks the server to warm the 8 grid cells around the request +# (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's +# snapping math in JS). One worker drains the queue so warms never stack; the +# hard guarantee matches prefetch=1 — a cell with no cached archive is skipped, +# so no weather-API quota is ever spent, and reverse_geocode itself paces the +# at-most-one Nominatim call per never-labeled cell (~1/s policy). + +_WARM_TTL = 3600.0 # don't re-enqueue a cell within the hour +_WARM_SEEN: dict[str, float] = {} # cell_id -> last enqueue time +_warm_queue: "queue.Queue[dict]" = queue.Queue() + + +def _warm_cell(cell: dict) -> None: + """Materialize the history-derived slices for one cell — the same rows the + prefetch=1 bundle serves. Never fetches weather upstream.""" + history = climate.load_cached_history(cell) + if history is None or history.empty: + return + place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) + token = views.history_token(history) + start_ts, end_ts = views.cal_span(history, None, None, 24) + cal_key = views.calendar_key(start_ts, end_ts, 24) + if store.get_payload("calendar", cell["id"], cal_key, token) is None: + store.put_payload("calendar", cell["id"], cal_key, token, + views.build_calendar(cell, history, start_ts, end_ts, 24, place)) + last = pd.Timestamp(history["date"].max()).normalize() + day_key = views.day_key(last) + if store.get_payload("day", cell["id"], day_key, token) is None: + store.put_payload("day", cell["id"], day_key, token, + views.build_day(cell, history, last, place)) + + +def _enqueue_neighbor_warming(cell: dict) -> None: + now = time.time() + for n in grid.neighbors(cell): + if now - _WARM_SEEN.get(n["id"], 0.0) < _WARM_TTL: + continue + _WARM_SEEN[n["id"]] = now + _warm_queue.put(n) + + +def _neighbor_warmer() -> None: + while True: + cell = _warm_queue.get() + try: + _warm_cell(cell) + except Exception: # noqa: BLE001 - warming is best-effort, never fatal + pass + finally: + _warm_queue.task_done() + + @contextlib.asynccontextmanager async def _lifespan(app): # Warm the local place-name index (/suggest's typo tolerance) in the # background; the app boots and serves fine without it — suggestions just # fall back to the upstream geocoder until it's ready. A server-startup # hook (not import time) so offline importers (tests, the migrate script's - # dependencies) don't kick off a GeoNames download. + # dependencies) don't kick off a GeoNames download. The neighbor warmer is + # also a server concern: enqueues before startup just wait in the queue. places.start_loading() + threading.Thread(target=_neighbor_warmer, name="neighbor-warmer", daemon=True).start() yield @@ -362,6 +420,9 @@ def api_cell( lon: float = Query(..., ge=-180, le=180), prefetch: int = Query(0, ge=0, le=1, description="1 = warm-only: never fetch weather upstream; 204 for a cold cell"), + neighbors: int = Query(0, ge=0, le=1, + description="1 = also warm the 8 surrounding cells in the background " + "(warm-only: cold neighbors are skipped, no upstream quota)"), ): """One bundle carrying every view's payload for a cell, so the frontend warms all views with a single request instead of four. @@ -397,6 +458,9 @@ def api_cell( cid = cell["id"] last = pd.Timestamp(history["date"].max()).normalize() + if neighbors: + _enqueue_neighbor_warming(cell) + with run.phase("reverse_geocode"): place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) diff --git a/grid.py b/grid.py index c038ca6..ef0f823 100644 --- a/grid.py +++ b/grid.py @@ -64,6 +64,26 @@ def snap(lat: float, lon: float) -> dict: return _cell(i, j) +def neighbors(cell: dict) -> list[dict]: + """The up-to-8 cells surrounding one cell. Steps one cell width from the + center and re-snaps, so adjacent rows — whose longitude step differs — + resolve to whichever cell actually contains the stepped point. Rows past + the poles are skipped; longitude wraps across the antimeridian (both via + snap). Deduplicated (near the poles steps can collapse onto one cell).""" + out: dict[str, dict] = {} + for di in (-1, 0, 1): + for dj in (-1, 0, 1): + if not di and not dj: + continue + lat = cell["center_lat"] + di * LAT_STEP + if abs(lat) > 90.0: + continue + n = snap(lat, cell["center_lon"] + dj * cell["lon_step"]) + if n["id"] != cell["id"]: + out[n["id"]] = n + return list(out.values()) + + def from_id(cell_id: str) -> dict: """Rebuild the full cell dict from a cache id ("i_j") — the inverse of snap(). Lets offline tooling (the migrate script) recover a cell from its parquet diff --git a/tests/test_api.py b/tests/test_api.py index 409af38..2af6df1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -192,3 +192,44 @@ def test_pages_serve_with_origin_filled_in(client): assert "text/html" in r.headers["content-type"] assert "__ORIGIN__" not in r.text assert client.head("/thermograph/calendar").status_code == 200 + + +def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch): + import app as appmod + monkeypatch.setattr(appmod, "_WARM_SEEN", {}) + # No lifespan in TestClient without a context manager, so no worker drains + # the queue — enqueued cells just accumulate for inspection. + monkeypatch.setattr(appmod, "_warm_queue", __import__("queue").Queue()) + r = client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1}) + assert r.status_code == 200 + assert appmod._warm_queue.qsize() == 8 + assert len(appmod._WARM_SEEN) == 8 + # Same spot again within the TTL: nothing new enqueued. + client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1}, + headers={"If-None-Match": r.headers["etag"]}) + assert appmod._warm_queue.qsize() == 8 + + +def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history): + import app as appmod + import grid + import views + cell = grid.snap(-33.87, 151.21) + appmod._warm_cell(cell) + token = views.history_token(history) + start_ts, end_ts = views.cal_span(history, None, None, 24) + assert tmp_store.get_payload("calendar", cell["id"], + views.calendar_key(start_ts, end_ts, 24), token) is not None + import pandas as pd + last = pd.Timestamp(history["date"].max()).normalize() + assert tmp_store.get_payload("day", cell["id"], views.day_key(last), token) is not None + + +def test_warm_cell_never_fetches_upstream(client, monkeypatch): + import app as appmod + def boom(cell): + raise AssertionError("warming must never fetch weather upstream") + monkeypatch.setattr(climate, "get_history", boom) + monkeypatch.setattr(climate, "get_recent_forecast", boom) + monkeypatch.setattr(climate, "load_cached_history", lambda cell: None) + appmod._warm_cell({"id": "1_1", "center_lat": 0.03, "center_lon": 0.03}) # cold: no-op diff --git a/tests/test_grid.py b/tests/test_grid.py index c1b85da..268ca99 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -59,3 +59,28 @@ def test_lon_step_clamps_near_poles(): # cos(89°) ~ 0.017 would blow the step up; the 0.05 clamp caps it. assert grid._lon_step(89.0) == pytest.approx(grid.LAT_STEP / 0.05) assert not math.isinf(grid._lon_step(90.0)) + + +def test_neighbors_mid_latitude(): + cell = grid.snap(47.6062, -122.3321) + ns = grid.neighbors(cell) + assert len(ns) == 8 + ids = {n["id"] for n in ns} + assert cell["id"] not in ids and len(ids) == 8 + # Every neighbor is at most one row away and shares a border region. + i = int(cell["id"].split("_")[0]) + assert all(abs(int(n["id"].split("_")[0]) - i) <= 1 for n in ns) + + +def test_neighbors_skip_rows_past_the_pole(): + cell = grid.snap(89.99, 10.0) # topmost row + ns = grid.neighbors(cell) + assert 0 < len(ns) < 8 # no row above the pole + assert all(-90 <= n["center_lat"] <= 90 for n in ns) + + +def test_neighbors_wrap_across_antimeridian(): + cell = grid.snap(10.0, 179.999) + ns = grid.neighbors(cell) + assert len(ns) == 8 + assert any(n["center_lon"] < 0 for n in ns) # some sit past the date line