35b58300cb
Three operator decisions change v1's premises, so the design is now two tiers with separate flags: Tier A (THERMOGRAPH_EVENTS) is v1 unchanged -- anonymous hourly aggregates, no identifier, nothing stored on the device, no consent needed, runs for everyone. It stays the primary signal precisely so a poor consent rate cannot take the numbers down with it, and so Tier B's rates can be calibrated against it to measure the consent bias rather than ignoring it. Tier B (THERMOGRAPH_EVENT_SESSIONS) adds an ephemeral per-tab id -- 16 random bytes in sessionStorage, rotating on a 30-minute idle and a 2-hour absolute cap, capped at 200 events, never linked to another session, device, or to the account (api_event does not read the auth cookie and there is no user column). Rows land in a 30-day hypertable with minute-granularity timestamps and an in-session sequence number instead of precise clock times. Storing an identifier engages ePrivacy Art. 5(3) and analytics is not strictly necessary, so it is gated on opt-in consent: an equal-weight banner that mints nothing before "Allow", one-click withdrawal in every footer that drops the live id immediately, and GPC/DNT treated as a refusal already given. THERMOGRAPH_SEARCH_MISS captures zero-result search text server-side in api_suggest only -- normalised, rejected outright on any personal-data smell, stored as a per-day count, and pruned below a three-person floor after a week. The privacy page is rewritten rather than deferred: "no per-visitor identifier" becomes false the moment Tier B ships. Tests assert the load-bearing promises so the copy and the code cannot drift apart silently. Raw-IP truncation is NOT implemented here (the logging pipeline owns it) but UI-EVENTS.md states what the app side must do, and records that Caddy's default JSON log already stores full request URIs -- so every search query is in Loki with the client IP today, which the search-miss mitigations depend on fixing. UI-EVENTS.md carries the v1-to-v2 diff, the consent reasoning including why legitimate interest is not available, and an explicit argument that the session identifier is the wrong trade at this traffic volume.
359 lines
16 KiB
Python
359 lines
16 KiB
Python
"""Route-level tests over the FastAPI app with the weather/geocode layer faked —
|
|
they exercise the real routing, validation, derived-store and ETag plumbing, and
|
|
would catch wiring regressions (e.g. a handler calling a deleted helper)."""
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
from web import app as appmod
|
|
from data import climate
|
|
from data import places
|
|
|
|
|
|
@pytest.fixture
|
|
def client(monkeypatch, history, recent):
|
|
monkeypatch.setattr(climate, "get_history",
|
|
lambda cell: (history.clone(), {"cached": True, "cache_age_days": 3}))
|
|
monkeypatch.setattr(climate, "get_recent_forecast", lambda cell: recent.clone())
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone())
|
|
monkeypatch.setattr(climate, "recent_stamp", lambda cell_id: "rs-test")
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: "Testville, Washington")
|
|
return TestClient(appmod.app)
|
|
|
|
|
|
Q = {"lat": 47.6062, "lon": -122.3321}
|
|
|
|
|
|
def test_place_serves_any_point_worldwide(client):
|
|
for q in (Q, {"lat": 48.8566, "lon": 2.3522}, {"lat": -33.8688, "lon": 151.2093}):
|
|
r = client.get("/thermograph/api/v2/place", params=q)
|
|
assert r.status_code == 200
|
|
assert r.json()["place"] == "Testville, Washington"
|
|
assert set(r.json()["cell"]) == {"center_lat", "center_lon"}
|
|
|
|
|
|
def test_grade_shape_and_conditional_revalidation(client, history):
|
|
r = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["place"] == "Testville, Washington"
|
|
assert body["climatology"]["tmax"] is not None
|
|
days = [d["date"] for d in body["recent"]]
|
|
assert days == sorted(days, reverse=True) # newest first
|
|
assert body["recent"][0]["tmax"]["grade"]
|
|
|
|
etag = r.headers["etag"]
|
|
r304 = client.get("/thermograph/api/v2/grade", params=Q,
|
|
headers={"If-None-Match": etag})
|
|
assert r304.status_code == 304 and r304.headers["etag"] == etag
|
|
|
|
# Without the validator the derived store replays the exact same bytes.
|
|
r2 = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert r2.status_code == 200 and r2.content == r.content
|
|
|
|
|
|
def test_grade_is_aliased_across_api_versions(client):
|
|
for prefix in ("api", "api/v1", "api/v2"):
|
|
assert client.get(f"/thermograph/{prefix}/grade", params=Q).status_code == 200
|
|
|
|
|
|
def test_query_validation_rejects_out_of_range(client):
|
|
assert client.get("/thermograph/api/v2/grade",
|
|
params={"lat": 999, "lon": 0}).status_code == 422
|
|
assert client.get("/thermograph/api/v2/grade",
|
|
params={**Q, "days": 0}).status_code == 422
|
|
|
|
|
|
def test_api_version_reports_backend_contract(client):
|
|
r = client.get("/thermograph/api/version")
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert {"backend_version", "min_frontend", "payload_ver"} <= body.keys()
|
|
assert body["backend_version"] == "2"
|
|
|
|
|
|
def test_day_detail_and_ladders(client, history):
|
|
r = client.get("/thermograph/api/v2/day", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["latest"] == history["date"].max().isoformat()
|
|
tmax = body["detail"]["metrics"]["tmax"]
|
|
assert tmax["ladder"]["tiers"][0]["c"] == "rec-hot"
|
|
assert tmax["obs"]["grade"]
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["grade", "calendar", "day", "forecast", "cell"])
|
|
def test_every_data_route_maps_rate_limit_to_503(client, monkeypatch, path):
|
|
def rate_limited(cell):
|
|
raise climate.WeatherUnavailable(climate.limit_message(False))
|
|
monkeypatch.setattr(climate, "get_history", rate_limited)
|
|
# A distinct spot per route: a store row cached by an earlier test would
|
|
# otherwise be checked only after the (failing) history fetch anyway.
|
|
r = client.get(f"/thermograph/api/v2/{path}", params={"lat": 51.5, "lon": -0.1})
|
|
assert r.status_code == 503
|
|
assert "rate-limited" in r.json()["detail"]
|
|
|
|
|
|
def test_upstream_failure_classification(client, monkeypatch):
|
|
q = {"lat": 52.52, "lon": 13.4}
|
|
|
|
def daily_quota(cell):
|
|
raise climate.WeatherUnavailable(climate.limit_message(True), daily=True)
|
|
monkeypatch.setattr(climate, "get_history", daily_quota)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 503 and "tomorrow" in r.json()["detail"]
|
|
|
|
# A raw upstream 429 that no fetcher classified still maps to a clean 503.
|
|
def raw_429(cell):
|
|
e = RuntimeError("upstream said no")
|
|
e.response = type("R", (), {"status_code": 429})()
|
|
raise e
|
|
monkeypatch.setattr(climate, "get_history", raw_429)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 503 and "rate-limited" in r.json()["detail"]
|
|
|
|
# Anything else is a genuine upstream fault: 502 with the raw error.
|
|
def boom(cell):
|
|
raise RuntimeError("parquet cache corrupted")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 502 and "parquet cache corrupted" in r.json()["detail"]
|
|
|
|
|
|
def test_cell_prefetch_never_fetches_upstream(client, monkeypatch):
|
|
def boom(cell):
|
|
raise AssertionError("prefetch=1 must never fetch weather upstream")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
|
r = client.get("/thermograph/api/v2/cell", params={**Q, "prefetch": 1})
|
|
assert r.status_code == 204 # cold cell: no body, no quota spent
|
|
|
|
|
|
def test_cell_prefetch_builds_history_slices_only(client):
|
|
r = client.get("/thermograph/api/v2/cell", params={"lat": 10.0, "lon": 10.0, "prefetch": 1})
|
|
assert r.status_code == 200
|
|
assert set(r.json()["slices"]) == {"calendar", "day"}
|
|
|
|
|
|
def test_calendar_placeless_payload_is_not_persisted(client, monkeypatch):
|
|
q = {"lat": -10.0, "lon": 20.0, "months": 2}
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: None)
|
|
r = client.get("/thermograph/api/v2/calendar", params=q)
|
|
assert r.status_code == 200 and r.json()["place"] is None
|
|
# The placeless payload wasn't cached, so the next request retries the
|
|
# label instead of replaying bare coordinates for the life of the token.
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: "Resolved, Now")
|
|
assert client.get("/thermograph/api/v2/calendar", params=q).json()["place"] == "Resolved, Now"
|
|
|
|
|
|
def test_calendar_compact_range(client, history):
|
|
r = client.get("/thermograph/api/v2/calendar", params={**Q, "months": 2})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["range"]["end"] == history["date"].max().isoformat()
|
|
day = body["days"][0]
|
|
assert {"date", "dsr", "tmax", "tmin", "precip"} <= set(day)
|
|
assert set(day["tmax"]) == {"v", "pct", "c", "g"}
|
|
|
|
|
|
def test_forecast_grades_future_days(client):
|
|
r = client.get("/thermograph/api/v2/forecast", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["forecast"] is True
|
|
days = [d["date"] for d in body["recent"]]
|
|
assert days and days == sorted(days, reverse=True) # furthest-out first
|
|
assert min(days) > body["target_date"] # strictly future
|
|
|
|
|
|
def test_cell_bundle_matches_per_view_payloads(client):
|
|
r = client.get("/thermograph/api/v2/cell", params=Q)
|
|
assert r.status_code == 200
|
|
slices = r.json()["slices"]
|
|
assert set(slices) == {"calendar", "grade", "forecast", "day"}
|
|
for s in slices.values():
|
|
assert s["etag"] and s["data"]
|
|
# The grade slice must be byte-for-byte what /grade serves (same store row).
|
|
grade = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert grade.json() == slices["grade"]["data"]
|
|
assert grade.headers["etag"] == slices["grade"]["etag"]
|
|
|
|
r304 = client.get("/thermograph/api/v2/cell", params=Q,
|
|
headers={"If-None-Match": r.headers["etag"]})
|
|
assert r304.status_code == 304
|
|
|
|
|
|
_LOCAL_HIT = [{"name": "Seattle", "admin1": "Washington", "country": "United States",
|
|
"country_code": "US", "lat": 47.6, "lon": -122.33, "population": 737015,
|
|
"match": "prefix"}]
|
|
|
|
|
|
def test_geocode_local_first_skips_network(client, monkeypatch):
|
|
"""A local GeoNames hit answers /geocode with no outbound Nominatim call, and
|
|
the internal prefix/fuzzy tag is stripped from the wire shape."""
|
|
monkeypatch.setattr(places, "search", lambda q, limit=5: list(_LOCAL_HIT))
|
|
|
|
def _boom(*a, **k):
|
|
raise AssertionError("Nominatim must not be called when the local index hits")
|
|
monkeypatch.setattr(climate, "geocode_nominatim", _boom)
|
|
|
|
r = client.get("/thermograph/api/v2/geocode", params={"q": "seattle"})
|
|
assert r.status_code == 200
|
|
res = r.json()["results"]
|
|
assert res[0]["name"] == "Seattle"
|
|
assert "match" not in res[0]
|
|
|
|
|
|
def test_geocode_falls_back_to_nominatim_on_local_miss(client, monkeypatch):
|
|
"""A local miss (or a still-loading index) falls back to Nominatim /search —
|
|
the path that covers neighbourhoods/postcodes the cities dump lacks."""
|
|
monkeypatch.setattr(places, "search", lambda q, limit=5: []) # genuine miss
|
|
nom = [{"name": "West Seattle", "admin1": "Washington", "country": "United States",
|
|
"country_code": "US", "lat": 47.57, "lon": -122.38, "population": None}]
|
|
called = []
|
|
monkeypatch.setattr(climate, "geocode_nominatim",
|
|
lambda q, count=5: (called.append(q), nom)[1])
|
|
|
|
r = client.get("/thermograph/api/v2/geocode", params={"q": "west seattle"})
|
|
assert r.status_code == 200
|
|
assert r.json()["results"][0]["name"] == "West Seattle"
|
|
assert called == ["west seattle"]
|
|
|
|
|
|
def test_suggest_is_local_only_no_network(client, monkeypatch):
|
|
"""Autocomplete is served purely from the local index — even a reachable
|
|
Nominatim must never be hit per keystroke."""
|
|
monkeypatch.setattr(places, "search", lambda q, limit=5: list(_LOCAL_HIT))
|
|
|
|
def _boom(*a, **k):
|
|
raise AssertionError("/suggest must not make an outbound geocoder call")
|
|
monkeypatch.setattr(climate, "geocode_nominatim", _boom)
|
|
|
|
r = client.get("/thermograph/api/v2/suggest", params={"q": "seattle"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["results"][0]["name"] == "Seattle"
|
|
assert body["corrected"] is None
|
|
|
|
|
|
def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch):
|
|
from web import app as appmod
|
|
monkeypatch.setattr(appmod, "_WARM_SEEN", {})
|
|
# No lifespan in TestClient without a context manager, so no worker drains
|
|
# the queue — enqueued cells just accumulate for inspection.
|
|
monkeypatch.setattr(appmod, "_warm_queue", __import__("queue").Queue())
|
|
r = client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1})
|
|
assert r.status_code == 200
|
|
assert appmod._warm_queue.qsize() == 8
|
|
assert len(appmod._WARM_SEEN) == 8
|
|
# Same spot again within the TTL: nothing new enqueued.
|
|
client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1},
|
|
headers={"If-None-Match": r.headers["etag"]})
|
|
assert appmod._warm_queue.qsize() == 8
|
|
|
|
|
|
def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history):
|
|
from web import app as appmod
|
|
from data import grid
|
|
from api import payloads
|
|
cell = grid.snap(-33.87, 151.21)
|
|
appmod._warm_cell(cell)
|
|
token = payloads.history_token(history)
|
|
start_ts, end_ts = payloads.cal_span(history, None, None, 24)
|
|
assert tmp_store.get_payload("calendar", cell["id"],
|
|
payloads.calendar_key(start_ts, end_ts, 24), token) is not None
|
|
last = history["date"].max()
|
|
assert tmp_store.get_payload("day", cell["id"], payloads.day_key(last), token) is not None
|
|
|
|
|
|
def test_warm_cell_never_fetches_upstream(client, monkeypatch):
|
|
from web import app as appmod
|
|
def boom(cell):
|
|
raise AssertionError("warming must never fetch weather upstream")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
|
appmod._warm_cell({"id": "1_1", "center_lat": 0.03, "center_lon": 0.03}) # cold: no-op
|
|
|
|
|
|
def _score_history(years=45, seed=5):
|
|
import datetime
|
|
import numpy as np
|
|
import polars as pl
|
|
end = datetime.date(2026, 7, 11)
|
|
start = datetime.date(end.year - years, end.month, end.day)
|
|
dates = [start + datetime.timedelta(days=i) for i in range((end - start).days + 1)]
|
|
n = len(dates)
|
|
rng = np.random.default_rng(seed)
|
|
doy = np.array([d.timetuple().tm_yday for d in dates])
|
|
tmax = 55 + 30 * np.sin((doy - 100) / 366.0 * 2 * np.pi) + rng.normal(0, 8, n)
|
|
return pl.DataFrame({
|
|
"date": dates, "tmax": np.round(tmax, 1), "tmin": np.round(tmax - 15, 1),
|
|
"feels": np.round(tmax + 1, 1), "humid": np.round(np.clip(12 + rng.normal(0, 3, n), 1, None), 1),
|
|
"wetbulb": np.round(tmax - 12, 1), "wind": np.round(np.clip(8 + rng.normal(0, 3, n), 0, None), 1),
|
|
"gust": np.round(np.clip(16 + rng.normal(0, 5, n), 0, None), 1),
|
|
"precip": np.where(rng.random(n) < 0.3, 0.2, 0.0),
|
|
}).with_columns(pl.col("date").dt.ordinal_day().cast(pl.Int16).alias("doy"))
|
|
|
|
|
|
@pytest.fixture
|
|
def score_client(monkeypatch):
|
|
hist = _score_history()
|
|
monkeypatch.setattr(climate, "get_history",
|
|
lambda cell: (hist.clone(), {"cached": True, "cache_age_days": 3}))
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: "Testville, Washington")
|
|
return TestClient(appmod.app)
|
|
|
|
|
|
def test_score_shape_and_conditional_revalidation(score_client):
|
|
r = score_client.get("/thermograph/api/v2/score", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["place"] == "Testville, Washington"
|
|
s = body["scores"]
|
|
assert set(s["slices"]) == {"annual", "djf", "mam", "jja", "son"}
|
|
assert s["slices"]["annual"]["overall"]["score"] is not None
|
|
|
|
etag = r.headers["etag"]
|
|
r304 = score_client.get("/thermograph/api/v2/score", params=Q,
|
|
headers={"If-None-Match": etag})
|
|
assert r304.status_code == 304 and r304.headers["etag"] == etag
|
|
|
|
# Second GET replays the exact same bytes from the derived store.
|
|
r2 = score_client.get("/thermograph/api/v2/score", params=Q)
|
|
assert r2.status_code == 200 and r2.content == r.content
|
|
|
|
|
|
# --- failed-search capture (server-side only) ----------------------------------
|
|
|
|
def test_zero_result_search_records_the_query_text_server_side(client, monkeypatch):
|
|
"""Decision 2: the miss text is captured in the handler, never sent from the
|
|
browser. Only misses — a query that found something is never stored."""
|
|
from core import events
|
|
seen = []
|
|
monkeypatch.setattr(events, "ENABLED", True)
|
|
monkeypatch.setattr(events, "SEARCH_MISS_ENABLED", True)
|
|
monkeypatch.setattr(events, "_upsert", lambda *a: None)
|
|
monkeypatch.setattr(events, "_upsert_search_miss", lambda q: seen.append(q))
|
|
monkeypatch.setattr(events.audit, "log_activity", lambda *a: None)
|
|
monkeypatch.setattr(places, "suggest", lambda q, up, lim: ([], None))
|
|
client.get("/thermograph/api/v2/suggest", params={"q": "Atlantis"})
|
|
assert seen == ["atlantis"]
|
|
|
|
monkeypatch.setattr(places, "suggest",
|
|
lambda q, up, lim: ([{"name": "Seattle"}], None))
|
|
client.get("/thermograph/api/v2/suggest", params={"q": "Seattle"})
|
|
assert seen == ["atlantis"] # a hit is never stored as text
|
|
|
|
|
|
def test_search_miss_text_is_not_written_to_the_access_log(client, monkeypatch):
|
|
"""The app's own access log records url.path, never the query string — so the
|
|
search text cannot leak into it. (Caddy's log is a separate concern: it logs
|
|
the full URI, which is why the miss capture must stay server-side and why the
|
|
Caddy log format needs the query stripped -- see UI-EVENTS.md.)"""
|
|
from core import audit
|
|
lines = []
|
|
monkeypatch.setattr(audit, "log_access", lambda rec: lines.append(rec))
|
|
monkeypatch.setattr(places, "suggest", lambda q, up, lim: ([], None))
|
|
client.get("/thermograph/api/v2/suggest", params={"q": "somewhere-secret"})
|
|
assert lines and all("somewhere-secret" not in str(v) for rec in lines for v in rec.values())
|