c3bacdce0c
* Migrate backend dataframe layer from pandas to polars Replace pandas with polars across the backend, dropping both pandas and its pyarrow parquet engine from the dependency set. numpy stays (the grading percentile math is unchanged). - climate.py: parquet IO, source→frame mappings, cache read/topup on polars. New _normalize_read casts the cached `date` column to pl.Date (older files were written by pandas as datetime64[ns]); frames now unify missing values as null so the grading boundary drops them consistently across sources. - grading.py: keep the numpy percentile core; swap the frame→numpy bridge to .to_numpy()/.drop_nulls(), day-of-year/year to polars dt expressions, and the per-row loop to iter_rows(named=True). - views.py: filter/anti-join/concat replace boolean-mask, isin and pd.concat; scalar dates are stdlib datetime.date; a local _months_before helper replaces DateOffset(months=) for the calendar-range default. - app.py, migrate.py: request-date parsing uses datetime.date, removing pandas from the endpoint and migrate layers entirely. - The date column is pl.Date end to end, eliminating the pandas normalize() calls and comparing cleanly against stdlib dates. Payloads are unchanged: calendar, day, grade and forecast responses are byte-for-byte identical to the pandas implementation on the same cached record. Tests ported to polars fixtures, with added coverage for the combined feels-like fallback, calendar month-offset (month-end/leap), and the concat/dedup "fresher source wins" rule. * Port notify.py to polars after merging dev's account system Merge origin/dev (accounts + notification subscriptions) and carry the pandas→ polars migration into the newly added notify.py, which the merge brought in still using pandas — with pandas removed from requirements this broke its import. - notify.py: _candidate_rows filters/sorts the recent bundle with polars expressions and returns iter_rows dicts; date scalars are datetime.date; history/recent emptiness via is_empty(). - test_notify.py: synthetic history/rows built with polars + datetime.
233 lines
10 KiB
Python
233 lines
10 KiB
Python
"""Route-level tests over the FastAPI app with the weather/geocode layer faked —
|
|
they exercise the real routing, validation, derived-store and ETag plumbing, and
|
|
would catch wiring regressions (e.g. a handler calling a deleted helper)."""
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import app as appmod
|
|
import climate
|
|
|
|
|
|
@pytest.fixture
|
|
def client(monkeypatch, history, recent):
|
|
monkeypatch.setattr(climate, "get_history",
|
|
lambda cell: (history.clone(), {"cached": True, "cache_age_days": 3}))
|
|
monkeypatch.setattr(climate, "get_recent_forecast", lambda cell: recent.clone())
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone())
|
|
monkeypatch.setattr(climate, "recent_stamp", lambda cell_id: "rs-test")
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: "Testville, Washington")
|
|
return TestClient(appmod.app)
|
|
|
|
|
|
Q = {"lat": 47.6062, "lon": -122.3321}
|
|
|
|
|
|
def test_place_serves_any_point_worldwide(client):
|
|
for q in (Q, {"lat": 48.8566, "lon": 2.3522}, {"lat": -33.8688, "lon": 151.2093}):
|
|
r = client.get("/thermograph/api/v2/place", params=q)
|
|
assert r.status_code == 200
|
|
assert r.json()["place"] == "Testville, Washington"
|
|
assert set(r.json()["cell"]) == {"center_lat", "center_lon"}
|
|
|
|
|
|
def test_grade_shape_and_conditional_revalidation(client, history):
|
|
r = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["place"] == "Testville, Washington"
|
|
assert body["climatology"]["tmax"] is not None
|
|
days = [d["date"] for d in body["recent"]]
|
|
assert days == sorted(days, reverse=True) # newest first
|
|
assert body["recent"][0]["tmax"]["grade"]
|
|
|
|
etag = r.headers["etag"]
|
|
r304 = client.get("/thermograph/api/v2/grade", params=Q,
|
|
headers={"If-None-Match": etag})
|
|
assert r304.status_code == 304 and r304.headers["etag"] == etag
|
|
|
|
# Without the validator the derived store replays the exact same bytes.
|
|
r2 = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert r2.status_code == 200 and r2.content == r.content
|
|
|
|
|
|
def test_grade_is_aliased_across_api_versions(client):
|
|
for prefix in ("api", "api/v1", "api/v2"):
|
|
assert client.get(f"/thermograph/{prefix}/grade", params=Q).status_code == 200
|
|
|
|
|
|
def test_query_validation_rejects_out_of_range(client):
|
|
assert client.get("/thermograph/api/v2/grade",
|
|
params={"lat": 999, "lon": 0}).status_code == 422
|
|
assert client.get("/thermograph/api/v2/grade",
|
|
params={**Q, "days": 0}).status_code == 422
|
|
|
|
|
|
def test_day_detail_and_ladders(client, history):
|
|
r = client.get("/thermograph/api/v2/day", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["latest"] == history["date"].max().isoformat()
|
|
tmax = body["detail"]["metrics"]["tmax"]
|
|
assert tmax["ladder"]["tiers"][0]["c"] == "rec-hot"
|
|
assert tmax["obs"]["grade"]
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["grade", "calendar", "day", "forecast", "cell"])
|
|
def test_every_data_route_maps_rate_limit_to_503(client, monkeypatch, path):
|
|
def rate_limited(cell):
|
|
raise climate.WeatherUnavailable(climate.limit_message(False))
|
|
monkeypatch.setattr(climate, "get_history", rate_limited)
|
|
# A distinct spot per route: a store row cached by an earlier test would
|
|
# otherwise be checked only after the (failing) history fetch anyway.
|
|
r = client.get(f"/thermograph/api/v2/{path}", params={"lat": 51.5, "lon": -0.1})
|
|
assert r.status_code == 503
|
|
assert "rate-limited" in r.json()["detail"]
|
|
|
|
|
|
def test_upstream_failure_classification(client, monkeypatch):
|
|
q = {"lat": 52.52, "lon": 13.4}
|
|
|
|
def daily_quota(cell):
|
|
raise climate.WeatherUnavailable(climate.limit_message(True), daily=True)
|
|
monkeypatch.setattr(climate, "get_history", daily_quota)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 503 and "tomorrow" in r.json()["detail"]
|
|
|
|
# A raw upstream 429 that no fetcher classified still maps to a clean 503.
|
|
def raw_429(cell):
|
|
e = RuntimeError("upstream said no")
|
|
e.response = type("R", (), {"status_code": 429})()
|
|
raise e
|
|
monkeypatch.setattr(climate, "get_history", raw_429)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 503 and "rate-limited" in r.json()["detail"]
|
|
|
|
# Anything else is a genuine upstream fault: 502 with the raw error.
|
|
def boom(cell):
|
|
raise RuntimeError("parquet cache corrupted")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
r = client.get("/thermograph/api/v2/grade", params=q)
|
|
assert r.status_code == 502 and "parquet cache corrupted" in r.json()["detail"]
|
|
|
|
|
|
def test_cell_prefetch_never_fetches_upstream(client, monkeypatch):
|
|
def boom(cell):
|
|
raise AssertionError("prefetch=1 must never fetch weather upstream")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
|
r = client.get("/thermograph/api/v2/cell", params={**Q, "prefetch": 1})
|
|
assert r.status_code == 204 # cold cell: no body, no quota spent
|
|
|
|
|
|
def test_cell_prefetch_builds_history_slices_only(client):
|
|
r = client.get("/thermograph/api/v2/cell", params={"lat": 10.0, "lon": 10.0, "prefetch": 1})
|
|
assert r.status_code == 200
|
|
assert set(r.json()["slices"]) == {"calendar", "day"}
|
|
|
|
|
|
def test_calendar_placeless_payload_is_not_persisted(client, monkeypatch):
|
|
q = {"lat": -10.0, "lon": 20.0, "months": 2}
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: None)
|
|
r = client.get("/thermograph/api/v2/calendar", params=q)
|
|
assert r.status_code == 200 and r.json()["place"] is None
|
|
# The placeless payload wasn't cached, so the next request retries the
|
|
# label instead of replaying bare coordinates for the life of the token.
|
|
monkeypatch.setattr(climate, "reverse_geocode", lambda lat, lon: "Resolved, Now")
|
|
assert client.get("/thermograph/api/v2/calendar", params=q).json()["place"] == "Resolved, Now"
|
|
|
|
|
|
def test_calendar_compact_range(client, history):
|
|
r = client.get("/thermograph/api/v2/calendar", params={**Q, "months": 2})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["range"]["end"] == history["date"].max().isoformat()
|
|
day = body["days"][0]
|
|
assert {"date", "dsr", "tmax", "tmin", "precip"} <= set(day)
|
|
assert set(day["tmax"]) == {"v", "pct", "c", "g"}
|
|
|
|
|
|
def test_forecast_grades_future_days(client):
|
|
r = client.get("/thermograph/api/v2/forecast", params=Q)
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["forecast"] is True
|
|
days = [d["date"] for d in body["recent"]]
|
|
assert days and days == sorted(days, reverse=True) # furthest-out first
|
|
assert min(days) > body["target_date"] # strictly future
|
|
|
|
|
|
def test_cell_bundle_matches_per_view_payloads(client):
|
|
r = client.get("/thermograph/api/v2/cell", params=Q)
|
|
assert r.status_code == 200
|
|
slices = r.json()["slices"]
|
|
assert set(slices) == {"calendar", "grade", "forecast", "day"}
|
|
for s in slices.values():
|
|
assert s["etag"] and s["data"]
|
|
# The grade slice must be byte-for-byte what /grade serves (same store row).
|
|
grade = client.get("/thermograph/api/v2/grade", params=Q)
|
|
assert grade.json() == slices["grade"]["data"]
|
|
assert grade.headers["etag"] == slices["grade"]["etag"]
|
|
|
|
r304 = client.get("/thermograph/api/v2/cell", params=Q,
|
|
headers={"If-None-Match": r.headers["etag"]})
|
|
assert r304.status_code == 304
|
|
|
|
|
|
def test_suggest_falls_back_to_upstream_geocoder(client, monkeypatch):
|
|
upstream = [{"name": "Seattle", "admin1": "Washington", "country": "United States",
|
|
"country_code": "US", "lat": 47.6, "lon": -122.33, "population": 737015}]
|
|
monkeypatch.setattr(climate, "geocode", lambda q, count=5: list(upstream))
|
|
r = client.get("/thermograph/api/v2/suggest", params={"q": "seattle-fallback-probe"})
|
|
assert r.status_code == 200
|
|
body = r.json()
|
|
assert body["results"][0]["name"] == "Seattle"
|
|
assert body["corrected"] is None
|
|
|
|
|
|
def test_pages_serve_with_origin_filled_in(client):
|
|
r = client.get("/thermograph/")
|
|
assert r.status_code == 200
|
|
assert "text/html" in r.headers["content-type"]
|
|
assert "__ORIGIN__" not in r.text
|
|
assert client.head("/thermograph/calendar").status_code == 200
|
|
|
|
|
|
def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch):
|
|
import app as appmod
|
|
monkeypatch.setattr(appmod, "_WARM_SEEN", {})
|
|
# No lifespan in TestClient without a context manager, so no worker drains
|
|
# the queue — enqueued cells just accumulate for inspection.
|
|
monkeypatch.setattr(appmod, "_warm_queue", __import__("queue").Queue())
|
|
r = client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1})
|
|
assert r.status_code == 200
|
|
assert appmod._warm_queue.qsize() == 8
|
|
assert len(appmod._WARM_SEEN) == 8
|
|
# Same spot again within the TTL: nothing new enqueued.
|
|
client.get("/thermograph/api/v2/cell", params={"lat": 35.0, "lon": 25.0, "neighbors": 1},
|
|
headers={"If-None-Match": r.headers["etag"]})
|
|
assert appmod._warm_queue.qsize() == 8
|
|
|
|
|
|
def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history):
|
|
import app as appmod
|
|
import grid
|
|
import views
|
|
cell = grid.snap(-33.87, 151.21)
|
|
appmod._warm_cell(cell)
|
|
token = views.history_token(history)
|
|
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
|
assert tmp_store.get_payload("calendar", cell["id"],
|
|
views.calendar_key(start_ts, end_ts, 24), token) is not None
|
|
last = history["date"].max()
|
|
assert tmp_store.get_payload("day", cell["id"], views.day_key(last), token) is not None
|
|
|
|
|
|
def test_warm_cell_never_fetches_upstream(client, monkeypatch):
|
|
import app as appmod
|
|
def boom(cell):
|
|
raise AssertionError("warming must never fetch weather upstream")
|
|
monkeypatch.setattr(climate, "get_history", boom)
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
|
appmod._warm_cell({"id": "1_1", "center_lat": 0.03, "center_lon": 0.03}) # cold: no-op
|