Extract payload builders into views.py; load places index at startup (#42)
app.py had grown three co-resident strata: HTTP/caching plumbing, payload
assembly, and search policy. This moves the payload layer — the four
build_* functions, cal_span clamping, hist_end, PAYLOAD_VER, NullRun and
their helpers — into a new views.py with no web dependencies (pure moves,
public names). app.py keeps routing, ETag/derived-store plumbing, and
page serving; migrate.py imports views instead of reaching into app's
privates.
That import previously constructed the whole FastAPI app and — because
places.start_loading() ran at import time — kicked off a background
GeoNames download from an offline batch script. The index load now hangs
off the app's lifespan hook, so it fires when a server starts, not when
the module is imported (tests, migrate).
New tests: cal_span clamping (months-back default, record bounds, 2-year
cap, inversion), builder shapes (grade window ordering, day obs from the
recent bundle, forecast future-only), a regression test that build_day
degrades to climatology when the recent fetch fails, and a subprocess
layering guard that importing views/migrate pulls in neither FastAPI nor
the app module and starts no index download.
2026-07-11 19:43:41 +00:00
|
|
|
"""The payload layer: pure builders, span clamping, and the layering guarantee
|
|
|
|
|
that offline callers (migrate) can use it without dragging in the web stack."""
|
|
|
|
|
import os
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
import climate
|
|
|
|
|
import views
|
|
|
|
|
|
|
|
|
|
CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_views_and_migrate_import_without_the_web_stack():
|
|
|
|
|
"""migrate.py must stay runnable offline: importing the payload layer may
|
|
|
|
|
not construct the FastAPI app or start the places-index download."""
|
|
|
|
|
code = ("import sys; import views, migrate; "
|
|
|
|
|
"assert 'fastapi' not in sys.modules, 'views/migrate pulled in FastAPI'; "
|
|
|
|
|
"assert 'app' not in sys.modules, 'views/migrate imported the web app'; "
|
|
|
|
|
"import places; assert places._load_started is False")
|
|
|
|
|
backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
subprocess.run([sys.executable, "-c", code], cwd=backend, check=True)
|
|
|
|
|
|
|
|
|
|
|
Single-source cache identity; shared fetch preamble and cache flow (#43)
The derived store's key/token formats existed in three places — each
endpoint, api_cell's slice assembly, and migrate.py — where any drift
would silently split the cache (endpoints missing rows the bundle wrote,
migrate materializing rows nobody reads). They are now defined once in
views.py (grade_key/calendar_key/day_key/forecast_key, history_token/
recent_token/day_token) and consumed everywhere, with a pinning test so
a format change is always deliberate.
The four data endpoints shared two copy-pasted sequences, now helpers:
- _fetch_history: history (+ optional recent bundle) fetch with upstream
failures mapped to clean HTTP errors and an empty record to 404.
- _cached_response: If-None-Match 304 / token-valid store replay /
build + persist + serve, with calendar's dont-persist-placeless rule
as an explicit flag.
Each endpoint is now its audit run + identity + a build callback (~10
lines); api_day's hourly-token special case moved into day_token. New
tests: identity format pins, rate-limit 503 parametrized across all five
data routes, prefetch=1 never touching upstream (cold 204, warm
history-only slices), and calendar's placeless-payload retry behavior.
2026-07-11 19:49:15 +00:00
|
|
|
# ---- cache identity --------------------------------------------------------------
|
|
|
|
|
# These formats are shared by the endpoints, the /cell bundle and migrate; pin
|
|
|
|
|
# them so a change is always deliberate (an accidental drift silently strands
|
|
|
|
|
# every existing derived row — change them only alongside a PAYLOAD_VER bump).
|
|
|
|
|
|
|
|
|
|
def test_cache_identity_formats_are_pinned(history):
|
|
|
|
|
t = pd.Timestamp("2026-06-15")
|
|
|
|
|
assert views.grade_key(t, 14, 7) == "2026-06-15:14:7"
|
|
|
|
|
assert views.calendar_key(t, pd.Timestamp("2026-06-20"), 24) == "2026-06-15:2026-06-20:24"
|
|
|
|
|
assert views.day_key(t) == "2026-06-15"
|
|
|
|
|
assert views.forecast_key(t, 7) == "2026-06-15:7"
|
|
|
|
|
assert views.history_token(history) == f"{views.PAYLOAD_VER}:{views.hist_end(history)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
|
|
|
|
|
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
|
|
|
|
|
assert views.recent_token(history, "1_2") == f"{views.history_token(history)}:stamp"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_day_token_expires_hourly_only_beyond_the_archive(history):
|
|
|
|
|
last = pd.Timestamp(history["date"].max()).normalize()
|
|
|
|
|
assert views.day_token(history, last) == views.history_token(history)
|
|
|
|
|
future = views.day_token(history, last + pd.Timedelta(days=1))
|
|
|
|
|
assert future.startswith(views.history_token(history) + ":h")
|
|
|
|
|
|
|
|
|
|
|
Extract payload builders into views.py; load places index at startup (#42)
app.py had grown three co-resident strata: HTTP/caching plumbing, payload
assembly, and search policy. This moves the payload layer — the four
build_* functions, cal_span clamping, hist_end, PAYLOAD_VER, NullRun and
their helpers — into a new views.py with no web dependencies (pure moves,
public names). app.py keeps routing, ETag/derived-store plumbing, and
page serving; migrate.py imports views instead of reaching into app's
privates.
That import previously constructed the whole FastAPI app and — because
places.start_loading() ran at import time — kicked off a background
GeoNames download from an offline batch script. The index load now hangs
off the app's lifespan hook, so it fires when a server starts, not when
the module is imported (tests, migrate).
New tests: cal_span clamping (months-back default, record bounds, 2-year
cap, inversion), builder shapes (grade window ordering, day obs from the
recent bundle, forecast future-only), a regression test that build_day
degrades to climatology when the recent fetch fails, and a subprocess
layering guard that importing views/migrate pulls in neither FastAPI nor
the app module and starts no index download.
2026-07-11 19:43:41 +00:00
|
|
|
# ---- cal_span clamping ---------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
def _hist(start, end):
|
|
|
|
|
df = pd.DataFrame({"date": pd.date_range(start, end, freq="D")})
|
|
|
|
|
return df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cal_span_defaults_to_months_back_same_day_of_month():
|
|
|
|
|
start_ts, end_ts = views.cal_span(_hist("2020-01-01", "2026-06-29"), None, None, 24)
|
|
|
|
|
assert end_ts == pd.Timestamp("2026-06-29")
|
|
|
|
|
assert start_ts == pd.Timestamp("2024-06-29")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cal_span_clamps_to_the_record():
|
|
|
|
|
h = _hist("2025-03-01", "2026-06-29") # record shorter than the 2-year cap
|
|
|
|
|
start_ts, end_ts = views.cal_span(h, "2020-01-01", None, 24)
|
|
|
|
|
assert start_ts == pd.Timestamp("2025-03-01") # can't start before the record
|
|
|
|
|
_, end_ts = views.cal_span(h, None, "2030-01-01", 24)
|
|
|
|
|
assert end_ts == pd.Timestamp("2026-06-29") # nor end past it
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cal_span_caps_at_two_years():
|
|
|
|
|
start_ts, end_ts = views.cal_span(_hist("2018-01-01", "2026-06-29"),
|
|
|
|
|
"2018-01-01", "2026-06-29", 24)
|
|
|
|
|
assert (end_ts - start_ts).days == views.CAL_MAX_SPAN_DAYS - 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cal_span_never_inverts():
|
|
|
|
|
start_ts, end_ts = views.cal_span(_hist("2020-01-01", "2026-06-29"),
|
|
|
|
|
"2026-06-01", "2021-01-01", 24)
|
|
|
|
|
assert start_ts == end_ts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---- builders -------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
def test_build_grade_window_and_shape(history, recent):
|
|
|
|
|
target = pd.Timestamp.today().normalize()
|
|
|
|
|
payload = views.build_grade(CELL, target, 14, history, recent,
|
|
|
|
|
{"cached": True}, "Testville")
|
|
|
|
|
assert payload["target_date"] == target.date().isoformat()
|
|
|
|
|
days = [d["date"] for d in payload["recent"]]
|
|
|
|
|
assert days == sorted(days, reverse=True) # newest first
|
|
|
|
|
assert payload["climatology"]["tmax"] is not None
|
|
|
|
|
assert all(d["dsr"] is not None for d in payload["recent"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_build_day_pulls_future_obs_from_recent(history, recent):
|
|
|
|
|
today = pd.Timestamp.today().normalize()
|
|
|
|
|
payload = views.build_day(CELL, history, today, "Testville", recent=recent)
|
|
|
|
|
assert payload["detail"]["date"] == today.date().isoformat()
|
|
|
|
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is not None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_build_day_survives_recent_fetch_failure(history, monkeypatch):
|
|
|
|
|
def boom(cell):
|
|
|
|
|
raise RuntimeError("upstream down")
|
|
|
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
|
|
|
today = pd.Timestamp.today().normalize()
|
|
|
|
|
payload = views.build_day(CELL, history, today, "Testville")
|
|
|
|
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is None # climatology only
|
|
|
|
|
assert payload["detail"]["metrics"]["tmax"]["ladder"] is not None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_build_forecast_only_future_days(history, recent):
|
|
|
|
|
today = pd.Timestamp.today().normalize()
|
|
|
|
|
payload = views.build_forecast(CELL, 7, history, recent, today, None)
|
|
|
|
|
days = [d["date"] for d in payload["recent"]]
|
|
|
|
|
assert days == sorted(days, reverse=True) # furthest-out first
|
|
|
|
|
assert min(days) > today.date().isoformat()
|
|
|
|
|
assert payload["forecast"] is True
|