85810c21c5
All checks were successful
PR build (required check) / changes (pull_request) Successful in 8s
secrets-guard / encrypted (pull_request) Successful in 6s
PR build (required check) / build-frontend (pull_request) Has been skipped
shell-lint / shellcheck (pull_request) Successful in 10s
PR build (required check) / validate-observability (pull_request) Has been skipped
PR build (required check) / build-backend (pull_request) Successful in 1m2s
PR build (required check) / gate (pull_request) Successful in 3s
Content SEO pages keyed derived-payload cache validity on
history_token = PAYLOAD_VER:hist_end, computed from the fully-loaded
~45-year archive. Every hourly tail top-up advanced hist_end and
invalidated the whole content cache for a cell, and computing the token
at all required loading the full history first — so even cache hits paid
the full load.
Add content_token(cell_id) = PAYLOAD_VER:CONTENT_VER:max_date, keyed on
the archive's newest DATE read cheaply without loading history:
climate_store.history_max_date does an indexed MAX(date) over the
(cell_id, date) PK on Postgres; climate.history_max_date dispatches to it
or to a single-column scan of the cached parquet on the dev backend. The
token survives intra-day top-ups and turns over only when the last
archived day advances (~1x/day), keeping content pages <=1 day stale.
Fail-soft: a store/DB error buckets to 'none' rather than raising.
CONTENT_VER ("c1") is a separate content-shape version so a content-only
change need not orphan every other kind's cache.
214 lines
10 KiB
Python
214 lines
10 KiB
Python
"""The payload layer: pure builders, span clamping, and the layering guarantee
|
|
that offline callers (migrate) can use it without dragging in the web stack."""
|
|
import datetime
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from data import climate
|
|
from api import payloads
|
|
|
|
CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
|
|
|
|
|
|
def test_payloads_and_migrate_import_without_the_web_stack():
|
|
"""migrate.py must stay runnable offline: importing the payload layer may
|
|
not construct the FastAPI app or start the places-index download."""
|
|
code = ("import sys; from api import payloads; import migrate; "
|
|
"assert 'fastapi' not in sys.modules, 'payloads/migrate pulled in FastAPI'; "
|
|
"assert 'web.app' not in sys.modules, 'payloads/migrate imported the web app'; "
|
|
"from data import places; assert places._load_started is False")
|
|
backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
subprocess.run([sys.executable, "-c", code], cwd=backend, check=True)
|
|
|
|
|
|
# ---- cache identity --------------------------------------------------------------
|
|
# These formats are shared by the endpoints, the /cell bundle and migrate; pin
|
|
# them so a change is always deliberate (an accidental drift silently strands
|
|
# every existing derived row — change them only alongside a PAYLOAD_VER bump).
|
|
|
|
def test_cache_identity_formats_are_pinned(history):
|
|
t = datetime.date(2026, 6, 15)
|
|
assert payloads.grade_key(t, 14, 7) == "2026-06-15:14:7"
|
|
assert payloads.calendar_key(t, datetime.date(2026, 6, 20), 24) == "2026-06-15:2026-06-20:24"
|
|
assert payloads.day_key(t) == "2026-06-15"
|
|
assert payloads.forecast_key(t, 7) == "2026-06-15:7"
|
|
assert payloads.history_token(history) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(history)}"
|
|
|
|
|
|
def test_content_token_is_stable_across_tail_topups(monkeypatch):
|
|
"""The content token keys on the archive's newest DATE, not its row count/mtime,
|
|
so hourly tail top-ups that don't change the max date leave it unchanged — while
|
|
a genuinely-advanced last day turns it over."""
|
|
date_box = {"v": "2026-06-15"}
|
|
monkeypatch.setattr(climate, "history_max_date", lambda cid: date_box["v"])
|
|
first = payloads.content_token("1_2")
|
|
assert first == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-15"
|
|
# Simulated hourly top-ups: same max date -> identical token every time.
|
|
assert payloads.content_token("1_2") == first
|
|
assert payloads.content_token("1_2") == first
|
|
# The archive's last day advances -> the token turns over.
|
|
date_box["v"] = "2026-06-16"
|
|
assert payloads.content_token("1_2") != first
|
|
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-16"
|
|
|
|
|
|
def test_content_token_fail_soft_buckets_to_none(monkeypatch):
|
|
"""A store/DB error (or an uncached cell) never raises — it buckets to 'none'."""
|
|
monkeypatch.setattr(climate, "history_max_date", lambda cid: None)
|
|
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
|
|
|
|
def boom(cid):
|
|
raise RuntimeError("store down")
|
|
monkeypatch.setattr(climate, "history_max_date", boom)
|
|
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
|
|
|
|
|
|
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
|
|
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
|
|
assert payloads.recent_token(history, "1_2") == f"{payloads.history_token(history)}:stamp"
|
|
|
|
|
|
def test_day_token_expires_hourly_only_beyond_the_archive(history):
|
|
last = history["date"].max()
|
|
assert payloads.day_token(history, last) == payloads.history_token(history)
|
|
future = payloads.day_token(history, last + datetime.timedelta(days=1))
|
|
assert future.startswith(payloads.history_token(history) + ":h")
|
|
|
|
|
|
# ---- cal_span clamping ---------------------------------------------------------
|
|
|
|
def _hist(start, end):
|
|
# cal_span only reads the record's min/max date, so two rows pin the span.
|
|
return pl.DataFrame({"date": [datetime.date.fromisoformat(start),
|
|
datetime.date.fromisoformat(end)]})
|
|
|
|
|
|
def test_cal_span_defaults_to_months_back_same_day_of_month():
|
|
start_ts, end_ts = payloads.cal_span(_hist("2020-01-01", "2026-06-29"), None, None, 24)
|
|
assert end_ts == datetime.date(2026, 6, 29)
|
|
assert start_ts == datetime.date(2024, 6, 29)
|
|
|
|
|
|
def test_cal_span_clamps_to_the_record():
|
|
h = _hist("2025-03-01", "2026-06-29") # record shorter than the 2-year cap
|
|
start_ts, end_ts = payloads.cal_span(h, "2020-01-01", None, 24)
|
|
assert start_ts == datetime.date(2025, 3, 1) # can't start before the record
|
|
_, end_ts = payloads.cal_span(h, None, "2030-01-01", 24)
|
|
assert end_ts == datetime.date(2026, 6, 29) # nor end past it
|
|
|
|
|
|
def test_cal_span_caps_at_two_years():
|
|
start_ts, end_ts = payloads.cal_span(_hist("2018-01-01", "2026-06-29"),
|
|
"2018-01-01", "2026-06-29", 24)
|
|
assert (end_ts - start_ts).days == payloads.CAL_MAX_SPAN_DAYS - 1
|
|
|
|
|
|
def test_cal_span_never_inverts():
|
|
start_ts, end_ts = payloads.cal_span(_hist("2020-01-01", "2026-06-29"),
|
|
"2026-06-01", "2021-01-01", 24)
|
|
assert start_ts == end_ts
|
|
|
|
|
|
def test_months_before_clamps_month_end_and_leap():
|
|
"""Calendar-aware month subtraction (replaces pandas DateOffset): land on the
|
|
same day-of-month, clamped to the target month's last valid day."""
|
|
assert payloads._months_before(datetime.date(2026, 3, 31), 1) == datetime.date(2026, 2, 28)
|
|
assert payloads._months_before(datetime.date(2024, 3, 31), 1) == datetime.date(2024, 2, 29)
|
|
assert payloads._months_before(datetime.date(2026, 1, 15), 14) == datetime.date(2024, 11, 15)
|
|
|
|
|
|
def test_attach_dry_streaks_prefers_the_fresher_source_on_shared_dates():
|
|
"""De-dup keeps the recent/forecast row over the archive one for a shared date
|
|
(concat archive-first, keep='last') — the fresher precip drives the streak."""
|
|
d = datetime.date(2026, 1, 1)
|
|
hist = pl.DataFrame({"date": [d], "precip": [1.0]}) # archive: it rained (streak resets)
|
|
rec = pl.DataFrame({"date": [d], "precip": [0.0]}) # fresher: dry (streak counts)
|
|
graded = [{"date": d.isoformat()}]
|
|
payloads._attach_dry_streaks(graded, hist, rec) # archive first, recent last
|
|
assert graded[0]["dsr"] == 1 # recent (dry) won
|
|
|
|
|
|
# ---- builders -------------------------------------------------------------------
|
|
|
|
def test_build_grade_window_and_shape(history, recent):
|
|
target = datetime.date.today()
|
|
payload = payloads.build_grade(CELL, target, 14, history, recent,
|
|
{"cached": True}, "Testville")
|
|
assert payload["target_date"] == target.isoformat()
|
|
days = [d["date"] for d in payload["recent"]]
|
|
assert days == sorted(days, reverse=True) # newest first
|
|
assert payload["climatology"]["tmax"] is not None
|
|
assert all(d["dsr"] is not None for d in payload["recent"])
|
|
|
|
|
|
def test_build_day_pulls_future_obs_from_recent(history, recent):
|
|
today = datetime.date.today()
|
|
payload = payloads.build_day(CELL, history, today, "Testville", recent=recent)
|
|
assert payload["detail"]["date"] == today.isoformat()
|
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is not None
|
|
|
|
|
|
def test_build_day_survives_recent_fetch_failure(history, monkeypatch):
|
|
def boom(cell):
|
|
raise RuntimeError("upstream down")
|
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
|
today = datetime.date.today()
|
|
payload = payloads.build_day(CELL, history, today, "Testville")
|
|
assert payload["detail"]["metrics"]["tmax"]["obs"] is None # climatology only
|
|
assert payload["detail"]["metrics"]["tmax"]["ladder"] is not None
|
|
|
|
|
|
def test_build_forecast_only_future_days(history, recent):
|
|
today = datetime.date.today()
|
|
payload = payloads.build_forecast(CELL, 7, history, recent, today, None)
|
|
days = [d["date"] for d in payload["recent"]]
|
|
assert days == sorted(days, reverse=True) # furthest-out first
|
|
assert min(days) > today.isoformat()
|
|
assert payload["forecast"] is True
|
|
|
|
|
|
# ---- build_score ---------------------------------------------------------------
|
|
|
|
def _full_history(years=45, seed=5):
|
|
"""A 45-year all-metric record — enough span for the climate score (the shared
|
|
20-year `history` fixture is intentionally below MIN_BASELINE_YEARS)."""
|
|
import numpy as np
|
|
end = datetime.date(2026, 7, 11)
|
|
start = datetime.date(end.year - years, end.month, end.day)
|
|
dates = [start + datetime.timedelta(days=i) for i in range((end - start).days + 1)]
|
|
n = len(dates)
|
|
rng = np.random.default_rng(seed)
|
|
doy = np.array([d.timetuple().tm_yday for d in dates])
|
|
tmax = 55 + 30 * np.sin((doy - 100) / 366.0 * 2 * np.pi) + rng.normal(0, 8, n)
|
|
return pl.DataFrame({
|
|
"date": dates, "tmax": np.round(tmax, 1), "tmin": np.round(tmax - 15, 1),
|
|
"feels": np.round(tmax + 1, 1), "humid": np.round(np.clip(12 + rng.normal(0, 3, n), 1, None), 1),
|
|
"wetbulb": np.round(tmax - 12, 1), "wind": np.round(np.clip(8 + rng.normal(0, 3, n), 0, None), 1),
|
|
"gust": np.round(np.clip(16 + rng.normal(0, 5, n), 0, None), 1),
|
|
"precip": np.where(rng.random(n) < 0.3, 0.2, 0.0),
|
|
}).with_columns(pl.col("date").dt.ordinal_day().cast(pl.Int16).alias("doy"))
|
|
|
|
|
|
def test_build_score_shape():
|
|
hist = _full_history()
|
|
payload = payloads.build_score(CELL, hist, "Testville")
|
|
assert payload["api_version"] == "v2"
|
|
assert payload["cell"] == CELL and payload["place"] == "Testville"
|
|
assert payload["latest"] == payloads.hist_end(hist)
|
|
s = payload["scores"]
|
|
assert set(s["slices"]) == {"annual", "djf", "mam", "jja", "son"}
|
|
ann = s["slices"]["annual"]
|
|
assert ann["overall"]["score"] is not None
|
|
assert ann["metrics"]["tmax"]["score"] is not None
|
|
assert ann["metrics"]["wetbulb"]["score"] is not None # derived metric flows through
|
|
|
|
|
|
def test_score_key_is_the_version():
|
|
assert payloads.score_key() == payloads.SCORE_VER
|
|
# Score payloads are history-only, so they ride the plain history token.
|
|
hist = _full_history()
|
|
assert payloads.history_token(hist) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(hist)}"
|