Pre-warm the SEO content derived-store off-request #49
5 changed files with 119 additions and 0 deletions
|
|
@ -73,6 +73,30 @@ def history_token(history) -> str:
|
|||
return f"{PAYLOAD_VER}:{hist_end(history)}"
|
||||
|
||||
|
||||
# The content-shape version for the SEO content pages (/climate/<city>[/month|
|
||||
# /records]). Kept separate from PAYLOAD_VER so a content-only shape change need
|
||||
# not orphan every other kind's cache, and vice-versa. Bump on change.
|
||||
CONTENT_VER = "c1"
|
||||
|
||||
|
||||
def content_token(cell_id: str) -> str:
|
||||
"""Validity for the SEO content-page payloads — cheap AND stable.
|
||||
|
||||
Unlike history_token, this never loads the ~45-year archive: it keys on the
|
||||
cell's newest archived DATE (climate.history_max_date — an indexed
|
||||
``MAX(date)`` on Postgres, a single-column read on the parquet backend), so it
|
||||
survives the hourly tail top-ups (which only refresh intra-day freshness) and
|
||||
turns over only when the archive's last day genuinely advances (≈1×/day). That
|
||||
keeps content pages ≤1 day stale — acceptable for SEO — while sparing every
|
||||
request the full-history load the old token forced even on a cache hit.
|
||||
Fail-soft: a store/DB error yields a 'none' bucket rather than raising."""
|
||||
try:
|
||||
max_date = climate.history_max_date(cell_id)
|
||||
except Exception: # noqa: BLE001 - token computation must never fail a request
|
||||
max_date = None
|
||||
return f"{PAYLOAD_VER}:{CONTENT_VER}:{max_date or 'none'}"
|
||||
|
||||
|
||||
def recent_token(history, cell_id: str) -> str:
|
||||
"""Validity for payloads that also grade the hourly recent/forecast bundle."""
|
||||
return f"{history_token(history)}:{climate.recent_stamp(cell_id)}"
|
||||
|
|
|
|||
|
|
@ -862,6 +862,30 @@ def recent_stamp(cell_id: str) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def history_max_date(cell_id: str) -> "str | None":
|
||||
"""ISO date (YYYY-MM-DD) of the cell's newest cached archived day, or None when
|
||||
nothing is cached — read WITHOUT loading the full multi-decade history.
|
||||
|
||||
Backs the content-page cache token (api/payloads.content_token). On Postgres
|
||||
it's an indexed ``MAX(date)`` over climate_history; on the parquet backend it's
|
||||
the max of the cached file's date column (a single columnar read via
|
||||
``scan_parquet``, not a full frame load). Fail-soft: any error reads as None."""
|
||||
if climate_store.is_postgres():
|
||||
return climate_store.history_max_date(cell_id)
|
||||
path = _cache_path(cell_id)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
val = pl.scan_parquet(path).select(pl.col("date").max()).collect().item()
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, datetime.datetime): # older files stored date as Datetime
|
||||
val = val.date()
|
||||
return val.isoformat()
|
||||
except Exception: # noqa: BLE001 - a corrupt/absent cache reads as no max date
|
||||
return None
|
||||
|
||||
|
||||
def get_recent_forecast(cell: dict) -> pl.DataFrame:
|
||||
"""Recent observations + forward forecast, with humidity as absolute humidity
|
||||
(g/m³). Thin wrapper over the raw loader (see below)."""
|
||||
|
|
|
|||
|
|
@ -188,6 +188,29 @@ def recent_synced_at(cell_id: str) -> float:
|
|||
return 0.0
|
||||
|
||||
|
||||
def history_max_date(cell_id: str) -> "str | None":
|
||||
"""The ISO date (YYYY-MM-DD) of the cell's newest archived day, or None when
|
||||
there is no cached history (or Postgres is off/unreachable).
|
||||
|
||||
Backs the content-page cache token (api/payloads.content_token), so it must be
|
||||
CHEAP — an indexed ``MAX(date)`` over the ``(cell_id, date)`` primary key (see
|
||||
the 0002 migration), never a full-history load. Fail-soft: any error reads as
|
||||
None, so the caller falls back to a 'none' bucket rather than raising."""
|
||||
try:
|
||||
with _conn() as conn:
|
||||
if conn is None:
|
||||
return None
|
||||
row = conn.execute(
|
||||
"SELECT MAX(date) FROM climate_history WHERE cell_id = %s",
|
||||
(cell_id,),
|
||||
).fetchone()
|
||||
if not row or row[0] is None:
|
||||
return None
|
||||
return row[0].isoformat()
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
def cols_version(cell_id: str) -> int:
|
||||
"""The stored schema version for a cell's history; 0 if absent."""
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -39,6 +39,34 @@ def test_cache_identity_formats_are_pinned(history):
|
|||
assert payloads.history_token(history) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(history)}"
|
||||
|
||||
|
||||
def test_content_token_is_stable_across_tail_topups(monkeypatch):
|
||||
"""The content token keys on the archive's newest DATE, not its row count/mtime,
|
||||
so hourly tail top-ups that don't change the max date leave it unchanged — while
|
||||
a genuinely-advanced last day turns it over."""
|
||||
date_box = {"v": "2026-06-15"}
|
||||
monkeypatch.setattr(climate, "history_max_date", lambda cid: date_box["v"])
|
||||
first = payloads.content_token("1_2")
|
||||
assert first == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-15"
|
||||
# Simulated hourly top-ups: same max date -> identical token every time.
|
||||
assert payloads.content_token("1_2") == first
|
||||
assert payloads.content_token("1_2") == first
|
||||
# The archive's last day advances -> the token turns over.
|
||||
date_box["v"] = "2026-06-16"
|
||||
assert payloads.content_token("1_2") != first
|
||||
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-16"
|
||||
|
||||
|
||||
def test_content_token_fail_soft_buckets_to_none(monkeypatch):
|
||||
"""A store/DB error (or an uncached cell) never raises — it buckets to 'none'."""
|
||||
monkeypatch.setattr(climate, "history_max_date", lambda cid: None)
|
||||
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
|
||||
|
||||
def boom(cid):
|
||||
raise RuntimeError("store down")
|
||||
monkeypatch.setattr(climate, "history_max_date", boom)
|
||||
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
|
||||
|
||||
|
||||
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
|
||||
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
|
||||
assert payloads.recent_token(history, "1_2") == f"{payloads.history_token(history)}:stamp"
|
||||
|
|
|
|||
|
|
@ -69,6 +69,17 @@ def test_recent_backend_roundtrips_through_parquet(monkeypatch, tmp_path):
|
|||
assert hit is not None and hit[0].height == 3
|
||||
|
||||
|
||||
def test_history_max_date_reads_parquet_tail_without_full_load(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
|
||||
cell_id = "9_10"
|
||||
assert climate.history_max_date(cell_id) is None # nothing cached -> None
|
||||
climate._write_history_backed(cell_id, _hist_frame(n=5)) # dates 1995-01-01..05
|
||||
assert climate.history_max_date(cell_id) == "1995-01-05"
|
||||
# A tail top-up that appends a newer day advances the max date.
|
||||
climate._write_history_backed(cell_id, _hist_frame(n=7))
|
||||
assert climate.history_max_date(cell_id) == "1995-01-07"
|
||||
|
||||
|
||||
# --- gated Postgres integration ---------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -140,6 +151,15 @@ def test_pg_recent_stamp_advances_only_on_write(pg_store):
|
|||
assert pg_store.history_synced_at("1_2") == 6000.0
|
||||
|
||||
|
||||
def test_pg_history_max_date(pg_store):
|
||||
assert pg_store.history_max_date("1_2") is None # no rows -> None
|
||||
pg_store.write_history("1_2", _hist_frame(n=5), 1000.0) # dates 1995-01-01..05
|
||||
assert pg_store.history_max_date("1_2") == "1995-01-05"
|
||||
# Another cell's rows must not leak into this cell's max.
|
||||
pg_store.write_history("3_4", _hist_frame(n=9), 1000.0)
|
||||
assert pg_store.history_max_date("1_2") == "1995-01-05"
|
||||
|
||||
|
||||
def test_pg_cols_version_gate(pg_store):
|
||||
pg_store.write_history("1_2", _hist_frame(n=2), 1000.0)
|
||||
assert pg_store.read_history("1_2") is not None
|
||||
|
|
|
|||
Loading…
Reference in a new issue