From a4fa72639fdd71a7ebcb422b32319d16c40cc55e Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:30:38 +0000 Subject: [PATCH] Add cheap, stable content-page cache token (#47) Content pages keyed cache validity on the full-history hist_end, invalidating ~1-2x/day and paying a 45yr load even to compute the token. Add content_token(cell_id) = PAYLOAD_VER:CONTENT_VER:max_date, backed by an indexed MAX(date) (climate_store.history_max_date / climate.history_max_date), so it survives intra-day top-ups and needs no full-history load. --- backend/api/payloads.py | 24 ++++++++++++++++++++ backend/data/climate.py | 24 ++++++++++++++++++++ backend/data/climate_store.py | 23 +++++++++++++++++++ backend/tests/api/test_payloads.py | 28 ++++++++++++++++++++++++ backend/tests/data/test_climate_store.py | 20 +++++++++++++++++ 5 files changed, 119 insertions(+) diff --git a/backend/api/payloads.py b/backend/api/payloads.py index 420ab50..adcf9aa 100644 --- a/backend/api/payloads.py +++ b/backend/api/payloads.py @@ -73,6 +73,30 @@ def history_token(history) -> str: return f"{PAYLOAD_VER}:{hist_end(history)}" +# The content-shape version for the SEO content pages (/climate/[/month| +# /records]). Kept separate from PAYLOAD_VER so a content-only shape change need +# not orphan every other kind's cache, and vice-versa. Bump on change. +CONTENT_VER = "c1" + + +def content_token(cell_id: str) -> str: + """Validity for the SEO content-page payloads — cheap AND stable. + + Unlike history_token, this never loads the ~45-year archive: it keys on the + cell's newest archived DATE (climate.history_max_date — an indexed + ``MAX(date)`` on Postgres, a single-column read on the parquet backend), so it + survives the hourly tail top-ups (which only refresh intra-day freshness) and + turns over only when the archive's last day genuinely advances (≈1×/day). That + keeps content pages ≤1 day stale — acceptable for SEO — while sparing every + request the full-history load the old token forced even on a cache hit. + Fail-soft: a store/DB error yields a 'none' bucket rather than raising.""" + try: + max_date = climate.history_max_date(cell_id) + except Exception: # noqa: BLE001 - token computation must never fail a request + max_date = None + return f"{PAYLOAD_VER}:{CONTENT_VER}:{max_date or 'none'}" + + def recent_token(history, cell_id: str) -> str: """Validity for payloads that also grade the hourly recent/forecast bundle.""" return f"{history_token(history)}:{climate.recent_stamp(cell_id)}" diff --git a/backend/data/climate.py b/backend/data/climate.py index 0a148ff..fcf4ff1 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -862,6 +862,30 @@ def recent_stamp(cell_id: str) -> int: return 0 +def history_max_date(cell_id: str) -> "str | None": + """ISO date (YYYY-MM-DD) of the cell's newest cached archived day, or None when + nothing is cached — read WITHOUT loading the full multi-decade history. + + Backs the content-page cache token (api/payloads.content_token). On Postgres + it's an indexed ``MAX(date)`` over climate_history; on the parquet backend it's + the max of the cached file's date column (a single columnar read via + ``scan_parquet``, not a full frame load). Fail-soft: any error reads as None.""" + if climate_store.is_postgres(): + return climate_store.history_max_date(cell_id) + path = _cache_path(cell_id) + if not os.path.exists(path): + return None + try: + val = pl.scan_parquet(path).select(pl.col("date").max()).collect().item() + if val is None: + return None + if isinstance(val, datetime.datetime): # older files stored date as Datetime + val = val.date() + return val.isoformat() + except Exception: # noqa: BLE001 - a corrupt/absent cache reads as no max date + return None + + def get_recent_forecast(cell: dict) -> pl.DataFrame: """Recent observations + forward forecast, with humidity as absolute humidity (g/m³). Thin wrapper over the raw loader (see below).""" diff --git a/backend/data/climate_store.py b/backend/data/climate_store.py index 19c883c..b6951fd 100644 --- a/backend/data/climate_store.py +++ b/backend/data/climate_store.py @@ -188,6 +188,29 @@ def recent_synced_at(cell_id: str) -> float: return 0.0 +def history_max_date(cell_id: str) -> "str | None": + """The ISO date (YYYY-MM-DD) of the cell's newest archived day, or None when + there is no cached history (or Postgres is off/unreachable). + + Backs the content-page cache token (api/payloads.content_token), so it must be + CHEAP — an indexed ``MAX(date)`` over the ``(cell_id, date)`` primary key (see + the 0002 migration), never a full-history load. Fail-soft: any error reads as + None, so the caller falls back to a 'none' bucket rather than raising.""" + try: + with _conn() as conn: + if conn is None: + return None + row = conn.execute( + "SELECT MAX(date) FROM climate_history WHERE cell_id = %s", + (cell_id,), + ).fetchone() + if not row or row[0] is None: + return None + return row[0].isoformat() + except Exception: # noqa: BLE001 + return None + + def cols_version(cell_id: str) -> int: """The stored schema version for a cell's history; 0 if absent.""" try: diff --git a/backend/tests/api/test_payloads.py b/backend/tests/api/test_payloads.py index 29ab153..0fdf428 100644 --- a/backend/tests/api/test_payloads.py +++ b/backend/tests/api/test_payloads.py @@ -39,6 +39,34 @@ def test_cache_identity_formats_are_pinned(history): assert payloads.history_token(history) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(history)}" +def test_content_token_is_stable_across_tail_topups(monkeypatch): + """The content token keys on the archive's newest DATE, not its row count/mtime, + so hourly tail top-ups that don't change the max date leave it unchanged — while + a genuinely-advanced last day turns it over.""" + date_box = {"v": "2026-06-15"} + monkeypatch.setattr(climate, "history_max_date", lambda cid: date_box["v"]) + first = payloads.content_token("1_2") + assert first == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-15" + # Simulated hourly top-ups: same max date -> identical token every time. + assert payloads.content_token("1_2") == first + assert payloads.content_token("1_2") == first + # The archive's last day advances -> the token turns over. + date_box["v"] = "2026-06-16" + assert payloads.content_token("1_2") != first + assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-16" + + +def test_content_token_fail_soft_buckets_to_none(monkeypatch): + """A store/DB error (or an uncached cell) never raises — it buckets to 'none'.""" + monkeypatch.setattr(climate, "history_max_date", lambda cid: None) + assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none" + + def boom(cid): + raise RuntimeError("store down") + monkeypatch.setattr(climate, "history_max_date", boom) + assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none" + + def test_recent_token_composes_history_and_stamp(history, monkeypatch): monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp") assert payloads.recent_token(history, "1_2") == f"{payloads.history_token(history)}:stamp" diff --git a/backend/tests/data/test_climate_store.py b/backend/tests/data/test_climate_store.py index a6fbcd3..09eaa49 100644 --- a/backend/tests/data/test_climate_store.py +++ b/backend/tests/data/test_climate_store.py @@ -69,6 +69,17 @@ def test_recent_backend_roundtrips_through_parquet(monkeypatch, tmp_path): assert hit is not None and hit[0].height == 3 +def test_history_max_date_reads_parquet_tail_without_full_load(monkeypatch, tmp_path): + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + cell_id = "9_10" + assert climate.history_max_date(cell_id) is None # nothing cached -> None + climate._write_history_backed(cell_id, _hist_frame(n=5)) # dates 1995-01-01..05 + assert climate.history_max_date(cell_id) == "1995-01-05" + # A tail top-up that appends a newer day advances the max date. + climate._write_history_backed(cell_id, _hist_frame(n=7)) + assert climate.history_max_date(cell_id) == "1995-01-07" + + # --- gated Postgres integration --------------------------------------------- @pytest.fixture @@ -140,6 +151,15 @@ def test_pg_recent_stamp_advances_only_on_write(pg_store): assert pg_store.history_synced_at("1_2") == 6000.0 +def test_pg_history_max_date(pg_store): + assert pg_store.history_max_date("1_2") is None # no rows -> None + pg_store.write_history("1_2", _hist_frame(n=5), 1000.0) # dates 1995-01-01..05 + assert pg_store.history_max_date("1_2") == "1995-01-05" + # Another cell's rows must not leak into this cell's max. + pg_store.write_history("3_4", _hist_frame(n=9), 1000.0) + assert pg_store.history_max_date("1_2") == "1995-01-05" + + def test_pg_cols_version_gate(pg_store): pg_store.write_history("1_2", _hist_frame(n=2), 1000.0) assert pg_store.read_history("1_2") is not None