Add cheap, stable content-page cache token (#47)
Some checks failed
Deploy backend to LAN dev server / deploy (push) Blocked by required conditions
secrets-guard / encrypted (push) Successful in 7s
shell-lint / shellcheck (push) Successful in 7s
Build + push backend image (Forgejo registry) / build-push (push) Has been cancelled
Deploy backend to LAN dev server / build (push) Has been cancelled

Content pages keyed cache validity on the full-history hist_end, invalidating ~1-2x/day and paying a 45yr load even to compute the token. Add content_token(cell_id) = PAYLOAD_VER:CONTENT_VER:max_date, backed by an indexed MAX(date) (climate_store.history_max_date / climate.history_max_date), so it survives intra-day top-ups and needs no full-history load.
This commit is contained in:
emi 2026-07-24 19:30:38 +00:00
parent 864d38d772
commit a4fa72639f
5 changed files with 119 additions and 0 deletions

View file

@ -73,6 +73,30 @@ def history_token(history) -> str:
return f"{PAYLOAD_VER}:{hist_end(history)}"
# The content-shape version for the SEO content pages (/climate/<city>[/month|
# /records]). Kept separate from PAYLOAD_VER so a content-only shape change need
# not orphan every other kind's cache, and vice-versa. Bump on change.
CONTENT_VER = "c1"
def content_token(cell_id: str) -> str:
"""Validity for the SEO content-page payloads — cheap AND stable.
Unlike history_token, this never loads the ~45-year archive: it keys on the
cell's newest archived DATE (climate.history_max_date — an indexed
``MAX(date)`` on Postgres, a single-column read on the parquet backend), so it
survives the hourly tail top-ups (which only refresh intra-day freshness) and
turns over only when the archive's last day genuinely advances (≈1×/day). That
keeps content pages 1 day stale acceptable for SEO while sparing every
request the full-history load the old token forced even on a cache hit.
Fail-soft: a store/DB error yields a 'none' bucket rather than raising."""
try:
max_date = climate.history_max_date(cell_id)
except Exception: # noqa: BLE001 - token computation must never fail a request
max_date = None
return f"{PAYLOAD_VER}:{CONTENT_VER}:{max_date or 'none'}"
def recent_token(history, cell_id: str) -> str:
"""Validity for payloads that also grade the hourly recent/forecast bundle."""
return f"{history_token(history)}:{climate.recent_stamp(cell_id)}"

View file

@ -862,6 +862,30 @@ def recent_stamp(cell_id: str) -> int:
return 0
def history_max_date(cell_id: str) -> "str | None":
"""ISO date (YYYY-MM-DD) of the cell's newest cached archived day, or None when
nothing is cached read WITHOUT loading the full multi-decade history.
Backs the content-page cache token (api/payloads.content_token). On Postgres
it's an indexed ``MAX(date)`` over climate_history; on the parquet backend it's
the max of the cached file's date column (a single columnar read via
``scan_parquet``, not a full frame load). Fail-soft: any error reads as None."""
if climate_store.is_postgres():
return climate_store.history_max_date(cell_id)
path = _cache_path(cell_id)
if not os.path.exists(path):
return None
try:
val = pl.scan_parquet(path).select(pl.col("date").max()).collect().item()
if val is None:
return None
if isinstance(val, datetime.datetime): # older files stored date as Datetime
val = val.date()
return val.isoformat()
except Exception: # noqa: BLE001 - a corrupt/absent cache reads as no max date
return None
def get_recent_forecast(cell: dict) -> pl.DataFrame:
"""Recent observations + forward forecast, with humidity as absolute humidity
(g/). Thin wrapper over the raw loader (see below)."""

View file

@ -188,6 +188,29 @@ def recent_synced_at(cell_id: str) -> float:
return 0.0
def history_max_date(cell_id: str) -> "str | None":
"""The ISO date (YYYY-MM-DD) of the cell's newest archived day, or None when
there is no cached history (or Postgres is off/unreachable).
Backs the content-page cache token (api/payloads.content_token), so it must be
CHEAP an indexed ``MAX(date)`` over the ``(cell_id, date)`` primary key (see
the 0002 migration), never a full-history load. Fail-soft: any error reads as
None, so the caller falls back to a 'none' bucket rather than raising."""
try:
with _conn() as conn:
if conn is None:
return None
row = conn.execute(
"SELECT MAX(date) FROM climate_history WHERE cell_id = %s",
(cell_id,),
).fetchone()
if not row or row[0] is None:
return None
return row[0].isoformat()
except Exception: # noqa: BLE001
return None
def cols_version(cell_id: str) -> int:
"""The stored schema version for a cell's history; 0 if absent."""
try:

View file

@ -39,6 +39,34 @@ def test_cache_identity_formats_are_pinned(history):
assert payloads.history_token(history) == f"{payloads.PAYLOAD_VER}:{payloads.hist_end(history)}"
def test_content_token_is_stable_across_tail_topups(monkeypatch):
"""The content token keys on the archive's newest DATE, not its row count/mtime,
so hourly tail top-ups that don't change the max date leave it unchanged — while
a genuinely-advanced last day turns it over."""
date_box = {"v": "2026-06-15"}
monkeypatch.setattr(climate, "history_max_date", lambda cid: date_box["v"])
first = payloads.content_token("1_2")
assert first == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-15"
# Simulated hourly top-ups: same max date -> identical token every time.
assert payloads.content_token("1_2") == first
assert payloads.content_token("1_2") == first
# The archive's last day advances -> the token turns over.
date_box["v"] = "2026-06-16"
assert payloads.content_token("1_2") != first
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:2026-06-16"
def test_content_token_fail_soft_buckets_to_none(monkeypatch):
"""A store/DB error (or an uncached cell) never raises — it buckets to 'none'."""
monkeypatch.setattr(climate, "history_max_date", lambda cid: None)
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
def boom(cid):
raise RuntimeError("store down")
monkeypatch.setattr(climate, "history_max_date", boom)
assert payloads.content_token("1_2") == f"{payloads.PAYLOAD_VER}:{payloads.CONTENT_VER}:none"
def test_recent_token_composes_history_and_stamp(history, monkeypatch):
monkeypatch.setattr(climate, "recent_stamp", lambda cid: "stamp")
assert payloads.recent_token(history, "1_2") == f"{payloads.history_token(history)}:stamp"

View file

@ -69,6 +69,17 @@ def test_recent_backend_roundtrips_through_parquet(monkeypatch, tmp_path):
assert hit is not None and hit[0].height == 3
def test_history_max_date_reads_parquet_tail_without_full_load(monkeypatch, tmp_path):
monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
cell_id = "9_10"
assert climate.history_max_date(cell_id) is None # nothing cached -> None
climate._write_history_backed(cell_id, _hist_frame(n=5)) # dates 1995-01-01..05
assert climate.history_max_date(cell_id) == "1995-01-05"
# A tail top-up that appends a newer day advances the max date.
climate._write_history_backed(cell_id, _hist_frame(n=7))
assert climate.history_max_date(cell_id) == "1995-01-07"
# --- gated Postgres integration ---------------------------------------------
@pytest.fixture
@ -140,6 +151,15 @@ def test_pg_recent_stamp_advances_only_on_write(pg_store):
assert pg_store.history_synced_at("1_2") == 6000.0
def test_pg_history_max_date(pg_store):
assert pg_store.history_max_date("1_2") is None # no rows -> None
pg_store.write_history("1_2", _hist_frame(n=5), 1000.0) # dates 1995-01-01..05
assert pg_store.history_max_date("1_2") == "1995-01-05"
# Another cell's rows must not leak into this cell's max.
pg_store.write_history("3_4", _hist_frame(n=9), 1000.0)
assert pg_store.history_max_date("1_2") == "1995-01-05"
def test_pg_cols_version_gate(pg_store):
pg_store.write_history("1_2", _hist_frame(n=2), 1000.0)
assert pg_store.read_history("1_2") is not None