From 63ba5ab44e8438c2c4face18c6ead7603b9aa6d0 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 18 Jul 2026 01:58:34 -0700 Subject: [PATCH] Stop the records strip showing stale readings as "right now" (#185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A card read "Shanghai · 30°C · Near Record hot · Low temp · 99th pct" while the Day page for the same cell said 26°C, 66th pct, Above Normal. Both were right about the climatology — the normals matched exactly — but the strip was grading a reading from several days earlier and presenting it as current. Two causes, both mine: - load_cached_recent_forecast deliberately ignores how stale the file is, and _grade_city took the newest row <= today with no age check at all. - Nothing keeps that cache current for the tracked cities. warm_cities.py skips any city whose archive is already cached, so it never re-fetches their forecast bundle, and the notifier only touches subscribed cells. So "zero upstream calls" was satisfied by reading a cache nothing refreshes. Fixes: - Drop readings older than MAX_OBSERVATION_AGE_DAYS (1) instead of ranking them. Cities span a day of timezones, so a current observation can still be dated yesterday in UTC — hence 1 rather than 0. - Top up the stalest cells' recent/forecast each pass, oldest first and capped (THERMOGRAPH_HOMEPAGE_REFRESH, default 40), the same shape as the notifier's archive-fetch budget. This is a real change: the sweep is no longer zero-cost upstream, because it cannot be and still say "right now". Documented in DEPLOY.md, and refresh(fetch=0) keeps a strictly cache-only rebuild. - Rank over the top THERMOGRAPH_HOMEPAGE_CITIES (250) rather than all ~1000. The strip shows ~12 cards; a smaller genuinely-fresh set beats a large stale one. - Show the observation date on a card when it isn't today's, so a yesterday reading can't silently pass for current. --- homepage.py | 91 ++++++++++++++++++++++++++++++++++++++++-- templates/home.html.j2 | 1 + tests/test_homepage.py | 35 +++++++++++++++- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/homepage.py b/homepage.py index 5c3436a..f729039 100644 --- a/homepage.py +++ b/homepage.py @@ -50,6 +50,29 @@ COLD_TAIL_MAX_PCT = 10.0 # The feed is stale (label it "as of " rather than claiming today) beyond this. STALE_AFTER_S = 6 * 3600 +# A reading older than this is not "right now" and is dropped rather than ranked. +# Cities span a day's worth of timezones, so a genuinely current observation can +# still be dated yesterday in UTC — hence 1 rather than 0. +MAX_OBSERVATION_AGE_DAYS = 1 + +# How many cities the strip ranks over. The strip shows ~12 cards; ranking over a +# smaller set that is actually FRESH beats ranking over every cached city when +# most of those are stale (see _refresh_recent below). +CANDIDATE_LIMIT = int(os.environ.get("THERMOGRAPH_HOMEPAGE_CITIES", "250") or 250) + +# Recent/forecast fetches one refresh pass may spend, oldest cache first. +# +# This is the one place the homepage costs upstream quota, and it is not +# optional: nothing else keeps the recent/forecast cache current for the tracked +# cities. warm_cities.py skips any city whose archive is already cached, so it +# never re-fetches their forecast bundle, and the notifier only touches cells +# somebody has subscribed to. Reading that cache without refreshing it is what +# made the strip present days-old readings as "right now". +# +# One fetch covers a whole cell's recent+forecast window. Set to 0 to disable +# (the strip then only ranks cities something else happened to refresh). +MAX_RF_REFRESH_PER_PASS = int(os.environ.get("THERMOGRAPH_HOMEPAGE_REFRESH", "40") or 0) + # Ranking looks at the two metrics whose percentile people read as "how unusual # was today" — the same pair grade_day's own `departure` score uses. RANK_METRICS = ("tmax", "tmin") @@ -64,6 +87,14 @@ def _seasonal_window_label(d: datetime.date) -> str: return f"{part}-{d.strftime('%B')}" +def _short_date(date_s: str) -> str: + """'2026-07-17' -> 'Jul 17'.""" + try: + return datetime.date.fromisoformat(date_s).strftime("%b %-d") + except ValueError: + return date_s + + def _pct_label(pct: float) -> str: """A percentile as an ordinal, clamped to 1..99. @@ -102,6 +133,16 @@ def _grade_city(city: dict, today: datetime.date) -> dict | None: if observed.is_empty(): return None row = observed.row(observed.height - 1, named=True) + + # The newest cached row can be days old — the cache is never refreshed for a + # city nobody visits. Grading it anyway is what put a reading from three days + # ago under a heading that says "right now", disagreeing with the Day page + # for the very same cell. + row_date = row["date"] + if hasattr(row_date, "toordinal"): + if (today.toordinal() - row_date.toordinal()) > MAX_OBSERVATION_AGE_DAYS: + return None + obs = {k: row[k] for k in OBS_COLS if k in row} try: @@ -141,6 +182,9 @@ def _grade_city(city: dict, today: datetime.date) -> dict | None: "lon": round(city["lon"], 4), "cell_id": cell["id"], "date": date_s, + # Shown on the card when the reading isn't today's, so a yesterday-in-UTC + # observation never silently passes for "right now". + "date_label": None if date_s == today.isoformat() else _short_date(date_s), "metric": metric, "metric_label": _METRIC_LABEL.get(metric, metric), "window_label": _seasonal_window_label( @@ -171,7 +215,7 @@ def build(limit: int | None = None) -> dict: graded: list[dict] = [] considered = 0 - for city in cities.all_cities()[: limit or None]: + for city in cities.all_cities()[: limit or CANDIDATE_LIMIT]: row = _grade_city(city, today) if row is None: continue @@ -209,10 +253,48 @@ def build(limit: int | None = None) -> dict: } -def refresh(limit: int | None = None) -> dict: +def _refresh_recent(limit: int | None = None, budget: int = 0) -> int: + """Top up the recent/forecast cache for the stalest candidate cities. + + Bounded and oldest-first, so each pass advances the set a little and no pass + can burst the forecast quota — the same shape as the notifier's archive-fetch + budget. Returns how many cells were refreshed. + """ + if budget <= 0: + return 0 + candidates = [] + for city in cities.all_cities()[: limit or CANDIDATE_LIMIT]: + cell = grid.snap(city["lat"], city["lon"]) + # Only cities whose archive is already warm can be graded at all, so + # there is no point refreshing a forecast we can't compare to anything. + if climate.load_cached_history(cell) is None: + continue + candidates.append((climate.recent_stamp(cell["id"]), cell)) + candidates.sort(key=lambda t: t[0]) + + done = 0 + for _, cell in candidates[:budget]: + try: + climate.get_recent_forecast(cell) + done += 1 + except Exception: # noqa: BLE001 - one failed cell must not stop the pass + continue + return done + + +def refresh(limit: int | None = None, fetch: int | None = None) -> dict: """Rebuild the feed and write it atomically, so a concurrent reader never - sees a half-written file.""" + sees a half-written file. + + ``fetch`` caps the recent/forecast top-ups this pass may spend; pass 0 for a + strictly cache-only rebuild (what the tests do). + """ + refreshed = _refresh_recent( + limit=limit, + budget=MAX_RF_REFRESH_PER_PASS if fetch is None else fetch, + ) feed = build(limit=limit) + feed["refreshed"] = refreshed path = os.path.abspath(FEED_PATH) os.makedirs(os.path.dirname(path), exist_ok=True) fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp") @@ -254,6 +336,7 @@ if __name__ == "__main__": import sys n = int(sys.argv[1]) if len(sys.argv) > 1 else None out = refresh(limit=n) - print(f"graded {out['considered']} cached cities, " + print(f"refreshed {out.get('refreshed', 0)} forecast caches; " + f"graded {out['considered']} cached cities, " f"{len(out['ranked'])} ranked, top: " f"{out['ranked'][0]['display'] if out['ranked'] else '—'}") diff --git a/templates/home.html.j2 b/templates/home.html.j2 index b9d2564..0b4ca39 100644 --- a/templates/home.html.j2 +++ b/templates/home.html.j2 @@ -122,6 +122,7 @@ {{ r.metric_label }} · {{ r.pct_label }} pct {%- if r.normal is not none %} · normal {{ temp(r.normal) }}{% endif %} + {%- if r.date_label %} · {{ r.date_label }}{% endif %} diff --git a/tests/test_homepage.py b/tests/test_homepage.py index 7952b74..dfe19de 100644 --- a/tests/test_homepage.py +++ b/tests/test_homepage.py @@ -127,13 +127,46 @@ def test_load_survives_missing_and_corrupt_file(monkeypatch, tmp_path): def test_refresh_writes_atomically(monkeypatch, tmp_path): monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "homepage.json")) monkeypatch.setattr(climate, "load_cached_history", lambda cell: None) - homepage.refresh(limit=3) + homepage.refresh(limit=3, fetch=0) written = json.loads((tmp_path / "homepage.json").read_text()) assert written["ranked"] == [] # No temp files left behind. assert [p.name for p in tmp_path.iterdir()] == ["homepage.json"] +def test_refresh_can_be_strictly_cache_only(monkeypatch, tmp_path): + """fetch=0 must spend no upstream request at all — that is the contract the + test suite and any offline rebuild rely on.""" + monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "homepage.json")) + + def boom(*a, **k): + raise AssertionError("refresh(fetch=0) must not fetch") + + monkeypatch.setattr(climate, "get_recent_forecast", boom) + monkeypatch.setattr(climate, "get_history", boom) + feed = homepage.refresh(limit=3, fetch=0) + assert feed["refreshed"] == 0 + + +def test_stale_observations_are_not_ranked(monkeypatch, history, recent): + """A reading older than a day is not "right now". The recent/forecast cache + goes stale for any city nobody visits, and grading it anyway is what showed a + days-old reading under the "Unusual right now" heading, disagreeing with the + Day page for the same cell.""" + import polars as pl + + stale = recent.clone() + old = datetime.date.today() - datetime.timedelta(days=5) + stale = stale.with_columns( + (pl.col("date") - pl.duration(days=(stale["date"].max() - old).days)).alias("date") + ) + monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone()) + monkeypatch.setattr(climate, "load_cached_recent_forecast", lambda cell: stale.clone()) + feed = homepage.build(limit=5) + assert feed["ranked"] == [], "stale readings must not be presented as current" + assert feed["considered"] == 0 + + def test_staleness(monkeypatch): today = datetime.date.today().isoformat() assert not homepage.is_stale({"date": today, "generated_at": time.time()})