Jinemi/thermograph
4
0
Fork 0
Code Issues 2 Pull requests 9 Projects Releases Packages Wiki Activity Actions

Notifier: fetch a subscribed cell's archive once when it's missing (#94)

Browse source 
A subscription to a city that had never been viewed had no cached ~45-year
archive, so the evaluator skipped it and it never fired. Now when a subscribed
cell has no cached archive, the pass fetches it once via get_history (which caches
it); every later pass reads it from cache and never re-fetches. A per-pass budget
(THERMOGRAPH_NOTIFY_ARCHIVE_FETCHES, default 4) caps how many missing archives one
pass will fetch so a burst of new subscriptions can't exhaust the archive quota,
and a rate-limited fetch just retries on the next pass. The recent/forecast bundle
keeps refreshing on its own hourly cadence.

Add integration tests covering fetch-when-missing and never-refetch-when-cached.
This commit is contained in:
Emi Griffith 2026-07-15 15:18:19 -07:00 committed by GitHub
parent d426fadad1
commit 3dc0a0cf5b
2 changed files with 101 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
notify.py
View file

@ -15,9 +15,11 @@ Two guards keep it quiet and cheap:
skipped entirely (Subscription.last_notified_at), so one alert = at most one
notification per week, as specified.
Quota safety: the loop reads history from the parquet cache only
(climate.load_cached_history) and treats a forecast-fetch failure as "skip this
cell", so it never spends archive quota or dies on an upstream rate limit.
Quota safety: a subscribed cell's ~45-year archive is read from the parquet cache
(climate.load_cached_history) and never re-fetched once present. A cell that has no
cached archive yet is fetched ONCE (budget-capped per pass) so a freshly-subscribed
city starts working; the recent/forecast bundle refreshes on its own hourly cadence.
A forecast-fetch failure just skips the cell, so a pass never dies on a rate limit.
"""
import datetime
import os
@ -39,6 +41,12 @@ INTERVAL = int(os.environ.get("THERMOGRAPH_NOTIFY_INTERVAL", "900")) # 15 min d
SESSION_TTL_SECONDS = int(os.environ.get("THERMOGRAPH_SESSION_TTL_DAYS", "30")) * 86400
LOOKBACK_DAYS = 3 # re-check the last few observed days (dedup makes it safe)
FORECAST_HORIZON_DAYS = 7
# The ~45-year archive is otherwise never fetched from the background loop. When a
# subscribed cell has no cached archive yet, fetch it ONCE (it then stays cached and
# is only ever read afterwards). This caps how many missing archives a single pass
# will fetch, so a burst of new subscriptions can't hammer the strict archive quota
# — the rest are picked up on later passes.
MAX_ARCHIVE_FETCHES_PER_PASS = int(os.environ.get("THERMOGRAPH_NOTIFY_ARCHIVE_FETCHES", "4"))
# Metrics whose low tail also counts as unusual when a subscription is two-sided
# (cold snaps, unusually calm/dry). Precipitation is one-directional (high only).
@ -137,15 +145,33 @@ def _first_trigger(sub: Subscription, rows, history, grade_cache):
return None
def _process_cell(session, cell_id, subs, today, now):
"""Evaluate every subscription on one cell, inserting any new notifications."""
def _process_cell(session, cell_id, subs, today, now, archive_budget):
"""Evaluate every subscription on one cell, inserting any new notifications.
`archive_budget` is a one-element list holding the remaining number of missing
archives this pass may fetch; it's decremented when this cell fetches one.
"""
try:
cell = grid.from_id(cell_id)
except Exception: # noqa: BLE001 - a malformed cell_id shouldn't kill the pass
return 0
history = climate.load_cached_history(cell)
if history is None or history.is_empty():
return 0 # nothing cached yet — never spend archive quota from here
# No cached archive for this subscribed cell. Grab it ONCE (get_history
# fetches + caches the full archive); every later pass reads it from cache
# above and never re-fetches. Budget-gated so a burst of new subscriptions
# can't exhaust the archive quota in a single pass.
if archive_budget[0] <= 0:
return 0
archive_budget[0] -= 1
try:
history, _ = climate.get_history(cell)
except climate.WeatherUnavailable:
return 0 # archive rate-limited — retry on a later pass
except Exception: # noqa: BLE001
return 0
if history is None or history.is_empty():
return 0
try:
recent = climate.get_recent_forecast(cell)
except climate.WeatherUnavailable:
@ -209,6 +235,7 @@ def run_pass() -> int:
now = time.time()
today = datetime.date.today()
created = 0
archive_budget = [MAX_ARCHIVE_FETCHES_PER_PASS] # missing archives to fetch this pass
with sync_session_maker() as session:
subs = session.execute(
select(Subscription).where(Subscription.active.is_(True))
@ -218,7 +245,7 @@ def run_pass() -> int:
by_cell.setdefault(sub.cell_id, []).append(sub)
for cell_id, cell_subs in by_cell.items():
try:
created += _process_cell(session, cell_id, cell_subs, today, now)
created += _process_cell(session, cell_id, cell_subs, today, now, archive_budget)
except Exception: # noqa: BLE001 - one bad cell must not abort the pass
session.rollback()
try:

69
tests/test_notify.py
View file

@ -1,12 +1,18 @@
"""Unit tests for the subscription evaluation engine's pure logic (no DB, no
network): trigger detection against a synthetic climatology and message wording."""
"""Tests for the subscription evaluation engine: trigger detection and message
wording (pure logic), plus an integration check that a full pass fetches a MISSING
archive once but never re-fetches a cached one (against conftest's throwaway DB)."""
import datetime
import types
import uuid
import numpy as np
import polars as pl
import climate
import db
import notify
from models import Notification, Subscription, User
from sqlalchemy import delete, select
def _history() -> pl.DataFrame:
@ -81,3 +87,62 @@ def test_compose_forecast_wording():
"2026-01-02", "tmin", "low", g)
assert title == "Nome: unusually cold night"
assert body.startswith("Forecast for 2026-01-02")
# --- integration: on-demand archive fetch during a full pass -----------------
def _recent_extreme(today):
dates = [today - datetime.timedelta(days=1), today]
return pl.DataFrame({
"date": dates,
"doy": np.array([d.timetuple().tm_yday for d in dates], dtype="int16"),
"tmax": [72.0, 115.0], # today's high is an extreme -> should trigger
"tmin": [48.0, 48.0],
"precip": [0.0, 0.0],
})
def _seed_single_subscription(cell_id="100_200"):
db.Base.metadata.create_all(db.sync_engine)
uid = uuid.uuid4()
with db.sync_session_maker() as s:
s.execute(delete(User)) # clean slate in the throwaway DB
s.commit()
s.add(User(id=uid, email="pass@example.com", hashed_password="x", is_active=True))
s.commit()
s.add(Subscription(user_id=uid, cell_id=cell_id, label="X", lat=1.0, lon=2.0,
threshold=95, metrics=["tmax"], kind="observed", two_sided=False))
s.commit()
return uid
def _user_notifications(uid):
with db.sync_session_maker() as s:
return s.execute(
select(Notification).where(Notification.user_id == uid)
).scalars().all()
def test_missing_archive_is_fetched_once(monkeypatch):
uid = _seed_single_subscription()
fetched = []
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
monkeypatch.setattr(climate, "get_history",
lambda cell: (fetched.append(cell["id"]) or (_history(), {})))
monkeypatch.setattr(climate, "get_recent_forecast",
lambda cell: _recent_extreme(datetime.date.today()))
notify.run_pass()
assert fetched, "a subscribed cell with no cached archive should be fetched once"
assert len(_user_notifications(uid)) == 1
def test_cached_archive_is_never_refetched(monkeypatch):
uid = _seed_single_subscription()
fetched = []
monkeypatch.setattr(climate, "load_cached_history", lambda cell: _history())
monkeypatch.setattr(climate, "get_history",
lambda cell: (fetched.append(cell["id"]) or (_history(), {})))
monkeypatch.setattr(climate, "get_recent_forecast",
lambda cell: _recent_extreme(datetime.date.today()))
notify.run_pass()
assert not fetched, "a cached archive must never be re-fetched by the evaluator"
assert len(_user_notifications(uid)) == 1