diff --git a/notify.py b/notify.py index 46fa58b..a9e6c2e 100644 --- a/notify.py +++ b/notify.py @@ -15,9 +15,11 @@ Two guards keep it quiet and cheap: skipped entirely (Subscription.last_notified_at), so one alert = at most one notification per week, as specified. -Quota safety: the loop reads history from the parquet cache only -(climate.load_cached_history) and treats a forecast-fetch failure as "skip this -cell", so it never spends archive quota or dies on an upstream rate limit. +Quota safety: a subscribed cell's ~45-year archive is read from the parquet cache +(climate.load_cached_history) and never re-fetched once present. A cell that has no +cached archive yet is fetched ONCE (budget-capped per pass) so a freshly-subscribed +city starts working; the recent/forecast bundle refreshes on its own hourly cadence. +A forecast-fetch failure just skips the cell, so a pass never dies on a rate limit. """ import datetime import os @@ -39,6 +41,12 @@ INTERVAL = int(os.environ.get("THERMOGRAPH_NOTIFY_INTERVAL", "900")) # 15 min d SESSION_TTL_SECONDS = int(os.environ.get("THERMOGRAPH_SESSION_TTL_DAYS", "30")) * 86400 LOOKBACK_DAYS = 3 # re-check the last few observed days (dedup makes it safe) FORECAST_HORIZON_DAYS = 7 +# The ~45-year archive is otherwise never fetched from the background loop. When a +# subscribed cell has no cached archive yet, fetch it ONCE (it then stays cached and +# is only ever read afterwards). This caps how many missing archives a single pass +# will fetch, so a burst of new subscriptions can't hammer the strict archive quota +# — the rest are picked up on later passes. +MAX_ARCHIVE_FETCHES_PER_PASS = int(os.environ.get("THERMOGRAPH_NOTIFY_ARCHIVE_FETCHES", "4")) # Metrics whose low tail also counts as unusual when a subscription is two-sided # (cold snaps, unusually calm/dry). Precipitation is one-directional (high only). @@ -137,15 +145,33 @@ def _first_trigger(sub: Subscription, rows, history, grade_cache): return None -def _process_cell(session, cell_id, subs, today, now): - """Evaluate every subscription on one cell, inserting any new notifications.""" +def _process_cell(session, cell_id, subs, today, now, archive_budget): + """Evaluate every subscription on one cell, inserting any new notifications. + + `archive_budget` is a one-element list holding the remaining number of missing + archives this pass may fetch; it's decremented when this cell fetches one. + """ try: cell = grid.from_id(cell_id) except Exception: # noqa: BLE001 - a malformed cell_id shouldn't kill the pass return 0 history = climate.load_cached_history(cell) if history is None or history.is_empty(): - return 0 # nothing cached yet — never spend archive quota from here + # No cached archive for this subscribed cell. Grab it ONCE (get_history + # fetches + caches the full archive); every later pass reads it from cache + # above and never re-fetches. Budget-gated so a burst of new subscriptions + # can't exhaust the archive quota in a single pass. + if archive_budget[0] <= 0: + return 0 + archive_budget[0] -= 1 + try: + history, _ = climate.get_history(cell) + except climate.WeatherUnavailable: + return 0 # archive rate-limited — retry on a later pass + except Exception: # noqa: BLE001 + return 0 + if history is None or history.is_empty(): + return 0 try: recent = climate.get_recent_forecast(cell) except climate.WeatherUnavailable: @@ -209,6 +235,7 @@ def run_pass() -> int: now = time.time() today = datetime.date.today() created = 0 + archive_budget = [MAX_ARCHIVE_FETCHES_PER_PASS] # missing archives to fetch this pass with sync_session_maker() as session: subs = session.execute( select(Subscription).where(Subscription.active.is_(True)) @@ -218,7 +245,7 @@ def run_pass() -> int: by_cell.setdefault(sub.cell_id, []).append(sub) for cell_id, cell_subs in by_cell.items(): try: - created += _process_cell(session, cell_id, cell_subs, today, now) + created += _process_cell(session, cell_id, cell_subs, today, now, archive_budget) except Exception: # noqa: BLE001 - one bad cell must not abort the pass session.rollback() try: diff --git a/tests/test_notify.py b/tests/test_notify.py index 1e7d656..da587b6 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -1,12 +1,18 @@ -"""Unit tests for the subscription evaluation engine's pure logic (no DB, no -network): trigger detection against a synthetic climatology and message wording.""" +"""Tests for the subscription evaluation engine: trigger detection and message +wording (pure logic), plus an integration check that a full pass fetches a MISSING +archive once but never re-fetches a cached one (against conftest's throwaway DB).""" import datetime import types +import uuid import numpy as np import polars as pl +import climate +import db import notify +from models import Notification, Subscription, User +from sqlalchemy import delete, select def _history() -> pl.DataFrame: @@ -81,3 +87,62 @@ def test_compose_forecast_wording(): "2026-01-02", "tmin", "low", g) assert title == "Nome: unusually cold night" assert body.startswith("Forecast for 2026-01-02") + + +# --- integration: on-demand archive fetch during a full pass ----------------- +def _recent_extreme(today): + dates = [today - datetime.timedelta(days=1), today] + return pl.DataFrame({ + "date": dates, + "doy": np.array([d.timetuple().tm_yday for d in dates], dtype="int16"), + "tmax": [72.0, 115.0], # today's high is an extreme -> should trigger + "tmin": [48.0, 48.0], + "precip": [0.0, 0.0], + }) + + +def _seed_single_subscription(cell_id="100_200"): + db.Base.metadata.create_all(db.sync_engine) + uid = uuid.uuid4() + with db.sync_session_maker() as s: + s.execute(delete(User)) # clean slate in the throwaway DB + s.commit() + s.add(User(id=uid, email="pass@example.com", hashed_password="x", is_active=True)) + s.commit() + s.add(Subscription(user_id=uid, cell_id=cell_id, label="X", lat=1.0, lon=2.0, + threshold=95, metrics=["tmax"], kind="observed", two_sided=False)) + s.commit() + return uid + + +def _user_notifications(uid): + with db.sync_session_maker() as s: + return s.execute( + select(Notification).where(Notification.user_id == uid) + ).scalars().all() + + +def test_missing_archive_is_fetched_once(monkeypatch): + uid = _seed_single_subscription() + fetched = [] + monkeypatch.setattr(climate, "load_cached_history", lambda cell: None) + monkeypatch.setattr(climate, "get_history", + lambda cell: (fetched.append(cell["id"]) or (_history(), {}))) + monkeypatch.setattr(climate, "get_recent_forecast", + lambda cell: _recent_extreme(datetime.date.today())) + notify.run_pass() + assert fetched, "a subscribed cell with no cached archive should be fetched once" + assert len(_user_notifications(uid)) == 1 + + +def test_cached_archive_is_never_refetched(monkeypatch): + uid = _seed_single_subscription() + fetched = [] + monkeypatch.setattr(climate, "load_cached_history", lambda cell: _history()) + monkeypatch.setattr(climate, "get_history", + lambda cell: (fetched.append(cell["id"]) or (_history(), {}))) + monkeypatch.setattr(climate, "get_recent_forecast", + lambda cell: _recent_extreme(datetime.date.today())) + notify.run_pass() + assert not fetched, "a cached archive must never be re-fetched by the evaluator" + assert len(_user_notifications(uid)) == 1