diff --git a/backend/data/climate.py b/backend/data/climate.py index 2bb1b2f..d7c1357 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -75,9 +75,10 @@ ARCHIVE_URL = os.environ.get("THERMOGRAPH_ARCHIVE_URL", # instance serves the same 0.1° resolution. Archive requests only, not forecast. ARCHIVE_MODEL = "era5_seamless" FORECAST_URL = "https://api.open-meteo.com/v1/forecast" -# Backup archive when Open-Meteo is unavailable (e.g. its daily rate limit). NASA -# POWER is free + keyless, global, daily from 1981. It lacks gusts + apparent temp, -# so gusts read as unavailable and "feels like" is computed from heat index/chill. +# NASA POWER is RETIRED from every serving path (2026-07-24): its MERRA-2 +# record ran ~1.3°F off the ERA5 lake/archive with location-dependent sign and +# carried no gusts, so serving it beside ERA5 sources skewed percentiles. Kept +# only as drift_check.py's comparison source. NASA_POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point" NASA_START = "19810101" NASA_FILL = -900.0 # POWER's missing-value sentinel is ~-999 @@ -710,15 +711,16 @@ def _read_history_cache(path): def _fetch_history_tail(cell: dict, start_date: str, end_date: str) -> pl.DataFrame: - """Fetch a recent history range to top up the cached tail — NASA POWER primary, - Open-Meteo archive fallback (mirroring the full-history source order). The - Open-Meteo fallback is skipped while it's in a rate-limit cooldown.""" - try: - return _fetch_history_nasa(cell, start=start_date, end=end_date) - except Exception: # noqa: BLE001 - NASA unavailable; try Open-Meteo unless it's cooling down - if time.time() < _archive_cooldown_until: - raise - return _fetch_history_range(cell, start_date, end_date) + """Fetch a recent history range to top up the cached tail — the Open-Meteo + archive, which is the same ERA5 family as the lake record it extends, so + topped-up days grade against the climatology without a source seam (NASA's + MERRA-2 days ran ~1.3°F off ERA5, sign varying by location). Raises while + the archive is in a rate-limit cooldown; the top-up is best-effort and + retries next interval.""" + if time.time() < _archive_cooldown_until: + raise WeatherUnavailable(limit_message(_archive_limit_daily), + daily=_archive_limit_daily) + return _fetch_history_range(cell, start_date, end_date) def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame: @@ -802,12 +804,14 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: def _serve_stale(): return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None} - # Fetch. The ERA5 lake is first when configured (true ERA5 with measured - # gusts, served from our own bucket — no third-party API in the path); - # unconfigured or missing-point cells fall through at zero cost. NASA - # POWER is next (keyless, independent of Open-Meteo; its missing gusts - # are filled from Meteostat inside _fetch_history_nasa), then Open-Meteo - # — still guarded by its rate-limit cooldown. Every source must return a + # Fetch. The ERA5 lake is first (true ERA5 with measured gusts, served + # from our own bucket — no third-party API in the path); unconfigured + # or missing-point cells fall through at zero cost. The backup is the + # Open-Meteo archive — the same ERA5 family as the lake, so a + # backup-sourced record grades consistently with lake-sourced + # neighbors — still guarded by its rate-limit cooldown. NASA POWER is + # retired from serving (its MERRA-2 record ran ~1.3°F off ERA5 with no + # gusts; kept only for drift_check). Every source must return a # plausibly-full span before being accepted and cached as a complete # record (a short/partial response is rejected rather than held # indefinitely). @@ -818,16 +822,8 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: if fetched.height >= MIN_ARCHIVE_DAYS: df = fetched source = "era5-lake" - except Exception: # noqa: BLE001 - lake unconfigured/miss; fall through to NASA + except Exception: # noqa: BLE001 - lake unconfigured/miss; fall through to Open-Meteo pass - if df is None: - try: - fetched = _fetch_history_nasa(cell) - if fetched.height >= MIN_ARCHIVE_DAYS: - df = fetched - source = "nasa-power" - except Exception: # noqa: BLE001 - primary unavailable; fall through to Open-Meteo - pass if df is None and time.time() >= _archive_cooldown_until: try: fetched = _fetch_history(cell) @@ -920,18 +916,21 @@ def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None": def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: - """Recent observations + forward forecast WITHOUT Open-Meteo: the recent observed - window from a NASA POWER range (measured), the forward days from MET Norway, - merged by date. Gusts — which neither source carries — are filled from Meteostat: - measured for the observed days, estimated from wind for the forecast days.""" + """Fallback recent+forecast bundle without the Open-Meteo *forecast* API: + the recent observed window from the Open-Meteo ARCHIVE (an independent + quota, and the same ERA5 family as the history record so the graded days + sit on a consistent baseline), the forward days from MET Norway with + Meteostat-estimated gusts. The archive lags a few days more than a live + feed, so this degraded path serves a slightly shorter observed window — + honest and consistent beats fresh and skewed.""" today = datetime.date.today() start = (today - datetime.timedelta(days=RECENT_PAST_DAYS)).isoformat() - end = (today - datetime.timedelta(days=RECENT_END_LAG_DAYS)).isoformat() - past = _fetch_history_nasa(cell, start=start, end=end) # measured + Meteostat gusts + end = (today - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).isoformat() + past = _fetch_history_range(cell, start, end) # ERA5 days, gusts included fwd = meteostat.fill_gusts( cell["center_lat"], cell["center_lon"], _fetch_forecast_metno(cell)) - # Forecast wins on any overlapping day (freshest model value for today); the NASA - # observed days fill the recent past MET Norway lacks. + # Forecast wins on any overlapping day (freshest model value); the archive + # days fill the recent past MET Norway lacks. return (pl.concat([past, fwd], how="diagonal_relaxed") .unique(subset="date", keep="last", maintain_order=True) .sort("date")) @@ -983,10 +982,12 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: def _load_recent_forecast(cell: dict) -> pl.DataFrame: """Recent observations AND the forward forecast for a cell. - The primary source is keyless and Open-Meteo-free: a NASA POWER recent-past range - plus the MET Norway forward forecast (see _fetch_recent_forecast). The Open-Meteo - forecast API is the fallback (skipped while it's in its own rate-limit cooldown, - see _note_forecast_rate_limit), then a stale cache. Cached per cell for + The primary source is the Open-Meteo forecast API (recent past + forward in + one call, gusts included, ERA5-consistent with the history record — see + _fetch_recent_forecast_om), skipped while it's in its own rate-limit + cooldown (_note_forecast_rate_limit). The fallback is Open-Meteo-forecast- + free: an archive recent-past range plus MET Norway forward days + (_fetch_recent_forecast). Then a stale cache. Cached per cell for FORECAST_TTL_HOURS so it follows model updates without per-hour upstream IO. """ cell_id = cell["id"] @@ -997,22 +998,23 @@ def _load_recent_forecast(cell: dict) -> pl.DataFrame: return _with_doy(cached) df = None - try: - df = _fetch_recent_forecast(cell) # NASA recent + MET forward (primary) - except Exception: # noqa: BLE001 - keyless primary unavailable; try Open-Meteo - pass - forecast_error = None - if df is None and time.time() >= _forecast_cooldown_until: - # Fall back to the Open-Meteo forecast API, skipped while it's in its own - # rate-limit cooldown so a brownout doesn't retry-storm the endpoint. + if time.time() >= _forecast_cooldown_until: try: - df = _fetch_recent_forecast_om(cell) + df = _fetch_recent_forecast_om(cell) # primary: one call, gusts, ERA5-consistent except Exception as e: # noqa: BLE001 if is_rate_limit(e): _note_forecast_rate_limit(e) forecast_error = e + if df is None: + # Fallback: archive past + MET forward — no Open-Meteo forecast quota. + try: + df = _fetch_recent_forecast(cell) + forecast_error = None + except Exception: # noqa: BLE001 - fallback also down; stale cache next + pass + if df is None: # Last resort: serve the stale cache if we have one, WITHOUT rewriting it, so # recent_stamp stays old and the next request retries upstream first (mirrors diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index de90ff3..9ca1932 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -165,12 +165,34 @@ def test_metno_to_frame_drops_days_without_diurnal_coverage(): assert row["tmax"] != row["tmin"] -def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_path): - """The recent/forecast bundle is built from a NASA POWER recent-past range plus the - MET Norway forward forecast, merged by date (Open-Meteo not consulted).""" +def test_recent_forecast_om_is_primary(monkeypatch, tmp_path): + """The Open-Meteo forecast bundle is the primary recent+forecast source + (one call, gusts included, ERA5-consistent); the archive+MET composite is + not consulted when it succeeds.""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0) cell = {"id": "rf_primary", "center_lat": 47.6, "center_lon": -122.3} + om = climate._to_frame(_om_daily(3)) + monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om) + def _fallback_should_not_run(c): raise AssertionError("composite fallback should not run") + monkeypatch.setattr(climate, "_fetch_recent_forecast", _fallback_should_not_run) + + df = climate._load_recent_forecast(cell) + assert df.height == om.height + + +def test_recent_forecast_fallback_merges_archive_past_and_metno_forward(monkeypatch, tmp_path): + """When the Open-Meteo forecast API is down, the bundle is built from an + Open-Meteo ARCHIVE recent-past range plus the MET Norway forward forecast — + NASA POWER is never consulted.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3} + + def _om_forecast_down(c): raise RuntimeError("forecast API down") + monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_forecast_down) past = climate._finalize_approximated(pl.DataFrame({ "date": [datetime.date(2026, 7, 10), datetime.date(2026, 7, 11)], "tmax": [70.0, 72.0], "tmin": [50.0, 51.0], "precip": [0.0, 0.1], @@ -181,11 +203,11 @@ def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_pat "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.0], "wind": [3.0, 4.0], "humid": [40.0, 45.0], })) - monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c, start, end: past) + monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: past) monkeypatch.setattr(climate, "_fetch_forecast_metno", lambda c: fwd) monkeypatch.setattr(climate.meteostat, "fill_gusts", lambda lat, lon, df: df) - def _om_should_not_run(c): raise AssertionError("Open-Meteo forecast should not run") - monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run) + def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) df = climate._load_recent_forecast(cell) assert df["date"].to_list() == [ @@ -193,21 +215,6 @@ def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_pat datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)] -def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path): - """When the NASA + MET Norway primary fails, the Open-Meteo forecast API serves.""" - monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) - monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0) - cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3} - - def _primary_down(c): raise RuntimeError("NASA + MET Norway down") - monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down) - om = climate._to_frame(_om_daily(3)) - monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om) - - df = climate._load_recent_forecast(cell) - assert df.height == om.height - - def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): """The Open-Meteo fallback must not grade the cell's in-progress local day: its daily high/low is only a partial aggregate of the hours elapsed so far (a cool @@ -402,9 +409,9 @@ def test_lake_is_first_history_source(monkeypatch, tmp_path): assert df.height == full.height -def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path): - """An unconfigured lake (or a point outside it) costs nothing: NASA serves - exactly as before the lake existed.""" +def test_lake_miss_falls_through_to_open_meteo(monkeypatch, tmp_path): + """An unconfigured lake (or a point outside it) costs nothing: the + Open-Meteo archive serves, and NASA — retired — is never consulted.""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) cell = {"id": "lake_miss", "center_lat": 47.6, "center_lon": -122.3} @@ -413,89 +420,60 @@ def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path): raise climate.era5lake.LakeUnavailable("no lake configured") monkeypatch.setattr(climate.era5lake, "fetch_history_lake", _no_lake) full = _full_history_frame() - monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full) - - df, meta = climate._load_history(cell) - assert meta["source"] == "nasa-power" - assert df.height == full.height - - -def test_nasa_is_primary_history_source(monkeypatch, tmp_path): - """NASA POWER is the primary history source: a full NASA frame is accepted and - Open-Meteo is never consulted.""" - monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) - monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) - cell = {"id": "nasa_primary", "center_lat": 47.6, "center_lon": -122.3} - - full = _full_history_frame() - monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full) - def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called") - monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run) - - df, meta = climate._load_history(cell) - assert meta["source"] == "nasa-power" - assert df.height == full.height - - -def test_falls_back_to_open_meteo_when_nasa_unavailable(monkeypatch, tmp_path): - """When NASA POWER fails, the load falls back to the Open-Meteo archive.""" - monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) - monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) - cell = {"id": "om_fallback", "center_lat": 47.6, "center_lon": -122.3} - - full = _full_history_frame() - def _nasa_down(c): raise RuntimeError("NASA POWER outage") - monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down) monkeypatch.setattr(climate, "_fetch_history", lambda c: full) + def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) df, meta = climate._load_history(cell) assert meta["source"] == "open-meteo" assert df.height == full.height -def test_short_nasa_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path): - """A too-short NASA response (a partial/unavailable point) is rejected rather than - cached as a complete record, and Open-Meteo serves instead.""" +def test_short_lake_record_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path): + """A too-short lake frame (a thin point that slipped past the client's own + gate) is rejected rather than cached as a complete record; the Open-Meteo + archive serves instead.""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) - cell = {"id": "short_nasa", "center_lat": 47.6, "center_lon": -122.3} + cell = {"id": "short_lake", "center_lat": 47.6, "center_lon": -122.3} short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold full = _full_history_frame() # >= MIN_ARCHIVE_DAYS assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height - monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: short) + monkeypatch.setattr(climate.era5lake, "fetch_history_lake", + lambda c, start=None: short) monkeypatch.setattr(climate, "_fetch_history", lambda c: full) df, meta = climate._load_history(cell) - assert meta["source"] == "open-meteo" # rejected the short NASA response + assert meta["source"] == "open-meteo" # rejected the short lake frame assert df.height == full.height -def test_history_tail_prefers_nasa(monkeypatch): - """The tail top-up fetches NASA POWER first; Open-Meteo's range fetch is not - touched when NASA succeeds.""" - cell = {"center_lat": 1.0, "center_lon": 2.0} - sentinel = object() - monkeypatch.setattr(climate, "_fetch_history_nasa", - lambda c, start=None, end=None: sentinel) - def _om_should_not_run(c, s, e): raise AssertionError("Open-Meteo tail should not run") - monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run) - - assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel - - -def test_history_tail_falls_back_to_open_meteo(monkeypatch): - """When NASA fails and Open-Meteo isn't cooling down, the tail falls back to the - Open-Meteo archive range.""" +def test_history_tail_uses_open_meteo_archive(monkeypatch): + """The tail top-up is an Open-Meteo archive range — ERA5-consistent with + the record it extends; NASA is retired and never consulted.""" monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) cell = {"center_lat": 1.0, "center_lon": 2.0} - def _nasa_down(c, start=None, end=None): raise RuntimeError("NASA down") - monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down) sentinel = object() monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: sentinel) + def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel + +def test_history_tail_raises_during_archive_cooldown(monkeypatch): + """While the archive is rate-limit cooling, the top-up raises (best-effort; + retried next interval) instead of hammering the endpoint.""" + import time as _time + monkeypatch.setattr(climate, "_archive_cooldown_until", _time.time() + 60) + cell = {"center_lat": 1.0, "center_lon": 2.0} + def _om_should_not_run(c, s, e): raise AssertionError("archive is cooling down") + monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run) + + with pytest.raises(climate.WeatherUnavailable): + climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") + # --- forecast cooldown (mirrors the archive path's, tracked separately) ----- def test_forecast_cooldown_skips_the_open_meteo_fallback(monkeypatch, tmp_path):