Retire NASA POWER from every serving path (#72)

backend/data/climate.py

@@ -75,9 +75,10 @@ ARCHIVE_URL = os.environ.get("THERMOGRAPH_ARCHIVE_URL",
 # instance serves the same 0.1° resolution. Archive requests only, not forecast.
 ARCHIVE_MODEL = "era5_seamless"
 FORECAST_URL = "https://api.open-meteo.com/v1/forecast"
-# Backup archive when Open-Meteo is unavailable (e.g. its daily rate limit). NASA
-# POWER is free + keyless, global, daily from 1981. It lacks gusts + apparent temp,
-# so gusts read as unavailable and "feels like" is computed from heat index/chill.
+# NASA POWER is RETIRED from every serving path (2026-07-24): its MERRA-2
+# record ran ~1.3°F off the ERA5 lake/archive with location-dependent sign and
+# carried no gusts, so serving it beside ERA5 sources skewed percentiles. Kept
+# only as drift_check.py's comparison source.
 NASA_POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point"
 NASA_START = "19810101"
 NASA_FILL = -900.0   # POWER's missing-value sentinel is ~-999
@@ -710,15 +711,16 @@ def _read_history_cache(path):
 
 
 def _fetch_history_tail(cell: dict, start_date: str, end_date: str) -> pl.DataFrame:
-    """Fetch a recent history range to top up the cached tail — NASA POWER primary,
-    Open-Meteo archive fallback (mirroring the full-history source order). The
-    Open-Meteo fallback is skipped while it's in a rate-limit cooldown."""
-    try:
-        return _fetch_history_nasa(cell, start=start_date, end=end_date)
-    except Exception:  # noqa: BLE001 - NASA unavailable; try Open-Meteo unless it's cooling down
-        if time.time() < _archive_cooldown_until:
-            raise
-        return _fetch_history_range(cell, start_date, end_date)
+    """Fetch a recent history range to top up the cached tail — the Open-Meteo
+    archive, which is the same ERA5 family as the lake record it extends, so
+    topped-up days grade against the climatology without a source seam (NASA's
+    MERRA-2 days ran ~1.3°F off ERA5, sign varying by location). Raises while
+    the archive is in a rate-limit cooldown; the top-up is best-effort and
+    retries next interval."""
+    if time.time() < _archive_cooldown_until:
+        raise WeatherUnavailable(limit_message(_archive_limit_daily),
+                                 daily=_archive_limit_daily)
+    return _fetch_history_range(cell, start_date, end_date)
 
 
 def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame:
@@ -802,12 +804,14 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]:
         def _serve_stale():
             return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None}
 
-        # Fetch. The ERA5 lake is first when configured (true ERA5 with measured
-        # gusts, served from our own bucket — no third-party API in the path);
-        # unconfigured or missing-point cells fall through at zero cost. NASA
-        # POWER is next (keyless, independent of Open-Meteo; its missing gusts
-        # are filled from Meteostat inside _fetch_history_nasa), then Open-Meteo
-        # — still guarded by its rate-limit cooldown. Every source must return a
+        # Fetch. The ERA5 lake is first (true ERA5 with measured gusts, served
+        # from our own bucket — no third-party API in the path); unconfigured
+        # or missing-point cells fall through at zero cost. The backup is the
+        # Open-Meteo archive — the same ERA5 family as the lake, so a
+        # backup-sourced record grades consistently with lake-sourced
+        # neighbors — still guarded by its rate-limit cooldown. NASA POWER is
+        # retired from serving (its MERRA-2 record ran ~1.3°F off ERA5 with no
+        # gusts; kept only for drift_check). Every source must return a
         # plausibly-full span before being accepted and cached as a complete
         # record (a short/partial response is rejected rather than held
         # indefinitely).
@@ -818,16 +822,8 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]:
             if fetched.height >= MIN_ARCHIVE_DAYS:
                 df = fetched
                 source = "era5-lake"
-        except Exception:  # noqa: BLE001 - lake unconfigured/miss; fall through to NASA
+        except Exception:  # noqa: BLE001 - lake unconfigured/miss; fall through to Open-Meteo
             pass
-        if df is None:
-            try:
-                fetched = _fetch_history_nasa(cell)
-                if fetched.height >= MIN_ARCHIVE_DAYS:
-                    df = fetched
-                    source = "nasa-power"
-            except Exception:  # noqa: BLE001 - primary unavailable; fall through to Open-Meteo
-                pass
         if df is None and time.time() >= _archive_cooldown_until:
             try:
                 fetched = _fetch_history(cell)
@@ -920,18 +916,21 @@ def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None":
 
 
 def _fetch_recent_forecast(cell: dict) -> pl.DataFrame:
-    """Recent observations + forward forecast WITHOUT Open-Meteo: the recent observed
-    window from a NASA POWER range (measured), the forward days from MET Norway,
-    merged by date. Gusts — which neither source carries — are filled from Meteostat:
-    measured for the observed days, estimated from wind for the forecast days."""
+    """Fallback recent+forecast bundle without the Open-Meteo *forecast* API:
+    the recent observed window from the Open-Meteo ARCHIVE (an independent
+    quota, and the same ERA5 family as the history record so the graded days
+    sit on a consistent baseline), the forward days from MET Norway with
+    Meteostat-estimated gusts. The archive lags a few days more than a live
+    feed, so this degraded path serves a slightly shorter observed window —
+    honest and consistent beats fresh and skewed."""
     today = datetime.date.today()
     start = (today - datetime.timedelta(days=RECENT_PAST_DAYS)).isoformat()
-    end = (today - datetime.timedelta(days=RECENT_END_LAG_DAYS)).isoformat()
-    past = _fetch_history_nasa(cell, start=start, end=end)   # measured + Meteostat gusts
+    end = (today - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).isoformat()
+    past = _fetch_history_range(cell, start, end)            # ERA5 days, gusts included
     fwd = meteostat.fill_gusts(
         cell["center_lat"], cell["center_lon"], _fetch_forecast_metno(cell))
-    # Forecast wins on any overlapping day (freshest model value for today); the NASA
-    # observed days fill the recent past MET Norway lacks.
+    # Forecast wins on any overlapping day (freshest model value); the archive
+    # days fill the recent past MET Norway lacks.
     return (pl.concat([past, fwd], how="diagonal_relaxed")
             .unique(subset="date", keep="last", maintain_order=True)
             .sort("date"))
@@ -983,10 +982,12 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame:
 def _load_recent_forecast(cell: dict) -> pl.DataFrame:
     """Recent observations AND the forward forecast for a cell.
 
-    The primary source is keyless and Open-Meteo-free: a NASA POWER recent-past range
-    plus the MET Norway forward forecast (see _fetch_recent_forecast). The Open-Meteo
-    forecast API is the fallback (skipped while it's in its own rate-limit cooldown,
-    see _note_forecast_rate_limit), then a stale cache. Cached per cell for
+    The primary source is the Open-Meteo forecast API (recent past + forward in
+    one call, gusts included, ERA5-consistent with the history record — see
+    _fetch_recent_forecast_om), skipped while it's in its own rate-limit
+    cooldown (_note_forecast_rate_limit). The fallback is Open-Meteo-forecast-
+    free: an archive recent-past range plus MET Norway forward days
+    (_fetch_recent_forecast). Then a stale cache. Cached per cell for
     FORECAST_TTL_HOURS so it follows model updates without per-hour upstream IO.
     """
     cell_id = cell["id"]
@@ -997,22 +998,23 @@ def _load_recent_forecast(cell: dict) -> pl.DataFrame:
             return _with_doy(cached)
 
     df = None
-    try:
-        df = _fetch_recent_forecast(cell)          # NASA recent + MET forward (primary)
-    except Exception:  # noqa: BLE001 - keyless primary unavailable; try Open-Meteo
-        pass
-
     forecast_error = None
-    if df is None and time.time() >= _forecast_cooldown_until:
-        # Fall back to the Open-Meteo forecast API, skipped while it's in its own
-        # rate-limit cooldown so a brownout doesn't retry-storm the endpoint.
+    if time.time() >= _forecast_cooldown_until:
         try:
-            df = _fetch_recent_forecast_om(cell)
+            df = _fetch_recent_forecast_om(cell)   # primary: one call, gusts, ERA5-consistent
         except Exception as e:  # noqa: BLE001
             if is_rate_limit(e):
                 _note_forecast_rate_limit(e)
             forecast_error = e
 
+    if df is None:
+        # Fallback: archive past + MET forward — no Open-Meteo forecast quota.
+        try:
+            df = _fetch_recent_forecast(cell)
+            forecast_error = None
+        except Exception:  # noqa: BLE001 - fallback also down; stale cache next
+            pass
+
     if df is None:
         # Last resort: serve the stale cache if we have one, WITHOUT rewriting it, so
         # recent_stamp stays old and the next request retries upstream first (mirrors

backend/tests/data/test_climate.py

@@ -165,12 +165,34 @@ def test_metno_to_frame_drops_days_without_diurnal_coverage():
     assert row["tmax"] != row["tmin"]
 
 
-def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_path):
-    """The recent/forecast bundle is built from a NASA POWER recent-past range plus the
-    MET Norway forward forecast, merged by date (Open-Meteo not consulted)."""
+def test_recent_forecast_om_is_primary(monkeypatch, tmp_path):
+    """The Open-Meteo forecast bundle is the primary recent+forecast source
+    (one call, gusts included, ERA5-consistent); the archive+MET composite is
+    not consulted when it succeeds."""
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
+    monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0)
     cell = {"id": "rf_primary", "center_lat": 47.6, "center_lon": -122.3}
 
+    om = climate._to_frame(_om_daily(3))
+    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om)
+    def _fallback_should_not_run(c): raise AssertionError("composite fallback should not run")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast", _fallback_should_not_run)
+
+    df = climate._load_recent_forecast(cell)
+    assert df.height == om.height
+
+
+def test_recent_forecast_fallback_merges_archive_past_and_metno_forward(monkeypatch, tmp_path):
+    """When the Open-Meteo forecast API is down, the bundle is built from an
+    Open-Meteo ARCHIVE recent-past range plus the MET Norway forward forecast —
+    NASA POWER is never consulted."""
+    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
+    monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0)
+    monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
+    cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3}
+
+    def _om_forecast_down(c): raise RuntimeError("forecast API down")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_forecast_down)
     past = climate._finalize_approximated(pl.DataFrame({
         "date": [datetime.date(2026, 7, 10), datetime.date(2026, 7, 11)],
         "tmax": [70.0, 72.0], "tmin": [50.0, 51.0], "precip": [0.0, 0.1],
@@ -181,11 +203,11 @@ def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_pat
         "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.0],
         "wind": [3.0, 4.0], "humid": [40.0, 45.0],
     }))
-    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c, start, end: past)
+    monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: past)
     monkeypatch.setattr(climate, "_fetch_forecast_metno", lambda c: fwd)
     monkeypatch.setattr(climate.meteostat, "fill_gusts", lambda lat, lon, df: df)
-    def _om_should_not_run(c): raise AssertionError("Open-Meteo forecast should not run")
-    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run)
+    def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving")
+    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run)
 
     df = climate._load_recent_forecast(cell)
     assert df["date"].to_list() == [
@@ -193,21 +215,6 @@ def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_pat
         datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)]
 
 
-def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path):
-    """When the NASA + MET Norway primary fails, the Open-Meteo forecast API serves."""
-    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
-    monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0)
-    cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3}
-
-    def _primary_down(c): raise RuntimeError("NASA + MET Norway down")
-    monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down)
-    om = climate._to_frame(_om_daily(3))
-    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om)
-
-    df = climate._load_recent_forecast(cell)
-    assert df.height == om.height
-
-
 def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch):
     """The Open-Meteo fallback must not grade the cell's in-progress local day: its
     daily high/low is only a partial aggregate of the hours elapsed so far (a cool
@@ -402,9 +409,9 @@ def test_lake_is_first_history_source(monkeypatch, tmp_path):
     assert df.height == full.height
 
 
-def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path):
-    """An unconfigured lake (or a point outside it) costs nothing: NASA serves
-    exactly as before the lake existed."""
+def test_lake_miss_falls_through_to_open_meteo(monkeypatch, tmp_path):
+    """An unconfigured lake (or a point outside it) costs nothing: the
+    Open-Meteo archive serves, and NASA — retired — is never consulted."""
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
     monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
     cell = {"id": "lake_miss", "center_lat": 47.6, "center_lon": -122.3}
@@ -413,89 +420,60 @@ def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path):
         raise climate.era5lake.LakeUnavailable("no lake configured")
     monkeypatch.setattr(climate.era5lake, "fetch_history_lake", _no_lake)
     full = _full_history_frame()
-    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full)
-
-    df, meta = climate._load_history(cell)
-    assert meta["source"] == "nasa-power"
-    assert df.height == full.height
-
-
-def test_nasa_is_primary_history_source(monkeypatch, tmp_path):
-    """NASA POWER is the primary history source: a full NASA frame is accepted and
-    Open-Meteo is never consulted."""
-    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
-    monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
-    cell = {"id": "nasa_primary", "center_lat": 47.6, "center_lon": -122.3}
-
-    full = _full_history_frame()
-    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full)
-    def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called")
-    monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run)
-
-    df, meta = climate._load_history(cell)
-    assert meta["source"] == "nasa-power"
-    assert df.height == full.height
-
-
-def test_falls_back_to_open_meteo_when_nasa_unavailable(monkeypatch, tmp_path):
-    """When NASA POWER fails, the load falls back to the Open-Meteo archive."""
-    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
-    monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
-    cell = {"id": "om_fallback", "center_lat": 47.6, "center_lon": -122.3}
-
-    full = _full_history_frame()
-    def _nasa_down(c): raise RuntimeError("NASA POWER outage")
-    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down)
     monkeypatch.setattr(climate, "_fetch_history", lambda c: full)
+    def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving")
+    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run)
 
     df, meta = climate._load_history(cell)
     assert meta["source"] == "open-meteo"
     assert df.height == full.height
 
 
-def test_short_nasa_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path):
-    """A too-short NASA response (a partial/unavailable point) is rejected rather than
-    cached as a complete record, and Open-Meteo serves instead."""
+def test_short_lake_record_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path):
+    """A too-short lake frame (a thin point that slipped past the client's own
+    gate) is rejected rather than cached as a complete record; the Open-Meteo
+    archive serves instead."""
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
     monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
-    cell = {"id": "short_nasa", "center_lat": 47.6, "center_lon": -122.3}
+    cell = {"id": "short_lake", "center_lat": 47.6, "center_lon": -122.3}
 
     short = climate._to_frame(_om_daily(3))                     # 2 usable days « threshold
     full = _full_history_frame()                                # >= MIN_ARCHIVE_DAYS
     assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height
-    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: short)
+    monkeypatch.setattr(climate.era5lake, "fetch_history_lake",
+                        lambda c, start=None: short)
     monkeypatch.setattr(climate, "_fetch_history", lambda c: full)
 
     df, meta = climate._load_history(cell)
-    assert meta["source"] == "open-meteo"      # rejected the short NASA response
+    assert meta["source"] == "open-meteo"      # rejected the short lake frame
     assert df.height == full.height
 
 
-def test_history_tail_prefers_nasa(monkeypatch):
-    """The tail top-up fetches NASA POWER first; Open-Meteo's range fetch is not
-    touched when NASA succeeds."""
-    cell = {"center_lat": 1.0, "center_lon": 2.0}
-    sentinel = object()
-    monkeypatch.setattr(climate, "_fetch_history_nasa",
-                        lambda c, start=None, end=None: sentinel)
-    def _om_should_not_run(c, s, e): raise AssertionError("Open-Meteo tail should not run")
-    monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run)
-
-    assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel
-
-
-def test_history_tail_falls_back_to_open_meteo(monkeypatch):
-    """When NASA fails and Open-Meteo isn't cooling down, the tail falls back to the
-    Open-Meteo archive range."""
+def test_history_tail_uses_open_meteo_archive(monkeypatch):
+    """The tail top-up is an Open-Meteo archive range — ERA5-consistent with
+    the record it extends; NASA is retired and never consulted."""
     monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
     cell = {"center_lat": 1.0, "center_lon": 2.0}
-    def _nasa_down(c, start=None, end=None): raise RuntimeError("NASA down")
-    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down)
     sentinel = object()
     monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: sentinel)
+    def _nasa_should_not_run(*a, **k): raise AssertionError("NASA is retired from serving")
+    monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run)
 
     assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel
 
+
+def test_history_tail_raises_during_archive_cooldown(monkeypatch):
+    """While the archive is rate-limit cooling, the top-up raises (best-effort;
+    retried next interval) instead of hammering the endpoint."""
+    import time as _time
+    monkeypatch.setattr(climate, "_archive_cooldown_until", _time.time() + 60)
+    cell = {"center_lat": 1.0, "center_lon": 2.0}
+    def _om_should_not_run(c, s, e): raise AssertionError("archive is cooling down")
+    monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run)
+
+    with pytest.raises(climate.WeatherUnavailable):
+        climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10")
+
 # --- forecast cooldown (mirrors the archive path's, tracked separately) -----
 
 def test_forecast_cooldown_skips_the_open_meteo_fallback(monkeypatch, tmp_path):