diff --git a/data/climate.py b/data/climate.py index dda822f..3fa340b 100644 --- a/data/climate.py +++ b/data/climate.py @@ -512,16 +512,22 @@ def _nasa_to_frame(param: dict) -> pl.DataFrame: return _finalize_approximated(df) -def _fetch_history_nasa(cell: dict) -> pl.DataFrame: - """Backup history fetch from NASA POWER (used when Open-Meteo is unavailable).""" - end = (datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).strftime("%Y%m%d") +def _fetch_history_nasa(cell: dict, start: str = NASA_START, + end: "str | None" = None) -> pl.DataFrame: + """Primary history fetch from NASA POWER (keyless, independent of Open-Meteo). + + `start`/`end` accept NASA's YYYYMMDD or ISO YYYY-MM-DD; `end` defaults to the + archive-latency cutoff. Serves both the full multi-decade pull and the tail + top-up range. NASA carries no gusts, so they're filled from Meteostat below.""" + if end is None: + end = (datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).strftime("%Y%m%d") params = { "parameters": "T2M_MAX,T2M_MIN,PRECTOTCORR,WS10M_MAX,RH2M", "community": "RE", "latitude": cell["center_lat"], "longitude": cell["center_lon"], - "start": NASA_START, - "end": end, + "start": start.replace("-", ""), + "end": end.replace("-", ""), "format": "JSON", } r = _request(NASA_POWER_URL, params, 180, phase="history_nasa") @@ -615,10 +621,21 @@ def _read_history_cache(path): return df, time.time() - os.path.getmtime(path) +def _fetch_history_tail(cell: dict, start_date: str, end_date: str) -> pl.DataFrame: + """Fetch a recent history range to top up the cached tail — NASA POWER primary, + Open-Meteo archive fallback (mirroring the full-history source order). The + Open-Meteo fallback is skipped while it's in a rate-limit cooldown.""" + try: + return _fetch_history_nasa(cell, start=start_date, end=end_date) + except Exception: # noqa: BLE001 - NASA unavailable; try Open-Meteo unless it's cooling down + if time.time() < _archive_cooldown_until: + raise + return _fetch_history_range(cell, start_date, end_date) + + def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame: """Append newly-available archive days to a cached record. Best-effort and serialized per cell; a small incremental fetch, not the full multi-decade pull.""" - global _archive_cooldown_until cell_id = cell["id"] with _cell_lock(cell_id): fresh = _read_history_backed(cell_id) # another thread may have just refreshed it @@ -626,15 +643,13 @@ def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame: df, age_s = fresh if age_s < HISTORY_TOPUP_INTERVAL: return df - if time.time() < _archive_cooldown_until: - return df expected = datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS) cached_max = df["date"].max() if cached_max >= expected: _touch_history_backed(cell_id) # tail already current — reset the hourly timer return df try: - recent = _fetch_history_range( + recent = _fetch_history_tail( cell, (cached_max + datetime.timedelta(days=1)).isoformat(), expected.isoformat()) except Exception as e: # noqa: BLE001 - keep the cached record on failure if is_rate_limit(e): @@ -699,30 +714,30 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: def _serve_stale(): return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None} - # Fetch. Open-Meteo is primary (richer: gusts + apparent temp); NASA POWER is - # the backup when Open-Meteo is unavailable (network error or its daily limit). - # Skip Open-Meteo entirely while it's in a rate-limit cooldown. + # Fetch. NASA POWER is the primary source (keyless, independent of Open-Meteo); + # the gusts it lacks are filled from Meteostat inside _fetch_history_nasa. + # Open-Meteo is the fallback when NASA is unavailable — still guarded by its + # rate-limit cooldown. Both sources must return a plausibly-full span before + # being accepted and cached as a complete record (a short/partial response is + # rejected rather than held indefinitely). df = None source = None - if time.time() >= _archive_cooldown_until: + try: + fetched = _fetch_history_nasa(cell) + if fetched.height >= MIN_ARCHIVE_DAYS: + df = fetched + source = "nasa-power" + except Exception: # noqa: BLE001 - primary unavailable; fall through to Open-Meteo + pass + if df is None and time.time() >= _archive_cooldown_until: try: fetched = _fetch_history(cell) - # Guard a self-hosted archive that isn't backfilled yet: an all-null or - # recent-only response reduces to a near-empty frame, which would - # otherwise be cached indefinitely as a complete record and never - # fall to NASA. Require a plausibly-full span before accepting it. if fetched.height >= MIN_ARCHIVE_DAYS: df = fetched source = "open-meteo" except Exception as e: # noqa: BLE001 if is_rate_limit(e): _note_rate_limit(e) - if df is None: - try: - df = _fetch_history_nasa(cell) - source = "nasa-power" - except Exception: # noqa: BLE001 - backup unavailable too - df = None if df is None: if stale is not None: return _serve_stale() diff --git a/tests/data/test_climate.py b/tests/data/test_climate.py index e8e5782..cfc0475 100644 --- a/tests/data/test_climate.py +++ b/tests/data/test_climate.py @@ -292,35 +292,78 @@ def _full_history_frame(n=4000): })) -def test_short_archive_is_rejected_and_falls_back_to_nasa(monkeypatch, tmp_path): - """A self-hosted archive that isn't backfilled yet returns too few days; the load - must reject it (not cache a near-empty history as complete) and use NASA instead.""" +def test_nasa_is_primary_history_source(monkeypatch, tmp_path): + """NASA POWER is the primary history source: a full NASA frame is accepted and + Open-Meteo is never consulted.""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) - cell = {"id": "short_cell", "center_lat": 47.6, "center_lon": -122.3} + cell = {"id": "nasa_primary", "center_lat": 47.6, "center_lon": -122.3} - short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold - full = _full_history_frame() # >= MIN_ARCHIVE_DAYS - assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height - monkeypatch.setattr(climate, "_fetch_history", lambda c: short) + full = _full_history_frame() monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full) + def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called") + monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run) df, meta = climate._load_history(cell) - assert meta["source"] == "nasa-power" # rejected the short archive + assert meta["source"] == "nasa-power" assert df.height == full.height -def test_full_archive_is_accepted(monkeypatch, tmp_path): - """A backfilled archive (enough days) is accepted and served as open-meteo.""" +def test_falls_back_to_open_meteo_when_nasa_unavailable(monkeypatch, tmp_path): + """When NASA POWER fails, the load falls back to the Open-Meteo archive.""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) - cell = {"id": "full_cell", "center_lat": 47.6, "center_lon": -122.3} + cell = {"id": "om_fallback", "center_lat": 47.6, "center_lon": -122.3} full = _full_history_frame() + def _nasa_down(c): raise RuntimeError("NASA POWER outage") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down) monkeypatch.setattr(climate, "_fetch_history", lambda c: full) - def _nasa_should_not_run(c): raise AssertionError("NASA should not be called") - monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) df, meta = climate._load_history(cell) assert meta["source"] == "open-meteo" assert df.height == full.height + + +def test_short_nasa_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path): + """A too-short NASA response (a partial/unavailable point) is rejected rather than + cached as a complete record, and Open-Meteo serves instead.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "short_nasa", "center_lat": 47.6, "center_lon": -122.3} + + short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold + full = _full_history_frame() # >= MIN_ARCHIVE_DAYS + assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height + monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: short) + monkeypatch.setattr(climate, "_fetch_history", lambda c: full) + + df, meta = climate._load_history(cell) + assert meta["source"] == "open-meteo" # rejected the short NASA response + assert df.height == full.height + + +def test_history_tail_prefers_nasa(monkeypatch): + """The tail top-up fetches NASA POWER first; Open-Meteo's range fetch is not + touched when NASA succeeds.""" + cell = {"center_lat": 1.0, "center_lon": 2.0} + sentinel = object() + monkeypatch.setattr(climate, "_fetch_history_nasa", + lambda c, start=None, end=None: sentinel) + def _om_should_not_run(c, s, e): raise AssertionError("Open-Meteo tail should not run") + monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run) + + assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel + + +def test_history_tail_falls_back_to_open_meteo(monkeypatch): + """When NASA fails and Open-Meteo isn't cooling down, the tail falls back to the + Open-Meteo archive range.""" + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"center_lat": 1.0, "center_lon": 2.0} + def _nasa_down(c, start=None, end=None): raise RuntimeError("NASA down") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down) + sentinel = object() + monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: sentinel) + + assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel