diff --git a/data/climate.py b/data/climate.py index 3a7fb54..b52a90c 100644 --- a/data/climate.py +++ b/data/climate.py @@ -23,6 +23,12 @@ CACHE_DIR = os.path.join(paths.DATA_DIR, "cache") MAX_ATTEMPTS = 3 # per upstream call, before giving up START_DATE = "1980-01-01" # ERA5 reaches back to 1940; 1980 = 45 yrs, fast + robust ARCHIVE_LATENCY_DAYS = 6 # ERA5 archive lags real time by a few days +# A real archive spans decades (~16k daily rows from START_DATE). A self-hosted +# Open-Meteo instance that isn't backfilled yet answers with all-null values or only +# its recent sync window, which _finalize_frame reduces to a near-empty frame. Below +# this many days we treat the archive as "not ready" — fall through to the NASA backup +# and do NOT cache the short result as a complete, indefinitely-held record. +MIN_ARCHIVE_DAYS = 3650 # ~10 yrs: far above any sync window, far below a full archive # History rarely changes, so the full multi-decade archive is fetched once and # cached indefinitely (refetched only to add new metric columns). Only the recent # tail is refreshed — a small incremental fetch, at most hourly. @@ -585,8 +591,14 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: source = None if time.time() >= _archive_cooldown_until: try: - df = _fetch_history(cell) - source = "open-meteo" + fetched = _fetch_history(cell) + # Guard a self-hosted archive that isn't backfilled yet: an all-null or + # recent-only response reduces to a near-empty frame, which would + # otherwise be cached indefinitely as a complete record and never + # fall to NASA. Require a plausibly-full span before accepting it. + if fetched.height >= MIN_ARCHIVE_DAYS: + df = fetched + source = "open-meteo" except Exception as e: # noqa: BLE001 if is_rate_limit(e): _note_rate_limit(e) diff --git a/tests/data/test_climate.py b/tests/data/test_climate.py index 4a43439..3084a7a 100644 --- a/tests/data/test_climate.py +++ b/tests/data/test_climate.py @@ -249,3 +249,49 @@ def test_daily_params_carry_no_model_by_default(): p = climate._om_daily_params({"center_lat": 1.0, "center_lon": 2.0}, past_days=7) assert "models" not in p assert climate.ARCHIVE_URL.endswith("/v1/archive") # default (no env override in tests) + + +def _full_history_frame(n=4000): + """A plausibly-full archive frame (>= MIN_ARCHIVE_DAYS rows), standard columns.""" + start = datetime.date(1990, 1, 1) + dates = [start + datetime.timedelta(days=i) for i in range(n)] + return climate._finalize_frame(pl.DataFrame({ + "date": dates, + "tmax": [70.0] * n, "tmin": [50.0] * n, "precip": [0.0] * n, + "wind": [5.0] * n, "gust": [8.0] * n, "humid": [60.0] * n, + "fmax": [70.0] * n, "fmin": [50.0] * n, + })) + + +def test_short_archive_is_rejected_and_falls_back_to_nasa(monkeypatch, tmp_path): + """A self-hosted archive that isn't backfilled yet returns too few days; the load + must reject it (not cache a near-empty history as complete) and use NASA instead.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "short_cell", "center_lat": 47.6, "center_lon": -122.3} + + short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold + full = _full_history_frame() # >= MIN_ARCHIVE_DAYS + assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height + monkeypatch.setattr(climate, "_fetch_history", lambda c: short) + monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full) + + df, meta = climate._load_history(cell) + assert meta["source"] == "nasa-power" # rejected the short archive + assert df.height == full.height + + +def test_full_archive_is_accepted(monkeypatch, tmp_path): + """A backfilled archive (enough days) is accepted and served as open-meteo.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "full_cell", "center_lat": 47.6, "center_lon": -122.3} + + full = _full_history_frame() + monkeypatch.setattr(climate, "_fetch_history", lambda c: full) + def _nasa_should_not_run(c): raise AssertionError("NASA should not be called") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) + + df, meta = climate._load_history(cell) + assert meta["source"] == "open-meteo" + assert df.height == full.height