Jinemi/thermograph
4
0
Fork 0
Code Issues 2 Pull requests 9 Projects Releases Packages Wiki Activity Actions 4

Flip history primary to NASA POWER, Open-Meteo to fallback

Browse source 
The live history path now tries NASA POWER first (keyless, independent of
Open-Meteo) and falls back to the Open-Meteo archive only when NASA is
unavailable — the reverse of before. Gusts NASA lacks are filled from Meteostat
inside _fetch_history_nasa (added in the prior change). Both sources must still
return a plausibly-full span (MIN_ARCHIVE_DAYS) before being cached as complete;
a short/partial response is rejected and the other source is tried.

_fetch_history_nasa now accepts a start/end range so it also serves the recent
tail top-up: _topup_tail fetches NASA-first via the new _fetch_history_tail
helper (Open-Meteo range as fallback, skipped during its rate-limit cooldown),
removing the last per-active-cell Open-Meteo call from the history path. The
Open-Meteo cooldown no longer gates the top-up, since NASA is not subject to it.

Open-Meteo remains wired as the fallback (deletion is a later step). Tests
updated to the new source order plus tail-fetch coverage.
This commit is contained in:
Emi Griffith 2026-07-22 22:02:30 -07:00
parent b4d8d67825
commit 30153dbc64
2 changed files with 95 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
data/climate.py
View file

@ -512,16 +512,22 @@ def _nasa_to_frame(param: dict) -> pl.DataFrame:
return _finalize_approximated(df)
def _fetch_history_nasa(cell: dict) -> pl.DataFrame:
"""Backup history fetch from NASA POWER (used when Open-Meteo is unavailable)."""
end = (datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).strftime("%Y%m%d")
def _fetch_history_nasa(cell: dict, start: str = NASA_START,
end: "str | None" = None) -> pl.DataFrame:
"""Primary history fetch from NASA POWER (keyless, independent of Open-Meteo).
`start`/`end` accept NASA's YYYYMMDD or ISO YYYY-MM-DD; `end` defaults to the
archive-latency cutoff. Serves both the full multi-decade pull and the tail
top-up range. NASA carries no gusts, so they're filled from Meteostat below."""
if end is None:
end = (datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)).strftime("%Y%m%d")
params = {
"parameters": "T2M_MAX,T2M_MIN,PRECTOTCORR,WS10M_MAX,RH2M",
"community": "RE",
"latitude": cell["center_lat"],
"longitude": cell["center_lon"],
"start": NASA_START,
"end": end,
"start": start.replace("-", ""),
"end": end.replace("-", ""),
"format": "JSON",
}
r = _request(NASA_POWER_URL, params, 180, phase="history_nasa")
@ -615,10 +621,21 @@ def _read_history_cache(path):
return df, time.time() - os.path.getmtime(path)
def _fetch_history_tail(cell: dict, start_date: str, end_date: str) -> pl.DataFrame:
"""Fetch a recent history range to top up the cached tail — NASA POWER primary,
Open-Meteo archive fallback (mirroring the full-history source order). The
Open-Meteo fallback is skipped while it's in a rate-limit cooldown."""
try:
return _fetch_history_nasa(cell, start=start_date, end=end_date)
except Exception: # noqa: BLE001 - NASA unavailable; try Open-Meteo unless it's cooling down
if time.time() < _archive_cooldown_until:
raise
return _fetch_history_range(cell, start_date, end_date)
def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame:
"""Append newly-available archive days to a cached record. Best-effort and
serialized per cell; a small incremental fetch, not the full multi-decade pull."""
global _archive_cooldown_until
cell_id = cell["id"]
with _cell_lock(cell_id):
fresh = _read_history_backed(cell_id) # another thread may have just refreshed it
@ -626,15 +643,13 @@ def _topup_tail(cell: dict, df: pl.DataFrame) -> pl.DataFrame:
df, age_s = fresh
if age_s < HISTORY_TOPUP_INTERVAL:
return df
if time.time() < _archive_cooldown_until:
return df
expected = datetime.date.today() - datetime.timedelta(days=ARCHIVE_LATENCY_DAYS)
cached_max = df["date"].max()
if cached_max >= expected:
_touch_history_backed(cell_id) # tail already current — reset the hourly timer
return df
try:
recent = _fetch_history_range(
recent = _fetch_history_tail(
cell, (cached_max + datetime.timedelta(days=1)).isoformat(), expected.isoformat())
except Exception as e: # noqa: BLE001 - keep the cached record on failure
if is_rate_limit(e):
@ -699,30 +714,30 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]:
def _serve_stale():
return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None}
# Fetch. Open-Meteo is primary (richer: gusts + apparent temp); NASA POWER is
# the backup when Open-Meteo is unavailable (network error or its daily limit).
# Skip Open-Meteo entirely while it's in a rate-limit cooldown.
# Fetch. NASA POWER is the primary source (keyless, independent of Open-Meteo);
# the gusts it lacks are filled from Meteostat inside _fetch_history_nasa.
# Open-Meteo is the fallback when NASA is unavailable — still guarded by its
# rate-limit cooldown. Both sources must return a plausibly-full span before
# being accepted and cached as a complete record (a short/partial response is
# rejected rather than held indefinitely).
df = None
source = None
if time.time() >= _archive_cooldown_until:
try:
fetched = _fetch_history_nasa(cell)
if fetched.height >= MIN_ARCHIVE_DAYS:
df = fetched
source = "nasa-power"
except Exception: # noqa: BLE001 - primary unavailable; fall through to Open-Meteo
pass
if df is None and time.time() >= _archive_cooldown_until:
try:
fetched = _fetch_history(cell)
# Guard a self-hosted archive that isn't backfilled yet: an all-null or
# recent-only response reduces to a near-empty frame, which would
# otherwise be cached indefinitely as a complete record and never
# fall to NASA. Require a plausibly-full span before accepting it.
if fetched.height >= MIN_ARCHIVE_DAYS:
df = fetched
source = "open-meteo"
except Exception as e: # noqa: BLE001
if is_rate_limit(e):
_note_rate_limit(e)
if df is None:
try:
df = _fetch_history_nasa(cell)
source = "nasa-power"
except Exception: # noqa: BLE001 - backup unavailable too
df = None
if df is None:
if stale is not None:
return _serve_stale()

71
tests/data/test_climate.py
View file

@ -292,35 +292,78 @@ def _full_history_frame(n=4000):
}))
def test_short_archive_is_rejected_and_falls_back_to_nasa(monkeypatch, tmp_path):
"""A self-hosted archive that isn't backfilled yet returns too few days; the load
must reject it (not cache a near-empty history as complete) and use NASA instead."""
def test_nasa_is_primary_history_source(monkeypatch, tmp_path):
"""NASA POWER is the primary history source: a full NASA frame is accepted and
Open-Meteo is never consulted."""
monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
cell = {"id": "short_cell", "center_lat": 47.6, "center_lon": -122.3}
cell = {"id": "nasa_primary", "center_lat": 47.6, "center_lon": -122.3}
short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold
full = _full_history_frame() # >= MIN_ARCHIVE_DAYS
assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height
monkeypatch.setattr(climate, "_fetch_history", lambda c: short)
full = _full_history_frame()
monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full)
def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called")
monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run)
df, meta = climate._load_history(cell)
assert meta["source"] == "nasa-power" # rejected the short archive
assert meta["source"] == "nasa-power"
assert df.height == full.height
def test_full_archive_is_accepted(monkeypatch, tmp_path):
"""A backfilled archive (enough days) is accepted and served as open-meteo."""
def test_falls_back_to_open_meteo_when_nasa_unavailable(monkeypatch, tmp_path):
"""When NASA POWER fails, the load falls back to the Open-Meteo archive."""
monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
cell = {"id": "full_cell", "center_lat": 47.6, "center_lon": -122.3}
cell = {"id": "om_fallback", "center_lat": 47.6, "center_lon": -122.3}
full = _full_history_frame()
def _nasa_down(c): raise RuntimeError("NASA POWER outage")
monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down)
monkeypatch.setattr(climate, "_fetch_history", lambda c: full)
def _nasa_should_not_run(c): raise AssertionError("NASA should not be called")
monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run)
df, meta = climate._load_history(cell)
assert meta["source"] == "open-meteo"
assert df.height == full.height
def test_short_nasa_is_rejected_and_falls_back_to_open_meteo(monkeypatch, tmp_path):
"""A too-short NASA response (a partial/unavailable point) is rejected rather than
cached as a complete record, and Open-Meteo serves instead."""
monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
cell = {"id": "short_nasa", "center_lat": 47.6, "center_lon": -122.3}
short = climate._to_frame(_om_daily(3)) # 2 usable days « threshold
full = _full_history_frame() # >= MIN_ARCHIVE_DAYS
assert short.height < climate.MIN_ARCHIVE_DAYS <= full.height
monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: short)
monkeypatch.setattr(climate, "_fetch_history", lambda c: full)
df, meta = climate._load_history(cell)
assert meta["source"] == "open-meteo" # rejected the short NASA response
assert df.height == full.height
def test_history_tail_prefers_nasa(monkeypatch):
"""The tail top-up fetches NASA POWER first; Open-Meteo's range fetch is not
touched when NASA succeeds."""
cell = {"center_lat": 1.0, "center_lon": 2.0}
sentinel = object()
monkeypatch.setattr(climate, "_fetch_history_nasa",
lambda c, start=None, end=None: sentinel)
def _om_should_not_run(c, s, e): raise AssertionError("Open-Meteo tail should not run")
monkeypatch.setattr(climate, "_fetch_history_range", _om_should_not_run)
assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel
def test_history_tail_falls_back_to_open_meteo(monkeypatch):
"""When NASA fails and Open-Meteo isn't cooling down, the tail falls back to the
Open-Meteo archive range."""
monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0)
cell = {"center_lat": 1.0, "center_lon": 2.0}
def _nasa_down(c, start=None, end=None): raise RuntimeError("NASA down")
monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_down)
sentinel = object()
monkeypatch.setattr(climate, "_fetch_history_range", lambda c, s, e: sentinel)
assert climate._fetch_history_tail(cell, "2026-06-01", "2026-06-10") is sentinel