From 6927a53ea08748e77d19dd00f285ac625a4b38cb Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Wed, 15 Jul 2026 22:30:56 -0700 Subject: [PATCH] Add MET Norway forecast fallback when Open-Meteo is unavailable (#115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forecast path had no backup — unlike history, which falls back to NASA POWER. When Open-Meteo's forecast API was rate-limited or down, the recent/forecast bundle (and every endpoint that grades future days) failed with a 503. Add MET Norway (yr.no) Locationforecast as a keyless, global forecast backup, mirroring the NASA POWER role for history: - _metno_to_frame: aggregates MET Norway's sub-daily timeseries into the daily schema, converting units (°C→°F, mm→in, m/s→mph) and deriving feels-like from the NWS heat index / wind chill (MET has no gusts or apparent temperature). Precip prefers the 1-hour block and falls back to the 6-hour block so the hourly→6-hourly resolution switch never double-counts. - _fetch_forecast_metno: the backup fetch, with the ToS-required identifying User-Agent and coordinates rounded to 4 decimals. - _load_recent_forecast: on any Open-Meteo forecast failure, try MET Norway before surfacing the error; a shared rate limit still raises the typed, daily-aware WeatherUnavailable. MET Norway is forecast-only (no recent past days), so it's a degraded-but-working fallback: the forecast / day-ahead views keep serving during an Open-Meteo outage. Tests cover the daily aggregation + unit conversion (incl. the no-double-count precip rule) and the fallback wiring. --- climate.py | 100 +++++++++++++++++++++++++++++++++++++++--- tests/test_climate.py | 57 ++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 7 deletions(-) diff --git a/climate.py b/climate.py index e5868a3..456bf4a 100644 --- a/climate.py +++ b/climate.py @@ -36,6 +36,16 @@ NASA_POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point" NASA_START = "19810101" NASA_FILL = -900.0 # POWER's missing-value sentinel is ~-999 +# Backup forward forecast when Open-Meteo's forecast API is unavailable. MET Norway +# (yr.no) is free + keyless + global, mirroring NASA POWER's role for history. It +# returns a sub-daily timeseries in metric units with no gusts or apparent temp, so +# we aggregate to daily, convert units, and derive feels-like like the NASA path. +# It is forecast-only — no recent past days — so it's a degraded-but-working fallback. +METNO_URL = "https://api.met.no/weatherapi/locationforecast/2.0/complete" +# MET Norway's ToS requires an identifying User-Agent (a missing/generic one is +# 403'd); include the app and a contact URL so they can reach us if usage misbehaves. +METNO_UA = "Thermograph/0.2 (+https://thermograph.org)" + DAILY_VARS = ( "temperature_2m_max,temperature_2m_min,precipitation_sum," "wind_speed_10m_max,wind_gusts_10m_max," @@ -350,6 +360,75 @@ def _fetch_history_nasa(cell: dict) -> pl.DataFrame: return _nasa_to_frame(r.json()["properties"]["parameter"]) +def _metno_to_frame(props: dict) -> pl.DataFrame: + """Map a MET Norway Locationforecast (yr.no) response to our daily schema. + + MET Norway returns a per-timestep timeseries (hourly near-term, then 6-hourly) + in metric units, with no gusts or apparent temperature, so we aggregate to daily + extremes/means, convert units (°C→°F, mm→in, m/s→mph), and approximate feels-like + from the NWS heat index / wind chill — the same treatment as the NASA POWER path.""" + # Aggregate the sub-daily steps into per-day values, keyed by (UTC) calendar day. + agg: dict[str, dict] = {} + for step in props.get("timeseries", []): + day = step["time"][:10] + data = step.get("data", {}) + inst = data.get("instant", {}).get("details", {}) + a = agg.setdefault(day, {"t": [], "rh": [], "wind": [], "precip": None}) + if (t := inst.get("air_temperature")) is not None: + a["t"].append(t) + if (rh := inst.get("relative_humidity")) is not None: + a["rh"].append(rh) + if (w := inst.get("wind_speed")) is not None: + a["wind"].append(w) + # Precip: prefer the 1-hour block (hourly near-term), else the 6-hour block + # (6-hourly far-term). Never add both — they overlap — so summing each step's + # chosen block avoids double counting across the resolution switch. + p = (data.get("next_1_hours") or {}).get("details", {}).get("precipitation_amount") + if p is None: + p = (data.get("next_6_hours") or {}).get("details", {}).get("precipitation_amount") + if p is not None: + a["precip"] = (a["precip"] or 0.0) + p + + dates = sorted(agg) + c2f = lambda c: c * 9.0 / 5.0 + 32.0 + df = pl.DataFrame({ + "date": dates, + "tmax": [c2f(max(agg[d]["t"])) if agg[d]["t"] else None for d in dates], + "tmin": [c2f(min(agg[d]["t"])) if agg[d]["t"] else None for d in dates], + "precip": [agg[d]["precip"] / 25.4 if agg[d]["precip"] is not None else None + for d in dates], + "wind": [max(agg[d]["wind"]) * 2.2369362920544 if agg[d]["wind"] else None + for d in dates], + "gust": [None] * len(dates), # MET Norway has no gusts + "humid": [sum(agg[d]["rh"]) / len(agg[d]["rh"]) if agg[d]["rh"] else None + for d in dates], + }) + df = df.with_columns( + pl.col("date").str.to_date(), + *[pl.col(c).cast(pl.Float64, strict=False) + for c in ("tmax", "tmin", "precip", "wind", "gust")], + ) + df = df.with_columns( + pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())), + pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())), + ) + return _finalize_frame(df) + + +def _fetch_forecast_metno(cell: dict) -> pl.DataFrame: + """Backup forward-forecast fetch from MET Norway / yr.no (used when Open-Meteo's + forecast API is unavailable). Coordinates are rounded to 4 decimals per MET's + ToS (improves their cache hit rate).""" + r = _request( + METNO_URL, + {"lat": round(cell["center_lat"], 4), "lon": round(cell["center_lon"], 4)}, + 60, + phase="forecast_metno", + headers={"User-Agent": METNO_UA}, + ) + return _metno_to_frame(r.json().get("properties", {})) + + def _fetch_history_range(cell: dict, start_date: str, end_date: str) -> pl.DataFrame: """Fetch just a date range of archive history (used to top up the recent tail).""" params = _om_daily_params(cell, start_date=start_date, end_date=end_date) @@ -530,14 +609,21 @@ def _load_recent_forecast(cell: dict) -> pl.DataFrame: } try: r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") + df = _to_frame(r.json()["daily"]) except Exception as e: # noqa: BLE001 - # Classify a forecast-API rate limit here so callers get the typed error - # (the archive path classifies its own inside _load_history). - if is_rate_limit(e): - daily = "daily" in _rate_limit_reason(e).lower() - raise WeatherUnavailable(limit_message(daily), daily=daily) from e - raise - df = _to_frame(r.json()["daily"]) + # Open-Meteo forecast unavailable (rate limit or outage). Fall back to MET + # Norway (yr.no) — global + keyless — mirroring the archive's NASA POWER + # backup. MET Norway is forecast-only (no recent past days), so this keeps + # the forecast / day-ahead views working in a degraded form. + try: + df = _fetch_forecast_metno(cell) + except Exception: # noqa: BLE001 - backup unavailable too; surface the original + # Classify the original Open-Meteo rate limit so callers get the typed + # error (the archive path classifies its own inside _load_history). + if is_rate_limit(e): + daily = "daily" in _rate_limit_reason(e).lower() + raise WeatherUnavailable(limit_message(daily), daily=daily) from e + raise _write_cache(df, path) return df diff --git a/tests/test_climate.py b/tests/test_climate.py index 6f2ebd0..857dab0 100644 --- a/tests/test_climate.py +++ b/tests/test_climate.py @@ -74,6 +74,63 @@ def test_nasa_to_frame_converts_units_and_fills(): assert row["fmax"] > row["tmax"] # ≥80°F: the NWS heat index applies +def _metno_step(time, t, rh, wind, p1=None, p6=None): + """One MET Norway timeseries step (instant details + optional precip blocks).""" + data = {"instant": {"details": { + "air_temperature": t, "relative_humidity": rh, "wind_speed": wind}}} + if p1 is not None: + data["next_1_hours"] = {"details": {"precipitation_amount": p1}} + if p6 is not None: + data["next_6_hours"] = {"details": {"precipitation_amount": p6}} + return {"time": time, "data": data} + + +def test_metno_to_frame_aggregates_daily_and_converts_units(): + props = {"timeseries": [ + # Day 1: two hourly steps. The first also carries a next_6_hours block, which + # must be IGNORED (next_1_hours wins) so precip isn't double-counted. + _metno_step("2026-07-16T00:00:00Z", 20.0, 50.0, 2.0, p1=0.5, p6=3.0), + _metno_step("2026-07-16T01:00:00Z", 25.0, 60.0, 4.0, p1=0.5), + # Day 2: a 6-hourly step (only next_6_hours) + an instant-only tail step. + _metno_step("2026-07-17T00:00:00Z", 10.0, 80.0, 10.0, p6=6.0), + _metno_step("2026-07-17T06:00:00Z", 12.0, 70.0, 8.0), + ]} + df = climate._metno_to_frame(props) + assert len(df) == 2 + d1 = df.filter(pl.col("date") == datetime.date(2026, 7, 16)).row(0, named=True) + assert d1["tmax"] == 77.0 and d1["tmin"] == 68.0 # 25°C / 20°C -> °F + assert round(d1["precip"], 4) == round(1.0 / 25.4, 4) # 0.5+0.5 mm (not +3.0) -> in + assert round(d1["wind"], 1) == 8.9 # max 4 m/s -> mph + assert d1["gust"] is None # MET Norway has no gusts + assert d1["feels"] is not None + d2 = df.filter(pl.col("date") == datetime.date(2026, 7, 17)).row(0, named=True) + assert round(d2["precip"], 4) == round(6.0 / 25.4, 4) # only the 6-hour block + + +def test_recent_forecast_falls_back_to_metno(monkeypatch, tmp_path): + """When Open-Meteo's forecast API fails, the MET Norway backup serves the frame.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + cell = {"id": "fallback_cell", "center_lat": 47.6062, "center_lon": -122.3321} + met = {"timeseries": [ + _metno_step("2026-07-16T00:00:00Z", 18.0, 55.0, 3.0, p1=0.0), + _metno_step("2026-07-17T00:00:00Z", 22.0, 45.0, 5.0, p6=1.0), + ]} + + class Resp: + def json(self): return {"properties": met} + + def fake_request(url, params, timeout, *, phase, headers=None, attempts=climate.MAX_ATTEMPTS): + if url == climate.FORECAST_URL: + raise RuntimeError("open-meteo forecast outage") + assert url == climate.METNO_URL + assert headers and headers.get("User-Agent"), "MET Norway needs a User-Agent" + return Resp() + + monkeypatch.setattr(climate, "_request", fake_request) + df = climate._load_recent_forecast(cell) + assert len(df) == 2 and df["date"].max() == datetime.date(2026, 7, 17) + + def test_om_daily_params_carries_the_window(): cell = {"center_lat": 47.6, "center_lon": -122.3} p = climate._om_daily_params(cell, start_date="2026-01-01", end_date="2026-02-01")