From d4a00099f2fd89990c88a22f8027e83a6391cfe0 Mon Sep 17 00:00:00 2001 From: emi Date: Sat, 25 Jul 2026 16:56:21 +0000 Subject: [PATCH] climate: stop dropping the current day from the Open-Meteo bundle (#92) --- backend/data/climate.py | 47 ++++++-------- backend/tests/data/test_climate.py | 98 ++++++++++++------------------ 2 files changed, 60 insertions(+), 85 deletions(-) diff --git a/backend/data/climate.py b/backend/data/climate.py index d7c1357..c08561e 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -936,29 +936,27 @@ def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: .sort("date")) -def _om_local_today(payload: dict) -> "datetime.date | None": - """The calendar date it is *now* at the cell, from the UTC offset Open-Meteo - reports for a ``timezone=auto`` request. None when the offset is absent, which - tells the caller to skip the in-progress-day guard rather than guess a date.""" - offset = payload.get("utc_offset_seconds") - if offset is None: - return None - return (datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(seconds=int(offset))).date() - - def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: - """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former - primary): recent past + forward days in one call. + """Primary recent+forecast bundle from the Open-Meteo forecast API: recent past + and forward days in one call, covering today + 7 (see FORECAST_DAYS). - The cell's in-progress *local* day is dropped from the bundle. Open-Meteo's - daily high/low for today aggregates only the hours elapsed so far, so grading it - reads a still-unfolding day as complete and produces spurious extremes (a cool - morning served as a record-low high, seen live at the 1st percentile). This is - the same failure the MET Norway path guards against with its diurnal-coverage - gate (see _metno_to_frame); the fallback needs the equivalent. Past days are - complete and future days are whole-day forecasts, so only today is excluded — - the day lands in the record once it is over.""" + The cell's in-progress *local* day is KEPT. An earlier revision dropped it on + the assumption that Open-Meteo's daily high/low for today aggregates only the + hours elapsed so far — true of the MET Norway series, which genuinely starts + mid-day and is gated on diurnal coverage (see _metno_to_frame), but NOT of this + endpoint: Open-Meteo backfills the rest of today from the model run, so today's + daily value spans the whole local day exactly as tomorrow's does. + + Verified against the live API at four cells spanning 01:00-19:00 local: the + reported daily max equals the max over all 24 hourly values, and diverges from + the elapsed-hours-only max wherever the day still has hours left (Los Angeles + at 09:16 local reported 93.9F, the whole-day figure, where the hours elapsed + topped out at 76.1F). Re-check it that way before reintroducing any exclusion. + + Dropping today cost every freshly-synced cell its current day — a hole between + a complete yesterday and a forecast tomorrow, which the UI renders as a missing + column. A day that genuinely cannot be graded, because it came back with no + high/low, is already dropped upstream by _to_frame.""" params = { "latitude": cell["center_lat"], "longitude": cell["center_lon"], @@ -971,12 +969,7 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: "forecast_days": FORECAST_DAYS, } r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") - payload = r.json() - df = _to_frame(payload["daily"]) - local_today = _om_local_today(payload) - if local_today is not None: - df = df.filter(pl.col("date") != local_today) - return df + return _to_frame(r.json()["daily"]) def _load_recent_forecast(cell: dict) -> pl.DataFrame: diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index edaae3f..b09ff11 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -215,20 +215,20 @@ def test_recent_forecast_fallback_merges_archive_past_and_metno_forward(monkeypa datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)] -def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): - """The Open-Meteo fallback must not grade the cell's in-progress local day: its - daily high/low is only a partial aggregate of the hours elapsed so far (a cool - morning would read as a record-low high). Past days and future forecast days - survive; today — per the UTC offset Open-Meteo reports for a timezone=auto - request — is dropped, matching the MET path's diurnal-coverage gate.""" - offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) +def test_recent_forecast_om_keeps_the_in_progress_local_day(monkeypatch): + """Today must survive the Open-Meteo bundle. Unlike the MET Norway series (which + starts mid-day and is gated on diurnal coverage), this endpoint backfills the + rest of today from the model run, so today's high/low is a whole-day value of + the same kind as tomorrow's. Dropping it left a hole between a complete + yesterday and a forecast tomorrow, which the UI renders as a missing column.""" + offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai cell) local_today = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=offset)).date() days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] daily = { "time": [d.isoformat() for d in days], - "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value - "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], + "temperature_2m_max": [80.0, 82.0, 76.1, 84.0], + "temperature_2m_min": [60.0, 61.0, 55.9, 62.0], "precipitation_sum": [0.0, 0.0, 0.0, 0.0], } payload = {"utc_offset_seconds": offset, "daily": daily} @@ -237,60 +237,42 @@ def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): def json(self): return payload monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + got = climate._fetch_recent_forecast_om({"center_lat": 54.9, "center_lon": 23.8}) + assert got["date"].to_list() == days # every day present, no gap at today + row = got.filter(pl.col("date") == local_today) + assert row["tmax"].item() == 76.1 # today's whole-day high, ungraded-down + assert row["tmin"].item() == 55.9 + + +def test_recent_forecast_om_still_drops_a_today_with_no_high_low(monkeypatch): + """The one case today is excluded: it came back with no high/low and so cannot be + graded at all. _to_frame enforces that for every day, today included.""" + offset = 3 * 3600 + local_today = (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=offset)).date() + days = [local_today + datetime.timedelta(days=n) for n in (-1, 0, 1)] + daily = { + "time": [d.isoformat() for d in days], + "temperature_2m_max": [82.0, None, 84.0], + "temperature_2m_min": [61.0, None, 62.0], + "precipitation_sum": [0.0, 0.0, 0.0], + } + payload = {"utc_offset_seconds": offset, "daily": daily} + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + got = climate._fetch_recent_forecast_om( {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() - assert local_today not in got # partial today dropped - assert local_today - datetime.timedelta(days=1) in got # yesterday kept - assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept - assert len(got) == 3 + assert local_today not in got + assert got == [local_today - datetime.timedelta(days=1), + local_today + datetime.timedelta(days=1)] def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): - """No UTC offset reported -> skip the in-progress-day guard rather than guess a - date, so the bundle passes through as before (only the usual null-day filter).""" - payload = {"daily": _om_daily(3)} # no utc_offset_seconds - - class Resp: - def json(self): return payload - monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) - - df = climate._fetch_recent_forecast_om({"center_lat": 1.0, "center_lon": 2.0}) - assert df.height == climate._to_frame(_om_daily(3)).height - - -def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): - """The Open-Meteo fallback must not grade the cell's in-progress local day: its - daily high/low is only a partial aggregate of the hours elapsed so far (a cool - morning would read as a record-low high). Past days and future forecast days - survive; today — per the UTC offset Open-Meteo reports for a timezone=auto - request — is dropped, matching the MET path's diurnal-coverage gate.""" - offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) - local_today = (datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(seconds=offset)).date() - days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] - daily = { - "time": [d.isoformat() for d in days], - "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value - "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], - "precipitation_sum": [0.0, 0.0, 0.0, 0.0], - } - payload = {"utc_offset_seconds": offset, "daily": daily} - - class Resp: - def json(self): return payload - monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) - - got = climate._fetch_recent_forecast_om( - {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() - assert local_today not in got # partial today dropped - assert local_today - datetime.timedelta(days=1) in got # yesterday kept - assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept - assert len(got) == 3 - - -def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): - """No UTC offset reported -> skip the in-progress-day guard rather than guess a - date, so the bundle passes through as before (only the usual null-day filter).""" + """No UTC offset reported changes nothing: the bundle passes through with only + the usual null-day filter.""" payload = {"daily": _om_daily(3)} # no utc_offset_seconds class Resp: