From b1e02181479cd23164e04cf369d7837513f17240 Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Sat, 25 Jul 2026 09:22:46 -0700 Subject: [PATCH] climate: stop dropping the current day from the Open-Meteo bundle The recent+forecast bundle excluded the cell's in-progress local day, on the premise that Open-Meteo's daily high/low for today aggregates only the hours elapsed so far. That is true of the MET Norway series -- which starts mid-day and is correctly gated on diurnal coverage in _metno_to_frame -- but not of the forecast endpoint, which backfills the rest of today from the model run. The guard was generalised across by analogy and never verified against the API. Checked against the live API at four cells between 01:00 and 19:00 local: the reported daily max always equals the max over all 24 hourly values, and differs from the elapsed-hours-only max whenever the day has hours left. Los Angeles at 09:16 local reported 93.9F -- the whole-day figure -- where the hours so far topped out at 76.1F. The cost was a hole in the middle of every freshly-synced cell's window: a complete yesterday, no today, a forecast tomorrow, which the daily strip renders as a missing column. 861 of 1004 prod cells had lost the 25th by the time this was diagnosed; the 143 that still had it were stale caches predating the change. FORECAST_DAYS is commented "today + 7 days ahead", which the drop contradicted. Today is kept unconditionally now. A day that truly cannot be graded, because it came back with no high/low, is already dropped upstream by _to_frame, so the local-today helper has no remaining caller and goes with it. Also removes a verbatim duplicate of both Open-Meteo day-window tests, which the main-into-dev reconciliation left in the file (the second copy shadowed the first, so only one of each was ever running). --- backend/data/climate.py | 47 ++++++-------- backend/tests/data/test_climate.py | 98 ++++++++++++------------------ 2 files changed, 60 insertions(+), 85 deletions(-) diff --git a/backend/data/climate.py b/backend/data/climate.py index d7c1357..c08561e 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -936,29 +936,27 @@ def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: .sort("date")) -def _om_local_today(payload: dict) -> "datetime.date | None": - """The calendar date it is *now* at the cell, from the UTC offset Open-Meteo - reports for a ``timezone=auto`` request. None when the offset is absent, which - tells the caller to skip the in-progress-day guard rather than guess a date.""" - offset = payload.get("utc_offset_seconds") - if offset is None: - return None - return (datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(seconds=int(offset))).date() - - def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: - """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former - primary): recent past + forward days in one call. + """Primary recent+forecast bundle from the Open-Meteo forecast API: recent past + and forward days in one call, covering today + 7 (see FORECAST_DAYS). - The cell's in-progress *local* day is dropped from the bundle. Open-Meteo's - daily high/low for today aggregates only the hours elapsed so far, so grading it - reads a still-unfolding day as complete and produces spurious extremes (a cool - morning served as a record-low high, seen live at the 1st percentile). This is - the same failure the MET Norway path guards against with its diurnal-coverage - gate (see _metno_to_frame); the fallback needs the equivalent. Past days are - complete and future days are whole-day forecasts, so only today is excluded — - the day lands in the record once it is over.""" + The cell's in-progress *local* day is KEPT. An earlier revision dropped it on + the assumption that Open-Meteo's daily high/low for today aggregates only the + hours elapsed so far — true of the MET Norway series, which genuinely starts + mid-day and is gated on diurnal coverage (see _metno_to_frame), but NOT of this + endpoint: Open-Meteo backfills the rest of today from the model run, so today's + daily value spans the whole local day exactly as tomorrow's does. + + Verified against the live API at four cells spanning 01:00-19:00 local: the + reported daily max equals the max over all 24 hourly values, and diverges from + the elapsed-hours-only max wherever the day still has hours left (Los Angeles + at 09:16 local reported 93.9F, the whole-day figure, where the hours elapsed + topped out at 76.1F). Re-check it that way before reintroducing any exclusion. + + Dropping today cost every freshly-synced cell its current day — a hole between + a complete yesterday and a forecast tomorrow, which the UI renders as a missing + column. A day that genuinely cannot be graded, because it came back with no + high/low, is already dropped upstream by _to_frame.""" params = { "latitude": cell["center_lat"], "longitude": cell["center_lon"], @@ -971,12 +969,7 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: "forecast_days": FORECAST_DAYS, } r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") - payload = r.json() - df = _to_frame(payload["daily"]) - local_today = _om_local_today(payload) - if local_today is not None: - df = df.filter(pl.col("date") != local_today) - return df + return _to_frame(r.json()["daily"]) def _load_recent_forecast(cell: dict) -> pl.DataFrame: diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index edaae3f..b09ff11 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -215,20 +215,20 @@ def test_recent_forecast_fallback_merges_archive_past_and_metno_forward(monkeypa datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)] -def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): - """The Open-Meteo fallback must not grade the cell's in-progress local day: its - daily high/low is only a partial aggregate of the hours elapsed so far (a cool - morning would read as a record-low high). Past days and future forecast days - survive; today — per the UTC offset Open-Meteo reports for a timezone=auto - request — is dropped, matching the MET path's diurnal-coverage gate.""" - offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) +def test_recent_forecast_om_keeps_the_in_progress_local_day(monkeypatch): + """Today must survive the Open-Meteo bundle. Unlike the MET Norway series (which + starts mid-day and is gated on diurnal coverage), this endpoint backfills the + rest of today from the model run, so today's high/low is a whole-day value of + the same kind as tomorrow's. Dropping it left a hole between a complete + yesterday and a forecast tomorrow, which the UI renders as a missing column.""" + offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai cell) local_today = (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=offset)).date() days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] daily = { "time": [d.isoformat() for d in days], - "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value - "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], + "temperature_2m_max": [80.0, 82.0, 76.1, 84.0], + "temperature_2m_min": [60.0, 61.0, 55.9, 62.0], "precipitation_sum": [0.0, 0.0, 0.0, 0.0], } payload = {"utc_offset_seconds": offset, "daily": daily} @@ -237,60 +237,42 @@ def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): def json(self): return payload monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + got = climate._fetch_recent_forecast_om({"center_lat": 54.9, "center_lon": 23.8}) + assert got["date"].to_list() == days # every day present, no gap at today + row = got.filter(pl.col("date") == local_today) + assert row["tmax"].item() == 76.1 # today's whole-day high, ungraded-down + assert row["tmin"].item() == 55.9 + + +def test_recent_forecast_om_still_drops_a_today_with_no_high_low(monkeypatch): + """The one case today is excluded: it came back with no high/low and so cannot be + graded at all. _to_frame enforces that for every day, today included.""" + offset = 3 * 3600 + local_today = (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=offset)).date() + days = [local_today + datetime.timedelta(days=n) for n in (-1, 0, 1)] + daily = { + "time": [d.isoformat() for d in days], + "temperature_2m_max": [82.0, None, 84.0], + "temperature_2m_min": [61.0, None, 62.0], + "precipitation_sum": [0.0, 0.0, 0.0], + } + payload = {"utc_offset_seconds": offset, "daily": daily} + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + got = climate._fetch_recent_forecast_om( {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() - assert local_today not in got # partial today dropped - assert local_today - datetime.timedelta(days=1) in got # yesterday kept - assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept - assert len(got) == 3 + assert local_today not in got + assert got == [local_today - datetime.timedelta(days=1), + local_today + datetime.timedelta(days=1)] def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): - """No UTC offset reported -> skip the in-progress-day guard rather than guess a - date, so the bundle passes through as before (only the usual null-day filter).""" - payload = {"daily": _om_daily(3)} # no utc_offset_seconds - - class Resp: - def json(self): return payload - monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) - - df = climate._fetch_recent_forecast_om({"center_lat": 1.0, "center_lon": 2.0}) - assert df.height == climate._to_frame(_om_daily(3)).height - - -def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): - """The Open-Meteo fallback must not grade the cell's in-progress local day: its - daily high/low is only a partial aggregate of the hours elapsed so far (a cool - morning would read as a record-low high). Past days and future forecast days - survive; today — per the UTC offset Open-Meteo reports for a timezone=auto - request — is dropped, matching the MET path's diurnal-coverage gate.""" - offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) - local_today = (datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(seconds=offset)).date() - days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] - daily = { - "time": [d.isoformat() for d in days], - "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value - "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], - "precipitation_sum": [0.0, 0.0, 0.0, 0.0], - } - payload = {"utc_offset_seconds": offset, "daily": daily} - - class Resp: - def json(self): return payload - monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) - - got = climate._fetch_recent_forecast_om( - {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() - assert local_today not in got # partial today dropped - assert local_today - datetime.timedelta(days=1) in got # yesterday kept - assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept - assert len(got) == 3 - - -def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): - """No UTC offset reported -> skip the in-progress-day guard rather than guess a - date, so the bundle passes through as before (only the usual null-day filter).""" + """No UTC offset reported changes nothing: the bundle passes through with only + the usual null-day filter.""" payload = {"daily": _om_daily(3)} # no utc_offset_seconds class Resp: