c3bacdce0c
* Migrate backend dataframe layer from pandas to polars Replace pandas with polars across the backend, dropping both pandas and its pyarrow parquet engine from the dependency set. numpy stays (the grading percentile math is unchanged). - climate.py: parquet IO, source→frame mappings, cache read/topup on polars. New _normalize_read casts the cached `date` column to pl.Date (older files were written by pandas as datetime64[ns]); frames now unify missing values as null so the grading boundary drops them consistently across sources. - grading.py: keep the numpy percentile core; swap the frame→numpy bridge to .to_numpy()/.drop_nulls(), day-of-year/year to polars dt expressions, and the per-row loop to iter_rows(named=True). - views.py: filter/anti-join/concat replace boolean-mask, isin and pd.concat; scalar dates are stdlib datetime.date; a local _months_before helper replaces DateOffset(months=) for the calendar-range default. - app.py, migrate.py: request-date parsing uses datetime.date, removing pandas from the endpoint and migrate layers entirely. - The date column is pl.Date end to end, eliminating the pandas normalize() calls and comparing cleanly against stdlib dates. Payloads are unchanged: calendar, day, grade and forecast responses are byte-for-byte identical to the pandas implementation on the same cached record. Tests ported to polars fixtures, with added coverage for the combined feels-like fallback, calendar month-offset (month-end/leap), and the concat/dedup "fresher source wins" rule. * Port notify.py to polars after merging dev's account system Merge origin/dev (accounts + notification subscriptions) and carry the pandas→ polars migration into the newly added notify.py, which the merge brought in still using pandas — with pandas removed from requirements this broke its import. - notify.py: _candidate_rows filters/sorts the recent bundle with polars expressions and returns iter_rows dicts; date scalars are datetime.date; history/recent emptiness via is_empty(). - test_notify.py: synthetic history/rows built with polars + datetime.
93 lines
4 KiB
Python
93 lines
4 KiB
Python
"""The source→frame mappings and cache-write path (pure parts of climate.py —
|
|
no network)."""
|
|
import datetime
|
|
|
|
import polars as pl
|
|
|
|
import climate
|
|
|
|
|
|
def _om_daily(n=3):
|
|
"""A minimal Open-Meteo daily response."""
|
|
return {
|
|
"time": [f"2026-06-{d:02d}" for d in range(1, n + 1)],
|
|
"temperature_2m_max": [80.0, 90.0, None],
|
|
"temperature_2m_min": [60.0, 70.0, 55.0],
|
|
"precipitation_sum": [0.0, 0.25, 0.1],
|
|
"wind_speed_10m_max": [10.0, 20.0, 5.0],
|
|
"wind_gusts_10m_max": [15.0, 30.0, 8.0],
|
|
"apparent_temperature_max": [82.0, 95.0, 70.0],
|
|
"apparent_temperature_min": [58.0, 68.0, 50.0],
|
|
"relative_humidity_2m_mean": [50.0, 60.0, 70.0],
|
|
}
|
|
|
|
|
|
def test_to_frame_schema_and_day_filter():
|
|
df = climate._to_frame(_om_daily())
|
|
# The None tmax day is dropped: a usable climate day needs a real high/low.
|
|
assert len(df) == 2
|
|
assert df["doy"].dtype == pl.Int16
|
|
assert set(df.columns) == {"date", "tmax", "tmin", "precip", "wind", "gust",
|
|
"humid", "fmax", "fmin", "feels", "doy"}
|
|
|
|
|
|
def test_to_frame_tolerates_missing_series():
|
|
daily = _om_daily()
|
|
del daily["wind_gusts_10m_max"], daily["relative_humidity_2m_mean"]
|
|
df = climate._to_frame(daily)
|
|
assert df["gust"].is_null().all() and df["humid"].is_null().all()
|
|
assert df["tmax"].is_not_null().all() # required series unaffected
|
|
|
|
|
|
def test_combined_feels_picks_the_extreme_side():
|
|
df = climate._to_frame(_om_daily())
|
|
# Day 2: fmax 95 is 30 past the 65°F comfort point vs fmin 68 only 3 below —
|
|
# the hot side wins; both days here are hot-side days.
|
|
assert df.filter(pl.col("date") == datetime.date(2026, 6, 2))["feels"].item() == 95.0
|
|
|
|
|
|
def test_combined_feels_falls_back_to_the_present_side():
|
|
"""When one apparent-temperature side is missing, `feels` uses whichever side
|
|
is present (the null-vs-value coalesce must reproduce the old NaN fallback)."""
|
|
daily = _om_daily()
|
|
daily["apparent_temperature_max"] = [None, None, None] # hot side absent
|
|
df = climate._to_frame(daily)
|
|
row = df.filter(pl.col("date") == datetime.date(2026, 6, 1)).row(0, named=True)
|
|
assert row["feels"] == 58.0 # falls back to apparent low
|
|
|
|
|
|
def test_nasa_to_frame_converts_units_and_fills():
|
|
param = {
|
|
"T2M_MAX": {"20260601": 30.0, "20260602": -999.0}, # °C; -999 = missing
|
|
"T2M_MIN": {"20260601": 20.0, "20260602": 15.0},
|
|
"PRECTOTCORR": {"20260601": 25.4, "20260602": 0.0}, # mm
|
|
"WS10M_MAX": {"20260601": 10.0, "20260602": 5.0}, # m/s
|
|
"RH2M": {"20260601": 50.0, "20260602": 60.0},
|
|
}
|
|
df = climate._nasa_to_frame(param)
|
|
assert len(df) == 1 # the missing-tmax day dropped
|
|
row = df.row(0, named=True)
|
|
assert row["tmax"] == 86.0 # 30°C
|
|
assert row["precip"] == 1.0 # 25.4mm = 1in
|
|
assert round(row["wind"], 1) == 22.4 # 10 m/s in mph
|
|
assert row["gust"] is None # POWER has no gusts (missing -> null)
|
|
assert row["fmax"] > row["tmax"] # ≥80°F: the NWS heat index applies
|
|
|
|
|
|
def test_om_daily_params_carries_the_window():
|
|
cell = {"center_lat": 47.6, "center_lon": -122.3}
|
|
p = climate._om_daily_params(cell, start_date="2026-01-01", end_date="2026-02-01")
|
|
assert p["latitude"] == 47.6 and p["daily"] == climate.DAILY_VARS
|
|
assert p["temperature_unit"] == "fahrenheit"
|
|
assert p["start_date"] == "2026-01-01" and p["end_date"] == "2026-02-01"
|
|
p2 = climate._om_daily_params(cell, past_days=25, forecast_days=8)
|
|
assert p2["past_days"] == 25 and "start_date" not in p2
|
|
|
|
|
|
def test_write_cache_strips_the_derived_doy(tmp_path):
|
|
df = climate._to_frame(_om_daily())
|
|
path = str(tmp_path / "cell.parquet")
|
|
climate._write_cache(df, path)
|
|
stored = pl.read_parquet(path)
|
|
assert "doy" not in stored.columns
|
|
assert len(stored) == len(df)
|