diff --git a/data/climate.py b/data/climate.py index 3507ad4..4964f41 100644 --- a/data/climate.py +++ b/data/climate.py @@ -71,11 +71,18 @@ DAILY_VARS = ( "relative_humidity_2m_mean" ) +# Optional metric columns beyond the required date/tmax/tmin/precip schema. A +# source that lacks one gets an all-null column in _finalize_frame, so every +# finalized frame has the same schema without each builder stubbing missing +# columns by hand. +OPTIONAL_COLS = ("wind", "gust", "humid", "fmax", "fmin") + # Columns added after the original tmax/tmin/precip schema. A cached parquet # missing any of these predates the wind/feels/humidity features (or the split # apparent high/low) and is refetched so the new metrics show up immediately -# instead of after the 30-day cache TTL. -NEW_COLS = ("wind", "gust", "feels", "humid", "fmax", "fmin") +# instead of after the 30-day cache TTL. `feels` is derived in _finalize_frame; +# the rest are the optional raw inputs, so this stays in lockstep with them. +NEW_COLS = (*OPTIONAL_COLS, "feels") # Thermoneutral baseline (°F). The combined "feels like" metric reports whichever # apparent-temperature extreme — the heat-index-driven daily max or the @@ -379,12 +386,31 @@ def _finalize_frame(df: pl.DataFrame) -> pl.DataFrame: usable climate day needs a real high/low; other columns may be missing and simply grade as None. Float NaN (from numpy-derived columns) is folded into null so the whole pipeline models "missing" one way — the grading boundary drops it.""" + # Backfill any optional metric a source didn't provide as an all-null column, so + # the finalized schema (and the NEW_COLS cache check) is uniform across sources. + absent = [pl.lit(None, dtype=pl.Float64).alias(c) + for c in OPTIONAL_COLS if c not in df.columns] + if absent: + df = df.with_columns(absent) df = df.with_columns(pl.col(pl.Float32, pl.Float64).fill_nan(None)) df = df.with_columns(_combined_feels_expr().alias("feels")) df = df.drop_nulls(subset=["tmax", "tmin"]) return _with_doy(df) +def _finalize_approximated(df: pl.DataFrame) -> pl.DataFrame: + """Finalize a source that lacks apparent temperature (NASA POWER, MET Norway): + approximate the felt high with the NWS heat index and the felt low with wind + chill (each falls back to the air temperature outside its regime), mirroring the + Open-Meteo fmax/fmin columns, then finalize. The numpy NaN these produce lands + in float columns and is folded to null by _finalize_frame.""" + df = df.with_columns( + pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())), + pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())), + ) + return _finalize_frame(df) + + def _to_frame(daily: dict) -> pl.DataFrame: n = len(daily["time"]) # Older upstream responses (or a narrowed variable set) may omit a series; fall @@ -459,23 +485,14 @@ def _nasa_to_frame(param: dict) -> pl.DataFrame: "tmin": col("T2M_MIN", c2f), "precip": col("PRECTOTCORR", lambda mm: mm / 25.4), "wind": col("WS10M_MAX", lambda ms: ms * 2.2369362920544), - "gust": [None] * len(dates), # POWER has no gusts "humid": col("RH2M", lambda v: v), - }) + }) # POWER has no gusts (backfilled null) df = df.with_columns( pl.col("date").str.to_date("%Y%m%d"), *[pl.col(c).cast(pl.Float64, strict=False) - for c in ("tmax", "tmin", "precip", "wind", "gust")], + for c in ("tmax", "tmin", "precip", "wind")], ) - # POWER has no apparent temperature; approximate the felt high with the NWS - # heat index and the felt low with NWS wind chill (each falls back to the air - # temperature outside its regime), mirroring the Open-Meteo columns. numpy NaN - # from these lands in float columns and is folded to null in _finalize_frame. - df = df.with_columns( - pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())), - pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())), - ) - return _finalize_frame(df) + return _finalize_approximated(df) def _fetch_history_nasa(cell: dict) -> pl.DataFrame: @@ -533,20 +550,15 @@ def _metno_to_frame(props: dict) -> pl.DataFrame: for d in dates], "wind": [max(agg[d]["wind"]) * 2.2369362920544 if agg[d]["wind"] else None for d in dates], - "gust": [None] * len(dates), # MET Norway has no gusts "humid": [sum(agg[d]["rh"]) / len(agg[d]["rh"]) if agg[d]["rh"] else None for d in dates], - }) + }) # MET Norway has no gusts (backfilled null) df = df.with_columns( pl.col("date").str.to_date(), *[pl.col(c).cast(pl.Float64, strict=False) - for c in ("tmax", "tmin", "precip", "wind", "gust")], + for c in ("tmax", "tmin", "precip", "wind")], ) - df = df.with_columns( - pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())), - pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())), - ) - return _finalize_frame(df) + return _finalize_approximated(df) def _fetch_forecast_metno(cell: dict) -> pl.DataFrame: diff --git a/tests/data/test_climate.py b/tests/data/test_climate.py index 3084a7a..74533cc 100644 --- a/tests/data/test_climate.py +++ b/tests/data/test_climate.py @@ -40,6 +40,19 @@ def test_to_frame_tolerates_missing_series(): assert df["tmax"].is_not_null().all() # required series unaffected +def test_finalize_frame_backfills_absent_optional_columns(): + # A source frame carrying only the required base columns is finalized into the + # full schema, every absent optional metric added as an all-null column. + bare = pl.DataFrame({ + "date": [datetime.date(2026, 6, 1), datetime.date(2026, 6, 2)], + "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.1], + }) + df = climate._finalize_frame(bare) + for c in climate.OPTIONAL_COLS: + assert c in df.columns and df[c].is_null().all() + assert all(c in df.columns for c in climate.NEW_COLS) # incl. derived `feels` + + def test_combined_feels_picks_the_extreme_side(): df = climate._to_frame(_om_daily()) # Day 2: fmax 95 is 30 past the 65°F comfort point vs fmin 68 only 3 below —