Port 3753bbc Refactor climate.py source->frame builders from monorepo (drift reconciliation)

Claude-Session: https://claude.ai/code/session_01RdARHDJaYC1wSQRTYM6t3Z

data/climate.py

@@ -71,11 +71,18 @@ DAILY_VARS = (
     "relative_humidity_2m_mean"
 )
 
+# Optional metric columns beyond the required date/tmax/tmin/precip schema. A
+# source that lacks one gets an all-null column in _finalize_frame, so every
+# finalized frame has the same schema without each builder stubbing missing
+# columns by hand.
+OPTIONAL_COLS = ("wind", "gust", "humid", "fmax", "fmin")
+
 # Columns added after the original tmax/tmin/precip schema. A cached parquet
 # missing any of these predates the wind/feels/humidity features (or the split
 # apparent high/low) and is refetched so the new metrics show up immediately
-# instead of after the 30-day cache TTL.
+# instead of after the 30-day cache TTL. `feels` is derived in _finalize_frame;
-NEW_COLS = ("wind", "gust", "feels", "humid", "fmax", "fmin")
+# the rest are the optional raw inputs, so this stays in lockstep with them.
+NEW_COLS = (*OPTIONAL_COLS, "feels")
 
 # Thermoneutral baseline (°F). The combined "feels like" metric reports whichever
 # apparent-temperature extreme — the heat-index-driven daily max or the
@@ -379,12 +386,31 @@ def _finalize_frame(df: pl.DataFrame) -> pl.DataFrame:
     usable climate day needs a real high/low; other columns may be missing and
     simply grade as None. Float NaN (from numpy-derived columns) is folded into null
     so the whole pipeline models "missing" one way — the grading boundary drops it."""
+    # Backfill any optional metric a source didn't provide as an all-null column, so
+    # the finalized schema (and the NEW_COLS cache check) is uniform across sources.
+    absent = [pl.lit(None, dtype=pl.Float64).alias(c)
+              for c in OPTIONAL_COLS if c not in df.columns]
+    if absent:
+        df = df.with_columns(absent)
     df = df.with_columns(pl.col(pl.Float32, pl.Float64).fill_nan(None))
     df = df.with_columns(_combined_feels_expr().alias("feels"))
     df = df.drop_nulls(subset=["tmax", "tmin"])
     return _with_doy(df)
 
 
+def _finalize_approximated(df: pl.DataFrame) -> pl.DataFrame:
+    """Finalize a source that lacks apparent temperature (NASA POWER, MET Norway):
+    approximate the felt high with the NWS heat index and the felt low with wind
+    chill (each falls back to the air temperature outside its regime), mirroring the
+    Open-Meteo fmax/fmin columns, then finalize. The numpy NaN these produce lands
+    in float columns and is folded to null by _finalize_frame."""
+    df = df.with_columns(
+        pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
+        pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
+    )
+    return _finalize_frame(df)
+
+
 def _to_frame(daily: dict) -> pl.DataFrame:
     n = len(daily["time"])
     # Older upstream responses (or a narrowed variable set) may omit a series; fall
@@ -459,23 +485,14 @@ def _nasa_to_frame(param: dict) -> pl.DataFrame:
         "tmin": col("T2M_MIN", c2f),
         "precip": col("PRECTOTCORR", lambda mm: mm / 25.4),
         "wind": col("WS10M_MAX", lambda ms: ms * 2.2369362920544),
-        "gust": [None] * len(dates),          # POWER has no gusts
         "humid": col("RH2M", lambda v: v),
-    })
+    })                                        # POWER has no gusts (backfilled null)
     df = df.with_columns(
         pl.col("date").str.to_date("%Y%m%d"),
         *[pl.col(c).cast(pl.Float64, strict=False)
-          for c in ("tmax", "tmin", "precip", "wind", "gust")],
+          for c in ("tmax", "tmin", "precip", "wind")],
     )
-    # POWER has no apparent temperature; approximate the felt high with the NWS
+    return _finalize_approximated(df)
-    # heat index and the felt low with NWS wind chill (each falls back to the air
-    # temperature outside its regime), mirroring the Open-Meteo columns. numpy NaN
-    # from these lands in float columns and is folded to null in _finalize_frame.
-    df = df.with_columns(
-        pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
-        pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
-    )
-    return _finalize_frame(df)
 
 
 def _fetch_history_nasa(cell: dict) -> pl.DataFrame:
@@ -533,20 +550,15 @@ def _metno_to_frame(props: dict) -> pl.DataFrame:
                    for d in dates],
         "wind": [max(agg[d]["wind"]) * 2.2369362920544 if agg[d]["wind"] else None
                  for d in dates],
-        "gust": [None] * len(dates),          # MET Norway has no gusts
         "humid": [sum(agg[d]["rh"]) / len(agg[d]["rh"]) if agg[d]["rh"] else None
                   for d in dates],
-    })
+    })                                        # MET Norway has no gusts (backfilled null)
     df = df.with_columns(
         pl.col("date").str.to_date(),
         *[pl.col(c).cast(pl.Float64, strict=False)
-          for c in ("tmax", "tmin", "precip", "wind", "gust")],
+          for c in ("tmax", "tmin", "precip", "wind")],
     )
-    df = df.with_columns(
+    return _finalize_approximated(df)
-        pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
-        pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
-    )
-    return _finalize_frame(df)
 
 
 def _fetch_forecast_metno(cell: dict) -> pl.DataFrame:

tests/data/test_climate.py

@@ -40,6 +40,19 @@ def test_to_frame_tolerates_missing_series():
     assert df["tmax"].is_not_null().all()   # required series unaffected
 
 
+def test_finalize_frame_backfills_absent_optional_columns():
+    # A source frame carrying only the required base columns is finalized into the
+    # full schema, every absent optional metric added as an all-null column.
+    bare = pl.DataFrame({
+        "date": [datetime.date(2026, 6, 1), datetime.date(2026, 6, 2)],
+        "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.1],
+    })
+    df = climate._finalize_frame(bare)
+    for c in climate.OPTIONAL_COLS:
+        assert c in df.columns and df[c].is_null().all()
+    assert all(c in df.columns for c in climate.NEW_COLS)   # incl. derived `feels`
+
+
 def test_combined_feels_picks_the_extreme_side():
     df = climate._to_frame(_om_daily())
     # Day 2: fmax 95 is 30 past the 65°F comfort point vs fmin 68 only 3 below —