Jinemi/thermograph
4
0
Fork 0
Code Issues 2 Pull requests 9 Projects Releases Packages Wiki Activity Actions

Port 3753bbc Refactor climate.py source->frame builders from monorepo (drift reconciliation)

Browse source 
Claude-Session: https://claude.ai/code/session_01RdARHDJaYC1wSQRTYM6t3Z
This commit is contained in:
Emi Griffith 2026-07-22 12:07:54 -07:00
parent f830bce4d9
commit b1869af70a
2 changed files with 47 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
data/climate.py
View file

@ -71,11 +71,18 @@ DAILY_VARS = (
"relative_humidity_2m_mean"
)
# Optional metric columns beyond the required date/tmax/tmin/precip schema. A
# source that lacks one gets an all-null column in _finalize_frame, so every
# finalized frame has the same schema without each builder stubbing missing
# columns by hand.
OPTIONAL_COLS = ("wind", "gust", "humid", "fmax", "fmin")
# Columns added after the original tmax/tmin/precip schema. A cached parquet
# missing any of these predates the wind/feels/humidity features (or the split
# apparent high/low) and is refetched so the new metrics show up immediately
# instead of after the 30-day cache TTL.
NEW_COLS = ("wind", "gust", "feels", "humid", "fmax", "fmin")
# instead of after the 30-day cache TTL. `feels` is derived in _finalize_frame;
# the rest are the optional raw inputs, so this stays in lockstep with them.
NEW_COLS = (*OPTIONAL_COLS, "feels")
# Thermoneutral baseline (°F). The combined "feels like" metric reports whichever
# apparent-temperature extreme — the heat-index-driven daily max or the
@ -379,12 +386,31 @@ def _finalize_frame(df: pl.DataFrame) -> pl.DataFrame:
usable climate day needs a real high/low; other columns may be missing and
simply grade as None. Float NaN (from numpy-derived columns) is folded into null
so the whole pipeline models "missing" one way — the grading boundary drops it."""
# Backfill any optional metric a source didn't provide as an all-null column, so
# the finalized schema (and the NEW_COLS cache check) is uniform across sources.
absent = [pl.lit(None, dtype=pl.Float64).alias(c)
for c in OPTIONAL_COLS if c not in df.columns]
if absent:
df = df.with_columns(absent)
df = df.with_columns(pl.col(pl.Float32, pl.Float64).fill_nan(None))
df = df.with_columns(_combined_feels_expr().alias("feels"))
df = df.drop_nulls(subset=["tmax", "tmin"])
return _with_doy(df)
def _finalize_approximated(df: pl.DataFrame) -> pl.DataFrame:
"""Finalize a source that lacks apparent temperature (NASA POWER, MET Norway):
approximate the felt high with the NWS heat index and the felt low with wind
chill (each falls back to the air temperature outside its regime), mirroring the
Open-Meteo fmax/fmin columns, then finalize. The numpy NaN these produce lands
in float columns and is folded to null by _finalize_frame."""
df = df.with_columns(
pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
)
return _finalize_frame(df)
def _to_frame(daily: dict) -> pl.DataFrame:
n = len(daily["time"])
# Older upstream responses (or a narrowed variable set) may omit a series; fall
@ -459,23 +485,14 @@ def _nasa_to_frame(param: dict) -> pl.DataFrame:
"tmin": col("T2M_MIN", c2f),
"precip": col("PRECTOTCORR", lambda mm: mm / 25.4),
"wind": col("WS10M_MAX", lambda ms: ms * 2.2369362920544),
"gust": [None] * len(dates), # POWER has no gusts
"humid": col("RH2M", lambda v: v),
})
}) # POWER has no gusts (backfilled null)
df = df.with_columns(
pl.col("date").str.to_date("%Y%m%d"),
*[pl.col(c).cast(pl.Float64, strict=False)
for c in ("tmax", "tmin", "precip", "wind", "gust")],
for c in ("tmax", "tmin", "precip", "wind")],
)
# POWER has no apparent temperature; approximate the felt high with the NWS
# heat index and the felt low with NWS wind chill (each falls back to the air
# temperature outside its regime), mirroring the Open-Meteo columns. numpy NaN
# from these lands in float columns and is folded to null in _finalize_frame.
df = df.with_columns(
pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
)
return _finalize_frame(df)
return _finalize_approximated(df)
def _fetch_history_nasa(cell: dict) -> pl.DataFrame:
@ -533,20 +550,15 @@ def _metno_to_frame(props: dict) -> pl.DataFrame:
for d in dates],
"wind": [max(agg[d]["wind"]) * 2.2369362920544 if agg[d]["wind"] else None
for d in dates],
"gust": [None] * len(dates), # MET Norway has no gusts
"humid": [sum(agg[d]["rh"]) / len(agg[d]["rh"]) if agg[d]["rh"] else None
for d in dates],
})
}) # MET Norway has no gusts (backfilled null)
df = df.with_columns(
pl.col("date").str.to_date(),
*[pl.col(c).cast(pl.Float64, strict=False)
for c in ("tmax", "tmin", "precip", "wind", "gust")],
for c in ("tmax", "tmin", "precip", "wind")],
)
df = df.with_columns(
pl.Series("fmax", _heat_index(df["tmax"].to_numpy(), df["humid"].to_numpy())),
pl.Series("fmin", _wind_chill(df["tmin"].to_numpy(), df["wind"].to_numpy())),
)
return _finalize_frame(df)
return _finalize_approximated(df)
def _fetch_forecast_metno(cell: dict) -> pl.DataFrame:

13
tests/data/test_climate.py
View file

@ -40,6 +40,19 @@ def test_to_frame_tolerates_missing_series():
assert df["tmax"].is_not_null().all() # required series unaffected
def test_finalize_frame_backfills_absent_optional_columns():
# A source frame carrying only the required base columns is finalized into the
# full schema, every absent optional metric added as an all-null column.
bare = pl.DataFrame({
"date": [datetime.date(2026, 6, 1), datetime.date(2026, 6, 2)],
"tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.1],
})
df = climate._finalize_frame(bare)
for c in climate.OPTIONAL_COLS:
assert c in df.columns and df[c].is_null().all()
assert all(c in df.columns for c in climate.NEW_COLS) # incl. derived `feels`
def test_combined_feels_picks_the_extreme_side():
df = climate._to_frame(_om_daily())
# Day 2: fmax 95 is 30 past the 65°F comfort point vs fmin 68 only 3 below —