24d8cd9ba1
seed_era5.py backfills the curated-city cells with true ERA5 data from the Earthmover Icechunk ERA5 archive on AWS Open Data (anonymous, keyless), so high-traffic pages keep ERA5 fidelity now that NASA POWER is the live primary. It aggregates the hourly ERA5 point series to the daily store schema in polars (no pandas), derives feels-like via the existing heat-index/wind-chill path, and writes straight to the history store via climate._write_history_backed, bypassing the live fetch. ERA5 carries real gusts (i10fg), so seeded cells keep measured gusts rather than the Meteostat fallback. The icechunk/xarray/zarr stack is isolated in requirements-seed.txt (not the app image or CI) and lazy-imported, so the module and its unit-tested hourly->daily transform load without those deps. The S3 access layer targets a young API and is verified via `--dry-run` on the seed box, not in tests. Idempotent by default (skips already-cached cells; --overwrite to reseed).
72 lines
2.9 KiB
Python
72 lines
2.9 KiB
Python
"""Unit tests for the ERA5 seed's hourly→daily transform (hermetic — no network, no
|
|
icechunk/xarray). The S3 access layer (_open_store/fetch_point) is deliberately not
|
|
tested here; it must be verified with `seed_era5.py --dry-run` on the seed box."""
|
|
import datetime
|
|
import math
|
|
|
|
import polars as pl
|
|
import pytest
|
|
|
|
import seed_era5
|
|
|
|
_MS_TO_MPH = 2.2369362920544
|
|
_M_TO_IN = 39.37007874
|
|
|
|
|
|
def _rh(t_c, td_c):
|
|
es = math.exp(17.625 * t_c / (243.04 + t_c))
|
|
e = math.exp(17.625 * td_c / (243.04 + td_c))
|
|
return min(100.0, 100.0 * e / es)
|
|
|
|
|
|
def _hourly(rows):
|
|
"""rows: list of (time, t2m_K, d2m_K, tp_m, u10, v10, i10fg)."""
|
|
cols = ["time", "t2m", "d2m", "tp", "u10", "v10", "i10fg"]
|
|
return pl.DataFrame({c: [r[i] for r in rows] for i, c in enumerate(cols)})
|
|
|
|
|
|
def test_hourly_to_daily_aggregates_and_converts():
|
|
d1 = datetime.datetime(2020, 1, 1)
|
|
d2 = datetime.datetime(2020, 1, 2)
|
|
hourly = _hourly([
|
|
# day 1: 10°C sat, then 20°C / 10°C dew; winds 5 m/s then calm; gusts 10/12
|
|
(d1.replace(hour=0), 283.15, 283.15, 0.001, 3.0, 4.0, 10.0),
|
|
(d1.replace(hour=1), 293.15, 283.15, 0.002, 0.0, 0.0, 12.0),
|
|
# day 2: 0°C sat; wind 10 m/s; gust 20
|
|
(d2.replace(hour=0), 273.15, 273.15, 0.0, 6.0, 8.0, 20.0),
|
|
])
|
|
daily = seed_era5.hourly_to_daily(hourly).sort("date")
|
|
assert daily["date"].to_list() == [datetime.date(2020, 1, 1), datetime.date(2020, 1, 2)]
|
|
|
|
r0 = daily.row(0, named=True)
|
|
assert r0["tmax"] == pytest.approx(68.0) # 20°C
|
|
assert r0["tmin"] == pytest.approx(50.0) # 10°C
|
|
assert r0["precip"] == pytest.approx(0.003 * _M_TO_IN)
|
|
assert r0["wind"] == pytest.approx(5.0 * _MS_TO_MPH) # max(5, 0)
|
|
assert r0["gust"] == pytest.approx(12.0 * _MS_TO_MPH) # max(10, 12)
|
|
assert r0["humid"] == pytest.approx((_rh(10, 10) + _rh(20, 10)) / 2)
|
|
|
|
r1 = daily.row(1, named=True)
|
|
assert r1["tmin"] == pytest.approx(32.0) # 0°C
|
|
assert r1["wind"] == pytest.approx(10.0 * _MS_TO_MPH) # sqrt(6²+8²)
|
|
|
|
|
|
def test_saturated_air_reads_full_humidity():
|
|
hourly = _hourly([(datetime.datetime(2021, 6, 1), 293.15, 293.15, 0.0, 1.0, 1.0, 2.0)])
|
|
daily = seed_era5.hourly_to_daily(hourly)
|
|
assert daily.row(0, named=True)["humid"] == pytest.approx(100.0)
|
|
|
|
|
|
def test_finalize_produces_the_store_schema():
|
|
"""The daily frame, run through climate._finalize_approximated (as seed_cell does),
|
|
yields the persisted columns incl. derived feels."""
|
|
from data import climate
|
|
hourly = _hourly([
|
|
(datetime.datetime(2019, 7, 1, 0), 300.0, 290.0, 0.0, 2.0, 2.0, 5.0),
|
|
(datetime.datetime(2019, 7, 1, 1), 305.0, 291.0, 0.001, 3.0, 1.0, 7.0),
|
|
])
|
|
frame = climate._finalize_approximated(seed_era5.hourly_to_daily(hourly))
|
|
for col in ("date", "tmax", "tmin", "precip", "wind", "gust", "humid",
|
|
"fmax", "fmin", "feels"):
|
|
assert col in frame.columns
|
|
assert frame.height == 1
|