c3bacdce0c
* Migrate backend dataframe layer from pandas to polars Replace pandas with polars across the backend, dropping both pandas and its pyarrow parquet engine from the dependency set. numpy stays (the grading percentile math is unchanged). - climate.py: parquet IO, source→frame mappings, cache read/topup on polars. New _normalize_read casts the cached `date` column to pl.Date (older files were written by pandas as datetime64[ns]); frames now unify missing values as null so the grading boundary drops them consistently across sources. - grading.py: keep the numpy percentile core; swap the frame→numpy bridge to .to_numpy()/.drop_nulls(), day-of-year/year to polars dt expressions, and the per-row loop to iter_rows(named=True). - views.py: filter/anti-join/concat replace boolean-mask, isin and pd.concat; scalar dates are stdlib datetime.date; a local _months_before helper replaces DateOffset(months=) for the calendar-range default. - app.py, migrate.py: request-date parsing uses datetime.date, removing pandas from the endpoint and migrate layers entirely. - The date column is pl.Date end to end, eliminating the pandas normalize() calls and comparing cleanly against stdlib dates. Payloads are unchanged: calendar, day, grade and forecast responses are byte-for-byte identical to the pandas implementation on the same cached record. Tests ported to polars fixtures, with added coverage for the combined feels-like fallback, calendar month-offset (month-end/leap), and the concat/dedup "fresher source wins" rule. * Port notify.py to polars after merging dev's account system Merge origin/dev (accounts + notification subscriptions) and carry the pandas→ polars migration into the newly added notify.py, which the merge brought in still using pandas — with pandas removed from requirements this broke its import. - notify.py: _candidate_rows filters/sorts the recent bundle with polars expressions and returns iter_rows dicts; date scalars are datetime.date; history/recent emptiness via is_empty(). - test_notify.py: synthetic history/rows built with polars + datetime.
89 lines
3.6 KiB
Python
89 lines
3.6 KiB
Python
"""Backfill the SQLite derived store from the existing parquet cell caches.
|
|
|
|
For every cell with a cached archive record, materialize the payloads that are
|
|
derivable offline — the default 2-year calendar and the latest-day detail — plus
|
|
its reverse-geocode label, so a server restarted (or freshly deployed) onto an
|
|
existing parquet cache serves those views from the database immediately instead
|
|
of regrading each cell on first touch.
|
|
|
|
Idempotent and resumable: cells whose derived rows already match the current
|
|
validity token are skipped, so re-running after an interruption (or after new
|
|
archive days arrive) only does the missing work. Never fetches weather data —
|
|
history comes strictly from the parquet cache (cells without one are skipped and
|
|
will materialize lazily on first request, as always). Reverse geocoding makes at
|
|
most one Nominatim call per unlabeled cell, throttled to ~1/s per the service's
|
|
policy. Safe to run alongside a live server: writes go through the same WAL
|
|
store the server reads, and readers fall back to recomputing on any miss.
|
|
|
|
Run via `make migrate`.
|
|
"""
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import climate
|
|
import grid
|
|
import store
|
|
import views
|
|
|
|
|
|
def migrate() -> int:
|
|
if not os.path.isdir(climate.CACHE_DIR):
|
|
print("No parquet cache directory yet — nothing to migrate.")
|
|
return 0
|
|
cells = sorted(
|
|
f[: -len(".parquet")]
|
|
for f in os.listdir(climate.CACHE_DIR)
|
|
if f.endswith(".parquet") and not f.endswith("_rf.parquet")
|
|
)
|
|
built = current = skipped = 0
|
|
for cell_id in cells:
|
|
try:
|
|
cell = grid.from_id(cell_id)
|
|
except ValueError:
|
|
print(f" {cell_id}: unrecognized cache filename — skipped")
|
|
skipped += 1
|
|
continue
|
|
history = climate.load_cached_history(cell)
|
|
if history is None or history.is_empty():
|
|
# Pre-current-schema record; the server refetches it lazily on first
|
|
# touch (see climate.NEW_COLS) — nothing to materialize offline.
|
|
print(f" {cell_id}: no schema-complete record — skipped")
|
|
skipped += 1
|
|
continue
|
|
|
|
token = views.history_token(history)
|
|
start_ts, end_ts = views.cal_span(history, None, None, 24)
|
|
cal_key = views.calendar_key(start_ts, end_ts, 24)
|
|
last = history["date"].max()
|
|
day_key = views.day_key(last)
|
|
|
|
have_cal = store.get_payload("calendar", cell_id, cal_key, token) is not None
|
|
have_day = store.get_payload("day", cell_id, day_key, token) is not None
|
|
if have_cal and have_day:
|
|
current += 1
|
|
continue
|
|
|
|
found, place = climate.reverse_geocode_cached(cell["center_lat"], cell["center_lon"])
|
|
if not found:
|
|
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
|
|
time.sleep(1.1) # Nominatim usage policy: at most ~1 request/second
|
|
|
|
if not have_cal:
|
|
store.put_payload("calendar", cell_id, cal_key, token,
|
|
views.build_calendar(cell, history, start_ts, end_ts, 24, place))
|
|
if not have_day:
|
|
store.put_payload("day", cell_id, day_key, token,
|
|
views.build_day(cell, history, last, place))
|
|
built += 1
|
|
print(f" {cell_id}: materialized ({place or 'no label'})")
|
|
|
|
s = store.stats()
|
|
print(f"{built} cell(s) materialized, {current} already current, {skipped} skipped"
|
|
f" · derived rows: {s['derived']} · revgeo: {s['revgeo']}"
|
|
f" · db: {s['bytes'] / 1e6:.1f} MB → {s['db_path']}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(migrate())
|