diff --git a/.forgejo/workflows/backend-build-push.yml b/.forgejo/workflows/backend-build-push.yml index b502978..514aaca 100644 --- a/.forgejo/workflows/backend-build-push.yml +++ b/.forgejo/workflows/backend-build-push.yml @@ -65,6 +65,12 @@ jobs: IMAGE_PATH: ${{ github.repository }}/backend steps: - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 - name: Install Docker CLI run: | @@ -77,7 +83,13 @@ jobs: host="${REGISTRY#https://}"; host="${host#http://}" path="$(printf '%s' "$IMAGE_PATH" | tr '[:upper:]' '[:lower:]')" ref="$host/$path" - sha_tag="$ref:sha-${GITHUB_SHA:0:12}" + # Key the tag to the last commit that touched backend/ -- the SAME key + # every backend deploy workflow computes -- so build and deploy always + # agree even when the branch tip is another domain's commit. (A tag + # keyed to the push tip breaks the moment an infra-only commit lands: + # deploys go looking for an image no build ever produced.) + domain_sha="$(git log -1 --format=%H -- backend/)" + sha_tag="$ref:sha-${domain_sha:0:12}" echo "host=$host" >> "$GITHUB_OUTPUT" echo "sha_tag=$sha_tag" >> "$GITHUB_OUTPUT" if [[ "$GITHUB_REF" == refs/tags/v* ]]; then diff --git a/.forgejo/workflows/backend-deploy-dev.yml b/.forgejo/workflows/backend-deploy-dev.yml index cee669a..88227f1 100644 --- a/.forgejo/workflows/backend-deploy-dev.yml +++ b/.forgejo/workflows/backend-deploy-dev.yml @@ -51,12 +51,20 @@ jobs: cancel-in-progress: false steps: - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 - name: Compute image tag id: tag - # Same 12-hex truncation the beta/prod deploys use -- must match - # backend-build-push.yml's `sha-${GITHUB_SHA:0:12}` tag exactly. - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + # Keyed to the last commit that touched backend/ -- must match + # backend-build-push.yml's tag key exactly (see its comment). + run: | + domain_sha="$(git log -1 --format=%H -- backend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy to the LAN dev server # thermograph-lan is host-native (the runner job runs directly on the diff --git a/.forgejo/workflows/backend-deploy-prod.yml b/.forgejo/workflows/backend-deploy-prod.yml index 9df7155..c4fba2c 100644 --- a/.forgejo/workflows/backend-deploy-prod.yml +++ b/.forgejo/workflows/backend-deploy-prod.yml @@ -8,7 +8,7 @@ name: Deploy backend to prod VPS # frontend-deploy-prod.yml -- a backend change ships to prod without touching # frontend and vice versa, exactly as in the split era. # -# The tag MUST match backend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12 +# The tag MUST match backend-build-push.yml's last-backend-commit key (12 # hex) -- see backend-deploy.yml for why it's truncated here. deploy.sh # retries the pull because the build for this push may still be in flight. @@ -26,9 +26,19 @@ jobs: deploy: runs-on: docker steps: - - name: Compute image tag + - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 + + - name: Compute image tag (last backend-touching commit) id: tag - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + run: | + domain_sha="$(git log -1 --format=%H -- backend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy backend over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 diff --git a/.forgejo/workflows/backend-deploy.yml b/.forgejo/workflows/backend-deploy.yml index d2cbbf7..0ad3999 100644 --- a/.forgejo/workflows/backend-deploy.yml +++ b/.forgejo/workflows/backend-deploy.yml @@ -35,9 +35,19 @@ jobs: deploy: runs-on: docker steps: - - name: Compute image tag + - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 + + - name: Compute image tag (last backend-touching commit) id: tag - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + run: | + domain_sha="$(git log -1 --format=%H -- backend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy backend over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 diff --git a/.forgejo/workflows/frontend-build-push.yml b/.forgejo/workflows/frontend-build-push.yml index df6f568..cf38190 100644 --- a/.forgejo/workflows/frontend-build-push.yml +++ b/.forgejo/workflows/frontend-build-push.yml @@ -65,6 +65,12 @@ jobs: IMAGE_PATH: ${{ github.repository }}/frontend steps: - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 - name: Install Docker CLI run: | @@ -77,7 +83,13 @@ jobs: host="${REGISTRY#https://}"; host="${host#http://}" path="$(printf '%s' "$IMAGE_PATH" | tr '[:upper:]' '[:lower:]')" ref="$host/$path" - sha_tag="$ref:sha-${GITHUB_SHA:0:12}" + # Key the tag to the last commit that touched frontend/ -- the SAME key + # every frontend deploy workflow computes -- so build and deploy always + # agree even when the branch tip is another domain's commit. (A tag + # keyed to the push tip breaks the moment an infra-only commit lands: + # deploys go looking for an image no build ever produced.) + domain_sha="$(git log -1 --format=%H -- frontend/)" + sha_tag="$ref:sha-${domain_sha:0:12}" echo "host=$host" >> "$GITHUB_OUTPUT" echo "sha_tag=$sha_tag" >> "$GITHUB_OUTPUT" if [[ "$GITHUB_REF" == refs/tags/v* ]]; then diff --git a/.forgejo/workflows/frontend-deploy-dev.yml b/.forgejo/workflows/frontend-deploy-dev.yml index 2b745a8..0269777 100644 --- a/.forgejo/workflows/frontend-deploy-dev.yml +++ b/.forgejo/workflows/frontend-deploy-dev.yml @@ -51,12 +51,20 @@ jobs: cancel-in-progress: false steps: - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 - name: Compute image tag id: tag - # Same 12-hex truncation the beta/prod deploys use -- must match - # frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` tag exactly. - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + # Keyed to the last commit that touched frontend/ -- must match + # frontend-build-push.yml's tag key exactly (see its comment). + run: | + domain_sha="$(git log -1 --format=%H -- frontend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy to the LAN dev server # thermograph-lan is host-native (the runner job runs directly on the diff --git a/.forgejo/workflows/frontend-deploy-prod.yml b/.forgejo/workflows/frontend-deploy-prod.yml index 7a707d2..e483273 100644 --- a/.forgejo/workflows/frontend-deploy-prod.yml +++ b/.forgejo/workflows/frontend-deploy-prod.yml @@ -8,7 +8,7 @@ name: Deploy frontend to prod VPS # backend-deploy-prod.yml -- a frontend change ships to prod without touching # backend and vice versa, exactly as in the split era. # -# The tag MUST match frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12 +# The tag MUST match frontend-build-push.yml's last-frontend-commit key (12 # hex) -- see frontend-deploy.yml for why it's truncated here. deploy.sh # retries the pull because the build for this push may still be in flight. @@ -31,9 +31,19 @@ jobs: deploy: runs-on: docker steps: - - name: Compute image tag + - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 + + - name: Compute image tag (last frontend-touching commit) id: tag - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + run: | + domain_sha="$(git log -1 --format=%H -- frontend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy frontend over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 diff --git a/.forgejo/workflows/frontend-deploy.yml b/.forgejo/workflows/frontend-deploy.yml index 75c511a..bbdad62 100644 --- a/.forgejo/workflows/frontend-deploy.yml +++ b/.forgejo/workflows/frontend-deploy.yml @@ -14,7 +14,7 @@ name: Deploy frontend to beta VPS # referenced by full GitHub URL because it isn't mirrored in Forgejo's default # action registry. # -# The tag MUST match frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12 +# The tag MUST match frontend-build-push.yml's last-frontend-commit key (12 # hex), not the full 40-char ${{ github.sha }} -- default Actions expressions # have no substring function, so the compute step below truncates it. # deploy.sh also retries the pull for ~5 min because Forgejo `needs:` can't @@ -40,9 +40,19 @@ jobs: deploy: runs-on: docker steps: - - name: Compute image tag + - uses: actions/checkout@v4 + # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS + # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is + # often an unrelated domain's commit, and a depth-1 clone can't see + # past it to find the real key. + with: + fetch-depth: 0 + + - name: Compute image tag (last frontend-touching commit) id: tag - run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" + run: | + domain_sha="$(git log -1 --format=%H -- frontend/)" + echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT" - name: Deploy frontend over SSH uses: https://github.com/appleboy/ssh-action@v1.2.0 diff --git a/CUTOVER-NOTES.md b/CUTOVER-NOTES.md index 00753b6..845e21c 100644 --- a/CUTOVER-NOTES.md +++ b/CUTOVER-NOTES.md @@ -69,10 +69,42 @@ deploy. a `uses: ./.forgejo/workflows/build.yml` + `with:` job must pass its input. If paths aren't honored, fall back to the pr-build pattern (diff job + gated jobs) for the push workflows too. -3. **Land the in-flight split-repo branches** (backend/frontend - `reconcile-dev-with-main`, infra `phase5/discord-bot-flag`, …) in the split - repos, then re-subtree or graft the deltas — this assembly is `origin/main` - as of 2026-07-22 and goes stale as those merge. +3. **Branch drift**: the split repos' branch state as of 2026-07-22 evening is + fully migrated (see "Branch migration status" below) — re-run the check + (`git merge-tree --write-tree` per branch vs dev/main) if cutover slips and + upstream moves again. + +## Branch migration status (2026-07-22) + +Checked every split-repo branch by dry-run merge; migrated everything real: + +- **`dev` (this repo)** = `main` + backend/frontend `origin/dev` subtrees: the + in-image test CI feature (ported into root `build.yml` — backend full suite, + frontend unit tier; their subtree `build.yml` edits resolved as deleted) + + backend test scripts/Makefile/compose.test + frontend two-tier suite with + fixtures. Faithful to upstream: the feature is dev-only, so mono `main` + doesn't have it; promote with dev→main like the split flow. +- **`feature/geocode-local` → `feature/history-meteostat-gusts` → + `feature/nasa-primary-flip` → `feature/era5-seed`** — the unmerged backend + stack, rebuilt stacked on mono `dev`. ⚠️ Two conflicts were resolved here + that upstream has NOT resolved (the same conflicts exist merging those + branches into split-repo dev): `data/climate.py` (dev's hardened `_request` + signature combined with the gust-fill wrap) and `tests/data/test_climate.py` + (kept both appended test blocks). **Unverified** — tests weren't runnable + locally (Docker Hub blocked); run the in-image suite on these branches + before trusting them. +- **Stale (absorbed upstream, nothing migrated — safe to delete in the split + repos):** backend `ci-run-tests-in-image`, `perf/*`, + `phase5/discord-gateway-bot`, `port-discord-gateway-bot`, + `reconcile-dev-with-main`, `test-runner-and-smoke`; frontend + `ci-run-tests-in-image`, `reconcile-dev-with-main`; all three infra branches + (`ci-fix/render-secrets-owner`, `phase5/discord-bot-flag`, + `terraform-two-image-tags` — all landed in infra `main`); both observability + dashboard branches. +- **Dismissed, not stale-but-not-migrated:** frontend + `perf/frontend-responsiveness` (its "new" content would resurrect test + files dev reorganized into the unit tier) and `test-real-backend-harness` + (an older revision of dev's `tests/unit/test_rendering.py`). ## Cutover runbook diff --git a/backend/data/climate.py b/backend/data/climate.py index 909ad24..fc5e411 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -60,7 +60,8 @@ MIN_ARCHIVE_DAYS = 3650 # ~10 yrs: far above any sync window, far bel # cached indefinitely (refetched only to add new metric columns). Only the recent # tail is refreshed — a small incremental fetch, at most hourly. HISTORY_TOPUP_INTERVAL = 3600 # seconds between recent-tail refresh attempts -FORECAST_TTL_HOURS = 1 # refetch the forward forecast hourly to track updates +FORECAST_TTL_HOURS = 4 # refetch the recent+forecast bundle every ~4h (lower IO; + # daily-resolution forecast normals move slowly) # The archive (historical) endpoint. Overridable so it can point at a self-hosted # Open-Meteo instance (ERA5 in object storage) instead of the rate-limited public @@ -80,12 +81,13 @@ NASA_POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point" NASA_START = "19810101" NASA_FILL = -900.0 # POWER's missing-value sentinel is ~-999 -# Backup forward forecast when Open-Meteo's forecast API is unavailable. MET Norway -# (yr.no) is free + keyless + global, mirroring NASA POWER's role for history. It -# returns a sub-daily timeseries in metric units with no gusts or apparent temp, so -# we aggregate to daily, convert units, and derive feels-like like the NASA path. -# It is forecast-only — no recent past days — so it's a degraded-but-working fallback. -METNO_URL = "https://api.met.no/weatherapi/locationforecast/2.0/complete" +# Forward-forecast source (primary for the forward days). MET Norway (yr.no) is free +# + keyless + global. It returns a sub-daily timeseries in metric units with no gusts +# or apparent temp, so we aggregate to daily, convert units, derive feels-like like +# the NASA path, and fill gusts from Meteostat. It is forecast-only (no recent past), +# so the recent observed days come from a NASA POWER range instead. The /compact +# endpoint carries every field we use at a smaller payload than /complete. +METNO_URL = "https://api.met.no/weatherapi/locationforecast/2.0/compact" # MET Norway's ToS requires an identifying User-Agent (a missing/generic one is # 403'd); include the app and a contact URL so they can reach us if usage misbehaves. METNO_UA = "Thermograph/0.2 (+https://thermograph.org)" @@ -824,6 +826,10 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: RECENT_PAST_DAYS = 25 # recent observations window (covers the ~2-week graded view) FORECAST_DAYS = 8 # today + 7 days ahead +# NASA POWER near-real-time lags a couple of days, so the recent observed window ends +# here; the small today-1/today-2 gap before the MET Norway forecast begins grades as +# missing (drop_nulls), bracketed by history behind and forecast ahead. +RECENT_END_LAG_DAYS = 2 def _rf_cache_path(cell_id: str) -> str: @@ -865,20 +871,27 @@ def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None": return None -def _load_recent_forecast(cell: dict) -> pl.DataFrame: - """Recent observations AND the forward forecast in ONE forecast-API call. +def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: + """Recent observations + forward forecast WITHOUT Open-Meteo: the recent observed + window from a NASA POWER range (measured), the forward days from MET Norway, + merged by date. Gusts — which neither source carries — are filled from Meteostat: + measured for the observed days, estimated from wind for the forecast days.""" + today = datetime.date.today() + start = (today - datetime.timedelta(days=RECENT_PAST_DAYS)).isoformat() + end = (today - datetime.timedelta(days=RECENT_END_LAG_DAYS)).isoformat() + past = _fetch_history_nasa(cell, start=start, end=end) # measured + Meteostat gusts + fwd = meteostat.fill_gusts( + cell["center_lat"], cell["center_lon"], _fetch_forecast_metno(cell)) + # Forecast wins on any overlapping day (freshest model value for today); the NASA + # observed days fill the recent past MET Norway lacks. + return (pl.concat([past, fwd], how="diagonal_relaxed") + .unique(subset="date", keep="last", maintain_order=True) + .sort("date")) - Both the recent (past) view and the forecast (future) view slice from this, so - a cell needs just one upstream forecast request per hour (plus the ~monthly - archive fetch). Cached per cell for one hour so it still follows model updates. - """ - cell_id = cell["id"] - hit = _read_recent_backed(cell_id) - if hit is not None: - cached, age_s = hit - if age_s / 3600.0 < FORECAST_TTL_HOURS: - return _with_doy(cached) +def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: + """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former + primary): recent past + forward days in one call.""" params = { "latitude": cell["center_lat"], "longitude": cell["center_lon"], @@ -890,49 +903,59 @@ def _load_recent_forecast(cell: dict) -> pl.DataFrame: "past_days": RECENT_PAST_DAYS, "forecast_days": FORECAST_DAYS, } - # Skip Open-Meteo's forecast endpoint entirely while it's in its own - # rate-limit cooldown (see _note_forecast_rate_limit) -- mirrors - # _load_history's archive-cooldown check, so a forecast brownout doesn't - # retry-storm the endpoint from every subscribed cell and every live request - # independently; it falls straight through to the MET Norway backup instead. + r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") + return _to_frame(r.json()["daily"]) + + +def _load_recent_forecast(cell: dict) -> pl.DataFrame: + """Recent observations AND the forward forecast for a cell. + + The primary source is keyless and Open-Meteo-free: a NASA POWER recent-past range + plus the MET Norway forward forecast (see _fetch_recent_forecast). The Open-Meteo + forecast API is the fallback (skipped while it's in its own rate-limit cooldown, + see _note_forecast_rate_limit), then a stale cache. Cached per cell for + FORECAST_TTL_HOURS so it follows model updates without per-hour upstream IO. + """ + cell_id = cell["id"] + hit = _read_recent_backed(cell_id) + if hit is not None: + cached, age_s = hit + if age_s / 3600.0 < FORECAST_TTL_HOURS: + return _with_doy(cached) + df = None - primary_error = None - if time.time() >= _forecast_cooldown_until: + try: + df = _fetch_recent_forecast(cell) # NASA recent + MET forward (primary) + except Exception: # noqa: BLE001 - keyless primary unavailable; try Open-Meteo + pass + + forecast_error = None + if df is None and time.time() >= _forecast_cooldown_until: + # Fall back to the Open-Meteo forecast API, skipped while it's in its own + # rate-limit cooldown so a brownout doesn't retry-storm the endpoint. try: - r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") - df = _to_frame(r.json()["daily"]) + df = _fetch_recent_forecast_om(cell) except Exception as e: # noqa: BLE001 if is_rate_limit(e): _note_forecast_rate_limit(e) - primary_error = e + forecast_error = e if df is None: - # Open-Meteo forecast unavailable (rate limit, cooldown, or outage). Fall - # back to MET Norway (yr.no) — global + keyless — mirroring the archive's - # NASA POWER backup. MET Norway is forecast-only (no recent past days), - # so this keeps the forecast / day-ahead views working in a degraded form. - try: - df = _fetch_forecast_metno(cell) - except Exception: # noqa: BLE001 - backup unavailable too - # Last resort: serve the stale cache if we have one. An hours-old bundle - # (which still carries the recent observed days MET Norway lacks) beats a - # hard failure, mirroring _load_history's stale-serve. Return WITHOUT - # rewriting it, so recent_stamp stays old and the next request still - # retries upstream first rather than serving this as if it were fresh. - if hit is not None: - return _with_doy(hit[0]) - # No cache either: surface the original error, classifying an Open-Meteo - # rate limit as the typed, daily-aware WeatherUnavailable (the archive - # path classifies its own inside _load_history). primary_error is None - # when the primary fetch was skipped outright (cooldown active) -- - # report the cooldown itself in that case. - if primary_error is not None and is_rate_limit(primary_error): - daily = "daily" in _rate_limit_reason(primary_error).lower() - raise WeatherUnavailable(limit_message(daily), daily=daily) from primary_error - if primary_error is not None: - raise primary_error - raise WeatherUnavailable(limit_message(_forecast_limit_daily), - daily=_forecast_limit_daily) + # Last resort: serve the stale cache if we have one, WITHOUT rewriting it, so + # recent_stamp stays old and the next request retries upstream first (mirrors + # _load_history's stale-serve). + if hit is not None: + return _with_doy(hit[0]) + # No cache either: surface the error, classifying an Open-Meteo rate limit as + # the typed, daily-aware WeatherUnavailable. forecast_error is None when the + # fallback was skipped outright (cooldown active) — report the cooldown then. + if forecast_error is not None and is_rate_limit(forecast_error): + daily = "daily" in _rate_limit_reason(forecast_error).lower() + raise WeatherUnavailable(limit_message(daily), daily=daily) from forecast_error + if forecast_error is not None: + raise forecast_error + raise WeatherUnavailable(limit_message(_forecast_limit_daily), + daily=_forecast_limit_daily) _write_recent_backed(cell_id, df) return df diff --git a/backend/drift_check.py b/backend/drift_check.py new file mode 100644 index 0000000..b91df04 --- /dev/null +++ b/backend/drift_check.py @@ -0,0 +1,136 @@ +"""Value-drift check: quantify how much the migration moved the data, per variable +and by grade band, by comparing the two live history sources head-to-head. + +Open-Meteo is kept as a dormant fallback, so both sources are still fetchable: this +pulls NASA POWER (the new primary) and the Open-Meteo archive (ERA5) for a sample of +cells and reports, per variable: + - mean-absolute-difference / bias / max — the RAW value drift, and + - grade-band divergence — the share of (day, metric) whose graded band actually + changes between sources. +Since Open-Meteo IS ERA5, this doubles as a fidelity check for the ERA5 seed. + + python drift_check.py [--limit N] [--days N] # on a networked box + +Note a real property of the grading model worth reading the two numbers together: +grades are percentiles within each source's OWN distribution, so a uniform bias +between sources shifts MAD but leaves grades unchanged. High MAD with low grade +divergence means "different absolute values, same story" — usually fine. High grade +divergence is the signal that actually matters. + +The comparison logic (compare_values / grade_band_divergence) is unit-tested; the +fetch is not. +""" +import datetime +import sys + +import polars as pl + +from data import cities +from data import climate +from data import grading +from data import grid + +# Variables compared for raw drift (feels is derived; precip/wind/gust/humid optional). +_METRICS = ("tmax", "tmin", "precip", "wind", "gust", "humid", "feels") + + +def compare_values(a: pl.DataFrame, b: pl.DataFrame, metrics=_METRICS) -> dict: + """Per-variable raw-drift stats over the dates the two frames share: n aligned + days, mean-absolute-difference, mean bias (a - b), and max absolute difference. + Days where either source is null for a metric are dropped from that metric.""" + j = a.join(b, on="date", how="inner", suffix="_b") + out = {} + for m in metrics: + if m not in a.columns or m not in b.columns: + continue + d = j.select((pl.col(m) - pl.col(f"{m}_b")).alias("d")).drop_nulls() + if d.height == 0: + out[m] = {"n": 0, "mad": None, "bias": None, "max_abs": None} + continue + diff = d["d"] + out[m] = { + "n": d.height, + "mad": round(float(diff.abs().mean()), 3), + "bias": round(float(diff.mean()), 3), + "max_abs": round(float(diff.abs().max()), 3), + } + return out + + +def grade_band_divergence(a: pl.DataFrame, b: pl.DataFrame, start, end) -> dict: + """Per-metric share of days whose graded band differs between the two sources over + [start, end]. Each source's days are graded against its OWN ±7-day climatology + (grading.grade_range), so this measures whether the migration changes the story a + day tells, not just its raw value.""" + ga = {d["date"]: d for d in grading.grade_range(a, start, end)} + gb = {d["date"]: d for d in grading.grade_range(b, start, end)} + per: dict[str, list[int]] = {} + for date in ga.keys() & gb.keys(): + da, db = ga[date], gb[date] + for m in grading.CLIMO_METRICS: + va, vb = da.get(m), db.get(m) + if va and vb: + slot = per.setdefault(m, [0, 0]) # [compared, differ] + slot[0] += 1 + slot[1] += int(va["g"] != vb["g"]) + return {m: {"compared": c, "differ": d, "pct": round(100.0 * d / c, 1) if c else None} + for m, (c, d) in sorted(per.items())} + + +def check_cell(cell: dict, days: int) -> dict: + """Fetch both sources for a cell and compare them over the most recent `days`.""" + nasa = climate._fetch_history_nasa(cell) # new primary (+ Meteostat gusts) + om = climate._fetch_history(cell) # Open-Meteo archive (ERA5) + end = min(nasa["date"].max(), om["date"].max()) + start = end - datetime.timedelta(days=days) + return {"values": compare_values(nasa, om), + "grades": grade_band_divergence(nasa, om, start, end)} + + +def _mean(xs: list) -> "float | None": + xs = [x for x in xs if x is not None] + return round(sum(xs) / len(xs), 3) if xs else None + + +def main(limit: "int | None" = None, days: int = 365) -> None: + todo = cities.all_cities() + if limit: + todo = todo[:limit] + mad_by_metric: dict[str, list] = {} + grade_by_metric: dict[str, list] = {} + checked = failed = 0 + for i, c in enumerate(todo, 1): + cell = grid.snap(c["lat"], c["lon"]) + try: + rep = check_cell(cell, days) + except Exception as e: # noqa: BLE001 - one cell's outage shouldn't abort the sweep + failed += 1 + print(f"[{i}/{len(todo)}] FAILED {c['slug']}: {e}") + continue + checked += 1 + vals = ", ".join(f"{m} mad={s['mad']}" for m, s in rep["values"].items() + if s.get("mad") is not None) + grds = ", ".join(f"{m} {s['pct']}%" for m, s in rep["grades"].items()) + print(f"[{i}/{len(todo)}] {c['slug']} ({cell['id']})") + print(f" values: {vals}") + print(f" grade drift: {grds}") + for m, s in rep["values"].items(): + mad_by_metric.setdefault(m, []).append(s.get("mad")) + for m, s in rep["grades"].items(): + grade_by_metric.setdefault(m, []).append(s.get("pct")) + + print(f"\n=== summary over {checked} cells (failed={failed}) ===") + print("mean MAD (raw drift):") + for m in _METRICS: + if m in mad_by_metric: + print(f" {m:7s} {_mean(mad_by_metric[m])}") + print("mean grade-band divergence (share of days a grade changes):") + for m in sorted(grade_by_metric): + print(f" {m:7s} {_mean(grade_by_metric[m])}%") + + +if __name__ == "__main__": + argv = sys.argv[1:] + lim = int(argv[argv.index("--limit") + 1]) if "--limit" in argv else None + dys = int(argv[argv.index("--days") + 1]) if "--days" in argv else 365 + main(limit=lim, days=dys) diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index 6fab033..79c3d71 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -137,34 +137,53 @@ def test_metno_to_frame_aggregates_daily_and_converts_units(): assert round(d2["precip"], 4) == round(6.0 / 25.4, 4) # only the 6-hour block -def test_recent_forecast_falls_back_to_metno(monkeypatch, tmp_path): - """When Open-Meteo's forecast API fails, the MET Norway backup serves the frame.""" +def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_path): + """The recent/forecast bundle is built from a NASA POWER recent-past range plus the + MET Norway forward forecast, merged by date (Open-Meteo not consulted).""" monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) - cell = {"id": "fallback_cell", "center_lat": 47.6062, "center_lon": -122.3321} - met = {"timeseries": [ - _metno_step("2026-07-16T00:00:00Z", 18.0, 55.0, 3.0, p1=0.0), - _metno_step("2026-07-17T00:00:00Z", 22.0, 45.0, 5.0, p6=1.0), - ]} + cell = {"id": "rf_primary", "center_lat": 47.6, "center_lon": -122.3} - class Resp: - def json(self): return {"properties": met} + past = climate._finalize_approximated(pl.DataFrame({ + "date": [datetime.date(2026, 7, 10), datetime.date(2026, 7, 11)], + "tmax": [70.0, 72.0], "tmin": [50.0, 51.0], "precip": [0.0, 0.1], + "wind": [5.0, 6.0], "humid": [60.0, 55.0], + })) + fwd = climate._finalize_approximated(pl.DataFrame({ + "date": [datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)], + "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.0], + "wind": [3.0, 4.0], "humid": [40.0, 45.0], + })) + monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c, start, end: past) + monkeypatch.setattr(climate, "_fetch_forecast_metno", lambda c: fwd) + monkeypatch.setattr(climate.meteostat, "fill_gusts", lambda lat, lon, df: df) + def _om_should_not_run(c): raise AssertionError("Open-Meteo forecast should not run") + monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run) - def fake_request(url, params, timeout, *, phase, headers=None, attempts=climate.MAX_ATTEMPTS): - if url == climate.FORECAST_URL: - raise RuntimeError("open-meteo forecast outage") - assert url == climate.METNO_URL - assert headers and headers.get("User-Agent"), "MET Norway needs a User-Agent" - return Resp() - - monkeypatch.setattr(climate, "_request", fake_request) df = climate._load_recent_forecast(cell) - assert len(df) == 2 and df["date"].max() == datetime.date(2026, 7, 17) + assert df["date"].to_list() == [ + datetime.date(2026, 7, 10), datetime.date(2026, 7, 11), + datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)] + + +def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path): + """When the NASA + MET Norway primary fails, the Open-Meteo forecast API serves.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0) + cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3} + + def _primary_down(c): raise RuntimeError("NASA + MET Norway down") + monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down) + om = climate._to_frame(_om_daily(3)) + monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om) + + df = climate._load_recent_forecast(cell) + assert df.height == om.height def test_recent_forecast_serves_stale_cache_when_all_sources_fail(monkeypatch, tmp_path): - """With Open-Meteo AND MET Norway both down, an existing (stale) cache is served - rather than failing — and it is NOT rewritten, so its mtime stays old and the - next request still retries upstream first.""" + """With every source down (the NASA + MET Norway primary and the Open-Meteo + fallback), an existing (stale) cache is served rather than failing — and it is NOT + rewritten, so its mtime stays old and the next request still retries upstream first.""" import os import time monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) @@ -370,28 +389,23 @@ def test_history_tail_falls_back_to_open_meteo(monkeypatch): # --- forecast cooldown (mirrors the archive path's, tracked separately) ----- -def test_forecast_cooldown_skips_open_meteo_and_falls_to_metno(monkeypatch, tmp_path): - """While the forecast cooldown is active, the forecast fetch must not call - Open-Meteo at all -- straight to the MET Norway backup, mirroring - _load_history's archive-cooldown skip.""" +def test_forecast_cooldown_skips_the_open_meteo_fallback(monkeypatch, tmp_path): + """While the forecast cooldown is active, the Open-Meteo FALLBACK is skipped: with + the keyless primary (NASA + MET Norway) also down and no cache, the load surfaces + WeatherUnavailable without ever calling Open-Meteo.""" import time as time_mod monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) monkeypatch.setattr(climate, "_forecast_cooldown_until", time_mod.time() + 60) cell = {"id": "fc_cooldown_cell", "center_lat": 47.6, "center_lon": -122.3} - met = {"timeseries": [_metno_step("2026-07-16T00:00:00Z", 18.0, 55.0, 3.0, p1=0.0)]} + def _primary_down(c): raise RuntimeError("NASA + MET Norway down") + monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down) + def _om_should_not_run(c): + raise AssertionError("Open-Meteo fallback must not run during cooldown") + monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run) - class Resp: - def json(self): return {"properties": met} - - def fake_request(url, params, timeout, *, phase, headers=None, attempts=climate.MAX_ATTEMPTS): - assert url != climate.FORECAST_URL, "must not call Open-Meteo during cooldown" - assert url == climate.METNO_URL - return Resp() - - monkeypatch.setattr(climate, "_request", fake_request) - df = climate._load_recent_forecast(cell) - assert len(df) == 1 + with pytest.raises(climate.WeatherUnavailable): + climate._load_recent_forecast(cell) def test_forecast_429_sets_its_own_cooldown(monkeypatch, tmp_path): diff --git a/backend/tests/test_drift_check.py b/backend/tests/test_drift_check.py new file mode 100644 index 0000000..698dc95 --- /dev/null +++ b/backend/tests/test_drift_check.py @@ -0,0 +1,55 @@ +"""Unit tests for the drift-check comparison logic (hermetic — no network). The +source fetch (check_cell) is not tested here; it runs on a networked box.""" +import datetime +import math + +import polars as pl +import pytest + +import drift_check +from data import climate + +_D = [datetime.date(2024, 1, d) for d in (1, 2, 3)] + + +def test_compare_values_stats(): + a = pl.DataFrame({"date": _D, "tmax": [70.0, 80.0, 90.0], "wind": [5.0, 6.0, 7.0]}) + b = pl.DataFrame({"date": _D, "tmax": [72.0, 78.0, 90.0], "wind": [5.0, 6.0, 7.0]}) + out = drift_check.compare_values(a, b, metrics=("tmax", "wind")) + assert out["tmax"] == {"n": 3, "mad": pytest.approx(4 / 3, abs=1e-3), + "bias": pytest.approx(0.0), "max_abs": pytest.approx(2.0)} + assert out["wind"] == {"n": 3, "mad": 0.0, "bias": 0.0, "max_abs": 0.0} + + +def test_compare_values_skips_nulls_and_missing_metric(): + a = pl.DataFrame({"date": _D, "tmax": [70.0, 80.0, 90.0]}) + b = pl.DataFrame({"date": _D, "tmax": [71.0, None, 90.0]}) + out = drift_check.compare_values(a, b, metrics=("tmax", "gust")) + assert out["tmax"]["n"] == 2 # the null day is dropped + assert out["tmax"]["mad"] == pytest.approx(0.5) # |70-71| and |90-90| + assert "gust" not in out # metric absent from both frames + + +def _gradeable(n=800): + """A finalized, gradeable multi-year frame (seasonal temp swing so windows have a + real distribution).""" + start = datetime.date(2019, 1, 1) + dates = [start + datetime.timedelta(days=i) for i in range(n)] + tmax = [70.0 + 15.0 * math.sin(i / 58.0) for i in range(n)] + tmin = [t - 15.0 for t in tmax] + return climate._finalize_frame(pl.DataFrame({ + "date": dates, "tmax": tmax, "tmin": tmin, "precip": [0.0] * n, + "wind": [5.0] * n, "gust": [8.0] * n, "humid": [60.0] * n, + "fmax": tmax, "fmin": tmin, + })) + + +def test_grade_band_divergence_identical_is_zero(): + f = _gradeable() + end = f["date"].max() + start = end - datetime.timedelta(days=60) + div = drift_check.grade_band_divergence(f, f, start, end) + assert div # some metrics were compared + assert any(s["compared"] > 0 for s in div.values()) + assert all(s["differ"] == 0 for s in div.values()) # identical frames never diverge + assert all(s["pct"] == 0.0 for s in div.values())