Merge dev into the data-pipeline chain (CI tag-keying fixes)

.forgejo/workflows/backend-build-push.yml

@@ -65,6 +65,12 @@ jobs:
       IMAGE_PATH: ${{ github.repository }}/backend
     steps:
       - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
 
       - name: Install Docker CLI
         run: |
@@ -77,7 +83,13 @@ jobs:
           host="${REGISTRY#https://}"; host="${host#http://}"
           path="$(printf '%s' "$IMAGE_PATH" | tr '[:upper:]' '[:lower:]')"
           ref="$host/$path"
-          sha_tag="$ref:sha-${GITHUB_SHA:0:12}"
+          # Key the tag to the last commit that touched backend/ -- the SAME key
+          # every backend deploy workflow computes -- so build and deploy always
+          # agree even when the branch tip is another domain's commit. (A tag
+          # keyed to the push tip breaks the moment an infra-only commit lands:
+          # deploys go looking for an image no build ever produced.)
+          domain_sha="$(git log -1 --format=%H -- backend/)"
+          sha_tag="$ref:sha-${domain_sha:0:12}"
           echo "host=$host" >> "$GITHUB_OUTPUT"
           echo "sha_tag=$sha_tag" >> "$GITHUB_OUTPUT"
           if [[ "$GITHUB_REF" == refs/tags/v* ]]; then

.forgejo/workflows/backend-deploy-dev.yml

@@ -51,12 +51,20 @@ jobs:
       cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
 
       - name: Compute image tag
         id: tag
-        # Same 12-hex truncation the beta/prod deploys use -- must match
-        # backend-build-push.yml's `sha-${GITHUB_SHA:0:12}` tag exactly.
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        # Keyed to the last commit that touched backend/ -- must match
+        # backend-build-push.yml's tag key exactly (see its comment).
+        run: |
+          domain_sha="$(git log -1 --format=%H -- backend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy to the LAN dev server
         # thermograph-lan is host-native (the runner job runs directly on the

.forgejo/workflows/backend-deploy-prod.yml

@@ -8,7 +8,7 @@ name: Deploy backend to prod VPS
 # frontend-deploy-prod.yml -- a backend change ships to prod without touching
 # frontend and vice versa, exactly as in the split era.
 #
-# The tag MUST match backend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12
+# The tag MUST match backend-build-push.yml's last-backend-commit key (12
 # hex) -- see backend-deploy.yml for why it's truncated here. deploy.sh
 # retries the pull because the build for this push may still be in flight.
 
@@ -26,9 +26,19 @@ jobs:
   deploy:
     runs-on: docker
     steps:
-      - name: Compute image tag
+      - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
+
+      - name: Compute image tag (last backend-touching commit)
         id: tag
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        run: |
+          domain_sha="$(git log -1 --format=%H -- backend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy backend over SSH
         uses: https://github.com/appleboy/ssh-action@v1.2.0

.forgejo/workflows/backend-deploy.yml

@@ -35,9 +35,19 @@ jobs:
   deploy:
     runs-on: docker
     steps:
-      - name: Compute image tag
+      - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
+
+      - name: Compute image tag (last backend-touching commit)
         id: tag
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        run: |
+          domain_sha="$(git log -1 --format=%H -- backend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy backend over SSH
         uses: https://github.com/appleboy/ssh-action@v1.2.0

.forgejo/workflows/frontend-build-push.yml

@@ -65,6 +65,12 @@ jobs:
       IMAGE_PATH: ${{ github.repository }}/frontend
     steps:
       - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
 
       - name: Install Docker CLI
         run: |
@@ -77,7 +83,13 @@ jobs:
           host="${REGISTRY#https://}"; host="${host#http://}"
           path="$(printf '%s' "$IMAGE_PATH" | tr '[:upper:]' '[:lower:]')"
           ref="$host/$path"
-          sha_tag="$ref:sha-${GITHUB_SHA:0:12}"
+          # Key the tag to the last commit that touched frontend/ -- the SAME key
+          # every frontend deploy workflow computes -- so build and deploy always
+          # agree even when the branch tip is another domain's commit. (A tag
+          # keyed to the push tip breaks the moment an infra-only commit lands:
+          # deploys go looking for an image no build ever produced.)
+          domain_sha="$(git log -1 --format=%H -- frontend/)"
+          sha_tag="$ref:sha-${domain_sha:0:12}"
           echo "host=$host" >> "$GITHUB_OUTPUT"
           echo "sha_tag=$sha_tag" >> "$GITHUB_OUTPUT"
           if [[ "$GITHUB_REF" == refs/tags/v* ]]; then

.forgejo/workflows/frontend-deploy-dev.yml

@@ -51,12 +51,20 @@ jobs:
       cancel-in-progress: false
     steps:
       - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
 
       - name: Compute image tag
         id: tag
-        # Same 12-hex truncation the beta/prod deploys use -- must match
-        # frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` tag exactly.
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        # Keyed to the last commit that touched frontend/ -- must match
+        # frontend-build-push.yml's tag key exactly (see its comment).
+        run: |
+          domain_sha="$(git log -1 --format=%H -- frontend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy to the LAN dev server
         # thermograph-lan is host-native (the runner job runs directly on the

.forgejo/workflows/frontend-deploy-prod.yml

@@ -8,7 +8,7 @@ name: Deploy frontend to prod VPS
 # backend-deploy-prod.yml -- a frontend change ships to prod without touching
 # backend and vice versa, exactly as in the split era.
 #
-# The tag MUST match frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12
+# The tag MUST match frontend-build-push.yml's last-frontend-commit key (12
 # hex) -- see frontend-deploy.yml for why it's truncated here. deploy.sh
 # retries the pull because the build for this push may still be in flight.
 
@@ -31,9 +31,19 @@ jobs:
   deploy:
     runs-on: docker
     steps:
-      - name: Compute image tag
+      - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
+
+      - name: Compute image tag (last frontend-touching commit)
         id: tag
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        run: |
+          domain_sha="$(git log -1 --format=%H -- frontend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy frontend over SSH
         uses: https://github.com/appleboy/ssh-action@v1.2.0

.forgejo/workflows/frontend-deploy.yml

@@ -14,7 +14,7 @@ name: Deploy frontend to beta VPS
 # referenced by full GitHub URL because it isn't mirrored in Forgejo's default
 # action registry.
 #
-# The tag MUST match frontend-build-push.yml's `sha-${GITHUB_SHA:0:12}` (12
+# The tag MUST match frontend-build-push.yml's last-frontend-commit key (12
 # hex), not the full 40-char ${{ github.sha }} -- default Actions expressions
 # have no substring function, so the compute step below truncates it.
 # deploy.sh also retries the pull for ~5 min because Forgejo `needs:` can't
@@ -40,9 +40,19 @@ jobs:
   deploy:
     runs-on: docker
     steps:
-      - name: Compute image tag
+      - uses: actions/checkout@v4
+        # fetch-depth 0: the tag is keyed to the LAST COMMIT THAT TOUCHED THIS
+        # DOMAIN, not the branch tip -- in a path-filtered monorepo the tip is
+        # often an unrelated domain's commit, and a depth-1 clone can't see
+        # past it to find the real key.
+        with:
+          fetch-depth: 0
+
+      - name: Compute image tag (last frontend-touching commit)
         id: tag
-        run: echo "tag=sha-${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT"
+        run: |
+          domain_sha="$(git log -1 --format=%H -- frontend/)"
+          echo "tag=sha-${domain_sha:0:12}" >> "$GITHUB_OUTPUT"
 
       - name: Deploy frontend over SSH
         uses: https://github.com/appleboy/ssh-action@v1.2.0

CUTOVER-NOTES.md

@@ -69,10 +69,42 @@ deploy.
    a `uses: ./.forgejo/workflows/build.yml` + `with:` job must pass its input.
    If paths aren't honored, fall back to the pr-build pattern (diff job +
    gated jobs) for the push workflows too.
-3. **Land the in-flight split-repo branches** (backend/frontend
-   `reconcile-dev-with-main`, infra `phase5/discord-bot-flag`, …) in the split
-   repos, then re-subtree or graft the deltas — this assembly is `origin/main`
-   as of 2026-07-22 and goes stale as those merge.
+3. **Branch drift**: the split repos' branch state as of 2026-07-22 evening is
+   fully migrated (see "Branch migration status" below) — re-run the check
+   (`git merge-tree --write-tree` per branch vs dev/main) if cutover slips and
+   upstream moves again.
+
+## Branch migration status (2026-07-22)
+
+Checked every split-repo branch by dry-run merge; migrated everything real:
+
+- **`dev` (this repo)** = `main` + backend/frontend `origin/dev` subtrees: the
+  in-image test CI feature (ported into root `build.yml` — backend full suite,
+  frontend unit tier; their subtree `build.yml` edits resolved as deleted) +
+  backend test scripts/Makefile/compose.test + frontend two-tier suite with
+  fixtures. Faithful to upstream: the feature is dev-only, so mono `main`
+  doesn't have it; promote with dev→main like the split flow.
+- **`feature/geocode-local` → `feature/history-meteostat-gusts` →
+  `feature/nasa-primary-flip` → `feature/era5-seed`** — the unmerged backend
+  stack, rebuilt stacked on mono `dev`. ⚠️ Two conflicts were resolved here
+  that upstream has NOT resolved (the same conflicts exist merging those
+  branches into split-repo dev): `data/climate.py` (dev's hardened `_request`
+  signature combined with the gust-fill wrap) and `tests/data/test_climate.py`
+  (kept both appended test blocks). **Unverified** — tests weren't runnable
+  locally (Docker Hub blocked); run the in-image suite on these branches
+  before trusting them.
+- **Stale (absorbed upstream, nothing migrated — safe to delete in the split
+  repos):** backend `ci-run-tests-in-image`, `perf/*`,
+  `phase5/discord-gateway-bot`, `port-discord-gateway-bot`,
+  `reconcile-dev-with-main`, `test-runner-and-smoke`; frontend
+  `ci-run-tests-in-image`, `reconcile-dev-with-main`; all three infra branches
+  (`ci-fix/render-secrets-owner`, `phase5/discord-bot-flag`,
+  `terraform-two-image-tags` — all landed in infra `main`); both observability
+  dashboard branches.
+- **Dismissed, not stale-but-not-migrated:** frontend
+  `perf/frontend-responsiveness` (its "new" content would resurrect test
+  files dev reorganized into the unit tier) and `test-real-backend-harness`
+  (an older revision of dev's `tests/unit/test_rendering.py`).
 
 ## Cutover runbook
 

backend/data/climate.py

@@ -60,7 +60,8 @@ MIN_ARCHIVE_DAYS = 3650            # ~10 yrs: far above any sync window, far bel
 # cached indefinitely (refetched only to add new metric columns). Only the recent
 # tail is refreshed — a small incremental fetch, at most hourly.
 HISTORY_TOPUP_INTERVAL = 3600      # seconds between recent-tail refresh attempts
-FORECAST_TTL_HOURS = 1             # refetch the forward forecast hourly to track updates
+FORECAST_TTL_HOURS = 4             # refetch the recent+forecast bundle every ~4h (lower IO;
+                                   # daily-resolution forecast normals move slowly)
 
 # The archive (historical) endpoint. Overridable so it can point at a self-hosted
 # Open-Meteo instance (ERA5 in object storage) instead of the rate-limited public
@@ -80,12 +81,13 @@ NASA_POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point"
 NASA_START = "19810101"
 NASA_FILL = -900.0   # POWER's missing-value sentinel is ~-999
 
-# Backup forward forecast when Open-Meteo's forecast API is unavailable. MET Norway
-# (yr.no) is free + keyless + global, mirroring NASA POWER's role for history. It
-# returns a sub-daily timeseries in metric units with no gusts or apparent temp, so
-# we aggregate to daily, convert units, and derive feels-like like the NASA path.
-# It is forecast-only — no recent past days — so it's a degraded-but-working fallback.
-METNO_URL = "https://api.met.no/weatherapi/locationforecast/2.0/complete"
+# Forward-forecast source (primary for the forward days). MET Norway (yr.no) is free
+# + keyless + global. It returns a sub-daily timeseries in metric units with no gusts
+# or apparent temp, so we aggregate to daily, convert units, derive feels-like like
+# the NASA path, and fill gusts from Meteostat. It is forecast-only (no recent past),
+# so the recent observed days come from a NASA POWER range instead. The /compact
+# endpoint carries every field we use at a smaller payload than /complete.
+METNO_URL = "https://api.met.no/weatherapi/locationforecast/2.0/compact"
 # MET Norway's ToS requires an identifying User-Agent (a missing/generic one is
 # 403'd); include the app and a contact URL so they can reach us if usage misbehaves.
 METNO_UA = "Thermograph/0.2 (+https://thermograph.org)"
@@ -824,6 +826,10 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]:
 
 RECENT_PAST_DAYS = 25   # recent observations window (covers the ~2-week graded view)
 FORECAST_DAYS = 8       # today + 7 days ahead
+# NASA POWER near-real-time lags a couple of days, so the recent observed window ends
+# here; the small today-1/today-2 gap before the MET Norway forecast begins grades as
+# missing (drop_nulls), bracketed by history behind and forecast ahead.
+RECENT_END_LAG_DAYS = 2
 
 
 def _rf_cache_path(cell_id: str) -> str:
@@ -865,20 +871,27 @@ def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None":
         return None
 
 
-def _load_recent_forecast(cell: dict) -> pl.DataFrame:
-    """Recent observations AND the forward forecast in ONE forecast-API call.
+def _fetch_recent_forecast(cell: dict) -> pl.DataFrame:
+    """Recent observations + forward forecast WITHOUT Open-Meteo: the recent observed
+    window from a NASA POWER range (measured), the forward days from MET Norway,
+    merged by date. Gusts — which neither source carries — are filled from Meteostat:
+    measured for the observed days, estimated from wind for the forecast days."""
+    today = datetime.date.today()
+    start = (today - datetime.timedelta(days=RECENT_PAST_DAYS)).isoformat()
+    end = (today - datetime.timedelta(days=RECENT_END_LAG_DAYS)).isoformat()
+    past = _fetch_history_nasa(cell, start=start, end=end)   # measured + Meteostat gusts
+    fwd = meteostat.fill_gusts(
+        cell["center_lat"], cell["center_lon"], _fetch_forecast_metno(cell))
+    # Forecast wins on any overlapping day (freshest model value for today); the NASA
+    # observed days fill the recent past MET Norway lacks.
+    return (pl.concat([past, fwd], how="diagonal_relaxed")
+            .unique(subset="date", keep="last", maintain_order=True)
+            .sort("date"))
 
-    Both the recent (past) view and the forecast (future) view slice from this, so
-    a cell needs just one upstream forecast request per hour (plus the ~monthly
-    archive fetch). Cached per cell for one hour so it still follows model updates.
-    """
-    cell_id = cell["id"]
-    hit = _read_recent_backed(cell_id)
-    if hit is not None:
-        cached, age_s = hit
-        if age_s / 3600.0 < FORECAST_TTL_HOURS:
-            return _with_doy(cached)
 
+def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame:
+    """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former
+    primary): recent past + forward days in one call."""
     params = {
         "latitude": cell["center_lat"],
         "longitude": cell["center_lon"],
@@ -890,49 +903,59 @@ def _load_recent_forecast(cell: dict) -> pl.DataFrame:
         "past_days": RECENT_PAST_DAYS,
         "forecast_days": FORECAST_DAYS,
     }
-    # Skip Open-Meteo's forecast endpoint entirely while it's in its own
-    # rate-limit cooldown (see _note_forecast_rate_limit) -- mirrors
-    # _load_history's archive-cooldown check, so a forecast brownout doesn't
-    # retry-storm the endpoint from every subscribed cell and every live request
-    # independently; it falls straight through to the MET Norway backup instead.
+    r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch")
+    return _to_frame(r.json()["daily"])
+
+
+def _load_recent_forecast(cell: dict) -> pl.DataFrame:
+    """Recent observations AND the forward forecast for a cell.
+
+    The primary source is keyless and Open-Meteo-free: a NASA POWER recent-past range
+    plus the MET Norway forward forecast (see _fetch_recent_forecast). The Open-Meteo
+    forecast API is the fallback (skipped while it's in its own rate-limit cooldown,
+    see _note_forecast_rate_limit), then a stale cache. Cached per cell for
+    FORECAST_TTL_HOURS so it follows model updates without per-hour upstream IO.
+    """
+    cell_id = cell["id"]
+    hit = _read_recent_backed(cell_id)
+    if hit is not None:
+        cached, age_s = hit
+        if age_s / 3600.0 < FORECAST_TTL_HOURS:
+            return _with_doy(cached)
+
     df = None
-    primary_error = None
-    if time.time() >= _forecast_cooldown_until:
+    try:
+        df = _fetch_recent_forecast(cell)          # NASA recent + MET forward (primary)
+    except Exception:  # noqa: BLE001 - keyless primary unavailable; try Open-Meteo
+        pass
+
+    forecast_error = None
+    if df is None and time.time() >= _forecast_cooldown_until:
+        # Fall back to the Open-Meteo forecast API, skipped while it's in its own
+        # rate-limit cooldown so a brownout doesn't retry-storm the endpoint.
         try:
-            r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch")
-            df = _to_frame(r.json()["daily"])
+            df = _fetch_recent_forecast_om(cell)
         except Exception as e:  # noqa: BLE001
             if is_rate_limit(e):
                 _note_forecast_rate_limit(e)
-            primary_error = e
+            forecast_error = e
 
     if df is None:
-        # Open-Meteo forecast unavailable (rate limit, cooldown, or outage). Fall
-        # back to MET Norway (yr.no) — global + keyless — mirroring the archive's
-        # NASA POWER backup. MET Norway is forecast-only (no recent past days),
-        # so this keeps the forecast / day-ahead views working in a degraded form.
-        try:
-            df = _fetch_forecast_metno(cell)
-        except Exception:  # noqa: BLE001 - backup unavailable too
-            # Last resort: serve the stale cache if we have one. An hours-old bundle
-            # (which still carries the recent observed days MET Norway lacks) beats a
-            # hard failure, mirroring _load_history's stale-serve. Return WITHOUT
-            # rewriting it, so recent_stamp stays old and the next request still
-            # retries upstream first rather than serving this as if it were fresh.
-            if hit is not None:
-                return _with_doy(hit[0])
-            # No cache either: surface the original error, classifying an Open-Meteo
-            # rate limit as the typed, daily-aware WeatherUnavailable (the archive
-            # path classifies its own inside _load_history). primary_error is None
-            # when the primary fetch was skipped outright (cooldown active) --
-            # report the cooldown itself in that case.
-            if primary_error is not None and is_rate_limit(primary_error):
-                daily = "daily" in _rate_limit_reason(primary_error).lower()
-                raise WeatherUnavailable(limit_message(daily), daily=daily) from primary_error
-            if primary_error is not None:
-                raise primary_error
-            raise WeatherUnavailable(limit_message(_forecast_limit_daily),
-                                     daily=_forecast_limit_daily)
+        # Last resort: serve the stale cache if we have one, WITHOUT rewriting it, so
+        # recent_stamp stays old and the next request retries upstream first (mirrors
+        # _load_history's stale-serve).
+        if hit is not None:
+            return _with_doy(hit[0])
+        # No cache either: surface the error, classifying an Open-Meteo rate limit as
+        # the typed, daily-aware WeatherUnavailable. forecast_error is None when the
+        # fallback was skipped outright (cooldown active) — report the cooldown then.
+        if forecast_error is not None and is_rate_limit(forecast_error):
+            daily = "daily" in _rate_limit_reason(forecast_error).lower()
+            raise WeatherUnavailable(limit_message(daily), daily=daily) from forecast_error
+        if forecast_error is not None:
+            raise forecast_error
+        raise WeatherUnavailable(limit_message(_forecast_limit_daily),
+                                 daily=_forecast_limit_daily)
     _write_recent_backed(cell_id, df)
     return df
 

backend/drift_check.py

@@ -0,0 +1,136 @@
+"""Value-drift check: quantify how much the migration moved the data, per variable
+and by grade band, by comparing the two live history sources head-to-head.
+
+Open-Meteo is kept as a dormant fallback, so both sources are still fetchable: this
+pulls NASA POWER (the new primary) and the Open-Meteo archive (ERA5) for a sample of
+cells and reports, per variable:
+  - mean-absolute-difference / bias / max — the RAW value drift, and
+  - grade-band divergence — the share of (day, metric) whose graded band actually
+    changes between sources.
+Since Open-Meteo IS ERA5, this doubles as a fidelity check for the ERA5 seed.
+
+    python drift_check.py [--limit N] [--days N]   # on a networked box
+
+Note a real property of the grading model worth reading the two numbers together:
+grades are percentiles within each source's OWN distribution, so a uniform bias
+between sources shifts MAD but leaves grades unchanged. High MAD with low grade
+divergence means "different absolute values, same story" — usually fine. High grade
+divergence is the signal that actually matters.
+
+The comparison logic (compare_values / grade_band_divergence) is unit-tested; the
+fetch is not.
+"""
+import datetime
+import sys
+
+import polars as pl
+
+from data import cities
+from data import climate
+from data import grading
+from data import grid
+
+# Variables compared for raw drift (feels is derived; precip/wind/gust/humid optional).
+_METRICS = ("tmax", "tmin", "precip", "wind", "gust", "humid", "feels")
+
+
+def compare_values(a: pl.DataFrame, b: pl.DataFrame, metrics=_METRICS) -> dict:
+    """Per-variable raw-drift stats over the dates the two frames share: n aligned
+    days, mean-absolute-difference, mean bias (a - b), and max absolute difference.
+    Days where either source is null for a metric are dropped from that metric."""
+    j = a.join(b, on="date", how="inner", suffix="_b")
+    out = {}
+    for m in metrics:
+        if m not in a.columns or m not in b.columns:
+            continue
+        d = j.select((pl.col(m) - pl.col(f"{m}_b")).alias("d")).drop_nulls()
+        if d.height == 0:
+            out[m] = {"n": 0, "mad": None, "bias": None, "max_abs": None}
+            continue
+        diff = d["d"]
+        out[m] = {
+            "n": d.height,
+            "mad": round(float(diff.abs().mean()), 3),
+            "bias": round(float(diff.mean()), 3),
+            "max_abs": round(float(diff.abs().max()), 3),
+        }
+    return out
+
+
+def grade_band_divergence(a: pl.DataFrame, b: pl.DataFrame, start, end) -> dict:
+    """Per-metric share of days whose graded band differs between the two sources over
+    [start, end]. Each source's days are graded against its OWN ±7-day climatology
+    (grading.grade_range), so this measures whether the migration changes the story a
+    day tells, not just its raw value."""
+    ga = {d["date"]: d for d in grading.grade_range(a, start, end)}
+    gb = {d["date"]: d for d in grading.grade_range(b, start, end)}
+    per: dict[str, list[int]] = {}
+    for date in ga.keys() & gb.keys():
+        da, db = ga[date], gb[date]
+        for m in grading.CLIMO_METRICS:
+            va, vb = da.get(m), db.get(m)
+            if va and vb:
+                slot = per.setdefault(m, [0, 0])   # [compared, differ]
+                slot[0] += 1
+                slot[1] += int(va["g"] != vb["g"])
+    return {m: {"compared": c, "differ": d, "pct": round(100.0 * d / c, 1) if c else None}
+            for m, (c, d) in sorted(per.items())}
+
+
+def check_cell(cell: dict, days: int) -> dict:
+    """Fetch both sources for a cell and compare them over the most recent `days`."""
+    nasa = climate._fetch_history_nasa(cell)      # new primary (+ Meteostat gusts)
+    om = climate._fetch_history(cell)             # Open-Meteo archive (ERA5)
+    end = min(nasa["date"].max(), om["date"].max())
+    start = end - datetime.timedelta(days=days)
+    return {"values": compare_values(nasa, om),
+            "grades": grade_band_divergence(nasa, om, start, end)}
+
+
+def _mean(xs: list) -> "float | None":
+    xs = [x for x in xs if x is not None]
+    return round(sum(xs) / len(xs), 3) if xs else None
+
+
+def main(limit: "int | None" = None, days: int = 365) -> None:
+    todo = cities.all_cities()
+    if limit:
+        todo = todo[:limit]
+    mad_by_metric: dict[str, list] = {}
+    grade_by_metric: dict[str, list] = {}
+    checked = failed = 0
+    for i, c in enumerate(todo, 1):
+        cell = grid.snap(c["lat"], c["lon"])
+        try:
+            rep = check_cell(cell, days)
+        except Exception as e:  # noqa: BLE001 - one cell's outage shouldn't abort the sweep
+            failed += 1
+            print(f"[{i}/{len(todo)}] FAILED {c['slug']}: {e}")
+            continue
+        checked += 1
+        vals = ", ".join(f"{m} mad={s['mad']}" for m, s in rep["values"].items()
+                         if s.get("mad") is not None)
+        grds = ", ".join(f"{m} {s['pct']}%" for m, s in rep["grades"].items())
+        print(f"[{i}/{len(todo)}] {c['slug']} ({cell['id']})")
+        print(f"    values: {vals}")
+        print(f"    grade drift: {grds}")
+        for m, s in rep["values"].items():
+            mad_by_metric.setdefault(m, []).append(s.get("mad"))
+        for m, s in rep["grades"].items():
+            grade_by_metric.setdefault(m, []).append(s.get("pct"))
+
+    print(f"\n=== summary over {checked} cells (failed={failed}) ===")
+    print("mean MAD (raw drift):")
+    for m in _METRICS:
+        if m in mad_by_metric:
+            print(f"  {m:7s} {_mean(mad_by_metric[m])}")
+    print("mean grade-band divergence (share of days a grade changes):")
+    for m in sorted(grade_by_metric):
+        print(f"  {m:7s} {_mean(grade_by_metric[m])}%")
+
+
+if __name__ == "__main__":
+    argv = sys.argv[1:]
+    lim = int(argv[argv.index("--limit") + 1]) if "--limit" in argv else None
+    dys = int(argv[argv.index("--days") + 1]) if "--days" in argv else 365
+    main(limit=lim, days=dys)

backend/tests/data/test_climate.py

@@ -137,34 +137,53 @@ def test_metno_to_frame_aggregates_daily_and_converts_units():
     assert round(d2["precip"], 4) == round(6.0 / 25.4, 4)     # only the 6-hour block
 
 
-def test_recent_forecast_falls_back_to_metno(monkeypatch, tmp_path):
-    """When Open-Meteo's forecast API fails, the MET Norway backup serves the frame."""
+def test_recent_forecast_merges_nasa_past_and_metno_forward(monkeypatch, tmp_path):
+    """The recent/forecast bundle is built from a NASA POWER recent-past range plus the
+    MET Norway forward forecast, merged by date (Open-Meteo not consulted)."""
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
-    cell = {"id": "fallback_cell", "center_lat": 47.6062, "center_lon": -122.3321}
-    met = {"timeseries": [
-        _metno_step("2026-07-16T00:00:00Z", 18.0, 55.0, 3.0, p1=0.0),
-        _metno_step("2026-07-17T00:00:00Z", 22.0, 45.0, 5.0, p6=1.0),
-    ]}
+    cell = {"id": "rf_primary", "center_lat": 47.6, "center_lon": -122.3}
 
-    class Resp:
-        def json(self): return {"properties": met}
+    past = climate._finalize_approximated(pl.DataFrame({
+        "date": [datetime.date(2026, 7, 10), datetime.date(2026, 7, 11)],
+        "tmax": [70.0, 72.0], "tmin": [50.0, 51.0], "precip": [0.0, 0.1],
+        "wind": [5.0, 6.0], "humid": [60.0, 55.0],
+    }))
+    fwd = climate._finalize_approximated(pl.DataFrame({
+        "date": [datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)],
+        "tmax": [80.0, 82.0], "tmin": [60.0, 61.0], "precip": [0.0, 0.0],
+        "wind": [3.0, 4.0], "humid": [40.0, 45.0],
+    }))
+    monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c, start, end: past)
+    monkeypatch.setattr(climate, "_fetch_forecast_metno", lambda c: fwd)
+    monkeypatch.setattr(climate.meteostat, "fill_gusts", lambda lat, lon, df: df)
+    def _om_should_not_run(c): raise AssertionError("Open-Meteo forecast should not run")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run)
 
-    def fake_request(url, params, timeout, *, phase, headers=None, attempts=climate.MAX_ATTEMPTS):
-        if url == climate.FORECAST_URL:
-            raise RuntimeError("open-meteo forecast outage")
-        assert url == climate.METNO_URL
-        assert headers and headers.get("User-Agent"), "MET Norway needs a User-Agent"
-        return Resp()
-
-    monkeypatch.setattr(climate, "_request", fake_request)
     df = climate._load_recent_forecast(cell)
-    assert len(df) == 2 and df["date"].max() == datetime.date(2026, 7, 17)
+    assert df["date"].to_list() == [
+        datetime.date(2026, 7, 10), datetime.date(2026, 7, 11),
+        datetime.date(2026, 7, 16), datetime.date(2026, 7, 17)]
+
+
+def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path):
+    """When the NASA + MET Norway primary fails, the Open-Meteo forecast API serves."""
+    monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
+    monkeypatch.setattr(climate, "_forecast_cooldown_until", 0.0)
+    cell = {"id": "rf_fallback", "center_lat": 47.6, "center_lon": -122.3}
+
+    def _primary_down(c): raise RuntimeError("NASA + MET Norway down")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down)
+    om = climate._to_frame(_om_daily(3))
+    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", lambda c: om)
+
+    df = climate._load_recent_forecast(cell)
+    assert df.height == om.height
 
 
 def test_recent_forecast_serves_stale_cache_when_all_sources_fail(monkeypatch, tmp_path):
-    """With Open-Meteo AND MET Norway both down, an existing (stale) cache is served
-    rather than failing — and it is NOT rewritten, so its mtime stays old and the
-    next request still retries upstream first."""
+    """With every source down (the NASA + MET Norway primary and the Open-Meteo
+    fallback), an existing (stale) cache is served rather than failing — and it is NOT
+    rewritten, so its mtime stays old and the next request still retries upstream first."""
     import os
     import time
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
@@ -370,28 +389,23 @@ def test_history_tail_falls_back_to_open_meteo(monkeypatch):
 
 # --- forecast cooldown (mirrors the archive path's, tracked separately) -----
 
-def test_forecast_cooldown_skips_open_meteo_and_falls_to_metno(monkeypatch, tmp_path):
-    """While the forecast cooldown is active, the forecast fetch must not call
-    Open-Meteo at all -- straight to the MET Norway backup, mirroring
-    _load_history's archive-cooldown skip."""
+def test_forecast_cooldown_skips_the_open_meteo_fallback(monkeypatch, tmp_path):
+    """While the forecast cooldown is active, the Open-Meteo FALLBACK is skipped: with
+    the keyless primary (NASA + MET Norway) also down and no cache, the load surfaces
+    WeatherUnavailable without ever calling Open-Meteo."""
     import time as time_mod
     monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path))
     monkeypatch.setattr(climate, "_forecast_cooldown_until", time_mod.time() + 60)
     cell = {"id": "fc_cooldown_cell", "center_lat": 47.6, "center_lon": -122.3}
 
-    met = {"timeseries": [_metno_step("2026-07-16T00:00:00Z", 18.0, 55.0, 3.0, p1=0.0)]}
+    def _primary_down(c): raise RuntimeError("NASA + MET Norway down")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast", _primary_down)
+    def _om_should_not_run(c):
+        raise AssertionError("Open-Meteo fallback must not run during cooldown")
+    monkeypatch.setattr(climate, "_fetch_recent_forecast_om", _om_should_not_run)
 
-    class Resp:
-        def json(self): return {"properties": met}
-
-    def fake_request(url, params, timeout, *, phase, headers=None, attempts=climate.MAX_ATTEMPTS):
-        assert url != climate.FORECAST_URL, "must not call Open-Meteo during cooldown"
-        assert url == climate.METNO_URL
-        return Resp()
-
-    monkeypatch.setattr(climate, "_request", fake_request)
-    df = climate._load_recent_forecast(cell)
-    assert len(df) == 1
+    with pytest.raises(climate.WeatherUnavailable):
+        climate._load_recent_forecast(cell)
 
 
 def test_forecast_429_sets_its_own_cooldown(monkeypatch, tmp_path):

backend/tests/test_drift_check.py

@@ -0,0 +1,55 @@
+"""Unit tests for the drift-check comparison logic (hermetic — no network). The
+source fetch (check_cell) is not tested here; it runs on a networked box."""
+import datetime
+import math
+
+import polars as pl
+import pytest
+
+import drift_check
+from data import climate
+
+_D = [datetime.date(2024, 1, d) for d in (1, 2, 3)]
+
+
+def test_compare_values_stats():
+    a = pl.DataFrame({"date": _D, "tmax": [70.0, 80.0, 90.0], "wind": [5.0, 6.0, 7.0]})
+    b = pl.DataFrame({"date": _D, "tmax": [72.0, 78.0, 90.0], "wind": [5.0, 6.0, 7.0]})
+    out = drift_check.compare_values(a, b, metrics=("tmax", "wind"))
+    assert out["tmax"] == {"n": 3, "mad": pytest.approx(4 / 3, abs=1e-3),
+                           "bias": pytest.approx(0.0), "max_abs": pytest.approx(2.0)}
+    assert out["wind"] == {"n": 3, "mad": 0.0, "bias": 0.0, "max_abs": 0.0}
+
+
+def test_compare_values_skips_nulls_and_missing_metric():
+    a = pl.DataFrame({"date": _D, "tmax": [70.0, 80.0, 90.0]})
+    b = pl.DataFrame({"date": _D, "tmax": [71.0, None, 90.0]})
+    out = drift_check.compare_values(a, b, metrics=("tmax", "gust"))
+    assert out["tmax"]["n"] == 2                     # the null day is dropped
+    assert out["tmax"]["mad"] == pytest.approx(0.5)  # |70-71| and |90-90|
+    assert "gust" not in out                         # metric absent from both frames
+
+
+def _gradeable(n=800):
+    """A finalized, gradeable multi-year frame (seasonal temp swing so windows have a
+    real distribution)."""
+    start = datetime.date(2019, 1, 1)
+    dates = [start + datetime.timedelta(days=i) for i in range(n)]
+    tmax = [70.0 + 15.0 * math.sin(i / 58.0) for i in range(n)]
+    tmin = [t - 15.0 for t in tmax]
+    return climate._finalize_frame(pl.DataFrame({
+        "date": dates, "tmax": tmax, "tmin": tmin, "precip": [0.0] * n,
+        "wind": [5.0] * n, "gust": [8.0] * n, "humid": [60.0] * n,
+        "fmax": tmax, "fmin": tmin,
+    }))
+
+
+def test_grade_band_divergence_identical_is_zero():
+    f = _gradeable()
+    end = f["date"].max()
+    start = end - datetime.timedelta(days=60)
+    div = drift_check.grade_band_divergence(f, f, start, end)
+    assert div                                        # some metrics were compared
+    assert any(s["compared"] > 0 for s in div.values())
+    assert all(s["differ"] == 0 for s in div.values())   # identical frames never diverge
+    assert all(s["pct"] == 0.0 for s in div.values())