From 7ed375f32e249d2275c0d9afe9163346f639850f Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Fri, 24 Jul 2026 15:57:15 -0700 Subject: [PATCH 1/3] key-gaps: the key regex was blind to digits, inventing missing secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `grep -oE "^[A-Z_]+="` cannot match a key name containing a digit. This estate has exactly six such names — all the *_S3_* keys — so a prod audit returned 26 keys against 32 real ones, and reported the S3 and lake credentials as missing from both live hosts. They were present the whole time. The phantom was independently reproduced twice by re-running the same pattern, which is what made it convincing, and it was briefly recorded as one of two root causes of the lake being unqueryable (#56). That issue has one cause: the missing duckdb-lake image. An audit that under-reports is worse than no audit. A missing-secret finding sends someone to provision a credential that already exists, and in a rotation tool it would justify writing over one. Fixed to `^[A-Z][A-Z0-9_]*=` in both the skill and key_gaps.py's docstring. The match still stops at the `=`, so no value is read — that property is the reason this grep exists rather than a parser. Centralis's secrets_gaps carried the same bug and is fixed separately, with a regression test. Claude-Session: https://claude.ai/code/session_0182KTMrsTHJc3TcewCatJFY --- .claude/worktrees/city-resolver | 1 + .claude/worktrees/thermograph-mentions | 1 + infra/.claude/skills/key-gaps/SKILL.md | 14 ++++++++++++-- infra/.claude/skills/key-gaps/key_gaps.py | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 160000 .claude/worktrees/city-resolver create mode 160000 .claude/worktrees/thermograph-mentions diff --git a/.claude/worktrees/city-resolver b/.claude/worktrees/city-resolver new file mode 160000 index 0000000..1b862aa --- /dev/null +++ b/.claude/worktrees/city-resolver @@ -0,0 +1 @@ +Subproject commit 1b862aa049403c02e8e9c4ef14dfc8c7a55b5aa2 diff --git a/.claude/worktrees/thermograph-mentions b/.claude/worktrees/thermograph-mentions new file mode 160000 index 0000000..ce4350c --- /dev/null +++ b/.claude/worktrees/thermograph-mentions @@ -0,0 +1 @@ +Subproject commit ce4350c64ecd6d6aaa89bdd965ea93f63b463295 diff --git a/infra/.claude/skills/key-gaps/SKILL.md b/infra/.claude/skills/key-gaps/SKILL.md index 3896d72..f243540 100644 --- a/infra/.claude/skills/key-gaps/SKILL.md +++ b/infra/.claude/skills/key-gaps/SKILL.md @@ -34,8 +34,18 @@ Gather the key names over SSH (never the values), then audit. Hosts/keys per INF K=~/.ssh/thermograph_agent_ed25519 # sudo: /etc/thermograph.env is root-owned (0640). On prod `agent` can read it # directly too, but sudo works uniformly on both boxes. -ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z_]+=" /etc/thermograph.env' > /tmp/prod.keys # prod -ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z_]+=" /etc/thermograph.env' > /tmp/beta.keys # beta +# +# The character class must allow DIGITS. This was `^[A-Z_]+=` until 2026-07-24, +# which silently skipped every key whose name contains a digit — in this estate +# that is all six *_S3_* keys. Prod audited as 26 keys against 32 real ones, and +# the audit reported the S3 credentials as missing from both hosts. They were +# present the whole time, and the phantom was independently "confirmed" twice by +# people re-running the same pattern. An audit that under-reports is worse than +# no audit: the invented finding sends someone to provision a credential that +# already exists, and it was briefly recorded as a root cause of a real bug. +# The match still stops at the `=`, so no value is ever read. +ssh -i $K agent@169.58.46.181 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/prod.keys # prod +ssh -i $K agent@75.119.132.91 'sudo grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env' > /tmp/beta.keys # beta python3 .claude/skills/key-gaps/key_gaps.py prod=/tmp/prod.keys beta=/tmp/beta.keys ``` diff --git a/infra/.claude/skills/key-gaps/key_gaps.py b/infra/.claude/skills/key-gaps/key_gaps.py index 775f7ab..a32df73 100755 --- a/infra/.claude/skills/key-gaps/key_gaps.py +++ b/infra/.claude/skills/key-gaps/key_gaps.py @@ -17,7 +17,7 @@ It reads only KEY NAMES, never values — safe to run anywhere. Sources per envi * a SOPS-encrypted YAML (deploy/secrets/.yaml) — keys are plaintext even when encrypted, so no age key or decryption is needed; or * a plain key-list file (one KEY per line, or KEY=... lines) — e.g. the output of - `ssh 'grep -oE "^[A-Z_]+=" /etc/thermograph.env'` for a live audit. + `ssh 'grep -oE "^[A-Z][A-Z0-9_]*=" /etc/thermograph.env'` for a live audit. Usage: key_gaps.py prod=deploy/secrets/prod.yaml beta=deploy/secrets/beta.yaml -- 2.45.2 From f4313b5d877f7f6991264056f6140ad68e01140a Mon Sep 17 00:00:00 2001 From: Emi Griffith Date: Fri, 24 Jul 2026 15:57:34 -0700 Subject: [PATCH 2/3] Drop accidentally-committed worktrees; ignore .claude/worktrees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit swept in .claude/worktrees/city-resolver and .claude/worktrees/thermograph-mentions as embedded git repositories. Those are other Claude sessions' live checkouts and have no business in this tree. Ignoring the directory so `git add -A` cannot do it again — the repo already has a documented history of parallel sessions colliding through shared checkouts, and this is the same hazard wearing a different hat. Claude-Session: https://claude.ai/code/session_0182KTMrsTHJc3TcewCatJFY --- .claude/worktrees/city-resolver | 1 - .claude/worktrees/thermograph-mentions | 1 - .gitignore | 4 ++++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 160000 .claude/worktrees/city-resolver delete mode 160000 .claude/worktrees/thermograph-mentions create mode 100644 .gitignore diff --git a/.claude/worktrees/city-resolver b/.claude/worktrees/city-resolver deleted file mode 160000 index 1b862aa..0000000 --- a/.claude/worktrees/city-resolver +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1b862aa049403c02e8e9c4ef14dfc8c7a55b5aa2 diff --git a/.claude/worktrees/thermograph-mentions b/.claude/worktrees/thermograph-mentions deleted file mode 160000 index ce4350c..0000000 --- a/.claude/worktrees/thermograph-mentions +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce4350c64ecd6d6aaa89bdd965ea93f63b463295 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5aee0f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ + +# Claude Code worktrees. These are other sessions' checkouts living inside this +# repo; `git add -A` will otherwise commit them as embedded git repositories. +.claude/worktrees/ -- 2.45.2 From 6df4968679d0080ff3f48d68fef7169876bf2c7f Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 23:13:36 +0000 Subject: [PATCH 3/3] Frontend QA batch: date/TZ, https origin, date-422, VAPID rotation, trace-precip, partial-day gate (#70) --- backend/api/content_routes.py | 18 +++++++- backend/data/climate.py | 29 ++++++++++++- backend/data/grading.py | 29 +++++++------ backend/notifications/push.py | 13 +++--- backend/tests/api/test_content_routes.py | 22 ++++++++++ backend/tests/data/test_climate.py | 43 ++++++++++++++++++++ backend/tests/data/test_grading.py | 6 ++- backend/tests/notifications/test_push.py | 52 ++++++++++++++++++++++-- backend/tests/web/test_api.py | 12 ++++++ backend/web/app.py | 18 +++++++- frontend/content.py | 18 +++++++- frontend/static/app.js | 11 +++-- frontend/static/calendar.js | 6 +-- frontend/static/chart.js | 10 +++-- frontend/static/day.js | 18 +++++--- frontend/static/push-client.js | 34 ++++++++++++++-- frontend/static/shared.js | 24 +++++++++-- frontend/tests/unit/test_rendering.py | 45 ++++++++++++++++++++ 18 files changed, 357 insertions(+), 51 deletions(-) diff --git a/backend/api/content_routes.py b/backend/api/content_routes.py index cd682b1..ef01df2 100644 --- a/backend/api/content_routes.py +++ b/backend/api/content_routes.py @@ -6,6 +6,7 @@ pattern web/app.py's own endpoints use. """ import hashlib import os +from urllib.parse import urlsplit from fastapi import APIRouter, HTTPException, Request, Response @@ -84,9 +85,24 @@ def _load_history(cell): return history +# The configured public origin is the source of truth for the scheme. Caddy +# terminates TLS and reverse-proxies plain HTTP, so x-forwarded-proto (and +# request.url.scheme) read "http" even though the public site is HTTPS-only -- +# and this origin is folded into the jsonld "url" the payloads carry. Trusting +# the forwarded scheme makes that url (and the canonical/og the frontend builds +# from the same origin it forwards here) emit http://, which 308-redirects to +# https://. So a request that arrives on the configured public host is answered +# with the configured public scheme; localhost/LAN dev never matches that host +# and keeps the observed scheme, leaving plain-HTTP development unaffected. +_PUBLIC = urlsplit(os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")) + + def _origin(request: Request) -> str: - proto = request.headers.get("x-forwarded-proto") or request.url.scheme host = request.headers.get("host") or request.url.netloc + if _PUBLIC.scheme and _PUBLIC.netloc and host == _PUBLIC.netloc: + proto = _PUBLIC.scheme + else: + proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}" diff --git a/backend/data/climate.py b/backend/data/climate.py index b1c5e00..2bb1b2f 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -937,9 +937,29 @@ def _fetch_recent_forecast(cell: dict) -> pl.DataFrame: .sort("date")) +def _om_local_today(payload: dict) -> "datetime.date | None": + """The calendar date it is *now* at the cell, from the UTC offset Open-Meteo + reports for a ``timezone=auto`` request. None when the offset is absent, which + tells the caller to skip the in-progress-day guard rather than guess a date.""" + offset = payload.get("utc_offset_seconds") + if offset is None: + return None + return (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=int(offset))).date() + + def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: """Fallback recent+forecast bundle from the Open-Meteo forecast API (the former - primary): recent past + forward days in one call.""" + primary): recent past + forward days in one call. + + The cell's in-progress *local* day is dropped from the bundle. Open-Meteo's + daily high/low for today aggregates only the hours elapsed so far, so grading it + reads a still-unfolding day as complete and produces spurious extremes (a cool + morning served as a record-low high, seen live at the 1st percentile). This is + the same failure the MET Norway path guards against with its diurnal-coverage + gate (see _metno_to_frame); the fallback needs the equivalent. Past days are + complete and future days are whole-day forecasts, so only today is excluded — + the day lands in the record once it is over.""" params = { "latitude": cell["center_lat"], "longitude": cell["center_lon"], @@ -952,7 +972,12 @@ def _fetch_recent_forecast_om(cell: dict) -> pl.DataFrame: "forecast_days": FORECAST_DAYS, } r = _request(FORECAST_URL, params, 60, phase="recent_forecast_fetch") - return _to_frame(r.json()["daily"]) + payload = r.json() + df = _to_frame(payload["daily"]) + local_today = _om_local_today(payload) + if local_today is not None: + df = df.filter(pl.col("date") != local_today) + return df def _load_recent_forecast(cell: dict) -> pl.DataFrame: diff --git a/backend/data/grading.py b/backend/data/grading.py index dfd34df..a160eb1 100644 --- a/backend/data/grading.py +++ b/backend/data/grading.py @@ -279,8 +279,10 @@ def all_time_records(df: pl.DataFrame) -> dict: def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: - """Longest run of consecutive days without measurable rain, and the ISO date the - streak began. A dry day is precip < RAIN_THRESHOLD; null precip counts as dry.""" + """Longest run of consecutive days without any rain, and the ISO date the streak + began. A dry day has no rain at all (precip <= 0); null precip counts as dry. The + threshold matches _grade_precip's dry/rain split — any measurable rain, however + slight, is a rain day that breaks the streak (not the 0.01" rain-frequency line).""" if "precip" not in df.columns: return (0, None) d = df.select(["date", "precip"]).sort("date") @@ -288,7 +290,7 @@ def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: best_len, best_start = 0, None cur_len, cur_start = 0, None for dt, p in zip(dates, precips): - wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p >= RAIN_THRESHOLD + wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p > 0 if wet: cur_len, cur_start = 0, None else: @@ -305,11 +307,12 @@ def longest_dry_streak(df: pl.DataFrame) -> tuple[int, str | None]: def _band_stats(samples: np.ndarray) -> dict | None: if samples.size == 0: return None - # Percentiles for the chart's nested "normal" fan, matching the 9 tier bounds: - # p40-p60 is the Normal band; p25/p75, p10/p90 and p1/p99 mark the successive - # Below/Above Normal, Low/High, Very Low/High and Near-Record edges. - p1, p10, p25, p40, p50, p60, p75, p90, p99 = np.percentile( - samples, [1, 10, 25, 40, 50, 60, 75, 90, 99] + # Percentiles for the chart's nested "normal" fan. p40-p60 is the Normal band; + # p25/p75, p10/p90 and p1/p99 mark the successive Below/Above Normal, Low/High, + # Very Low/High and Near-Record edges. p95 additionally splits the rain fan's top + # region into Very Heavy (90-95) and Severe (95-99); unused by the temperature fan. + p1, p10, p25, p40, p50, p60, p75, p90, p95, p99 = np.percentile( + samples, [1, 10, 25, 40, 50, 60, 75, 90, 95, 99] ) return { "p1": round(float(p1), 1), @@ -320,6 +323,7 @@ def _band_stats(samples: np.ndarray) -> dict | None: "p60": round(float(p60), 1), "p75": round(float(p75), 1), "p90": round(float(p90), 1), + "p95": round(float(p95), 1), "p99": round(float(p99), 1), } @@ -352,13 +356,14 @@ def _grade_precip(samples: np.ndarray, value) -> dict | None: def dry_streaks(dates, precips) -> dict[str, int]: - """Map each date (ISO string) to days since the last measurable rain, walking a - chronological precip series. Missing precip counts as a dry day. `dates` is an - iterable of ``datetime.date`` (a polars Date column's ``.to_list()``).""" + """Map each date (ISO string) to days since the last day with any rain, walking a + chronological precip series. Any measurable rain (precip > 0) resets the count, so + this matches _grade_precip's dry/rain split; missing precip counts as a dry day. + `dates` is an iterable of ``datetime.date`` (a polars Date column's ``.to_list()``).""" out: dict[str, int] = {} streak = 0 for d, p in zip(dates, precips): - wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p >= RAIN_THRESHOLD + wet = p is not None and not (isinstance(p, float) and np.isnan(p)) and p > 0 streak = 0 if wet else streak + 1 out[_as_date(d).isoformat()] = streak return out diff --git a/backend/notifications/push.py b/backend/notifications/push.py index 594607f..d2807e7 100644 --- a/backend/notifications/push.py +++ b/backend/notifications/push.py @@ -37,7 +37,7 @@ log = logging.getLogger("thermograph.push") _DATA_DIR = paths.DATA_DIR _VAPID_PATH = os.environ.get("THERMOGRAPH_VAPID_FILE") or os.path.join(_DATA_DIR, "vapid.json") # The VAPID "sub" claim — a contact the push service can reach about our traffic. -_CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.app") +_CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.org") # pywebpush 2.0.0 forwards `timeout` straight to requests.post with no default of # its own — omit the kwarg here and the send blocks with NO timeout at all (the @@ -174,10 +174,13 @@ def send(subscription_info: dict, payload: dict) -> str: return "ok" except WebPushException as e: status = getattr(getattr(e, "response", None), "status_code", None) - if status in (404, 410): - return "gone" # endpoint retired — caller should delete it - # 401/403 = VAPID key mismatch, etc. Record it where it's visible (the errors - # JSONL / dashboard), not just the system journal. + if status in (401, 403, 404, 410): + # 404/410 = endpoint retired; 401/403 = VAPID mismatch (e.g. after a key + # rotation) — the row was minted under a key we can no longer sign for and + # will never authenticate again. All are permanently dead: caller deletes. + return "gone" + # Anything else (rate limits, 5xx, malformed payload) may be transient — keep + # the row and record it where it's visible (the errors JSONL / dashboard). log.warning("web push failed (status=%s): %s", status, e) audit.log_event("error", {"phase": "push", "status": status, "endpoint": (subscription_info.get("endpoint") or "")[:120]}) diff --git a/backend/tests/api/test_content_routes.py b/backend/tests/api/test_content_routes.py index 236601a..e172b9b 100644 --- a/backend/tests/api/test_content_routes.py +++ b/backend/tests/api/test_content_routes.py @@ -118,6 +118,28 @@ def test_records_payload_shape(client): assert body["canonical_path"] == "/climate/testville/records" +# --- https origin (jsonld url) ----------------------------------------------- + +def test_public_host_jsonld_url_is_https(client): + """The public site is HTTPS-only behind Caddy, which proxies plain HTTP, so + x-forwarded-proto reads "http". A request on the configured public host + (THERMOGRAPH_BASE_URL defaults to https://thermograph.org) must still fold an + https:// origin into the payload's jsonld url; otherwise the frontend renders + an http:// canonical/JSON-LD that 308-redirects.""" + import json as _json + hdr = {"host": "thermograph.org", "x-forwarded-proto": "http"} + for path in ("city/testville", "city/testville/records"): + s = _json.dumps(client.get(f"{BASE}/{path}", headers=hdr).json()) + assert "https://thermograph.org/thermograph/climate/testville" in s + assert "http://thermograph.org" not in s + + +def test_non_public_host_jsonld_keeps_forwarded_scheme(client): + hdr = {"host": "192.168.1.10:8000", "x-forwarded-proto": "http"} + body = client.get(f"{BASE}/city/testville", headers=hdr).json() + assert body["jsonld"]["@graph"][0]["url"].startswith("http://192.168.1.10:8000/") + + # --- 404s -------------------------------------------------------------------- def test_unknown_slug_is_404(client, counts): diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index fab8bad..de90ff3 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -208,6 +208,49 @@ def test_recent_forecast_falls_back_to_open_meteo(monkeypatch, tmp_path): assert df.height == om.height +def test_recent_forecast_om_drops_the_in_progress_local_day(monkeypatch): + """The Open-Meteo fallback must not grade the cell's in-progress local day: its + daily high/low is only a partial aggregate of the hours elapsed so far (a cool + morning would read as a record-low high). Past days and future forecast days + survive; today — per the UTC offset Open-Meteo reports for a timezone=auto + request — is dropped, matching the MET path's diurnal-coverage gate.""" + offset = 3 * 3600 # UTC+3, e.g. Europe/Vilnius (the reported Ringaudai incident) + local_today = (datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=offset)).date() + days = [local_today + datetime.timedelta(days=n) for n in (-2, -1, 0, 1)] + daily = { + "time": [d.isoformat() for d in days], + "temperature_2m_max": [80.0, 82.0, 61.0, 84.0], # today's 61 is the partial value + "temperature_2m_min": [60.0, 61.0, 57.0, 62.0], + "precipitation_sum": [0.0, 0.0, 0.0, 0.0], + } + payload = {"utc_offset_seconds": offset, "daily": daily} + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + + got = climate._fetch_recent_forecast_om( + {"center_lat": 54.9, "center_lon": 23.8})["date"].to_list() + assert local_today not in got # partial today dropped + assert local_today - datetime.timedelta(days=1) in got # yesterday kept + assert local_today + datetime.timedelta(days=1) in got # tomorrow's forecast kept + assert len(got) == 3 + + +def test_recent_forecast_om_keeps_all_days_without_a_utc_offset(monkeypatch): + """No UTC offset reported -> skip the in-progress-day guard rather than guess a + date, so the bundle passes through as before (only the usual null-day filter).""" + payload = {"daily": _om_daily(3)} # no utc_offset_seconds + + class Resp: + def json(self): return payload + monkeypatch.setattr(climate, "_request", lambda *a, **k: Resp()) + + df = climate._fetch_recent_forecast_om({"center_lat": 1.0, "center_lon": 2.0}) + assert df.height == climate._to_frame(_om_daily(3)).height + + def test_recent_forecast_serves_stale_cache_when_all_sources_fail(monkeypatch, tmp_path): """With every source down (the NASA + MET Norway primary and the Open-Meteo fallback), an existing (stale) cache is served rather than failing — and it is NOT diff --git a/backend/tests/data/test_grading.py b/backend/tests/data/test_grading.py index cfe7d28..a2f512a 100644 --- a/backend/tests/data/test_grading.py +++ b/backend/tests/data/test_grading.py @@ -113,7 +113,9 @@ def test_dry_streaks_walk(): dates = [datetime.date(2024, 1, 1) + datetime.timedelta(days=i) for i in range(5)] precips = [0.5, 0.0, float("nan"), 0.02, 0.005] out = grading.dry_streaks(dates, precips) - assert list(out.values()) == [0, 1, 2, 0, 1] # NaN counts as dry + # Any rain > 0 breaks the streak (matching the dry/rain grading split), so the + # 0.005" trace day resets to 0; only exact 0.0 and NaN count as dry. + assert list(out.values()) == [0, 1, 2, 0, 0] # ---- range + day grading over a synthetic record -------------------------------- @@ -141,7 +143,7 @@ def test_grade_day_normals_and_departure(history): result = grading.grade_day(history, target, {"tmax": row["tmax"], "tmin": row["tmin"], "precip": row["precip"]}) assert set(result["normals"]["tmax"]) == {"p1", "p10", "p25", "p40", "p50", "p60", - "p75", "p90", "p99"} + "p75", "p90", "p95", "p99"} expected = max(abs(result["tmax"]["percentile"] - 50), abs(result["tmin"]["percentile"] - 50)) assert result["departure"] == round(expected, 1) diff --git a/backend/tests/notifications/test_push.py b/backend/tests/notifications/test_push.py index 422583f..5450778 100644 --- a/backend/tests/notifications/test_push.py +++ b/backend/tests/notifications/test_push.py @@ -1,11 +1,16 @@ """VAPID key resolution: the atomic first-writer-wins claim of the keypair file (push.py's `_claim_file`), and that `_load()` caches whatever that settles on -rather than its own local generation when it loses the race. +rather than its own local generation when it loses the race. Also `send()`'s +mapping of push-service responses onto 'ok'/'gone'/'error'. -No network — `send()`'s pywebpush call isn't exercised here (that's notify.py's -`test_push_dispatched_on_new_notification` etc., which stub `push.send` itself).""" +No network — where `send()` is exercised the pywebpush call is stubbed; the +dispatch path (notify.py's `test_push_dispatched_on_new_notification` etc.) stubs +`push.send` itself.""" import json +import pytest +from pywebpush import WebPushException + from notifications import push @@ -94,3 +99,44 @@ def test_load_prefers_env_over_file(monkeypatch, tmp_path): result = push._load() assert result == {"private_key": "priv-env", "public_key": "pub-env"} + + +# --- send() response mapping ------------------------------------------------ +_SUB = {"endpoint": "https://push.example.com/ep-1", "keys": {"p256dh": "BKEY", "auth": "YXV0aA"}} + + +class _Resp: + def __init__(self, status_code): + self.status_code = status_code + + +def _raise_status(status): + def _webpush(**kwargs): + raise WebPushException("boom", response=_Resp(status)) + return _webpush + + +@pytest.mark.parametrize("status", [401, 403, 404, 410]) +def test_send_prunes_permanently_dead_endpoints(monkeypatch, status): + # 404/410 = endpoint retired; 401/403 = VAPID key mismatch (e.g. after a key + # rotation). All are permanently dead, so send() reports 'gone' and the caller + # deletes the row — otherwise a rotated key leaves dead subscriptions forever. + monkeypatch.setattr(push, "webpush", _raise_status(status)) + assert push.send(_SUB, {"hello": "world"}) == "gone" + + +@pytest.mark.parametrize("status", [429, 500, 502]) +def test_send_keeps_row_on_transient_failure(monkeypatch, status): + # Rate limits / 5xx may recover; keep the row and report it as an error. + monkeypatch.setattr(push, "webpush", _raise_status(status)) + assert push.send(_SUB, {"hello": "world"}) == "error" + + +def test_send_ok(monkeypatch): + monkeypatch.setattr(push, "webpush", lambda **kwargs: None) + assert push.send(_SUB, {"hello": "world"}) == "ok" + + +def test_default_contact_is_the_org_domain(): + # The VAPID "sub" contact must be a domain we actually own; .app was a typo. + assert push._CONTACT == "mailto:admin@thermograph.org" diff --git a/backend/tests/web/test_api.py b/backend/tests/web/test_api.py index f6893fa..1048001 100644 --- a/backend/tests/web/test_api.py +++ b/backend/tests/web/test_api.py @@ -71,6 +71,18 @@ def test_api_version_reports_backend_contract(client): assert body["backend_version"] == "2" +def test_malformed_date_is_rejected(client): + # A malformed or non-calendar date must be a 422, not a 500: fromisoformat + # raises ValueError on these and the handler used to leak it as a crash. + for bad in ("notadate", "2026-13-40", "2026-02-30"): + for route in ("grade", "day"): + r = client.get(f"/thermograph/api/v2/{route}", params={**Q, "date": bad}) + assert r.status_code == 422, (route, bad, r.status_code) + # A valid date still works. + assert client.get("/thermograph/api/v2/grade", + params={**Q, "date": "2026-07-11"}).status_code == 200 + + def test_day_detail_and_ladders(client, history): r = client.get("/thermograph/api/v2/day", params=Q) assert r.status_code == 200 diff --git a/backend/web/app.py b/backend/web/app.py index b8a5d35..5949033 100644 --- a/backend/web/app.py +++ b/backend/web/app.py @@ -79,6 +79,20 @@ def _weather_fetch_error(e) -> HTTPException: return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}") +def _parse_target_date(date: str | None, default: datetime.date) -> datetime.date: + """Parse the ``date`` query param (YYYY-MM-DD) into a date, falling back to + ``default`` when it's absent. A malformed or non-calendar date (e.g. 2026-13-40) + is a client error, not a crash: surface it as a 422 rather than let + fromisoformat's ValueError become a 500.""" + if not date: + return default + try: + return datetime.date.fromisoformat(date[:10]) + except ValueError: + raise HTTPException(status_code=422, + detail=f"invalid date: {date!r} (expected YYYY-MM-DD)") + + # --- background neighbor warming --------------------------------------------- # /cell?neighbors=1 asks the server to warm the 8 grid cells around the request # (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's @@ -492,7 +506,7 @@ def api_grade( description="days to grade after the target (observed, or forecast when future; " "the forecast reaches ~7 days out so future days cap there)"), ): - target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today() + target = _parse_target_date(date, datetime.date.today()) cell = grid.snap(lat, lon) with audit.RunAudit( @@ -580,7 +594,7 @@ def api_day( ) as run: history, cache_meta, _ = _fetch_history(run, cell) last = history["date"].max() - target = datetime.date.fromisoformat(date[:10]) if date else last + target = _parse_target_date(date, last) run.set(target_date=target.isoformat()) def build(place): diff --git a/frontend/content.py b/frontend/content.py index 6981b25..7295362 100644 --- a/frontend/content.py +++ b/frontend/content.py @@ -13,6 +13,7 @@ import hashlib import json import logging import os +from urllib.parse import urlsplit import httpx from fastapi import HTTPException, Request, Response @@ -56,13 +57,28 @@ _log = logging.getLogger(__name__) # --- helpers ------------------------------------------------------------- +# The configured public origin is the source of truth for the scheme. Caddy +# terminates TLS and reverse-proxies plain HTTP, so x-forwarded-proto (and +# request.url.scheme) read "http" even though the public site is HTTPS-only. +# Trusting that scheme makes every canonical / og:url / og:image / sitemap +# emit http://, which 308-redirects to https:// -- self-conflicting canonicals +# and a sitemap full of redirecting URLs. So a request that arrives on the +# configured public host is answered with the configured public scheme (https on +# prod/beta, both of which set THERMOGRAPH_BASE_URL per host); localhost/LAN dev +# never matches it and keeps the observed scheme, so plain-HTTP dev is unchanged. +_PUBLIC = urlsplit(os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org")) + + def _origin(request: Request) -> str: # x-forwarded-host takes precedence over host: when reached via backend's # internal proxy fallback (no Caddy in front -- see _proxy_to_frontend in # backend/web/app.py), Host is the internal hop's own address, not the # browser-facing one. - proto = request.headers.get("x-forwarded-proto") or request.url.scheme host = request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc + if _PUBLIC.scheme and _PUBLIC.netloc and host == _PUBLIC.netloc: + proto = _PUBLIC.scheme + else: + proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}" diff --git a/frontend/static/app.js b/frontend/static/app.js index 47fa6be..4b21646 100644 --- a/frontend/static/app.js +++ b/frontend/static/app.js @@ -8,7 +8,7 @@ import { W, PW, H, PL, PR, plotTop, plotBot, xLabY, setChartWidth, chartPalette, tempChart, precipChart, dryChart, attachChartHover } from "./chart.js"; import { track } from "./digest.js"; import { TIER_COLORS, SCALE_TEMP, drynessColor, pctOrd, esc, todayISO, placeLabel, - tierKeySegs, GUIDE_LINK, fmtPrecip, fmtWind, fmtHumid } from "./shared.js"; + tierKeySegs, GUIDE_LINK, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid } from "./shared.js"; let selected = null; // {lat, lon} @@ -323,11 +323,14 @@ function precipCell(d, isFc, isToday) { const dd = ` data-date="${d.date}"`; const cls = "rd-c" + (isFc ? " rd-fc" : isToday ? " rd-today" : ""); if (!g) return `—`; - if (g.value === 0 && d.dsr != null && d.dsr > 0) { - return `${d.dsr}d`; + // Only a genuinely dry day (no rain at all) labels the streak; any rain day — + // down to sub-0.01" trace that rounds to 0 — shows its depth, tinted by tier. + if (g.class === "dry" && d.dsr != null && d.dsr > 0) { + return `${d.dsr}d`; } const col = TIER_COLORS[g.class] || ""; - return `${cRain(g.value)}`; + const amt = g.class && g.class !== "dry" ? fmtPrecipTier(g.value, g.class, false) : cRain(g.value); + return `${amt}`; } // Compact "graded days" table — a ROW per metric, a COLUMN per day (newest first). diff --git a/frontend/static/calendar.js b/frontend/static/calendar.js index ed3f848..c2b19bd 100644 --- a/frontend/static/calendar.js +++ b/frontend/static/calendar.js @@ -7,7 +7,7 @@ import { uv } from "./account.js"; // header sign-in entry + notification bell import { TTL, chunkedFetch, prefetchViews } from "./cache.js"; import { initFindButton, setFindLabel } from "./mappicker.js"; import { TIER_COLORS, PRECIP_COLORS, SCALE_TEMP, SCALE_RAIN, drynessColor, pctOrd, - placeLabel, fmtPrecip, fmtWind, fmtHumid, MONTHS, isoOfDate, monthStart, monthEnd, + placeLabel, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid, MONTHS, isoOfDate, monthStart, monthEnd, CHUNK_MONTHS, buildChunks, clickOpensPicker, metricBuckets, distStrip, seasonFilterDropdown, seasonSummaryText, syncSeasonChecks, applySeasonMonthChange, initSeasonExpand, allMonths, monthsToMask, maskToMonths, @@ -42,7 +42,7 @@ const SKY_WORDS = [ // The shared weather summary, adapted to the calendar's compact day records // (dsr included, so a long-dry no-rain day reads "dry" rather than "clear"). const weatherType = (rec) => - wxType(rec.tmax && rec.tmax.v, rec.precip && rec.precip.v, rec.dsr); + wxType(rec.tmax && rec.tmax.v, rec.precip && rec.precip.v, rec.dsr, rec.precip && rec.precip.c); let selected = null; // {lat, lon} let data = null; // last /api/v2/calendar response @@ -704,7 +704,7 @@ function attachHover(byDate) { ["humid", line("humid", "Humid", rec.humid, fmtHumid, tempColor(rec.humid))], ["wind", line("wind", "Wind", rec.wind, fmtWind, tempColor(rec.wind))], ["gust", line("gust", "Gust", rec.gust, fmtWind, tempColor(rec.gust))], - ["precip", line("precip", "Precip", rec.precip, fmtPrecip, precipColor)], + ["precip", line("precip", "Precip", rec.precip, (v) => fmtPrecipTier(v, rec.precip && rec.precip.c), precipColor)], ]; if (dsrStr) { const rc = metric === "dsr" ? " tt-r-active" : ""; diff --git a/frontend/static/chart.js b/frontend/static/chart.js index d48dfae..6facfcd 100644 --- a/frontend/static/chart.js +++ b/frontend/static/chart.js @@ -64,7 +64,7 @@ const PCT_FALLBACK = { p1: ["p1", "p10", "min", "p50"], p10: ["p10", "p50"], p25: ["p25", "p30", "p50"], p40: ["p40", "p30", "p50"], p60: ["p60", "p70", "p50"], p75: ["p75", "p70", "p50"], - p90: ["p90", "p50"], p99: ["p99", "p90", "max", "p50"], + p90: ["p90", "p50"], p95: ["p95", "p90", "p50"], p99: ["p99", "p90", "max", "p50"], }; const pget = (o, k) => { for (const kk of (PCT_FALLBACK[k] || [k])) if (o && o[kk] != null) return o[kk]; @@ -93,8 +93,12 @@ const TEMP_FAN = [ ["p90", "p99", "very-hot"], ["p75", "p90", "hot"], ["p60", "p75", "warm"], ["p40", "p60", "normal"], ["p25", "p40", "cool"], ["p10", "p25", "cold"], ["p1", "p10", "very-cold"], ]; +// Eight rain-intensity tiers (post-split): Trace / Light / Brisk / Typical / Heavy +// (60-90) / Very Heavy (90-95) / Severe (95-99) / Extreme. Each band maps a rain-day +// percentile range to its calendar tier colour; the p75 vertex inside the single +// Heavy tier keeps the fan following the p75 contour. const RAIN_FAN = [ - ["p90", "p99", "wet-8"], ["p75", "p90", "wet-7"], ["p60", "p75", "wet-6"], + ["p95", "p99", "wet-8"], ["p90", "p95", "wet-7"], ["p75", "p90", "wet-6"], ["p60", "p75", "wet-6"], ["p40", "p60", "wet-5"], ["p25", "p40", "wet-4"], ["p10", "p25", "wet-3"], ["p1", "p10", "wet-2"], ]; @@ -212,7 +216,7 @@ export function precipChart(days, n, xFor, C) { getPct: (d) => (d.precip ? d.precip.percentile : null), // Rain days take their intensity-tier color; no-rain days warm with the dry // streak (tan→red) so a dry spell reads as dry, matching the Dry chart. - dotColor: (d) => (d.precip && d.precip.value > 0 ? (TIER_COLORS[d.precip.class] || C.precip) : (drynessColor(d.dsr) || C.precip)), + dotColor: (d) => (d.precip && d.precip.class && d.precip.class !== "dry" ? (TIER_COLORS[d.precip.class] || C.precip) : (drynessColor(d.dsr) || C.precip)), fmtAxis: (v) => fmtPrecip(v, false), fmtLabel: (v) => fmtPrecip(v, false), labelOn: (v) => v > 0, // only label rain days; dry days rest on the baseline diff --git a/frontend/static/day.js b/frontend/static/day.js index c07aec3..3760323 100644 --- a/frontend/static/day.js +++ b/frontend/static/day.js @@ -6,8 +6,8 @@ import { fmtTemp, onUnitChange } from "./units.js"; import { uv } from "./account.js"; // header sign-in entry + notification bell, and the API-version resolver import { getJSON, TTL, prefetchViews } from "./cache.js"; import { initFindButton, setFindLabel } from "./mappicker.js"; -import { TIER_COLORS, PRECIP_COLORS, DRY_COLOR, pctOrd, fmtPrecip, fmtWind, fmtHumid, - todayISO, weatherType, placeLabel } from "./shared.js"; +import { TIER_COLORS, PRECIP_COLORS, DRY_COLOR, pctOrd, fmtPrecip, fmtPrecipTier, fmtWind, fmtHumid, + todayISO, isoOfDate, weatherType, placeLabel } from "./shared.js"; // Color a tier the same way the calendar cell would be colored for it. const tierColor = (c) => (c === "dry" ? DRY_COLOR : TIER_COLORS[c] || PRECIP_COLORS[c] || ""); @@ -44,7 +44,9 @@ function stepDay(delta) { if (!curDate) return; const d = new Date(curDate + "T00:00:00"); d.setDate(d.getDate() + delta); - const iso = d.toISOString().slice(0, 10); + // Format in LOCAL time — curDate was parsed as local midnight, so toISOString() + // (UTC) would shift the result a day for UTC± viewers and desync the today guard. + const iso = isoOfDate(d); if (iso > todayISO()) return; // don't step past today curDate = iso; fetchDay(); @@ -115,7 +117,8 @@ function render(data) { // Weather summary from the observed high + precip (omitted for days with no obs yet). const th = d.metrics.tmax.obs ? d.metrics.tmax.obs.value : null; const pr = d.metrics.precip.obs ? d.metrics.precip.obs.value : null; - const wt = th != null || pr != null ? weatherType(th, pr) : null; + const prCls = d.metrics.precip.obs ? d.metrics.precip.obs.class : null; + const wt = th != null || pr != null ? weatherType(th, pr, undefined, prCls) : null; dayHead.innerHTML = `

${nice}

${wt ? `

${wt.icon} ${wt.text}

` : ""} @@ -155,11 +158,14 @@ function ladderCard(title, m, fmt, kind) { if (!m || !m.ladder) return ""; const obs = m.obs; const activeClass = obs ? obs.class : null; + // The observed precip depth reads "trace" for a sub-0.01" rain day (rounds to 0 + // but is graded a rain tier); ladder tier ranges keep the plain numeric formatter. + const obsFmt = kind === "precip" ? ((v) => fmtPrecipTier(v, activeClass)) : fmt; let summary; if (obs) { const pct = obs.percentile == null ? "" : ` · ${pctOrd(obs.percentile)} pct`; - summary = `${fmt(obs.value)} + summary = `${obsFmt(obs.value)} ${obs.grade}${pct}`; } else { summary = `No observation for this day yet`; @@ -182,7 +188,7 @@ function ladderCard(title, m, fmt, kind) { else if (t.lo == null) val = `< ${fmt(t.hi)}`; // bottom tier: strictly below p1 else val = fmtRange(fmt, t.lo, t.hi); const marker = t.c === activeClass && obs - ? `◀ ${bareVal(fmt, obs.value)}` : ""; + ? `◀ ${bareVal(obsFmt, obs.value)}` : ""; return `
${t.label} diff --git a/frontend/static/push-client.js b/frontend/static/push-client.js index 8e61f8b..f9081b1 100644 --- a/frontend/static/push-client.js +++ b/frontend/static/push-client.js @@ -36,6 +36,21 @@ async function registration() { return navigator.serviceWorker.ready; } +// Does an existing subscription's baked-in applicationServerKey still match the +// server's current VAPID public key? After a key rotation it won't: the browser +// keeps signing pushes the server can no longer authenticate, so we must replace +// the subscription. `options.applicationServerKey` is an ArrayBuffer of the raw +// key bytes (or null on browsers that don't expose it — treat that as a mismatch +// and re-subscribe rather than leaving a possibly-stale subscription in place). +function applicationServerKeyMatches(sub, serverKey) { + const current = sub.options && sub.options.applicationServerKey; + if (!current) return false; + const a = new Uint8Array(current); + if (a.length !== serverKey.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== serverKey[i]) return false; + return true; +} + // Is this device currently subscribed? (a PushSubscription exists locally) export async function isEnabled() { if (!supported()) return false; @@ -60,13 +75,24 @@ export async function enable() { const reg = await registration(); let sub = await reg.pushManager.getSubscription(); + + // Always resolve the server's current VAPID key: it's needed to subscribe, and + // — when a subscription already exists — to detect a key rotation. A subscription + // minted under an old key silently stops receiving pushes, so on a mismatch we + // drop it and re-subscribe with the new key. Matching key = happy path, untouched. + const keyRes = await apiFetch(uv("push/vapid-key")); + if (!keyRes.ok) throw new Error("Couldn't fetch the server key."); + const { key } = await keyRes.json(); + const serverKey = urlB64ToUint8Array(key); + + if (sub && !applicationServerKeyMatches(sub, serverKey)) { + await sub.unsubscribe(); + sub = null; + } if (!sub) { - const res = await apiFetch(uv("push/vapid-key")); - if (!res.ok) throw new Error("Couldn't fetch the server key."); - const { key } = await res.json(); sub = await reg.pushManager.subscribe({ userVisibleOnly: true, - applicationServerKey: urlB64ToUint8Array(key), + applicationServerKey: serverKey, }); } const res = await apiFetch(uv("push/subscribe"), { method: "POST", json: sub.toJSON() }); diff --git a/frontend/static/shared.js b/frontend/static/shared.js index 423c5ea..a6e5224 100644 --- a/frontend/static/shared.js +++ b/frontend/static/shared.js @@ -257,7 +257,10 @@ export const pctOrd = (n) => { if (!Number.isFinite(v)) return "—"; return ord(Math.min(99, Math.max(1, Math.round(v)))); }; -export const todayISO = () => new Date().toISOString().slice(0, 10); +// Today in the viewer's LOCAL zone (see isoOfDate below). A UTC date rolls a day +// early/late for UTC± viewers past local midnight, putting "today" and the +// date-picker max out of reach of the day they're actually living in. +export const todayISO = () => isoOfDate(new Date()); export const esc = (s) => String(s).replace(/&/g, "&").replace(//g, ">"); // The heading label for a graded response: the resolved place name, or the // cell-center coordinates while (or if) no name resolves. @@ -464,13 +467,28 @@ export const WX_ICONS = { snow: WX(``), }; +// A graded precip depth for display, given its grade class. Reanalysis "trace" +// rain — any measurable precip below 0.01" — is graded as a rain tier but rounds to +// 0.00" / 0 mm, so its depth prints as "trace" rather than a bone-dry "0.00" that +// would contradict the rain grade (and the now-reset dry streak). Genuinely dry +// days (class "dry") and real, roundable depths format as usual. +export function fmtPrecipTier(v, cls, withUnit = true) { + if (cls && cls !== "dry" && v != null && parseFloat(fmtPrecip(v, false)) === 0) + return "trace"; + return fmtPrecip(v, withUnit); +} + // Plain-language "what was the day like" descriptor from the raw values: a // temperature word (from the daily high, °F) crossed with a sky/precip word // (precip in inches — falling as snow at/below freezing). `dsr` is optional: // when a long dry streak is known, a no-rain day reads "dry" instead of // "clear" (the calendar passes it; the day page doesn't track streaks). -export function weatherType(t, p, dsr) { - const wet = p != null && p >= 0.01; +// `precipCls` is the precip grade class when known: a rain tier means it rained +// even when the rounded depth reads 0 (trace), so it decides wet/dry ahead of the +// depth — matching the dry/rain grading split (precip > 0). Without it, any +// positive depth counts as wet. +export function weatherType(t, p, dsr, precipCls) { + const wet = precipCls != null ? precipCls !== "dry" : (p != null && p > 0); const freezing = t != null && t <= 34; const temp = t == null ? "" : t >= 95 ? "Scorching" : t >= 85 ? "Hot" : t >= 72 ? "Warm" : diff --git a/frontend/tests/unit/test_rendering.py b/frontend/tests/unit/test_rendering.py index 496580e..a10f9f8 100644 --- a/frontend/tests/unit/test_rendering.py +++ b/frontend/tests/unit/test_rendering.py @@ -29,6 +29,51 @@ def test_sitemap_lists_city_urls(client): assert f"/climate/{SLUG}/records" in r.text +# The public site is HTTPS-only behind Caddy, which terminates TLS and proxies +# plain HTTP -- so x-forwarded-proto / request.url.scheme read "http". A request +# arriving on the configured public host (THERMOGRAPH_BASE_URL defaults to +# https://thermograph.org in the test env) must still emit https:// in every +# frontend-built absolute URL, or canonicals self-conflict and the sitemap lists +# redirecting URLs. +_PUBLIC = {"host": "thermograph.org", "x-forwarded-proto": "http"} + + +def test_public_host_canonical_and_og_are_https(client): + b = client.get(f"{B}/climate/{SLUG}", headers=_PUBLIC).text + assert f'https://thermograph.org{B}/climate/{SLUG}" in body + assert "http://thermograph.org" not in body + + +def test_public_host_robots_sitemap_is_https(client): + body = client.get(f"{B}/robots.txt", headers=_PUBLIC).text + assert f"Sitemap: https://thermograph.org{B}/sitemap.xml" in body + + +def test_non_public_host_keeps_forwarded_scheme(client): + # A LAN/dev host never matches the configured public host, so the observed + # (plain-http) scheme is preserved -- http development is unaffected. + b = client.get(f"{B}/climate/{SLUG}", + headers={"host": "192.168.1.10:8000", "x-forwarded-proto": "http"}).text + assert f'