diff --git a/api/content_payloads.py b/api/content_payloads.py new file mode 100644 index 0000000..c95a8aa --- /dev/null +++ b/api/content_payloads.py @@ -0,0 +1,355 @@ +"""Payload builders for the SSR content API (backend/api/content_routes.py). + +These mirror backend/web/content.py's context builders — same underlying +grading.climatology()/all_time_records()/etc. calls — but return plain JSON-safe +dicts (raw floats, no Markup-wrapped HTML spans, no ContextVar-based unit +rendering) instead of pre-rendered strings, so a caller doesn't need to be a +Jinja/HTML consumer. Transitional duplication with content.py's own logic is +expected here: Stage 3 of the repo-split rewrites content.py to call these +endpoints via HTTP instead of computing in-process, at which point content.py's +copies of this logic go away and this becomes the only implementation. +""" +import datetime + +import polars as pl + +from api.payloads import OBS_COLS +from data import cities +from data import city_events +from data import grading + +MONTHS = ["january", "february", "march", "april", "may", "june", + "july", "august", "september", "october", "november", "december"] +MONTHS_TITLE = [m.capitalize() for m in MONTHS] +MONTH_INDEX = {m: i + 1 for i, m in enumerate(MONTHS)} + +SEASONS = [ + ("Winter", "Summer", [12, 1, 2], "Dec–Feb"), + ("Spring", "Autumn", [3, 4, 5], "Mar–May"), + ("Summer", "Winter", [6, 7, 8], "Jun–Aug"), + ("Autumn", "Spring", [9, 10, 11], "Sep–Nov"), +] + +METRIC_LABELS = [ + ("tmax", "High"), ("tmin", "Low"), ("feels", "Feels-like"), + ("humid", "Humidity"), ("wind", "Wind"), ("gust", "Gust"), ("precip", "Precip"), +] +_TEMP_METRICS = {"tmax", "tmin", "feels"} + +# Countries that actually use Fahrenheit. Mirrors content.py's F_COUNTRIES / +# frontend/units.js's F_REGIONS — a test asserts all three stay identical. +F_COUNTRIES = frozenset({"US", "PR", "GU", "VI", "AS", "MP", "UM", + "BS", "BZ", "KY", "PW", "FM", "MH", "LR"}) + +# 12 city chips for the homepage, geographically spread — mirrors content.py's +# HOME_CITY_SLUGS exactly (a test asserts the two lists stay identical). +HOME_CITY_SLUGS = ( + "new-york-city-new-york-us", "london-england-gb", "tokyo-jp", + "sydney-new-south-wales-au", "sao-paulo-br", "lagos-ng", + "mumbai-maharashtra-in", "mexico-city-mx", "cairo-eg", + "toronto-ontario-ca", "berlin-state-of-berlin-de", "seattle-washington-us", +) + + +def unit_for_country(code: str | None) -> str: + return "F" if (code or "").upper() in F_COUNTRIES else "C" + + +def _month_doy(month_idx: int) -> int: + return datetime.date(2001, month_idx, 15).timetuple().tm_yday + + +def _titled(text: str, suffix: str = " · Thermograph", limit: int = 60) -> str: + return text + suffix if len(text) + len(suffix) <= limit else text + + +def _clamp_desc(text: str, limit: int = 155) -> str: + text = " ".join(text.split()) + if len(text) <= limit: + return text + return text[:limit].rsplit(" ", 1)[0].rstrip(",;—-") + "…" + + +def _temp_text(f, unit: str) -> str: + if f is None: + return "—" + v = round((f - 32) * 5 / 9) if unit == "C" else round(f) + return f"{v}°{unit}" + + +def hub_payload() -> dict: + groups = cities.by_country() + return { + "n_cities": sum(len(v) for v in groups.values()), + "n_countries": len(groups), + "countries": [ + {"country": country, + "cities": [{"slug": c["slug"], "name": c["name"], + "display": cities.display_name(c)} for c in group]} + for country, group in groups.items() + ], + } + + +def _monthly_normals_raw(history) -> list[dict]: + """One row per month: raw high/low/precip means + the range-strip bounds.""" + rows = [] + for i, name in enumerate(MONTHS_TITLE, start=1): + clim = grading.climatology(history, _month_doy(i)) + tmax, tmin, precip = clim.get("tmax"), clim.get("tmin"), clim.get("precip") + rows.append({ + "name": name, "slug": MONTHS[i - 1], + "high_f": tmax["mean"] if tmax else None, + "low_f": tmin["mean"] if tmin else None, + "range_lo_f": tmin["p10"] if tmin else None, + "range_hi_f": tmax["p90"] if tmax else None, + "precip_f": precip["mean"] if precip else None, + }) + return rows + + +def _extreme_raw(metric_rec, key: str) -> dict | None: + if not metric_rec: + return None + return {"value_f": metric_rec[key], "date": metric_rec[f"{key}_date"]} + + +def _period_records_raw(history, months: list[int]) -> dict: + if len(months) == 1: + sub = history.filter(pl.col("date").dt.month() == months[0]) + else: + sub = history.filter(pl.col("date").dt.month().is_in(months)) + rec = grading.all_time_records(sub) if not sub.is_empty() else {} + tmax, tmin = rec.get("tmax"), rec.get("tmin") + return { + "high": {"warm": _extreme_raw(tmax, "max"), "cold": _extreme_raw(tmax, "min")}, + "low": {"warm": _extreme_raw(tmin, "max"), "cold": _extreme_raw(tmin, "min")}, + } + + +def _monthly_records_raw(history) -> list[dict]: + return [ + {"name": name, "slug": MONTHS[i - 1], **_period_records_raw(history, [i])} + for i, name in enumerate(MONTHS_TITLE, start=1) + ] + + +def _seasonal_records_raw(history, lat: float) -> list[dict]: + south = lat < 0 + return [ + {"name": (south_lbl if south else north_lbl), "span": span, + **_period_records_raw(history, months)} + for north_lbl, south_lbl, months, span in SEASONS + ] + + +def _today_vs_normal_raw(history, recent) -> dict | None: + if recent is None or recent.is_empty() or "date" not in recent.columns: + return None + today = datetime.date.today() + observed = recent.filter(pl.col("date") <= today).sort("date") + if observed.is_empty(): + return None + row = observed.row(observed.height - 1, named=True) + obs = {k: row[k] for k in OBS_COLS if k in row} + graded = grading.grade_day(history, row["date"], obs) + cards = [] + for key, label in METRIC_LABELS: + g = graded.get(key) + if not g: + continue + cards.append({ + "label": label, "metric": key, "value_f": g.get("value"), + "percentile": g.get("percentile"), "grade": g.get("grade"), "cls": g.get("class"), + }) + date = row["date"] + return {"date": date.isoformat() if hasattr(date, "isoformat") else str(date), "cards": cards} + + +def city_payload(request_origin: str, base: str, city: dict, history, recent=None) -> dict: + """The full /api/v2/content/city/{slug} payload. `recent` is the caller's + already-fetched recent/forecast bundle (or None to skip today_vs_normal).""" + display = cities.display_name(city) + title = cities.title_name(city) + years = history["date"].dt.year() + year_range = [int(years.min()), int(years.max())] + months = _monthly_normals_raw(history) + warmest = max((m for m in months if m["high_f"] is not None), key=lambda m: m["high_f"], default=None) + coldest = min((m for m in months if m["low_f"] is not None), key=lambda m: m["low_f"], default=None) + wettest = max((m for m in months if m["precip_f"] is not None), key=lambda m: m["precip_f"], default=None) + records = grading.all_time_records(history) + unit = unit_for_country(city.get("country_code")) + flavor = cities.flavor(city["slug"]) + event = city_events.get(city["slug"]) + today = _today_vs_normal_raw(history, recent) if recent is not None else None + + breadcrumb = [{"name": "Home", "href": f"{base}/"}, {"name": "Climate", "href": f"{base}/climate"}] + if city.get("country"): + breadcrumb.append({"name": city["country"], "href": None}) + breadcrumb.append({"name": city["name"], "href": None}) + + page_url = f"{request_origin}{base}/climate/{city['slug']}" + n_years = year_range[1] - year_range[0] + jsonld = { + "@context": "https://schema.org", + "@graph": [ + {"@type": "Dataset", + "name": f"{display} climate normals and records", + "description": f"Average temperatures, precipitation and record highs and lows for " + f"{display}, from ~{n_years} years of daily climate history.", + "url": page_url, + "temporalCoverage": f"{year_range[0]}/{year_range[1]}", + "spatialCoverage": {"@type": "Place", "name": display, + "geo": {"@type": "GeoCoordinates", + "latitude": city["lat"], "longitude": city["lon"]}}, + "creator": {"@type": "Organization", "name": "Thermograph"}, + "isBasedOn": "https://open-meteo.com/ (ERA5 reanalysis)"}, + ], + } + return { + "city": city, "display": display, "title": title, + "year_range": year_range, "n_years": n_years, + "months": months, + "warmest_month_slug": warmest["slug"] if warmest else None, + "coldest_month_slug": coldest["slug"] if coldest else None, + "wettest_month_slug": wettest["slug"] if wettest else None, + "all_time_records": records, + "today_vs_normal": today, + "flavor": flavor, "event": event, "default_unit": unit, + "breadcrumb": breadcrumb, + "canonical_path": f"/climate/{city['slug']}", + "page_title": _titled(f"{title} climate: daily normals, records & how unusual it is now"), + "page_description": _clamp_desc( + f"{title} averages highs of {_temp_text(warmest['high_f'], unit) if warmest else '—'} in " + f"{warmest['name'] if warmest else 'summer'} and lows of " + f"{_temp_text(coldest['low_f'], unit) if coldest else '—'} in " + f"{coldest['name'] if coldest else 'winter'}. " + f"Every day graded against {n_years} years of local history."), + "jsonld": jsonld, + } + + +def month_payload(city: dict, history, month_idx: int) -> dict: + title = cities.title_name(city) + month_name = MONTHS_TITLE[month_idx - 1] + month_slug = MONTHS[month_idx - 1] + years = history["date"].dt.year() + year_range = [int(years.min()), int(years.max())] + + clim = grading.climatology(history, _month_doy(month_idx)) + tmax, tmin, precip = clim.get("tmax"), clim.get("tmin"), clim.get("precip") + mdf = history.filter(pl.col("date").dt.month() == month_idx) + mrec = grading.all_time_records(mdf) if not mdf.is_empty() else {} + + records = [] + for key, label in METRIC_LABELS: + r = mrec.get(key) + if not r: + continue + if key == "precip": + totals = (mdf.filter(pl.col("precip").is_not_null()) + .group_by(pl.col("date").dt.year().alias("yr")) + .agg(pl.col("precip").sum().alias("total"), + pl.len().alias("days")) + .filter(pl.col("days") >= 26) + .sort("total")) + if totals.is_empty(): + continue + lo, hi = totals.row(0, named=True), totals.row(totals.height - 1, named=True) + records.append({"label": label, + "high_f": hi["total"], "high_date": str(hi["yr"]), "high_tag": "Wettest", + "low_f": lo["total"], "low_date": str(lo["yr"]), "low_tag": "Driest"}) + else: + records.append({"label": label, + "high_f": r["max"], "high_date": r["max_date"], "high_tag": "Highest", + "low_f": r["min"], "low_date": r["min_date"], "low_tag": "Lowest"}) + + prev_i = 12 if month_idx == 1 else month_idx - 1 + next_i = 1 if month_idx == 12 else month_idx + 1 + n_years = year_range[1] - year_range[0] + unit = unit_for_country(city.get("country_code")) + return { + "month_name": month_name, "month_slug": month_slug, + "year_range": year_range, "n_years": n_years, + "avg_high_f": tmax["mean"] if tmax else None, + "avg_low_f": tmin["mean"] if tmin else None, + "typical_high_range_f": [tmax["p10"], tmax["p90"]] if tmax else None, + "typical_low_range_f": [tmin["p10"], tmin["p90"]] if tmin else None, + "avg_precip_f": precip["mean"] if precip else None, + "records": records, + "prev": {"name": MONTHS_TITLE[prev_i - 1], "slug": MONTHS[prev_i - 1]}, + "next": {"name": MONTHS_TITLE[next_i - 1], "slug": MONTHS[next_i - 1]}, + "canonical_path": f"/climate/{city['slug']}/{month_slug}", + "page_title": _titled(f"{title} in {month_name}: normal weather & records"), + "page_description": _clamp_desc( + f"{title} averages {_temp_text(tmax['mean'] if tmax else None, unit)} highs and " + f"{_temp_text(tmin['mean'] if tmin else None, unit)} lows in {month_name}. " + f"Every day graded against {n_years} years of local history."), + } + + +def records_payload(city: dict, history) -> dict: + title = cities.title_name(city) + years = history["date"].dt.year() + year_range = [int(years.min()), int(years.max())] + n_years = year_range[1] - year_range[0] + rec = grading.all_time_records(history) + hi_rec = (rec.get("tmax") or {}).get("max") + hi_date = (rec.get("tmax") or {}).get("max_date") + lo_rec = (rec.get("tmin") or {}).get("min") + lo_date = (rec.get("tmin") or {}).get("min_date") + unit = unit_for_country(city.get("country_code")) + + rows = [] + for key, label in METRIC_LABELS: + r = rec.get(key) + if not r: + continue + if key == "precip": + # "Record low" rain is meaningless (it's just 0) -- the low side is + # the longest dry streak instead, which is a day count + start date, + # not a precip value, so low_f/low_date stay None here. + days, dry_start = grading.longest_dry_streak(history) + low_f, low_date = None, dry_start or None + dry_streak_days = days or None + else: + low_f, low_date, dry_streak_days = r["min"], r["min_date"], None + rows.append({ + "label": label, + "high_f": r["max"], "high_date": r["max_date"], + "low_f": low_f, "low_date": low_date, + "dry_streak_days": dry_streak_days, + }) + hemisphere = "Southern" if city["lat"] < 0 else "Northern" + return { + "year_range": year_range, "n_years": n_years, + "rows": rows, + "monthly": _monthly_records_raw(history), + "seasonal": _seasonal_records_raw(history, city["lat"]), + "hemisphere": hemisphere, + "canonical_path": f"/climate/{city['slug']}/records", + "page_title": _titled(f"{title} weather records: hottest & coldest days since {year_range[0]}"), + "page_description": _clamp_desc( + f"{title}'s hottest day hit {_temp_text(hi_rec, unit)}" + f"{f' ({hi_date})' if hi_date else ''}; its coldest fell to " + f"{_temp_text(lo_rec, unit)}{f' ({lo_date})' if lo_date else ''}. " + f"Every day graded against {n_years} years of local history."), + } + + +def home_payload() -> dict: + from api import homepage + feed = homepage.load() + stale = bool(feed) and homepage.is_stale(feed) + ranked, unusual = [], None + if feed: + ranked = feed.get("ranked") or [] + pick = (feed.get("picks") or {}).get("extreme") + if pick: + unusual = dict(pick, is_default=True) + home_cities = [] + for slug in HOME_CITY_SLUGS: + city = cities.get(slug) + if city: + home_cities.append({"slug": slug, "name": city["name"]}) + return {"unusual": unusual, "stale": stale, "ranked": ranked, "cities": home_cities} diff --git a/api/content_routes.py b/api/content_routes.py new file mode 100644 index 0000000..6f4015e --- /dev/null +++ b/api/content_routes.py @@ -0,0 +1,143 @@ +"""SSR content JSON API — the data a frontend SSR layer needs to render the +climate hub / per-city / month / records / homepage surfaces, without importing +backend code in-process. See backend/api/content_payloads.py for the builders; +this module is just FastAPI route wiring + the same derived-store/ETag caching +pattern web/app.py's own endpoints use. +""" +import hashlib +import os + +from fastapi import APIRouter, HTTPException, Request, Response + +from api import content_payloads as payloads +from api import sitemap as sitemap_mod +from api.payloads import history_token +from data import climate +from data import cities +from data import grid +from data import store + +router = APIRouter(tags=["content"]) + +# Same convention as web/app.py and web/content.py: the deployment's base path, +# used only for the breadcrumb hrefs and jsonld URL this endpoint returns. +_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") +BASE = f"/{_BASE}" if _BASE else "" + + +def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str: + h = hashlib.sha1(f"{kind}:{cell_id}:{key}:{token}".encode()).hexdigest()[:20] + return f'W/"{h}"' + + +def _not_modified(request: Request, etag: str) -> bool: + inm = request.headers.get("if-none-match") + if not inm: + return False + tags = {t.strip() for t in inm.split(",")} + return "*" in tags or etag in tags or etag.removeprefix("W/") in tags + + +def _json_response(body: bytes, etag: str) -> Response: + return Response(content=body, media_type="application/json", headers={"ETag": etag}) + + +def _cached(request: Request, kind: str, cell_id: str, key: str, token: str, build) -> Response: + """Same shape as web/app.py's _cached_response: If-None-Match -> empty 304; + a token-valid store row -> replay it; otherwise build, persist, serve. + store.get_payload/put_payload both deal in already-encoded JSON bytes, not + the payload dict -- put_payload returns the exact bytes it persisted so the + caller can serve them without re-serializing.""" + etag = _etag_for(kind, cell_id, key, token) + if _not_modified(request, etag): + return Response(status_code=304, headers={"ETag": etag}) + body = store.get_payload(kind, cell_id, key, token) + if body is not None: + return _json_response(body, etag) + payload = build() + return _json_response(store.put_payload(kind, cell_id, key, token, payload), etag) + + +def _resolve_city(slug: str): + """(city, cell, history) for a slug, or raise 404 (unknown) / 503 (warming) -- + mirrors web/content.py's _resolve_city.""" + city = cities.get(slug) + if city is None: + raise HTTPException(status_code=404, detail="Unknown city.") + cell = grid.snap(city["lat"], city["lon"]) + history = climate.load_cached_history(cell) + if history is None or history.is_empty(): + try: + history, _ = climate.get_history(cell) + except Exception: # noqa: BLE001 + history = None + if history is None or history.is_empty(): + raise HTTPException(status_code=503, detail="Climate data is warming up; please retry shortly.") + return city, cell, history + + +def _origin(request: Request) -> str: + proto = request.headers.get("x-forwarded-proto") or request.url.scheme + host = request.headers.get("host") or request.url.netloc + return f"{proto}://{host}" + + +@router.get("/content/hub") +def content_hub(request: Request): + return payloads.hub_payload() + + +@router.get("/content/sitemap") +def content_sitemap(request: Request): + return [{"path": p, "changefreq": cf, "priority": pr} for p, cf, pr in sitemap_mod.sitemap_entries()] + + +@router.get("/content/indexnow-key") +def content_indexnow_key(request: Request): + import indexnow + return {"key": indexnow.key()} + + +@router.get("/content/home") +def content_home(request: Request): + return payloads.home_payload() + + +@router.get("/content/city/{slug}") +def content_city(request: Request, slug: str): + city, cell, history = _resolve_city(slug) + recent = None + try: + recent = climate.get_recent_forecast(cell) + except Exception: # noqa: BLE001 - today_vs_normal degrades to None + pass + token = history_token(history) + + def build(): + return payloads.city_payload(_origin(request), BASE, city, history, recent) + + return _cached(request, "content-city", cell["id"], slug, token, build) + + +@router.get("/content/city/{slug}/month/{month}") +def content_city_month(request: Request, slug: str, month: str): + if month not in payloads.MONTH_INDEX: + raise HTTPException(status_code=404, detail="Unknown month.") + city, cell, history = _resolve_city(slug) + token = history_token(history) + + def build(): + return payloads.month_payload(city, history, payloads.MONTH_INDEX[month]) + + return _cached(request, "content-month", cell["id"], f"{slug}:{month}", token, build) + + +@router.get("/content/city/{slug}/records") +def content_city_records(request: Request, slug: str): + city, cell, history = _resolve_city(slug) + token = history_token(history) + + def build(): + return payloads.records_payload(city, history) + + return _cached(request, "content-records", cell["id"], slug, token, build) diff --git a/tests/web/test_content.py b/tests/web/test_content.py index 4c85ee9..5ecd04f 100644 --- a/tests/web/test_content.py +++ b/tests/web/test_content.py @@ -583,3 +583,78 @@ def test_city_page_percentiles_are_real_ordinals(client): # No float ordinals, no impossible ones, no "1th"/"2th"/"3th". for bad in re.findall(r"\d+\.\d+(?:st|nd|rd|th)|100th|\b0th|\b\d*[123]th", html): raise AssertionError(f"malformed percentile ordinal: {bad!r}") + + +# --- content API parity (repo-split Stage 2) ---------------------------------- +# The new /api/v2/content/* endpoints (backend/api/content_routes.py) recompute +# the same numbers this file's SSR pages render, independently (content.py's +# Markup-wrapped spans vs. the API's raw floats). These assert the two never +# drift apart while both exist side by side -- see backend/api/content_payloads.py. + +def _data_temp_fs(html: str) -> set[float]: + return {round(float(v), 1) for v in re.findall(r'data-temp-f="(-?[\d.]+)"', html)} + + +def test_content_api_city_matches_rendered_html(client): + html = client.get(f"{B}/climate/{SLUG}").text + rendered = _data_temp_fs(html) + + j = client.get(f"{B}/api/v2/content/city/{SLUG}").json() + assert j["canonical_path"] == f"/climate/{SLUG}" + assert j["city"]["slug"] == SLUG + + # Every month's high/low the API reports must appear somewhere in the + # page's client-convertible spans -- same underlying grading.climatology() + # call, so a real drift here means one side silently diverged. + for m in j["months"]: + if m["high_f"] is not None: + assert round(m["high_f"], 1) in rendered, f"{m['slug']} high_f not on page" + if m["low_f"] is not None: + assert round(m["low_f"], 1) in rendered, f"{m['slug']} low_f not on page" + + # All-time tmax/tmin extremes likewise. + rec = j["all_time_records"] + assert round(rec["tmax"]["max"], 1) in rendered + assert round(rec["tmin"]["min"], 1) in rendered + + +def test_content_api_month_matches_rendered_html(client): + html = client.get(f"{B}/climate/{SLUG}/july").text + rendered = _data_temp_fs(html) + + j = client.get(f"{B}/api/v2/content/city/{SLUG}/month/july").json() + assert j["month_slug"] == "july" + if j["avg_high_f"] is not None: + assert round(j["avg_high_f"], 1) in rendered + if j["avg_low_f"] is not None: + assert round(j["avg_low_f"], 1) in rendered + + +def test_content_api_records_matches_rendered_html(client): + html = client.get(f"{B}/climate/{SLUG}/records").text + rendered = _data_temp_fs(html) + + j = client.get(f"{B}/api/v2/content/city/{SLUG}/records").json() + for row in j["rows"]: + if row["label"] == "Precip": + continue # precip isn't a data-temp-f span + assert round(row["high_f"], 1) in rendered + if row["low_f"] is not None: + assert round(row["low_f"], 1) in rendered + + +def test_content_api_unknown_city_404s(client): + assert client.get(f"{B}/api/v2/content/city/not-a-real-city").status_code == 404 + + +def test_content_api_unknown_month_404s(client): + assert client.get(f"{B}/api/v2/content/city/{SLUG}/month/notamonth").status_code == 404 + + +def test_content_api_home_city_slugs_match_content_py(): + """Two hand-maintained copies (content.py's HOME_CITY_SLUGS and + content_payloads.py's) -- pin them together like the other parity tests + in this file.""" + from web import content + from api import content_payloads + assert content.HOME_CITY_SLUGS == content_payloads.HOME_CITY_SLUGS diff --git a/web/app.py b/web/app.py index 81efddc..e4d0943 100644 --- a/web/app.py +++ b/web/app.py @@ -17,6 +17,7 @@ from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles from accounts import api_accounts +from api import content_routes from core import audit from data import climate from web import content @@ -716,6 +717,11 @@ v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False) app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible) app.include_router(v1, prefix=f"{BASE}/api/v1") app.include_router(v2, prefix=f"{BASE}/api/v2") +# SSR content API (repo-split Stage 2) -- purely additive, not yet consumed by +# anything in this repo; content.py still renders its own SSR pages in-process. +# The eventual frontend SSR service will call these instead. See +# backend/api/content_routes.py. +app.include_router(content_routes.router, prefix=f"{BASE}/api/v2") # --- Discord slash commands (HTTP interactions) ----------------------------