thermograph/gen_flavor.py

"""Offline seeder for backend/cities_flavor.json — a short descriptive blurb per
city, so the crawlable /climate pages carry unique editorial text (not just the
templated stats) and read as distinct pages. Source: Wikipedia's free REST summary
API (no key). Text is CC BY-SA; the city page attributes it with a link.

    python gen_flavor.py [--limit N] [--workers 8]

Each city is matched by name (then "name, country" / "name, admin1"), and the match
is validated by comparing Wikipedia's article coordinates to the city's known
lat/lon — so we don't attach the wrong "Springfield". Re-run to refresh; existing
entries for cities no longer in cities.json are dropped.
"""
import json
import math
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote

import httpx

from data import cities
import paths

OUT_PATH = paths.CITIES_FLAVOR_JSON
UA = "ThermographBot/1.0 (https://thermograph.org; climate comparison tool)"
SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"


def _km(lat1, lon1, lat2, lon2) -> float:
    dlat = (lat2 - lat1) * 111.0
    dlon = (lon2 - lon1) * 111.0 * math.cos(math.radians(lat1))
    return math.hypot(dlat, dlon)


def _trim(text: str, maxlen: int = 300) -> str:
    text = " ".join(text.split())
    if len(text) <= maxlen:
        return text
    cut = text[:maxlen]
    i = cut.rfind(". ")
    return (cut[:i + 1] if i > 80 else cut.rstrip() + "…")


def _summary(client: httpx.Client, title: str) -> dict | None:
    # Retry through transient throttling (429/5xx/network) so a bulk run doesn't
    # silently drop cities to rate limits.
    url = SUMMARY + quote(title, safe="")
    for attempt in range(4):
        try:
            r = client.get(url, timeout=20, follow_redirects=True)
            if r.status_code == 200:
                return r.json()
            if r.status_code == 404:
                return None                      # no such article — don't retry
            # 429 / 5xx: back off and retry
        except Exception:  # noqa: BLE001
            pass
        time.sleep(0.6 * (attempt + 1))
    return None


def blurb_for(client: httpx.Client, city: dict) -> dict | None:
    candidates = [city["name"]]
    if city.get("country"):
        candidates.append(f"{city['name']}, {city['country']}")
    if city.get("admin1") and city["admin1"] != city["name"]:
        candidates.append(f"{city['name']}, {city['admin1']}")
    for title in candidates:
        d = _summary(client, title)
        if not d or d.get("type") == "disambiguation" or not d.get("extract"):
            continue
        coords = d.get("coordinates")
        if coords and _km(city["lat"], city["lon"], coords["lat"], coords["lon"]) <= 75:
            url = (d.get("content_urls", {}).get("desktop", {}) or {}).get("page")
            return {"extract": _trim(d["extract"]), "url": url, "title": d.get("title")}
    return None


def build(todo: list[dict], workers: int = 5) -> dict:
    """Fetch a blurb for each city in `todo`; returns {slug: {...}} for the hits."""
    out: dict[str, dict] = {}
    with httpx.Client(headers={"user-agent": UA}) as client:
        def work(c):
            return (c["slug"], blurb_for(client, c))
        with ThreadPoolExecutor(max_workers=workers) as ex:
            for i, (slug, b) in enumerate(ex.map(work, todo), 1):
                if b:
                    out[slug] = b
                if i % 100 == 0:
                    print(f"  {i}/{len(todo)} processed, {len(out)} new blurbs")
    return out


def main() -> None:
    args = sys.argv[1:]
    limit = int(args[args.index("--limit") + 1]) if "--limit" in args else None
    workers = int(args[args.index("--workers") + 1]) if "--workers" in args else 5
    full = "--full" in args   # re-fetch everything, else only cities missing a blurb

    # Incremental by default: keep existing blurbs, fetch only the missing ones, and
    # prune entries for cities no longer in cities.json.
    existing: dict[str, dict] = {}
    if not full:
        try:
            with open(OUT_PATH, encoding="utf-8") as f:
                existing = json.load(f)
        except (OSError, ValueError):
            existing = {}
    all_cities = cities.all_cities()
    all_slugs = {c["slug"] for c in all_cities}
    todo = [c for c in all_cities if full or c["slug"] not in existing]
    if limit:
        todo = todo[:limit]
    print(f"fetching {len(todo)} cities ({'full rebuild' if full else 'missing only'})")

    fetched = build(todo, workers=workers)
    merged = {s: v for s, v in {**existing, **fetched}.items() if s in all_slugs}
    with open(OUT_PATH, "w", encoding="utf-8") as f:
        json.dump(merged, f, ensure_ascii=False, sort_keys=True, indent=0, separators=(",", ":"))
        f.write("\n")
    print(f"wrote {len(merged)} blurbs ({len(fetched)} newly fetched) -> {OUT_PATH}")


if __name__ == "__main__":
    main()