7adcb3f77b
Add a third tier to gen_cities.py: the top-population cities from the remaining English-official countries plus countries where >35% speak English (Eurobarometer/ EF), that weren't already chosen. cities.json grows to 1000 (500 global + 250 core-English + 250 extended), pulling in the biggest cities of India, Nigeria, Pakistan, the Philippines, plus Amsterdam/Stockholm/Nairobi/Tel Aviv, etc. gen_flavor.py is now incremental (fetches only cities missing a blurb, prunes stale ones; --full to rebuild); cities_flavor.json refreshed to cover 942/1000.
126 lines
4.8 KiB
Python
126 lines
4.8 KiB
Python
"""Offline seeder for backend/cities_flavor.json — a short descriptive blurb per
|
|
city, so the crawlable /climate pages carry unique editorial text (not just the
|
|
templated stats) and read as distinct pages. Source: Wikipedia's free REST summary
|
|
API (no key). Text is CC BY-SA; the city page attributes it with a link.
|
|
|
|
python gen_flavor.py [--limit N] [--workers 8]
|
|
|
|
Each city is matched by name (then "name, country" / "name, admin1"), and the match
|
|
is validated by comparing Wikipedia's article coordinates to the city's known
|
|
lat/lon — so we don't attach the wrong "Springfield". Re-run to refresh; existing
|
|
entries for cities no longer in cities.json are dropped.
|
|
"""
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
|
|
import cities
|
|
|
|
OUT_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json")
|
|
UA = "ThermographBot/1.0 (https://thermograph.org; climate comparison tool)"
|
|
SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"
|
|
|
|
|
|
def _km(lat1, lon1, lat2, lon2) -> float:
|
|
dlat = (lat2 - lat1) * 111.0
|
|
dlon = (lon2 - lon1) * 111.0 * math.cos(math.radians(lat1))
|
|
return math.hypot(dlat, dlon)
|
|
|
|
|
|
def _trim(text: str, maxlen: int = 300) -> str:
|
|
text = " ".join(text.split())
|
|
if len(text) <= maxlen:
|
|
return text
|
|
cut = text[:maxlen]
|
|
i = cut.rfind(". ")
|
|
return (cut[:i + 1] if i > 80 else cut.rstrip() + "…")
|
|
|
|
|
|
def _summary(client: httpx.Client, title: str) -> dict | None:
|
|
# Retry through transient throttling (429/5xx/network) so a bulk run doesn't
|
|
# silently drop cities to rate limits.
|
|
url = SUMMARY + quote(title, safe="")
|
|
for attempt in range(4):
|
|
try:
|
|
r = client.get(url, timeout=20, follow_redirects=True)
|
|
if r.status_code == 200:
|
|
return r.json()
|
|
if r.status_code == 404:
|
|
return None # no such article — don't retry
|
|
# 429 / 5xx: back off and retry
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
time.sleep(0.6 * (attempt + 1))
|
|
return None
|
|
|
|
|
|
def blurb_for(client: httpx.Client, city: dict) -> dict | None:
|
|
candidates = [city["name"]]
|
|
if city.get("country"):
|
|
candidates.append(f"{city['name']}, {city['country']}")
|
|
if city.get("admin1") and city["admin1"] != city["name"]:
|
|
candidates.append(f"{city['name']}, {city['admin1']}")
|
|
for title in candidates:
|
|
d = _summary(client, title)
|
|
if not d or d.get("type") == "disambiguation" or not d.get("extract"):
|
|
continue
|
|
coords = d.get("coordinates")
|
|
if coords and _km(city["lat"], city["lon"], coords["lat"], coords["lon"]) <= 75:
|
|
url = (d.get("content_urls", {}).get("desktop", {}) or {}).get("page")
|
|
return {"extract": _trim(d["extract"]), "url": url, "title": d.get("title")}
|
|
return None
|
|
|
|
|
|
def build(todo: list[dict], workers: int = 5) -> dict:
|
|
"""Fetch a blurb for each city in `todo`; returns {slug: {...}} for the hits."""
|
|
out: dict[str, dict] = {}
|
|
with httpx.Client(headers={"user-agent": UA}) as client:
|
|
def work(c):
|
|
return (c["slug"], blurb_for(client, c))
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
for i, (slug, b) in enumerate(ex.map(work, todo), 1):
|
|
if b:
|
|
out[slug] = b
|
|
if i % 100 == 0:
|
|
print(f" {i}/{len(todo)} processed, {len(out)} new blurbs")
|
|
return out
|
|
|
|
|
|
def main() -> None:
|
|
args = sys.argv[1:]
|
|
limit = int(args[args.index("--limit") + 1]) if "--limit" in args else None
|
|
workers = int(args[args.index("--workers") + 1]) if "--workers" in args else 5
|
|
full = "--full" in args # re-fetch everything, else only cities missing a blurb
|
|
|
|
# Incremental by default: keep existing blurbs, fetch only the missing ones, and
|
|
# prune entries for cities no longer in cities.json.
|
|
existing: dict[str, dict] = {}
|
|
if not full:
|
|
try:
|
|
with open(OUT_PATH, encoding="utf-8") as f:
|
|
existing = json.load(f)
|
|
except (OSError, ValueError):
|
|
existing = {}
|
|
all_cities = cities.all_cities()
|
|
all_slugs = {c["slug"] for c in all_cities}
|
|
todo = [c for c in all_cities if full or c["slug"] not in existing]
|
|
if limit:
|
|
todo = todo[:limit]
|
|
print(f"fetching {len(todo)} cities ({'full rebuild' if full else 'missing only'})")
|
|
|
|
fetched = build(todo, workers=workers)
|
|
merged = {s: v for s, v in {**existing, **fetched}.items() if s in all_slugs}
|
|
with open(OUT_PATH, "w", encoding="utf-8") as f:
|
|
json.dump(merged, f, ensure_ascii=False, sort_keys=True, indent=0, separators=(",", ":"))
|
|
f.write("\n")
|
|
print(f"wrote {len(merged)} blurbs ({len(fetched)} newly fetched) -> {OUT_PATH}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|