c3a11ee994
Add unique editorial content so the programmatic pages don't read as templated:
- gen_flavor.py seeds backend/cities_flavor.json with a short descriptive blurb per
city from Wikipedia's free REST summary API (no key), validating each match by
comparing article coordinates to the city's lat/lon so the wrong 'Springfield'
never attaches. Retries + modest concurrency; ~700/750 cities get a blurb, the
rest render without one. Text is CC BY-SA, attributed with a 'via Wikipedia' link.
- City pages show the blurb under the intro and a travel callout that deep-links to
the compare page with the city pre-loaded (/compare#loc=lat,lon) — the visitor
just adds their own city. Month pages get the same seasonal 'visiting in {month}?'
compare link. Both add unique per-page text and internal links.
- cities.py gains flavor(slug); tests cover the blurb + attribution + compare CTA.
110 lines
4 KiB
Python
110 lines
4 KiB
Python
"""Offline seeder for backend/cities_flavor.json — a short descriptive blurb per
|
|
city, so the crawlable /climate pages carry unique editorial text (not just the
|
|
templated stats) and read as distinct pages. Source: Wikipedia's free REST summary
|
|
API (no key). Text is CC BY-SA; the city page attributes it with a link.
|
|
|
|
python gen_flavor.py [--limit N] [--workers 8]
|
|
|
|
Each city is matched by name (then "name, country" / "name, admin1"), and the match
|
|
is validated by comparing Wikipedia's article coordinates to the city's known
|
|
lat/lon — so we don't attach the wrong "Springfield". Re-run to refresh; existing
|
|
entries for cities no longer in cities.json are dropped.
|
|
"""
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
|
|
import cities
|
|
|
|
OUT_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json")
|
|
UA = "ThermographBot/1.0 (https://thermograph.org; climate comparison tool)"
|
|
SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"
|
|
|
|
|
|
def _km(lat1, lon1, lat2, lon2) -> float:
|
|
dlat = (lat2 - lat1) * 111.0
|
|
dlon = (lon2 - lon1) * 111.0 * math.cos(math.radians(lat1))
|
|
return math.hypot(dlat, dlon)
|
|
|
|
|
|
def _trim(text: str, maxlen: int = 300) -> str:
|
|
text = " ".join(text.split())
|
|
if len(text) <= maxlen:
|
|
return text
|
|
cut = text[:maxlen]
|
|
i = cut.rfind(". ")
|
|
return (cut[:i + 1] if i > 80 else cut.rstrip() + "…")
|
|
|
|
|
|
def _summary(client: httpx.Client, title: str) -> dict | None:
|
|
# Retry through transient throttling (429/5xx/network) so a bulk run doesn't
|
|
# silently drop cities to rate limits.
|
|
url = SUMMARY + quote(title, safe="")
|
|
for attempt in range(4):
|
|
try:
|
|
r = client.get(url, timeout=20, follow_redirects=True)
|
|
if r.status_code == 200:
|
|
return r.json()
|
|
if r.status_code == 404:
|
|
return None # no such article — don't retry
|
|
# 429 / 5xx: back off and retry
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
time.sleep(0.6 * (attempt + 1))
|
|
return None
|
|
|
|
|
|
def blurb_for(client: httpx.Client, city: dict) -> dict | None:
|
|
candidates = [city["name"]]
|
|
if city.get("country"):
|
|
candidates.append(f"{city['name']}, {city['country']}")
|
|
if city.get("admin1") and city["admin1"] != city["name"]:
|
|
candidates.append(f"{city['name']}, {city['admin1']}")
|
|
for title in candidates:
|
|
d = _summary(client, title)
|
|
if not d or d.get("type") == "disambiguation" or not d.get("extract"):
|
|
continue
|
|
coords = d.get("coordinates")
|
|
if coords and _km(city["lat"], city["lon"], coords["lat"], coords["lon"]) <= 75:
|
|
url = (d.get("content_urls", {}).get("desktop", {}) or {}).get("page")
|
|
return {"extract": _trim(d["extract"]), "url": url, "title": d.get("title")}
|
|
return None
|
|
|
|
|
|
def build(limit: int | None = None, workers: int = 8) -> dict:
|
|
todo = cities.all_cities()
|
|
if limit:
|
|
todo = todo[:limit]
|
|
out: dict[str, dict] = {}
|
|
with httpx.Client(headers={"user-agent": UA}) as client:
|
|
def work(c):
|
|
b = blurb_for(client, c)
|
|
return (c["slug"], b)
|
|
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
for i, (slug, b) in enumerate(ex.map(work, todo), 1):
|
|
if b:
|
|
out[slug] = b
|
|
if i % 100 == 0:
|
|
print(f" {i}/{len(todo)} processed, {len(out)} with blurbs")
|
|
return out
|
|
|
|
|
|
def main() -> None:
|
|
args = sys.argv[1:]
|
|
limit = int(args[args.index("--limit") + 1]) if "--limit" in args else None
|
|
workers = int(args[args.index("--workers") + 1]) if "--workers" in args else 8
|
|
flavor = build(limit=limit, workers=workers)
|
|
with open(OUT_PATH, "w", encoding="utf-8") as f:
|
|
json.dump(flavor, f, ensure_ascii=False, sort_keys=True, indent=0, separators=(",", ":"))
|
|
f.write("\n")
|
|
print(f"wrote {len(flavor)} blurbs -> {OUT_PATH}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|