SEO: expand city set to 1000 (extended English-speaking / high-proficiency) (#99)

Add a third tier to gen_cities.py: the top-population cities from the remaining
English-official countries plus countries where >35% speak English (Eurobarometer/
EF), that weren't already chosen. cities.json grows to 1000 (500 global + 250
core-English + 250 extended), pulling in the biggest cities of India, Nigeria,
Pakistan, the Philippines, plus Amsterdam/Stockholm/Nairobi/Tel Aviv, etc.

gen_flavor.py is now incremental (fetches only cities missing a blurb, prunes stale
ones; --full to rebuild); cities_flavor.json refreshed to cover 942/1000.

gen_cities.py

@@ -31,11 +31,22 @@ def slugify(*parts: str) -> str:
     return re.sub(r"-{2,}", "-", text)
 
 
-# Countries where English is the primary/official language of web search. Used to
-# top up the population-ranked global list (which skews to Asia) with the
-# high-search-demand English-market cities that would otherwise be missed.
+# Core English-speaking countries — the first English top-up tier.
 ENGLISH_CC = {"US", "GB", "CA", "AU", "NZ", "IE", "ZA"}
 
+# A second English top-up tier: the remaining countries where English is an official
+# language, plus countries where >35% of the population speaks English (Eurobarometer
+# 2012 "can hold a conversation in English" / EF EPI). Both drive English-language
+# search demand. Editable — this is a judgment call, not a hard rule.
+ENGLISH_EXTENDED_CC = {
+    # English official (beyond the core seven)
+    "IN", "PK", "PH", "SG", "HK", "MY", "LK", "PG", "FJ",
+    "NG", "KE", "GH", "UG", "TZ", "ZW", "ZM", "MW", "BW", "NA", "RW", "SL", "LR", "MU", "SS", "SZ", "LS", "GM", "SC",
+    "JM", "TT", "BB", "BS", "BZ", "GY", "GD", "LC", "VC", "AG", "DM", "KN", "MT",
+    # >35% English proficiency (non-official)
+    "NL", "SE", "DK", "NO", "IS", "FI", "DE", "AT", "BE", "CH", "LU", "CY", "SI", "GR", "EE", "LV", "LT", "FR", "IL",
+}
+
 
 def _to_city(e, seen_slugs: set[str]) -> dict | None:
     name, admin1, country, cc = e[_NAME], e[_ADMIN1], e[_COUNTRY], e[_CC]
@@ -59,9 +70,12 @@ def _to_city(e, seen_slugs: set[str]) -> dict | None:
     }
 
 
-def build(n_global: int = 500, n_english: int = 250) -> list[dict]:
-    """Top n_global cities worldwide by population, then up to n_english more from
-    English-speaking countries that weren't already in that global set."""
+def build(n_global: int = 500, n_english: int = 250, n_extended: int = 250) -> list[dict]:
+    """Three tiers, population-descending, de-duplicated:
+      1. top n_global cities worldwide,
+      2. up to n_english more from core English-speaking countries,
+      3. up to n_extended more from the remaining English-official + >35%-English
+         countries — all not already chosen."""
     places._load()  # synchronous parse; fills places._data (entries are pop-desc)
     if not places._data:
         raise SystemExit("GeoNames index failed to load (see logs); cannot generate cities.")
@@ -69,38 +83,42 @@ def build(n_global: int = 500, n_english: int = 250) -> list[dict]:
 
     out: list[dict] = []
     seen_slugs: set[str] = set()
-    for e in entries:
-        if len(out) >= n_global:
-            break
-        c = _to_city(e, seen_slugs)
-        if c:
-            out.append(c)
+    chosen_ids: set = set()   # (name, admin1, cc) already added, so tiers don't overlap
 
-    # Identity of the cities already chosen, so the English top-up skips them.
-    chosen = {(c["name"], c["admin1"], c["country_code"]) for c in out}
-    added = 0
-    for e in entries:
-        if added >= n_english:
-            break
-        if e[_CC] not in ENGLISH_CC:
-            continue
-        if (e[_NAME], e[_ADMIN1], e[_CC]) in chosen:
-            continue
-        c = _to_city(e, seen_slugs)
-        if c:
-            out.append(c)
-            added += 1
+    def add_from(pred, limit: int) -> int:
+        added = 0
+        for e in entries:
+            if added >= limit:
+                break
+            if not pred(e):
+                continue
+            ident = (e[_NAME], e[_ADMIN1], e[_CC])
+            if ident in chosen_ids:
+                continue
+            c = _to_city(e, seen_slugs)
+            if c:
+                out.append(c)
+                chosen_ids.add(ident)
+                added += 1
+        return added
+
+    add_from(lambda e: True, n_global)                              # tier 1: global
+    add_from(lambda e: e[_CC] in ENGLISH_CC, n_english)             # tier 2: core English
+    add_from(lambda e: e[_CC] in ENGLISH_EXTENDED_CC, n_extended)   # tier 3: extended English
     return out
 
 
 def main() -> None:
-    n_global = int(sys.argv[1]) if len(sys.argv) > 1 else 500
-    n_english = int(sys.argv[2]) if len(sys.argv) > 2 else 250
-    cities = build(n_global, n_english)
+    a = sys.argv[1:]
+    n_global = int(a[0]) if len(a) > 0 else 500
+    n_english = int(a[1]) if len(a) > 1 else 250
+    n_extended = int(a[2]) if len(a) > 2 else 250
+    cities = build(n_global, n_english, n_extended)
     with open(OUT_PATH, "w", encoding="utf-8") as f:
         json.dump(cities, f, ensure_ascii=False, indent=0, separators=(",", ":"))
         f.write("\n")
-    print(f"wrote {len(cities)} cities ({n_global} global + up to {n_english} English-market) -> {OUT_PATH}")
+    print(f"wrote {len(cities)} cities ({n_global} global + {n_english} core-English + "
+          f"{n_extended} extended-English) -> {OUT_PATH}")
     print("sample:", ", ".join(c["slug"] for c in cities[:8]))
 
 

gen_flavor.py

@@ -77,33 +77,49 @@ def blurb_for(client: httpx.Client, city: dict) -> dict | None:
     return None
 
 
-def build(limit: int | None = None, workers: int = 8) -> dict:
-    todo = cities.all_cities()
-    if limit:
-        todo = todo[:limit]
+def build(todo: list[dict], workers: int = 5) -> dict:
+    """Fetch a blurb for each city in `todo`; returns {slug: {...}} for the hits."""
     out: dict[str, dict] = {}
     with httpx.Client(headers={"user-agent": UA}) as client:
         def work(c):
-            b = blurb_for(client, c)
-            return (c["slug"], b)
+            return (c["slug"], blurb_for(client, c))
         with ThreadPoolExecutor(max_workers=workers) as ex:
             for i, (slug, b) in enumerate(ex.map(work, todo), 1):
                 if b:
                     out[slug] = b
                 if i % 100 == 0:
-                    print(f"  {i}/{len(todo)} processed, {len(out)} with blurbs")
+                    print(f"  {i}/{len(todo)} processed, {len(out)} new blurbs")
     return out
 
 
 def main() -> None:
     args = sys.argv[1:]
     limit = int(args[args.index("--limit") + 1]) if "--limit" in args else None
-    workers = int(args[args.index("--workers") + 1]) if "--workers" in args else 8
-    flavor = build(limit=limit, workers=workers)
+    workers = int(args[args.index("--workers") + 1]) if "--workers" in args else 5
+    full = "--full" in args   # re-fetch everything, else only cities missing a blurb
+
+    # Incremental by default: keep existing blurbs, fetch only the missing ones, and
+    # prune entries for cities no longer in cities.json.
+    existing: dict[str, dict] = {}
+    if not full:
+        try:
+            with open(OUT_PATH, encoding="utf-8") as f:
+                existing = json.load(f)
+        except (OSError, ValueError):
+            existing = {}
+    all_cities = cities.all_cities()
+    all_slugs = {c["slug"] for c in all_cities}
+    todo = [c for c in all_cities if full or c["slug"] not in existing]
+    if limit:
+        todo = todo[:limit]
+    print(f"fetching {len(todo)} cities ({'full rebuild' if full else 'missing only'})")
+
+    fetched = build(todo, workers=workers)
+    merged = {s: v for s, v in {**existing, **fetched}.items() if s in all_slugs}
     with open(OUT_PATH, "w", encoding="utf-8") as f:
-        json.dump(flavor, f, ensure_ascii=False, sort_keys=True, indent=0, separators=(",", ":"))
+        json.dump(merged, f, ensure_ascii=False, sort_keys=True, indent=0, separators=(",", ":"))
         f.write("\n")
-    print(f"wrote {len(flavor)} blurbs -> {OUT_PATH}")
+    print(f"wrote {len(merged)} blurbs ({len(fetched)} newly fetched) -> {OUT_PATH}")
 
 
 if __name__ == "__main__":