"""Pre-warm the archives for the curated city set (backend/cities.json) so the crawlable /climate pages render from cache and a search-engine crawl never bursts the archive API quota. Run at/after deploy: python warm_cities.py [--limit N] [--pace SECONDS] Idempotent: a cell whose archive is already cached is skipped. Fetches are paced (default 2s) to stay well under the archive API's rate limit. A cell that still has no cached archive when its page is first requested self-heals via get_history, so this is an optimization, not a hard dependency. Deploy launches this script detached on every deploy with no overlap check, so a slow previous run (still mid-warm) can still be going when the next deploy's invocation starts. Two overlapping runs would double-spend archive-fetch quota on the same cells and race climate._write_cache's parquet writes, so the CLI entry point below claims a single-instance flock (see LOCK_PATH) before calling main(). """ import os import sys import time from api import content_payloads from api import homepage from api import payloads as cache_ids from core import singleton from data import cities from data import climate from data import grid from data import store import paths # The deployment base path, derived exactly as api/content_routes.py does so the # breadcrumb hrefs / jsonld URLs the warmer bakes match what a live request bakes. _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") BASE = f"/{_BASE}" if _BASE else "" # core/singleton.py's flock guard, reused here for cross-*process* (not # cross-worker) exclusion: the first invocation holds this for its process # lifetime; a second one (an overlapping deploy) fails the non-blocking flock # and stands down immediately. LOCK_PATH = os.path.join(paths.DATA_DIR, "warm_cities.lock") def main(limit: int | None = None, pace: float = 2.0) -> None: todo = cities.all_cities() if limit: todo = todo[:limit] fetched = skipped = failed = 0 for i, c in enumerate(todo, 1): cell = grid.snap(c["lat"], c["lon"]) cached = climate.load_cached_history(cell) if cached is not None and not cached.is_empty(): skipped += 1 continue try: climate.get_history(cell) # fetch + cache the ~45-yr archive climate.get_recent_forecast(cell) # + the recent/forecast bundle (today block) fetched += 1 print(f"[{i}/{len(todo)}] warmed {c['slug']} ({cell['id']})") time.sleep(pace) except Exception as e: # noqa: BLE001 - keep going; the page self-heals later failed += 1 print(f"[{i}/{len(todo)}] FAILED {c['slug']}: {e}") time.sleep(pace) print(f"done: fetched={fetched} skipped(cached)={skipped} failed={failed}") # Rebuild the homepage's "unusual right now" feed from whatever is now warm. # Runs here as well as on the notifier loop so a fresh deploy has a populated # feed immediately, and so it still refreshes when the notifier is disabled. try: feed = homepage.refresh() print(f"homepage feed: {feed['considered']} cities graded, " f"{len(feed['ranked'])} ranked") except Exception as e: # noqa: BLE001 - the homepage renders without the feed print(f"homepage feed: FAILED {e}") # --- content derived-store pre-warm ------------------------------------------- # The SEO /climate/[/month|/records] pages render their JSON from the # derived store (api/content_routes.py), keyed by the cheap+stable content token # (payloads.content_token -> PAYLOAD_VER:CONTENT_VER:archive_last_date). That token # turns over only when a cell's archive gains a new day (~1x/day), so on the first # request after each daily advance the page would otherwise recompute a 45-year # payload cold. Pre-warming rebuilds those rows off-request — cache-only, so it # never spends upstream quota — so users and crawlers land on a warm cache. def _content_specs(slug: str, origin: str) -> "list[tuple[str, str, str, int | None]]": """The (store-kind, store-key, builder-tag, month_idx) tuples for one city, matching api/content_routes.py's kinds/keys exactly: content-city and content-records key on ``{slug}:{origin}`` (the origin is folded in because the payloads' jsonld.url is origin-qualified), content-month keys on ``{slug}:{month}`` for each of the 12 months.""" specs = [ ("content-city", f"{slug}:{origin}", "city", None), ("content-records", f"{slug}:{origin}", "records", None), ] for month, idx in content_payloads.MONTH_INDEX.items(): specs.append(("content-month", f"{slug}:{month}", "month", idx)) return specs def _build_content(tag: str, city: dict, history, recent, origin: str, month_idx: "int | None") -> dict: if tag == "city": return content_payloads.city_payload(origin, BASE, city, history, recent) if tag == "records": return content_payloads.records_payload(origin, BASE, city, history) return content_payloads.month_payload(BASE, city, history, month_idx) def warm_content(limit: int | None = None, origin: str | None = None, pace: float = 0.05) -> dict: """Rebuild any missing/stale content-page payloads in the derived store. For each curated city, compute the current content token (cheap — no history load) and check whether every content kind (city, 12 months, records) already has a row for that token. Cities that are wholly fresh are skipped *without* loading the archive, so a re-run after everything is warm is near-instant and only cells whose archive genuinely advanced (new token) do work — the same idempotent contract as ``main``. Cache-only, exactly like ``main``: the history and recent/forecast bundles are read from the cache; a cell with no cached archive is skipped rather than fetched, so warming never spends a single upstream request. ``limit`` caps the number of cities actually (re)built this call (not scanned), so a caller can bound one pass's wall-clock and rely on the idempotent skip to resume from where it left off on the next call. Returns a small counts dict. """ if origin is None: origin = os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org").rstrip("/") built = warmed = skipped = empty = failed = 0 for c in cities.all_cities(): if limit is not None and warmed >= limit: break slug = c["slug"] cell = grid.snap(c["lat"], c["lon"]) cell_id = cell["id"] token = cache_ids.content_token(cell_id) specs = _content_specs(slug, origin) # Cheap idempotency gate: which kinds are missing/stale for this token? # get_payload is a cheap keyed lookup; loading the 45-year archive is not, # so we defer that until we know at least one kind actually needs building. stale = [s for s in specs if store.get_payload(s[0], cell_id, s[1], token) is None] if not stale: skipped += 1 continue history = climate.load_cached_history(cell) if history is None or history.is_empty(): # No cached archive yet -> nothing to build from, and warming must not # fetch upstream. The cell self-heals on its first live request. empty += 1 continue # today_vs_normal on the city payload needs the recent/forecast bundle; # read it cache-only (never fetch) — None just drops that one block. recent = climate.load_cached_recent_forecast(cell) did_work = False for kind, key, tag, month_idx in stale: try: payload = _build_content(tag, c, history, recent, origin, month_idx) store.put_payload(kind, cell_id, key, token, payload) built += 1 did_work = True except Exception: # noqa: BLE001 - one bad payload must not abort the sweep failed += 1 if did_work: warmed += 1 if pace: time.sleep(pace) return {"warmed": warmed, "built": built, "skipped": skipped, "empty": empty, "failed": failed} if __name__ == "__main__": if not singleton.claim(LOCK_PATH): # Not an error: an overlapping run just means a previous deploy's warm is # still in flight. Exit 0 so the deploy script doesn't treat this as a # failure. print("warm_cities: another instance is already running -- exiting") sys.exit(0) args = sys.argv[1:] lim = int(args[args.index("--limit") + 1]) if "--limit" in args else None pc = float(args[args.index("--pace") + 1]) if "--pace" in args else 2.0 main(limit=lim, pace=pc)