thermograph/backend/warm_cities.py

"""Pre-warm the archives for the curated city set (backend/cities.json) so the
crawlable /climate pages render from cache and a search-engine crawl never bursts
the archive API quota. Run at/after deploy:

    python warm_cities.py [--limit N] [--pace SECONDS]

Idempotent: a cell whose archive is already cached is skipped. Fetches are paced
(default 2s) to stay well under the archive API's rate limit. A cell that still
has no cached archive when its page is first requested self-heals via get_history,
so this is an optimization, not a hard dependency.

Deploy launches this script detached on every deploy with no overlap check, so a
slow previous run (still mid-warm) can still be going when the next deploy's
invocation starts. Two overlapping runs would double-spend archive-fetch quota on
the same cells and race climate._write_cache's parquet writes, so the CLI entry
point below claims a single-instance flock (see LOCK_PATH) before calling main().
"""
import os
import sys
import time

from api import content_payloads
from api import homepage
from api import payloads as cache_ids
from core import singleton
from data import cities
from data import climate
from data import grid
from data import store
import paths

# The deployment base path, derived exactly as api/content_routes.py does so the
# breadcrumb hrefs / jsonld URLs the warmer bakes match what a live request bakes.
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""

# core/singleton.py's flock guard, reused here for cross-*process* (not
# cross-worker) exclusion: the first invocation holds this for its process
# lifetime; a second one (an overlapping deploy) fails the non-blocking flock
# and stands down immediately.
LOCK_PATH = os.path.join(paths.DATA_DIR, "warm_cities.lock")


def main(limit: int | None = None, pace: float = 2.0) -> None:
    todo = cities.all_cities()
    if limit:
        todo = todo[:limit]
    fetched = skipped = failed = 0
    for i, c in enumerate(todo, 1):
        cell = grid.snap(c["lat"], c["lon"])
        cached = climate.load_cached_history(cell)
        if cached is not None and not cached.is_empty():
            skipped += 1
            continue
        try:
            climate.get_history(cell)          # fetch + cache the ~45-yr archive
            climate.get_recent_forecast(cell)  # + the recent/forecast bundle (today block)
            fetched += 1
            print(f"[{i}/{len(todo)}] warmed {c['slug']} ({cell['id']})")
            time.sleep(pace)
        except Exception as e:  # noqa: BLE001 - keep going; the page self-heals later
            failed += 1
            print(f"[{i}/{len(todo)}] FAILED {c['slug']}: {e}")
            time.sleep(pace)
    print(f"done: fetched={fetched} skipped(cached)={skipped} failed={failed}")

    # Rebuild the homepage's "unusual right now" feed from whatever is now warm.
    # Runs here as well as on the notifier loop so a fresh deploy has a populated
    # feed immediately, and so it still refreshes when the notifier is disabled.
    try:
        feed = homepage.refresh()
        print(f"homepage feed: {feed['considered']} cities graded, "
              f"{len(feed['ranked'])} ranked")
    except Exception as e:  # noqa: BLE001 - the homepage renders without the feed
        print(f"homepage feed: FAILED {e}")


# --- content derived-store pre-warm -------------------------------------------
# The SEO /climate/<city>[/month|/records] pages render their JSON from the
# derived store (api/content_routes.py), keyed by the cheap+stable content token
# (payloads.content_token -> PAYLOAD_VER:CONTENT_VER:archive_last_date). That token
# turns over only when a cell's archive gains a new day (~1x/day), so on the first
# request after each daily advance the page would otherwise recompute a 45-year
# payload cold. Pre-warming rebuilds those rows off-request — cache-only, so it
# never spends upstream quota — so users and crawlers land on a warm cache.


def _content_specs(slug: str, origin: str) -> "list[tuple[str, str, str, int | None]]":
    """The (store-kind, store-key, builder-tag, month_idx) tuples for one city,
    matching api/content_routes.py's kinds/keys exactly: content-city and
    content-records key on ``{slug}:{origin}`` (the origin is folded in because the
    payloads' jsonld.url is origin-qualified), content-month keys on
    ``{slug}:{month}`` for each of the 12 months."""
    specs = [
        ("content-city", f"{slug}:{origin}", "city", None),
        ("content-records", f"{slug}:{origin}", "records", None),
    ]
    for month, idx in content_payloads.MONTH_INDEX.items():
        specs.append(("content-month", f"{slug}:{month}", "month", idx))
    return specs


def _build_content(tag: str, city: dict, history, recent, origin: str, month_idx: "int | None") -> dict:
    if tag == "city":
        return content_payloads.city_payload(origin, BASE, city, history, recent)
    if tag == "records":
        return content_payloads.records_payload(origin, BASE, city, history)
    return content_payloads.month_payload(BASE, city, history, month_idx)


def warm_content(limit: int | None = None, origin: str | None = None, pace: float = 0.05) -> dict:
    """Rebuild any missing/stale content-page payloads in the derived store.

    For each curated city, compute the current content token (cheap — no history
    load) and check whether every content kind (city, 12 months, records) already
    has a row for that token. Cities that are wholly fresh are skipped *without*
    loading the archive, so a re-run after everything is warm is near-instant and
    only cells whose archive genuinely advanced (new token) do work — the same
    idempotent contract as ``main``.

    Cache-only, exactly like ``main``: the history and recent/forecast bundles are
    read from the cache; a cell with no cached archive is skipped rather than
    fetched, so warming never spends a single upstream request.

    ``limit`` caps the number of cities actually (re)built this call (not scanned),
    so a caller can bound one pass's wall-clock and rely on the idempotent skip to
    resume from where it left off on the next call. Returns a small counts dict.
    """
    if origin is None:
        origin = os.environ.get("THERMOGRAPH_BASE_URL", "https://thermograph.org").rstrip("/")
    built = warmed = skipped = empty = failed = 0
    for c in cities.all_cities():
        if limit is not None and warmed >= limit:
            break
        slug = c["slug"]
        cell = grid.snap(c["lat"], c["lon"])
        cell_id = cell["id"]
        token = cache_ids.content_token(cell_id)
        specs = _content_specs(slug, origin)
        # Cheap idempotency gate: which kinds are missing/stale for this token?
        # get_payload is a cheap keyed lookup; loading the 45-year archive is not,
        # so we defer that until we know at least one kind actually needs building.
        stale = [s for s in specs if store.get_payload(s[0], cell_id, s[1], token) is None]
        if not stale:
            skipped += 1
            continue
        history = climate.load_cached_history(cell)
        if history is None or history.is_empty():
            # No cached archive yet -> nothing to build from, and warming must not
            # fetch upstream. The cell self-heals on its first live request.
            empty += 1
            continue
        # today_vs_normal on the city payload needs the recent/forecast bundle;
        # read it cache-only (never fetch) — None just drops that one block.
        recent = climate.load_cached_recent_forecast(cell)
        did_work = False
        for kind, key, tag, month_idx in stale:
            try:
                payload = _build_content(tag, c, history, recent, origin, month_idx)
                store.put_payload(kind, cell_id, key, token, payload)
                built += 1
                did_work = True
            except Exception:  # noqa: BLE001 - one bad payload must not abort the sweep
                failed += 1
        if did_work:
            warmed += 1
            if pace:
                time.sleep(pace)
    return {"warmed": warmed, "built": built, "skipped": skipped, "empty": empty, "failed": failed}


if __name__ == "__main__":
    if not singleton.claim(LOCK_PATH):
        # Not an error: an overlapping run just means a previous deploy's warm is
        # still in flight. Exit 0 so the deploy script doesn't treat this as a
        # failure.
        print("warm_cities: another instance is already running -- exiting")
        sys.exit(0)
    args = sys.argv[1:]
    lim = int(args[args.index("--limit") + 1]) if "--limit" in args else None
    pc = float(args[args.index("--pace") + 1]) if "--pace" in args else 2.0
    main(limit=lim, pace=pc)