diff --git a/accounts/__init__.py b/accounts/__init__.py new file mode 100644 index 0000000..b7da7eb --- /dev/null +++ b/accounts/__init__.py @@ -0,0 +1 @@ +"""accounts package.""" diff --git a/api_accounts.py b/accounts/api_accounts.py similarity index 97% rename from api_accounts.py rename to accounts/api_accounts.py index 07a6cab..beae8ef 100644 --- a/api_accounts.py +++ b/accounts/api_accounts.py @@ -12,12 +12,12 @@ from fastapi.concurrency import run_in_threadpool from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -import climate -import grid -import push -from db import get_async_session -from models import Notification, PushSubscription, Subscription -from schemas import ( +from data import climate +from data import grid +from notifications import push +from accounts.db import get_async_session +from accounts.models import Notification, PushSubscription, Subscription +from web.schemas import ( NotificationList, NotificationOut, PushSubscriptionIn, @@ -26,7 +26,7 @@ from schemas import ( SubscriptionOut, SubscriptionPatch, ) -from users import current_active_user +from accounts.users import current_active_user _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") BASE = f"/{_BASE}" if _BASE else "" diff --git a/db.py b/accounts/db.py similarity index 93% rename from db.py rename to accounts/db.py index 8f6390c..84bad3e 100644 --- a/db.py +++ b/accounts/db.py @@ -23,11 +23,11 @@ from sqlalchemy import create_engine, event from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import DeclarativeBase, sessionmaker +import paths + # Default to data/accounts.sqlite; THERMOGRAPH_ACCOUNTS_DB overrides it (tests point # this at a throwaway temp file to stay hermetic). -DB_PATH = os.environ.get("THERMOGRAPH_ACCOUNTS_DB") or os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "data", "accounts.sqlite") -) +DB_PATH = os.environ.get("THERMOGRAPH_ACCOUNTS_DB") or os.path.join(paths.DATA_DIR, "accounts.sqlite") os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) @@ -71,7 +71,7 @@ async def create_db_and_tables() -> None: Imported for its side effect of registering the mapped classes on Base.metadata before create_all runs. """ - import models # noqa: F401 (registers tables on Base.metadata) + from accounts import models # noqa: F401 (registers tables on Base.metadata) async with async_engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) diff --git a/models.py b/accounts/models.py similarity index 99% rename from models.py rename to accounts/models.py index d993916..4546e40 100644 --- a/models.py +++ b/accounts/models.py @@ -25,7 +25,7 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Mapped, mapped_column -from db import Base +from accounts.db import Base class User(SQLAlchemyBaseUserTableUUID, Base): diff --git a/users.py b/accounts/users.py similarity index 97% rename from users.py rename to accounts/users.py index ff0aed4..4531ef9 100644 --- a/users.py +++ b/accounts/users.py @@ -16,8 +16,8 @@ from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase from fastapi_users_db_sqlalchemy.access_token import SQLAlchemyAccessTokenDatabase from sqlalchemy.ext.asyncio import AsyncSession -from db import get_async_session -from models import AccessToken, User +from accounts.db import get_async_session +from accounts.models import AccessToken, User BASE = "/" + os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") # Only used to sign reset/verification tokens — features that stay dormant until diff --git a/app.py b/app.py index c109543..38e9d05 100644 --- a/app.py +++ b/app.py @@ -1,782 +1,7 @@ -"""Thermograph API — grade recent local weather against ~45 years of climatology.""" -import contextlib -import datetime -import functools -import hashlib -import hmac -import json -import mimetypes -import os -import queue -import threading -import time +"""Entry-point shim. -from fastapi import APIRouter, FastAPI, HTTPException, Query, Request, Response -from fastapi.middleware.gzip import GZipMiddleware -from fastapi.responses import RedirectResponse -from fastapi.staticfiles import StaticFiles - -import api_accounts -import audit -import climate -import content -import digest -import discord_interactions as discord_interactions_mod -import discord_link -import db -import grid -import metrics -import notify -import places -import singleton -import store -import users -import views -from schemas import UserCreate, UserRead, UserUpdate - -FRONTEND_DIR = os.path.join(os.path.dirname(__file__), "..", "frontend") - -# Everything (pages, assets, API) is served under this base path so the app can -# live at https:///thermograph/ behind a shared domain. Override with the -# THERMOGRAPH_BASE env var. The frontend uses base-relative URLs, so it follows -# this automatically without hardcoding the prefix. -# -# THERMOGRAPH_BASE=/ (or empty) means the app owns the whole domain: BASE becomes -# "" so every route sits at the domain root ("/", "/calendar", "/api/v2/…") with -# no prefix and no leading double slash. A non-empty value keeps the sub-path form -# ("/thermograph"). -_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") -BASE = f"/{_BASE}" if _BASE else "" - -# The PWA manifest is served as a static file; register its media type so -# StaticFiles labels it correctly (not in Python's default mimetypes map). -mimetypes.add_type("application/manifest+json", ".webmanifest") - - -def _weather_fetch_error(e) -> HTTPException: - """A clean, retryable 503 when weather data is temporarily unavailable; the - raw error as a 502 otherwise. The fetch layer classifies its own failures - (climate.WeatherUnavailable carries the user-facing text) so no message - parsing happens here; the is_rate_limit fallback covers a raw upstream 429 - from a fetcher that didn't classify it.""" - if isinstance(e, climate.WeatherUnavailable): - return HTTPException(status_code=503, detail=str(e)) - if climate.is_rate_limit(e): - return HTTPException(status_code=503, detail=climate.limit_message(False)) - return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}") - - -# --- background neighbor warming --------------------------------------------- -# /cell?neighbors=1 asks the server to warm the 8 grid cells around the request -# (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's -# snapping math in JS). One worker drains the queue so warms never stack; the -# hard guarantee matches prefetch=1 — a cell with no cached archive is skipped, -# so no weather-API quota is ever spent, and reverse_geocode itself paces the -# at-most-one Nominatim call per never-labeled cell (~1/s policy). - -_WARM_TTL = 3600.0 # don't re-enqueue a cell within the hour -_WARM_SEEN: dict[str, float] = {} # cell_id -> last enqueue time -_warm_queue: "queue.Queue[dict]" = queue.Queue() - - -def _warm_cell(cell: dict) -> None: - """Materialize the history-derived slices for one cell — the same rows the - prefetch=1 bundle serves. Never fetches weather upstream.""" - history = climate.load_cached_history(cell) - if history is None or history.is_empty(): - return - place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) - token = views.history_token(history) - start_ts, end_ts = views.cal_span(history, None, None, 24) - cal_key = views.calendar_key(start_ts, end_ts, 24) - if store.get_payload("calendar", cell["id"], cal_key, token) is None: - store.put_payload("calendar", cell["id"], cal_key, token, - views.build_calendar(cell, history, start_ts, end_ts, 24, place)) - last = history["date"].max() - day_key = views.day_key(last) - if store.get_payload("day", cell["id"], day_key, token) is None: - store.put_payload("day", cell["id"], day_key, token, - views.build_day(cell, history, last, place)) - - -def _enqueue_neighbor_warming(cell: dict) -> None: - now = time.time() - for n in grid.neighbors(cell): - if now - _WARM_SEEN.get(n["id"], 0.0) < _WARM_TTL: - continue - _WARM_SEEN[n["id"]] = now - _warm_queue.put(n) - - -def _neighbor_warmer() -> None: - while True: - cell = _warm_queue.get() - try: - _warm_cell(cell) - except Exception: # noqa: BLE001 - warming is best-effort, never fatal - pass - finally: - _warm_queue.task_done() - - -@contextlib.asynccontextmanager -async def _lifespan(app): - # Warm the local place-name index (/suggest's typo tolerance) in the - # background; the app boots and serves fine without it — suggestions just - # fall back to the upstream geocoder until it's ready. A server-startup - # hook (not import time) so offline importers (tests, the migrate script's - # dependencies) don't kick off a GeoNames download. The neighbor warmer is - # also a server concern: enqueues before startup just wait in the queue. - # Create the account DB tables (users/sessions/subscriptions/notifications) - # before serving — a fast no-op once they exist. - await db.create_db_and_tables() - places.start_loading() - threading.Thread(target=_neighbor_warmer, name="neighbor-warmer", daemon=True).start() - # Background subscription evaluator. Its periodic sweep fetches upstream on a timer - # (not per request), so with multiple workers it must run in exactly one — elect a - # leader via a lockfile (THERMOGRAPH_SINGLETON_LOCK). Unset (single worker / dev / - # tests) => always the leader. The neighbor warmer above stays per-worker: each - # drains its own request-fed queue. - if singleton.claim(os.environ.get("THERMOGRAPH_SINGLETON_LOCK")): - notify.start() - yield - notify.stop() - - -app = FastAPI(title="Thermograph", version="0.2.0", lifespan=_lifespan) - -# Compress every sizeable response (the 2-year calendar JSON shrinks ~6-8×). -# Applies to API JSON and static assets alike; tiny responses are left alone. -app.add_middleware(GZipMiddleware, minimum_size=1024) - - -def _client_ip(request) -> "str | None": - """The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us - (Caddy on prod sets it), otherwise the direct peer address (LAN dev).""" - xff = request.headers.get("x-forwarded-for") - if xff: - return xff.split(",")[0].strip() - return request.client.host if request.client else None - - -@app.middleware("http") -async def revalidate_static(request, call_next): - """Serve the frontend with no-cache (NOT no-store): browsers may keep a copy - but must revalidate it on every use. Starlette's FileResponse/StaticFiles - already emit ETag + Last-Modified, so an unchanged asset costs one conditional - request answered with an empty 304 instead of a full re-download — page-to-page - navigation stops re-transferring the JS/CSS/HTML while still picking up every - deploy immediately (no "my change isn't showing" bugs).""" - response = await call_next(request) - path = request.url.path - try: - cat = metrics.classify_inbound(path, BASE) - metrics.record_inbound(cat, response.status_code) - # Retain per-request client IPs for later analysis; skip static assets and the - # dashboard's own metrics polling to keep the log to real, meaningful traffic. - if cat not in ("static", "metrics", "event"): - audit.log_access({"ip": _client_ip(request), "method": request.method, - "path": path, "status": response.status_code, "cat": cat}) - except Exception: # noqa: BLE001 - never let instrumentation break a response - pass - pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/score", - f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy") - if path.endswith((".js", ".css", ".html")) or path in pages: - response.headers["Cache-Control"] = "no-cache" - return response - - -# --- derived-store plumbing -------------------------------------------------- -# Every graded payload is cached in SQLite under (kind, cell, key) and validated -# by a token that encodes what it was computed from (see store.py). The freshness -# drivers stay where they were — climate.get_history tops up the archive tail -# hourly and get_recent_forecast refetches hourly — and the tokens are derived -# from what those return, so a cached payload expires exactly when its inputs -# change and never before. The same tokens double as ETags: a client sending -# If-None-Match gets an empty 304 without the payload even being loaded. -# The payloads themselves are assembled in views.py, shared with the offline -# migrate script. - -def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str: - """Deterministic weak ETag from a payload's identity + validity token. Weak - because the same content may be served under different encodings (gzip).""" - h = hashlib.sha1(f"{kind}:{cell_id}:{key}:{token}".encode()).hexdigest()[:20] - return f'W/"{h}"' - - -def _not_modified(request: Request, etag: str) -> bool: - """Does the client already hold this exact payload? Answerable from the token - alone — no payload load needed.""" - inm = request.headers.get("if-none-match") - if not inm: - return False - tags = {t.strip() for t in inm.split(",")} - return "*" in tags or etag in tags or etag.removeprefix("W/") in tags - - -def _encode(payload: dict) -> bytes: - """Compact JSON bytes, matching store.put_payload's encoding (allow_nan=False - mirrors Starlette — a NaN from grading should fail loudly, not reach a client).""" - return json.dumps(payload, default=str, allow_nan=False, separators=(",", ":")).encode() - - -def _json_response(body: bytes, etag: str | None = None) -> Response: - headers = {"ETag": etag} if etag else {} - return Response(content=body, media_type="application/json", headers=headers) - - -def _fetch_history(run, cell, recent_too=False, recent_phase="recent"): - """The shared fetch preamble for every data route: the archive record (and - optionally the hourly recent/forecast bundle) with upstream failures mapped - to clean HTTP errors and an empty record to a 404.""" - try: - with run.phase("history"): - history, cache_meta = climate.get_history(cell) - recent = None - if recent_too: - with run.phase(recent_phase): - recent = climate.get_recent_forecast(cell) - except Exception as e: # noqa: BLE001 - raise _weather_fetch_error(e) - if history.is_empty(): - raise HTTPException(status_code=404, detail="No historical data for this cell.") - return history, cache_meta, recent - - -def _cached_response(request, run, kind, cell, key, token, build, cache_placeless=True): - """The shared derived-store flow behind every per-view endpoint: - If-None-Match -> empty 304; a token-valid store row -> replay its exact - bytes; otherwise resolve the place label, build the payload, persist, serve. - - ``build(place) -> payload`` does any endpoint-specific audit tagging itself. - ``cache_placeless=False`` skips persisting a payload whose place label - failed to resolve (a transient reverse-geocode miss) — otherwise the - bare-coordinates fallback would stick for the life of the token.""" - cid = cell["id"] - etag = _etag_for(kind, cid, key, token) - if _not_modified(request, etag): - run.set(run_type="cache", not_modified=True) - return Response(status_code=304, headers={"ETag": etag}) - body = store.get_payload(kind, cid, key, token) - if body is not None: - run.set(run_type="cache", history_source="store") - return _json_response(body, etag) - with run.phase("reverse_geocode"): - place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) - payload = build(place) - return _json_response( - store.put_payload(kind, cid, key, token, payload, - cache=cache_placeless or place is not None), etag) - - - -# --- endpoints ---------------------------------------------------------------- - -def api_geocode(q: str = Query(..., min_length=1)): - try: - return {"results": climate.geocode(q)} - except Exception as e: # noqa: BLE001 - surface upstream failures to the client - raise HTTPException(status_code=502, detail=f"geocoding failed: {e}") - - -_SUGGEST_LIMIT = 5 - - -@functools.lru_cache(maxsize=1024) -def _suggest_upstream(q: str) -> tuple: - """Open-Meteo lookup for /suggest, memoized per query string — type-ahead - re-asks the same prefixes constantly (backspacing, retyping). Failures - raise and are not cached, so a transient upstream error doesn't stick.""" - return tuple(climate.geocode(q, count=_SUGGEST_LIMIT)) - - -def api_suggest(q: str = Query(..., min_length=1)): - """Type-ahead location suggestions: the top 5 places for a (possibly - typo'd) query prefix; `corrected` reports the respelling that produced the - results ("pest seattle" → "west seattle"), or null. The ranking/typo policy - lives in places.suggest.""" - try: - results, corrected = places.suggest(q, _suggest_upstream, _SUGGEST_LIMIT) - except Exception as e: # noqa: BLE001 - nothing at all to serve - raise HTTPException(status_code=502, detail=f"geocoding failed: {e}") - return {"results": results, "corrected": corrected} - - -def api_place( - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), -): - """Best-effort neighbourhood/city label for a point — the same string the view - endpoints expose as ``place`` (snapped to the grid cell, reverse-geocoded and - cached). Resolved on its own so the compare page can show a location's name as - soon as it's added, before its full series loads.""" - cell = grid.snap(lat, lon) - with audit.RunAudit(endpoint="place", lat=round(lat, 4), lon=round(lon, 4), - cell_id=cell["id"]) as run: - with run.phase("reverse_geocode"): - place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) - return {"place": place, - "cell": {"center_lat": cell["center_lat"], "center_lon": cell["center_lon"]}} - - -def api_grade( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), - date: str | None = Query(None, description="target date YYYY-MM-DD (default today)"), - days: int = Query(14, ge=1, le=60, description="days of history to grade before the target"), - after: int = Query(14, ge=0, le=14, - description="days to grade after the target (observed, or forecast when future; " - "the forecast reaches ~7 days out so future days cap there)"), -): - target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today() - cell = grid.snap(lat, lon) - - with audit.RunAudit( - endpoint="grade", - lat=round(lat, 4), - lon=round(lon, 4), - target_date=target.isoformat(), - recent_days=days, - cell_id=cell["id"], - ) as run: - history, cache_meta, recent = _fetch_history(run, cell, recent_too=True) - return _cached_response( - request, run, "grade", cell, - views.grade_key(target, days, after), views.recent_token(history, cell["id"]), - lambda place: views.build_grade(cell, target, days, history, recent, - cache_meta, place, run, after=after)) - - -def api_calendar( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), - start: str | None = Query(None, description="first date YYYY-MM-DD (overrides months)"), - end: str | None = Query(None, description="last date YYYY-MM-DD (default = latest available)"), - months: int = Query(24, ge=1, le=24, description="months back to grade when no start given"), -): - """Grade every day over a date range (default: last 2 years) for the calendar view. - - A custom [start, end] range is supported and capped at ~2 years per request; - the frontend splits longer spans into successive 2-year chunks. Grades against - the already-cached history record (which reaches to ~6 days ago), so no extra - fetch is needed. The graded payload is cached in the derived store keyed by - the clamped span and validated by the record's end date, so a repeat span is - a database read — across restarts — until new archive days arrive. Returns a - compact per-day shape (see grading.grade_range). New in API v2. - """ - cell = grid.snap(lat, lon) - - with audit.RunAudit( - endpoint="calendar", - lat=round(lat, 4), - lon=round(lon, 4), - recent_days=months * 31, # approximate span graded - cell_id=cell["id"], - ) as run: - history, cache_meta, _ = _fetch_history(run, cell) - start_ts, end_ts = views.cal_span(history, start, end, months) - - def build(place): - payload = views.build_calendar(cell, history, start_ts, end_ts, months, place, run) - full = not cache_meta.get("cached", False) - run.set(run_type="full" if full else "partial", - history_source="fetch" if full else "cache") - return payload - - # Compare loads several cells at once, so a transient reverse-geocode miss - # is most likely here; cache_placeless=False keeps that miss un-persisted. - return _cached_response(request, run, "calendar", cell, - views.calendar_key(start_ts, end_ts, months), - views.history_token(history), build, - cache_placeless=False) - - -def api_day( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), - date: str | None = Query(None, description="target date YYYY-MM-DD (default = latest available)"), -): - """Full percentile breakdown for a single day: the value at every tier boundary - in that day-of-year's ±7-day window, plus where the observed values land. - - Powers the single-day detail page. Grades against the cached history record; - for a date newer than the cache it pulls the recent window for the observation - (those payloads expire hourly — the recent bundle's own cadence — while fully - archived days stay valid until the record itself advances). New in API v2. - """ - cell = grid.snap(lat, lon) - - with audit.RunAudit( - endpoint="day", - lat=round(lat, 4), - lon=round(lon, 4), - cell_id=cell["id"], - ) as run: - history, cache_meta, _ = _fetch_history(run, cell) - last = history["date"].max() - target = datetime.date.fromisoformat(date[:10]) if date else last - run.set(target_date=target.isoformat()) - - def build(place): - payload = views.build_day(cell, history, target, place, run) - full = not cache_meta.get("cached", False) - run.set(run_type="full" if full else "partial", - history_source="fetch" if full else "cache") - return payload - - return _cached_response(request, run, "day", cell, - views.day_key(target), views.day_token(history, target), - build) - - -def api_score( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), -): - """Climate-drift score for a location: how far the last 6 years have moved - from the full 45-year baseline, per metric, percentile category and season, - rolled into per-metric and overall scores. - - Derived purely from the archive record, so it is cached until the record's - tail advances (the hourly top-up); the scoring-math version is baked into the - cache key so a math change invalidates only score rows. New in API v2. - """ - cell = grid.snap(lat, lon) - - with audit.RunAudit(endpoint="score", lat=round(lat, 4), lon=round(lon, 4), - cell_id=cell["id"]) as run: - history, cache_meta, _ = _fetch_history(run, cell) - - def build(place): - payload = views.build_score(cell, history, place, run) - full = not cache_meta.get("cached", False) - run.set(run_type="full" if full else "partial", - history_source="fetch" if full else "cache") - return payload - - return _cached_response(request, run, "score", cell, - views.score_key(), views.history_token(history), build) - - -def api_forecast( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), - days: int = Query(7, ge=1, le=14, description="how many forecast days ahead to grade"), -): - """Grade the next `days` of forecast weather against local climatology. - - Same shape as /grade (so the frontend renders it identically), but `recent` - holds the forward forecast — furthest-out day first. The forecast is fetched - fresh and cached only ~1 hour (see climate.get_recent_forecast) to track - updates; the graded payload's validity is tied to that fetch stamp, so it - expires exactly when a new forecast lands. New in API v2. - """ - cell = grid.snap(lat, lon) - - with audit.RunAudit(endpoint="forecast", lat=round(lat, 4), lon=round(lon, 4), - recent_days=days, cell_id=cell["id"]) as run: - history, _, fc = _fetch_history(run, cell, recent_too=True, recent_phase="forecast") - today = datetime.date.today() - - def build(place): - payload = views.build_forecast(cell, days, history, fc, today, place, run) - run.set(run_type="partial") - return payload - - return _cached_response(request, run, "forecast", cell, - views.forecast_key(today, days), - views.recent_token(history, cell["id"]), build) - - -def api_cell( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lon: float = Query(..., ge=-180, le=180), - prefetch: int = Query(0, ge=0, le=1, - description="1 = warm-only: never fetch weather upstream; 204 for a cold cell"), - neighbors: int = Query(0, ge=0, le=1, - description="1 = also warm the 8 surrounding cells in the background " - "(warm-only: cold neighbors are skipped, no upstream quota)"), -): - """One bundle carrying every view's payload for a cell, so the frontend warms - all views with a single request instead of four. - - Each slice is the EXACT payload its per-view endpoint returns — built by the - same builders and cached under the same derived-store keys/tokens — paired - with the etag that endpoint would emit. The client seeds its per-view cache - from the slices and later revalidates each view individually with - If-None-Match, so the bundle and the per-view endpoints stay one cache. - - prefetch=1 is the neighbor-warming mode with a hard guarantee: it never - spends weather-API quota. A cell with no cached archive answers 204 (no - body), and only the history-derived slices (calendar + latest-day detail) - are built — grading recent/forecast days needs the hourly upstream bundle. - Reverse geocoding makes at most one Nominatim call for a never-labeled cell; - the client staggers neighbor prefetches to respect that service. New in API v2. - """ - cell = grid.snap(lat, lon) - today = datetime.date.today() - - with audit.RunAudit(endpoint="cell", lat=round(lat, 4), lon=round(lon, 4), - cell_id=cell["id"], prefetch=bool(prefetch)) as run: - recent = None - if prefetch: - history = climate.load_cached_history(cell) - if history is None or history.is_empty(): - run.set(run_type="cold-skip") - return Response(status_code=204) - cache_meta = {"cached": True} - else: - history, cache_meta, recent = _fetch_history(run, cell, recent_too=True) - - cid = cell["id"] - last = history["date"].max() - - if neighbors: - _enqueue_neighbor_warming(cell) - - with run.phase("reverse_geocode"): - place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) - - def slice_for(kind: str, key: str, token: str, build) -> dict: - """The endpoint's derived row (or build + store it), paired with the - etag that endpoint would emit for the same request.""" - etag = _etag_for(kind, cid, key, token) - data = store.get_json(kind, cid, key, token) - if data is None: - data = build() - store.put_payload(kind, cid, key, token, data) - return {"etag": etag, "data": data} - - slices = {} - hist_token = views.history_token(history) - # Calendar: the default last-24-months span — what the calendar view (and - # the cross-view prefetch) requests first. - start_ts, end_ts = views.cal_span(history, None, None, 24) - slices["calendar"] = slice_for( - "calendar", views.calendar_key(start_ts, end_ts, 24), hist_token, - lambda: views.build_calendar(cell, history, start_ts, end_ts, 24, place, run)) - - if prefetch: - # Latest archived day: its observation comes from history alone, so - # it's buildable without the (skipped) recent bundle. - slices["day"] = slice_for( - "day", views.day_key(last), hist_token, - lambda: views.build_day(cell, history, last, place, run)) - else: - rf_token = views.recent_token(history, cid) - slices["grade"] = slice_for( - "grade", views.grade_key(today, 14, 14), rf_token, - lambda: views.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14)) - slices["forecast"] = slice_for( - "forecast", views.forecast_key(today, 7), rf_token, - lambda: views.build_forecast(cell, 7, history, recent, today, place, run)) - slices["day"] = slice_for( - "day", views.day_key(today), views.day_token(history, today), - lambda: views.build_day(cell, history, today, place, run, recent=recent)) - - # The bundle's identity is the combination of its slices' identities. - etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""), - "|".join(s["etag"] for s in slices.values())) - if _not_modified(request, etag): - run.set(run_type="cache", not_modified=True) - return Response(status_code=304, headers={"ETag": etag}) - run.set(run_type="bundle", slices=sorted(slices)) - payload = { - "api_version": "v2", - "cell": cell, - "place": place, - "today": today.isoformat(), - "slices": slices, - } - # Not stored as its own derived row — the slices already are. - return _json_response(_encode(payload), etag) - - -# --- ops metrics (gated) --------------------------------------------------- -def _metrics_allowed(request: Request) -> bool: - """Guard the ops metrics: a valid token allows from anywhere (for SSH tunnels); - with no token it's direct-loopback-only, and any forwarded header (i.e. proxied - via Caddy) is refused — so prod ops data never leaks through the public domain.""" - token = os.environ.get("THERMOGRAPH_METRICS_TOKEN", "").strip() - if token: - supplied = request.headers.get("x-metrics-token") or request.query_params.get("token") or "" - if hmac.compare_digest(supplied, token): - return True - client = request.client.host if request.client else None - forwarded = any(h in request.headers for h in - ("x-forwarded-for", "x-forwarded-host", "x-forwarded-proto")) - return client in ("127.0.0.1", "::1") and not forwarded - - -def api_metrics(request: Request): - """Live in-process traffic counters + worker facts for the ops dashboard.""" - if not _metrics_allowed(request): - raise HTTPException(status_code=404) # 404, not 403 — don't reveal the route - snap = metrics.snapshot() - snap["warm_queue_depth"] = _warm_queue.qsize() - snap["warm_seen"] = len(_WARM_SEEN) - snap["threads"] = threading.active_count() - snap["thread_names"] = sorted(t.name for t in threading.enumerate()) - return snap - - -async def api_event(request: Request) -> Response: - """Record one product event (a tap on "use my location", a digest signup, …). - - Deliberately minimal: the body carries only an allowlisted event name. The - referrer is read from this request's own Referer header — a client-supplied - one would be forgeable and buys nothing — and is reduced to a bare domain. - No cookies, no per-visitor id, no full URLs. - - Always answers 204, whether the event was counted, rejected by the allowlist, - or dropped by the rate limiter, so probing the endpoint reveals nothing. That - also suits navigator.sendBeacon, which ignores the response. - """ - name = "" - try: - body = await request.json() - if isinstance(body, dict): - name = str(body.get("event") or "") - except Exception: # noqa: BLE001 - a junk body is just a no-op event - pass - if name: - metrics.record_event( - name, - referer=request.headers.get("referer"), - host=request.headers.get("host"), - ip=_client_ip(request), - ) - return Response(status_code=204) - - -# --- API versioning -------------------------------------------------------- -# Non-backward-compatible additions land in a new version; every version stays -# mounted and served simultaneously. v1 is the original contract (grade/geocode); -# v2 adds the calendar. The unversioned /api/* paths are kept as aliases of v1 so -# existing clients keep working. -v1 = APIRouter(tags=["v1"]) -v1.add_api_route("/geocode", api_geocode, methods=["GET"]) -v1.add_api_route("/grade", api_grade, methods=["GET"]) - -v2 = APIRouter(tags=["v2"]) -v2.add_api_route("/geocode", api_geocode, methods=["GET"]) -v2.add_api_route("/suggest", api_suggest, methods=["GET"]) -v2.add_api_route("/place", api_place, methods=["GET"]) -v2.add_api_route("/grade", api_grade, methods=["GET"]) -v2.add_api_route("/calendar", api_calendar, methods=["GET"]) -v2.add_api_route("/day", api_day, methods=["GET"]) -v2.add_api_route("/score", api_score, methods=["GET"]) -v2.add_api_route("/forecast", api_forecast, methods=["GET"]) -v2.add_api_route("/cell", api_cell, methods=["GET"]) -v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False) -v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False) - -app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible) -app.include_router(v1, prefix=f"{BASE}/api/v1") -app.include_router(v2, prefix=f"{BASE}/api/v2") - - -# --- Discord slash commands (HTTP interactions) ---------------------------- -async def discord_interactions(request: Request) -> Response: - """Discord posts each slash-command interaction here. Verification runs over the - RAW body, so this must read bytes, not request.json().""" - raw = await request.body() - status, payload = discord_interactions_mod.handle( - raw, - request.headers.get("x-signature-ed25519", ""), - request.headers.get("x-signature-timestamp", ""), - ) - return Response(json.dumps(payload), status_code=status, media_type="application/json") - -app.add_api_route(f"{BASE}/discord/interactions", discord_interactions, - methods=["POST"], include_in_schema=False) - -# --- accounts (fastapi-users) ---------------------------------------------- -# Library-provided routers: /auth/login + /auth/logout (cookie session), -# /auth/register (signup), and /users/me (session check / profile). All under the -# same v2 prefix as the rest of the API, so the frontend calls them base-relative. -app.include_router( - users.fastapi_users.get_auth_router(users.auth_backend), - prefix=f"{BASE}/api/v2/auth", tags=["auth"], -) -app.include_router( - users.fastapi_users.get_register_router(UserRead, UserCreate), - prefix=f"{BASE}/api/v2/auth", tags=["auth"], -) -app.include_router( - users.fastapi_users.get_users_router(UserRead, UserUpdate), - prefix=f"{BASE}/api/v2/users", tags=["users"], -) -# Subscriptions + notifications (authed, user-scoped). -app.include_router(api_accounts.router, prefix=f"{BASE}/api/v2") -# Discord account linking (OAuth2, authed). -app.include_router(discord_link.router, prefix=f"{BASE}/api/v2") - - -# --- static frontend (served under BASE) ----------------------------------- -def _page(name): - """Route handler for one HTML page. Serves the file with its __ORIGIN__ - placeholders (the link-preview/Open Graph tags) filled in as the request's - scheme://host + BASE — preview crawlers (Discord, Slack, …) need absolute - URLs, and the host differs between LAN and prod. The scheme comes from - X-Forwarded-Proto when a reverse proxy (Caddy) fronts the plain-HTTP - uvicorn; the proxy passes the original Host header through untouched.""" - path = os.path.join(FRONTEND_DIR, name) - - def route(request: Request): - with open(path, encoding="utf-8") as f: - html = f.read() - proto = request.headers.get("x-forwarded-proto") or request.url.scheme - host = request.headers.get("host") or request.url.netloc - html = html.replace("__ORIGIN__", f"{proto}://{host}{BASE}") - # Search-engine verification tags (same source as the SEO pages), so - # the homepage Google/Bing verify against carries them too. - verify = content.head_verify_html() - if verify: - html = html.replace("", f"\n {verify}", 1) - etag = f'W/"{hashlib.sha1(html.encode()).hexdigest()[:20]}"' - if _not_modified(request, etag): - return Response(status_code=304, headers={"ETag": etag}) - return Response(html, media_type="text/html", headers={"ETag": etag}) - return route - - -# The un-slashed base redirects to BASE/ so the frontend's relative asset URLs -# resolve correctly. Under a sub-path this keeps the app scoped to BASE (never -# claiming "/", so the domain root stays free for another app via the proxy). When -# BASE is "" the app owns the root: "/" is already the index route below, so there -# is no bare base to redirect (and "" is not a valid route path). -# Pages also answer HEAD — link-preview crawlers probe with it before fetching. -if BASE: - app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False) -# NOTE: "/" is not here — the homepage is server-rendered by content.register() -# below, so its hero, records strip and city links reach crawlers and no-JS -# readers as real HTML. -app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/score", _page("score.html"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False) - -# Monthly-digest signup (the footer form on every page). Answers the same generic -# message whatever the outcome, so it can't be probed for whether an address is -# already subscribed. -app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False) - -# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city / -# month / records / glossary / about / privacy) + robots.txt + sitemap.xml. -# Registered before the static mount so these explicit routes win. -content.register(app) - -# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset. -# Registered last so the explicit page routes above win. Mounts must start with -# "/", so at the root (BASE == "") mount at "/" — the earlier routes still win -# because Starlette matches in registration order. -app.mount(BASE or "/", StaticFiles(directory=FRONTEND_DIR), name="static") +The real application lives in ``web/app.py``. This re-export keeps the launch +target stable at ``app:app`` (run from ``backend/``), so the systemd units, +run.sh, and CI need no change after the move into packages. +""" +from web.app import app # noqa: F401 diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..2c410c1 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1 @@ +"""core package.""" diff --git a/audit.py b/core/audit.py similarity index 98% rename from audit.py rename to core/audit.py index 1c8fbcb..53835e9 100644 --- a/audit.py +++ b/core/audit.py @@ -23,7 +23,9 @@ import threading import time import uuid -LOG_ROOT = os.path.join(os.path.dirname(__file__), "..", "logs") +import paths + +LOG_ROOT = paths.LOGS_DIR AUDIT_DIR = os.path.join(LOG_ROOT, "audit") ERROR_DIR = os.path.join(LOG_ROOT, "errors") ACCESS_DIR = os.path.join(LOG_ROOT, "access") diff --git a/metrics.py b/core/metrics.py similarity index 100% rename from metrics.py rename to core/metrics.py diff --git a/singleton.py b/core/singleton.py similarity index 100% rename from singleton.py rename to core/singleton.py diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000..a8fa0e0 --- /dev/null +++ b/data/__init__.py @@ -0,0 +1 @@ +"""data package.""" diff --git a/cities.py b/data/cities.py similarity index 96% rename from cities.py rename to data/cities.py index 88d7ea5..2472519 100644 --- a/cities.py +++ b/data/cities.py @@ -3,8 +3,10 @@ climate pages. Loaded once, lazily; regenerate the JSON with gen_cities.py.""" import json import os -_PATH = os.path.join(os.path.dirname(__file__), "cities.json") -_FLAVOR_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json") +import paths + +_PATH = paths.CITIES_JSON +_FLAVOR_PATH = paths.CITIES_FLAVOR_JSON _CITIES: list[dict] | None = None _BY_SLUG: dict[str, dict] | None = None _FLAVOR: dict[str, dict] | None = None diff --git a/city_events.py b/data/city_events.py similarity index 100% rename from city_events.py rename to data/city_events.py diff --git a/climate.py b/data/climate.py similarity index 99% rename from climate.py rename to data/climate.py index 6c4d359..ca8dc4e 100644 --- a/climate.py +++ b/data/climate.py @@ -14,11 +14,12 @@ import httpx import numpy as np import polars as pl -import audit -import metrics -import store +from core import audit +from core import metrics +import paths +from data import store -CACHE_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "cache") +CACHE_DIR = os.path.join(paths.DATA_DIR, "cache") MAX_ATTEMPTS = 3 # per upstream call, before giving up START_DATE = "1980-01-01" # ERA5 reaches back to 1940; 1980 = 45 yrs, fast + robust ARCHIVE_LATENCY_DAYS = 6 # ERA5 archive lags real time by a few days diff --git a/grading.py b/data/grading.py similarity index 100% rename from grading.py rename to data/grading.py diff --git a/grid.py b/data/grid.py similarity index 100% rename from grid.py rename to data/grid.py diff --git a/places.py b/data/places.py similarity index 99% rename from places.py rename to data/places.py index 7aa7c76..c2faadb 100644 --- a/places.py +++ b/data/places.py @@ -28,9 +28,10 @@ import zipfile import httpx -import audit +from core import audit +import paths -GEO_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "geonames") +GEO_DIR = os.path.join(paths.DATA_DIR, "geonames") GEONAMES_URL = "https://download.geonames.org/export/dump/" # Which GeoNames cities dump to index. cities1000 (~170k places, population diff --git a/scoring.py b/data/scoring.py similarity index 99% rename from scoring.py rename to data/scoring.py index 6407fc7..1dae1bd 100644 --- a/scoring.py +++ b/data/scoring.py @@ -18,7 +18,7 @@ payload (``baseline_overlaps_recent``). import numpy as np import polars as pl -import grading +from data import grading RECENT_YEARS = 6 QS = (10, 25, 50, 75, 90) # the scored percentile categories diff --git a/store.py b/data/store.py similarity index 98% rename from store.py rename to data/store.py index 9a2365f..08b2d1a 100644 --- a/store.py +++ b/data/store.py @@ -26,7 +26,9 @@ import threading import time import zlib -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "data", "thermograph.sqlite") +import paths + +DB_PATH = os.path.join(paths.DATA_DIR, "thermograph.sqlite") _SCHEMA = """ CREATE TABLE IF NOT EXISTS derived ( diff --git a/gen_cities.py b/gen_cities.py index 2b75419..117dda9 100644 --- a/gen_cities.py +++ b/gen_cities.py @@ -15,9 +15,10 @@ import re import sys import unicodedata -import places +import paths +from data import places -OUT_PATH = os.path.join(os.path.dirname(__file__), "cities.json") +OUT_PATH = paths.CITIES_JSON # GeoNames entry tuple layout (see places._load): the fields we keep. _NAME, _ADMIN1, _COUNTRY, _CC, _LAT, _LON, _POP = 1, 2, 3, 4, 5, 6, 7 diff --git a/gen_flavor.py b/gen_flavor.py index 9fc1a76..5cfb2d2 100644 --- a/gen_flavor.py +++ b/gen_flavor.py @@ -20,9 +20,10 @@ from urllib.parse import quote import httpx -import cities +from data import cities +import paths -OUT_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json") +OUT_PATH = paths.CITIES_FLAVOR_JSON UA = "ThermographBot/1.0 (https://thermograph.org; climate comparison tool)" SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/" diff --git a/indexnow.py b/indexnow.py index 90e3b0a..a9323bd 100644 --- a/indexnow.py +++ b/indexnow.py @@ -25,9 +25,11 @@ from urllib.parse import urlparse import httpx +import paths + log = logging.getLogger("thermograph.indexnow") -_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data")) +_DATA_DIR = paths.DATA_DIR _KEY_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_KEY_FILE") or os.path.join(_DATA_DIR, "indexnow_key.txt") _STATE_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_STATE_FILE") or os.path.join(_DATA_DIR, "indexnow_state.txt") _ENDPOINT = "https://api.indexnow.org/indexnow" # a shared endpoint; it fans out to all participants @@ -98,7 +100,7 @@ def submit(urls, host: str, key_location: str, scheme: str = "https", timeout: f def submit_all(site_base_url: str, **kw) -> dict: """Submit every indexable page. ``site_base_url`` is the public origin + base path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``.""" - import content # lazy: avoids a content ↔ indexnow import cycle + from web import content # lazy: avoids a content ↔ indexnow import cycle site = site_base_url.rstrip("/") parsed = urlparse(site) urls = [site + path for path in content.public_paths()] @@ -110,7 +112,7 @@ def url_signature() -> str: """A stable fingerprint of the indexable URL set — changes only when pages are added or removed (e.g. a new city), not on code-only deploys. Used to skip resubmitting an unchanged set (see the --if-changed CLI flag).""" - import content + from web import content return hashlib.sha1("\n".join(content.public_paths()).encode()).hexdigest() diff --git a/migrate.py b/migrate.py index 862e37e..5e04626 100644 --- a/migrate.py +++ b/migrate.py @@ -21,10 +21,10 @@ import os import sys import time -import climate -import grid -import store -import views +from data import climate +from data import grid +from data import store +from web import views def migrate() -> int: diff --git a/notifications/__init__.py b/notifications/__init__.py new file mode 100644 index 0000000..351561e --- /dev/null +++ b/notifications/__init__.py @@ -0,0 +1 @@ +"""notifications package.""" diff --git a/digest.py b/notifications/digest.py similarity index 96% rename from digest.py rename to notifications/digest.py index 0bce6ec..9e557d7 100644 --- a/digest.py +++ b/notifications/digest.py @@ -16,10 +16,10 @@ from fastapi import Request, Response from fastapi.responses import JSONResponse from sqlalchemy import select -import mailer -import metrics -from db import async_session_maker -from models import PendingDigest +from notifications import mailer +from core import metrics +from accounts.db import async_session_maker +from accounts.models import PendingDigest # Deliberately permissive: real-world addresses defeat clever patterns, and the # confirmation email is the actual validator. This only rejects obvious junk. diff --git a/discord.py b/notifications/discord.py similarity index 99% rename from discord.py rename to notifications/discord.py index 2fd86a5..bb1fdfa 100644 --- a/discord.py +++ b/notifications/discord.py @@ -17,7 +17,7 @@ import time import httpx -import homepage +from web import homepage # The webhook URL IS the credential — env only, never the repo. Unset => disabled. WEBHOOK_URL = os.environ.get("THERMOGRAPH_DISCORD_WEBHOOK", "").strip() diff --git a/discord_interactions.py b/notifications/discord_interactions.py similarity index 98% rename from discord_interactions.py rename to notifications/discord_interactions.py index 1f0f951..f9b6605 100644 --- a/discord_interactions.py +++ b/notifications/discord_interactions.py @@ -21,9 +21,9 @@ import os from nacl.exceptions import BadSignatureError from nacl.signing import VerifyKey -import cities -import discord -import homepage +from data import cities +from notifications import discord +from web import homepage # App's Ed25519 public key (Developer Portal -> General Information). Unset => # every request fails verification, which is the safe default for an unconfigured diff --git a/discord_link.py b/notifications/discord_link.py similarity index 98% rename from discord_link.py rename to notifications/discord_link.py index b48bee9..c4feffd 100644 --- a/discord_link.py +++ b/notifications/discord_link.py @@ -29,9 +29,9 @@ from pydantic import BaseModel from sqlalchemy import update from sqlalchemy.ext.asyncio import AsyncSession -from db import get_async_session -from models import User -from users import SECRET, current_active_user +from accounts.db import get_async_session +from accounts.models import User +from accounts.users import SECRET, current_active_user router = APIRouter(tags=["discord"]) diff --git a/mailer.py b/notifications/mailer.py similarity index 100% rename from mailer.py rename to notifications/mailer.py diff --git a/notify.py b/notifications/notify.py similarity index 97% rename from notify.py rename to notifications/notify.py index dcd7610..08c8bde 100644 --- a/notify.py +++ b/notifications/notify.py @@ -29,16 +29,16 @@ import time import polars as pl from sqlalchemy import delete, select -import climate -import discord -import grading -import grid -import homepage -import metrics -import push -from db import sync_session_maker -from models import AccessToken, Notification, PushSubscription, Subscription, User -from views import OBS_COLS +from data import climate +from notifications import discord +from data import grading +from data import grid +from web import homepage +from core import metrics +from notifications import push +from accounts.db import sync_session_maker +from accounts.models import AccessToken, Notification, PushSubscription, Subscription, User +from web.views import OBS_COLS _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") BASE = f"/{_BASE}" if _BASE else "" diff --git a/push.py b/notifications/push.py similarity index 98% rename from push.py rename to notifications/push.py index 7b6c58a..0c40a51 100644 --- a/push.py +++ b/notifications/push.py @@ -25,7 +25,8 @@ import logging import os import threading -import audit +from core import audit +import paths from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import ec @@ -33,7 +34,7 @@ from pywebpush import WebPushException, webpush log = logging.getLogger("thermograph.push") -_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data")) +_DATA_DIR = paths.DATA_DIR _VAPID_PATH = os.environ.get("THERMOGRAPH_VAPID_FILE") or os.path.join(_DATA_DIR, "vapid.json") # The VAPID "sub" claim — a contact the push service can reach about our traffic. _CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.app") diff --git a/paths.py b/paths.py new file mode 100644 index 0000000..52f89f7 --- /dev/null +++ b/paths.py @@ -0,0 +1,42 @@ +"""Canonical filesystem locations, resolved once from the repo root. + +Every path the backend needs (the climate cache, the accounts DB, logs, the +Jinja templates, the frontend static dir, the bundled city data) is derived +here from the repository root rather than from each module's own ``__file__``. +That keeps the locations stable no matter how deep in the package tree a module +lives, so moving a module never silently re-points the cache or the accounts DB. + +Callers that support an environment override (e.g. ``THERMOGRAPH_ACCOUNTS_DB``) +keep doing so; these are only the defaults. +""" +import os + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _repo_root() -> str: + """Walk up from this file until a directory holds both ``backend/`` and + ``frontend/`` — the repository root, in every checkout (dev, prod, worktree).""" + d = _HERE + while True: + if os.path.isdir(os.path.join(d, "backend")) and os.path.isdir(os.path.join(d, "frontend")): + return d + parent = os.path.dirname(d) + if parent == d: + raise RuntimeError( + "cannot locate the repo root (no ancestor has sibling backend/ and frontend/)" + ) + d = parent + + +REPO_DIR = _repo_root() +BACKEND_DIR = os.path.join(REPO_DIR, "backend") +DATA_DIR = os.path.join(REPO_DIR, "data") +LOGS_DIR = os.path.join(REPO_DIR, "logs") +FRONTEND_DIR = os.path.join(REPO_DIR, "frontend") +TEMPLATES_DIR = os.path.join(BACKEND_DIR, "templates") + +# Bundled reference data shipped alongside the code (generated by gen_cities / +# gen_flavor), not runtime state — hence under backend/, not data/. +CITIES_JSON = os.path.join(BACKEND_DIR, "cities.json") +CITIES_FLAVOR_JSON = os.path.join(BACKEND_DIR, "cities_flavor.json") diff --git a/tests/accounts/test_api_accounts.py b/tests/accounts/test_api_accounts.py index efbc916..00412be 100644 --- a/tests/accounts/test_api_accounts.py +++ b/tests/accounts/test_api_accounts.py @@ -4,8 +4,8 @@ THERMOGRAPH_ACCOUNTS_DB at a temp file) so nothing is written into the repo.""" import pytest from fastapi.testclient import TestClient -import app as appmod -import db +from web import app as appmod +from accounts import db V2 = "/thermograph/api/v2" PW = "supersecret123" @@ -120,7 +120,7 @@ def test_push_requires_auth(): def test_push_subscribe_upsert_test_unsubscribe(monkeypatch): - import push + from notifications import push sent = [] monkeypatch.setattr(push, "send", lambda info, payload: sent.append(info["endpoint"]) or "ok") c = _client() @@ -142,7 +142,7 @@ def test_push_subscribe_upsert_test_unsubscribe(monkeypatch): def test_push_test_prunes_gone_endpoint(monkeypatch): - import push + from notifications import push monkeypatch.setattr(push, "send", lambda info, payload: "gone") c = _client() _login(c, "push-gone@example.com") @@ -156,7 +156,7 @@ def test_push_test_prunes_gone_endpoint(monkeypatch): def test_push_test_reports_failed_delivery(monkeypatch): # A rejected delivery (e.g. VAPID mismatch) must be reported, not silently dropped: # the endpoint keeps returning 202 but with sent:0 / failed:N so the client can tell. - import push + from notifications import push monkeypatch.setattr(push, "send", lambda info, payload: "error") c = _client() _login(c, "push-fail@example.com") diff --git a/tests/conftest.py b/tests/conftest.py index 8f7ae6f..6ffbc24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,7 @@ import pytest BACKEND = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, BACKEND) -import places # noqa: E402 +from data import places # noqa: E402 # app.py calls places.start_loading() at import; pretend it already ran so no # background GeoNames download starts. Tests build their own tiny index. @@ -31,13 +31,13 @@ os.environ.setdefault("THERMOGRAPH_ENABLE_NOTIFIER", "0") # Pin the IndexNow key so it isn't generated + written into the repo's data/ dir. os.environ.setdefault("THERMOGRAPH_INDEXNOW_KEY", "test0000indexnow0000key000000000") -import store # noqa: E402 +from data import store # noqa: E402 # Keep derived-store writes out of the repo's data/ folder. Individual store # tests re-point this per test; everything else shares one throwaway DB. store.DB_PATH = os.path.join(_TMP, "store.sqlite") -import audit # noqa: E402 +from core import audit # noqa: E402 audit.AUDIT_DIR = os.path.join(_TMP, "logs", "audit") audit.ERROR_DIR = os.path.join(_TMP, "logs", "errors") diff --git a/tests/core/test_metrics.py b/tests/core/test_metrics.py index e85721a..acbb69f 100644 --- a/tests/core/test_metrics.py +++ b/tests/core/test_metrics.py @@ -1,7 +1,7 @@ """Unit tests for the in-process traffic counters (metrics.py).""" import importlib -import metrics +from core import metrics def _fresh(): diff --git a/tests/core/test_singleton.py b/tests/core/test_singleton.py index a2c142b..f0f0ddb 100644 --- a/tests/core/test_singleton.py +++ b/tests/core/test_singleton.py @@ -3,7 +3,7 @@ import fcntl import importlib import os -import singleton +from core import singleton def _fresh(): diff --git a/tests/data/test_climate.py b/tests/data/test_climate.py index 8091594..929cd19 100644 --- a/tests/data/test_climate.py +++ b/tests/data/test_climate.py @@ -5,7 +5,7 @@ import datetime import polars as pl import pytest -import climate +from data import climate def _om_daily(n=3): diff --git a/tests/data/test_grading.py b/tests/data/test_grading.py index c45c31e..8fb7b88 100644 --- a/tests/data/test_grading.py +++ b/tests/data/test_grading.py @@ -4,7 +4,7 @@ import numpy as np import polars as pl import pytest -import grading +from data import grading # ---- empirical percentile ---------------------------------------------------- diff --git a/tests/data/test_grid.py b/tests/data/test_grid.py index 268ca99..e6703d2 100644 --- a/tests/data/test_grid.py +++ b/tests/data/test_grid.py @@ -2,7 +2,7 @@ import math import pytest -import grid +from data import grid def test_snap_is_deterministic_and_contains_point(): diff --git a/tests/data/test_places.py b/tests/data/test_places.py index 0691e90..9bc5946 100644 --- a/tests/data/test_places.py +++ b/tests/data/test_places.py @@ -1,6 +1,6 @@ import pytest -import places +from data import places def _entry(name, admin1, country, cc, lat, lon, pop): diff --git a/tests/data/test_scoring.py b/tests/data/test_scoring.py index e4a6fb8..d617d53 100644 --- a/tests/data/test_scoring.py +++ b/tests/data/test_scoring.py @@ -7,7 +7,7 @@ import numpy as np import polars as pl import pytest -import scoring +from data import scoring ALL_METRICS = ("tmax", "tmin", "feels", "humid", "wetbulb", "wind", "gust", "precip") diff --git a/tests/notifications/test_digest.py b/tests/notifications/test_digest.py index 5de1022..f297e26 100644 --- a/tests/notifications/test_digest.py +++ b/tests/notifications/test_digest.py @@ -3,12 +3,12 @@ import pytest import sqlalchemy as sa from fastapi.testclient import TestClient -import app as appmod -import db -import digest -import mailer -from db import async_session_maker -from models import PendingDigest +from web import app as appmod +from accounts import db +from notifications import digest +from notifications import mailer +from accounts.db import async_session_maker +from accounts.models import PendingDigest B = "/thermograph" @@ -104,7 +104,7 @@ def test_signup_accepts_plain_form_post(client): def test_signup_records_event(client): - import metrics + from core import metrics before = metrics.snapshot()["events"].get("home.digest_signup", {}) before_n = sum(sum(days.values()) for days in before.values()) client.post(f"{B}/digest", json={"email": "counted@example.com"}) diff --git a/tests/notifications/test_discord.py b/tests/notifications/test_discord.py index 184ddde..52c1fcb 100644 --- a/tests/notifications/test_discord.py +++ b/tests/notifications/test_discord.py @@ -5,8 +5,8 @@ import time import pytest -import discord -import notify +from notifications import discord +from notifications import notify def _feed(ranked, generated_at=None, date=None): diff --git a/tests/notifications/test_discord_dm.py b/tests/notifications/test_discord_dm.py index d85e36d..1152948 100644 --- a/tests/notifications/test_discord_dm.py +++ b/tests/notifications/test_discord_dm.py @@ -5,11 +5,11 @@ import types import pytest from fastapi.testclient import TestClient -import app as appmod -import db -import discord -import discord_link as dl -import notify +from web import app as appmod +from accounts import db +from notifications import discord +from notifications import discord_link as dl +from notifications import notify V2 = "/thermograph/api/v2" PW = "supersecret123" diff --git a/tests/notifications/test_discord_interactions.py b/tests/notifications/test_discord_interactions.py index 3d475ed..35c6334 100644 --- a/tests/notifications/test_discord_interactions.py +++ b/tests/notifications/test_discord_interactions.py @@ -8,9 +8,9 @@ import pytest from fastapi.testclient import TestClient from nacl.signing import SigningKey -import app as appmod -import discord_interactions as di -import homepage +from web import app as appmod +from notifications import discord_interactions as di +from web import homepage B = "/thermograph" diff --git a/tests/notifications/test_discord_link.py b/tests/notifications/test_discord_link.py index e75e599..ffaade0 100644 --- a/tests/notifications/test_discord_link.py +++ b/tests/notifications/test_discord_link.py @@ -3,9 +3,9 @@ mocked), unlink, and auth gating. Reuses the throwaway accounts DB from conftest import pytest from fastapi.testclient import TestClient -import app as appmod -import db -import discord_link as dl +from web import app as appmod +from accounts import db +from notifications import discord_link as dl V2 = "/thermograph/api/v2" PW = "supersecret123" diff --git a/tests/notifications/test_notify.py b/tests/notifications/test_notify.py index 9738fcb..8c22038 100644 --- a/tests/notifications/test_notify.py +++ b/tests/notifications/test_notify.py @@ -8,11 +8,11 @@ import uuid import numpy as np import polars as pl -import climate -import db -import notify -import push -from models import Notification, PushSubscription, Subscription, User +from data import climate +from accounts import db +from notifications import notify +from notifications import push +from accounts.models import Notification, PushSubscription, Subscription, User from sqlalchemy import delete, func, select diff --git a/tests/web/test_api.py b/tests/web/test_api.py index a5228cd..7500784 100644 --- a/tests/web/test_api.py +++ b/tests/web/test_api.py @@ -4,8 +4,8 @@ would catch wiring regressions (e.g. a handler calling a deleted helper).""" import pytest from fastapi.testclient import TestClient -import app as appmod -import climate +from web import app as appmod +from data import climate @pytest.fixture @@ -194,7 +194,7 @@ def test_pages_serve_with_origin_filled_in(client): def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch): - import app as appmod + from web import app as appmod monkeypatch.setattr(appmod, "_WARM_SEEN", {}) # No lifespan in TestClient without a context manager, so no worker drains # the queue — enqueued cells just accumulate for inspection. @@ -210,9 +210,9 @@ def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch): def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history): - import app as appmod - import grid - import views + from web import app as appmod + from data import grid + from web import views cell = grid.snap(-33.87, 151.21) appmod._warm_cell(cell) token = views.history_token(history) @@ -224,7 +224,7 @@ def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, h def test_warm_cell_never_fetches_upstream(client, monkeypatch): - import app as appmod + from web import app as appmod def boom(cell): raise AssertionError("warming must never fetch weather upstream") monkeypatch.setattr(climate, "get_history", boom) diff --git a/tests/web/test_content.py b/tests/web/test_content.py index f6b3db1..ecd9d8f 100644 --- a/tests/web/test_content.py +++ b/tests/web/test_content.py @@ -7,10 +7,10 @@ import re import pytest from fastapi.testclient import TestClient -import app as appmod -import cities -import climate -import grading +from web import app as appmod +from data import cities +from data import climate +from data import grading # BASE defaults to /thermograph in tests (THERMOGRAPH_BASE unset). B = "/thermograph" @@ -84,7 +84,7 @@ def test_city_404(client): def test_curated_events_have_valid_slugs(): - import city_events + from data import city_events slugs = set(cities.all_slugs()) assert len(city_events.EVENTS) >= 20 assert all(s in slugs for s in city_events.EVENTS), \ @@ -344,7 +344,7 @@ def test_wind_and_humidity_render_per_unit_system(): # The synthetic history carries only tmax/tmin/precip, so wind and humidity # never reach a rendered page in tests — exercise the renderers directly # rather than assert on a page that structurally cannot show them. - import content + from web import content with content._unit_scope("F"): assert content._wind_text(10) == "10 mph" assert '10 mph' == content._wind(10) @@ -362,7 +362,7 @@ def test_wind_and_humidity_render_per_unit_system(): def test_precip_precision_differs_by_unit(): # Rainfall is reported in whole millimetres; a tenth of a mm is below what # the source resolves. Inches keep two decimals. - import content + from web import content with content._unit_scope("F"): assert content._precip_text(0.04) == "0.04 in" with content._unit_scope("C"): @@ -373,7 +373,7 @@ def test_precip_precision_differs_by_unit(): def test_measure_conversion_constants_match_the_frontend(): # Same drift risk as the country list: two hand-maintained copies. import os - import content + from web import content units_js = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), os.pardir, "frontend", "units.js") with open(units_js, encoding="utf-8") as f: @@ -412,7 +412,7 @@ def test_f_country_list_matches_the_frontend(): # drifting apart, and drift means the server and the client disagree about # what a first-time visitor should see. import os - import content + from web import content units_js = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), os.pardir, "frontend", "units.js") with open(units_js, encoding="utf-8") as f: @@ -425,7 +425,7 @@ def test_celsius_rounding_matches_js(): # Python's round() goes to even, JS's Math.round goes up. Where they disagree # the server prints one number and climate.js repaints another — a flicker for # the visitor whose unit already matched. - import content + from web import content assert content._round_half_up(0.5) == 1 # round(0.5) would give 0 assert content._round_half_up(1.5) == 2 assert content._round_half_up(2.5) == 3 # round(2.5) would give 2 diff --git a/tests/web/test_homepage.py b/tests/web/test_homepage.py index ec76b61..7dd9393 100644 --- a/tests/web/test_homepage.py +++ b/tests/web/test_homepage.py @@ -13,11 +13,11 @@ import time import pytest from fastapi.testclient import TestClient -import app as appmod -import cities -import climate -import content -import homepage +from web import app as appmod +from data import cities +from data import climate +from web import content +from web import homepage B = "/thermograph" diff --git a/tests/web/test_views.py b/tests/web/test_views.py index e7e433b..a912db2 100644 --- a/tests/web/test_views.py +++ b/tests/web/test_views.py @@ -8,8 +8,8 @@ import sys import polars as pl import pytest -import climate -import views +from data import climate +from web import views CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377} @@ -17,10 +17,10 @@ CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377} def test_views_and_migrate_import_without_the_web_stack(): """migrate.py must stay runnable offline: importing the payload layer may not construct the FastAPI app or start the places-index download.""" - code = ("import sys; import views, migrate; " + code = ("import sys; from web import views; import migrate; " "assert 'fastapi' not in sys.modules, 'views/migrate pulled in FastAPI'; " - "assert 'app' not in sys.modules, 'views/migrate imported the web app'; " - "import places; assert places._load_started is False") + "assert 'web.app' not in sys.modules, 'views/migrate imported the web app'; " + "from data import places; assert places._load_started is False") backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) subprocess.run([sys.executable, "-c", code], cwd=backend, check=True) diff --git a/warm_cities.py b/warm_cities.py index 77a428c..a819767 100644 --- a/warm_cities.py +++ b/warm_cities.py @@ -12,10 +12,10 @@ so this is an optimization, not a hard dependency. import sys import time -import cities -import climate -import grid -import homepage +from data import cities +from data import climate +from data import grid +from web import homepage def main(limit: int | None = None, pace: float = 2.0) -> None: diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 0000000..0ae4c1c --- /dev/null +++ b/web/__init__.py @@ -0,0 +1 @@ +"""web package.""" diff --git a/web/app.py b/web/app.py new file mode 100644 index 0000000..e4a9ec3 --- /dev/null +++ b/web/app.py @@ -0,0 +1,783 @@ +"""Thermograph API — grade recent local weather against ~45 years of climatology.""" +import contextlib +import datetime +import functools +import hashlib +import hmac +import json +import mimetypes +import os +import queue +import threading +import time + +from fastapi import APIRouter, FastAPI, HTTPException, Query, Request, Response +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import RedirectResponse +from fastapi.staticfiles import StaticFiles + +from accounts import api_accounts +from core import audit +from data import climate +from web import content +from notifications import digest +from notifications import discord_interactions as discord_interactions_mod +from notifications import discord_link +from accounts import db +from data import grid +from core import metrics +from notifications import notify +import paths +from data import places +from core import singleton +from data import store +from accounts import users +from web import views +from web.schemas import UserCreate, UserRead, UserUpdate + +FRONTEND_DIR = paths.FRONTEND_DIR + +# Everything (pages, assets, API) is served under this base path so the app can +# live at https:///thermograph/ behind a shared domain. Override with the +# THERMOGRAPH_BASE env var. The frontend uses base-relative URLs, so it follows +# this automatically without hardcoding the prefix. +# +# THERMOGRAPH_BASE=/ (or empty) means the app owns the whole domain: BASE becomes +# "" so every route sits at the domain root ("/", "/calendar", "/api/v2/…") with +# no prefix and no leading double slash. A non-empty value keeps the sub-path form +# ("/thermograph"). +_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") +BASE = f"/{_BASE}" if _BASE else "" + +# The PWA manifest is served as a static file; register its media type so +# StaticFiles labels it correctly (not in Python's default mimetypes map). +mimetypes.add_type("application/manifest+json", ".webmanifest") + + +def _weather_fetch_error(e) -> HTTPException: + """A clean, retryable 503 when weather data is temporarily unavailable; the + raw error as a 502 otherwise. The fetch layer classifies its own failures + (climate.WeatherUnavailable carries the user-facing text) so no message + parsing happens here; the is_rate_limit fallback covers a raw upstream 429 + from a fetcher that didn't classify it.""" + if isinstance(e, climate.WeatherUnavailable): + return HTTPException(status_code=503, detail=str(e)) + if climate.is_rate_limit(e): + return HTTPException(status_code=503, detail=climate.limit_message(False)) + return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}") + + +# --- background neighbor warming --------------------------------------------- +# /cell?neighbors=1 asks the server to warm the 8 grid cells around the request +# (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's +# snapping math in JS). One worker drains the queue so warms never stack; the +# hard guarantee matches prefetch=1 — a cell with no cached archive is skipped, +# so no weather-API quota is ever spent, and reverse_geocode itself paces the +# at-most-one Nominatim call per never-labeled cell (~1/s policy). + +_WARM_TTL = 3600.0 # don't re-enqueue a cell within the hour +_WARM_SEEN: dict[str, float] = {} # cell_id -> last enqueue time +_warm_queue: "queue.Queue[dict]" = queue.Queue() + + +def _warm_cell(cell: dict) -> None: + """Materialize the history-derived slices for one cell — the same rows the + prefetch=1 bundle serves. Never fetches weather upstream.""" + history = climate.load_cached_history(cell) + if history is None or history.is_empty(): + return + place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) + token = views.history_token(history) + start_ts, end_ts = views.cal_span(history, None, None, 24) + cal_key = views.calendar_key(start_ts, end_ts, 24) + if store.get_payload("calendar", cell["id"], cal_key, token) is None: + store.put_payload("calendar", cell["id"], cal_key, token, + views.build_calendar(cell, history, start_ts, end_ts, 24, place)) + last = history["date"].max() + day_key = views.day_key(last) + if store.get_payload("day", cell["id"], day_key, token) is None: + store.put_payload("day", cell["id"], day_key, token, + views.build_day(cell, history, last, place)) + + +def _enqueue_neighbor_warming(cell: dict) -> None: + now = time.time() + for n in grid.neighbors(cell): + if now - _WARM_SEEN.get(n["id"], 0.0) < _WARM_TTL: + continue + _WARM_SEEN[n["id"]] = now + _warm_queue.put(n) + + +def _neighbor_warmer() -> None: + while True: + cell = _warm_queue.get() + try: + _warm_cell(cell) + except Exception: # noqa: BLE001 - warming is best-effort, never fatal + pass + finally: + _warm_queue.task_done() + + +@contextlib.asynccontextmanager +async def _lifespan(app): + # Warm the local place-name index (/suggest's typo tolerance) in the + # background; the app boots and serves fine without it — suggestions just + # fall back to the upstream geocoder until it's ready. A server-startup + # hook (not import time) so offline importers (tests, the migrate script's + # dependencies) don't kick off a GeoNames download. The neighbor warmer is + # also a server concern: enqueues before startup just wait in the queue. + # Create the account DB tables (users/sessions/subscriptions/notifications) + # before serving — a fast no-op once they exist. + await db.create_db_and_tables() + places.start_loading() + threading.Thread(target=_neighbor_warmer, name="neighbor-warmer", daemon=True).start() + # Background subscription evaluator. Its periodic sweep fetches upstream on a timer + # (not per request), so with multiple workers it must run in exactly one — elect a + # leader via a lockfile (THERMOGRAPH_SINGLETON_LOCK). Unset (single worker / dev / + # tests) => always the leader. The neighbor warmer above stays per-worker: each + # drains its own request-fed queue. + if singleton.claim(os.environ.get("THERMOGRAPH_SINGLETON_LOCK")): + notify.start() + yield + notify.stop() + + +app = FastAPI(title="Thermograph", version="0.2.0", lifespan=_lifespan) + +# Compress every sizeable response (the 2-year calendar JSON shrinks ~6-8×). +# Applies to API JSON and static assets alike; tiny responses are left alone. +app.add_middleware(GZipMiddleware, minimum_size=1024) + + +def _client_ip(request) -> "str | None": + """The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us + (Caddy on prod sets it), otherwise the direct peer address (LAN dev).""" + xff = request.headers.get("x-forwarded-for") + if xff: + return xff.split(",")[0].strip() + return request.client.host if request.client else None + + +@app.middleware("http") +async def revalidate_static(request, call_next): + """Serve the frontend with no-cache (NOT no-store): browsers may keep a copy + but must revalidate it on every use. Starlette's FileResponse/StaticFiles + already emit ETag + Last-Modified, so an unchanged asset costs one conditional + request answered with an empty 304 instead of a full re-download — page-to-page + navigation stops re-transferring the JS/CSS/HTML while still picking up every + deploy immediately (no "my change isn't showing" bugs).""" + response = await call_next(request) + path = request.url.path + try: + cat = metrics.classify_inbound(path, BASE) + metrics.record_inbound(cat, response.status_code) + # Retain per-request client IPs for later analysis; skip static assets and the + # dashboard's own metrics polling to keep the log to real, meaningful traffic. + if cat not in ("static", "metrics", "event"): + audit.log_access({"ip": _client_ip(request), "method": request.method, + "path": path, "status": response.status_code, "cat": cat}) + except Exception: # noqa: BLE001 - never let instrumentation break a response + pass + pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/score", + f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy") + if path.endswith((".js", ".css", ".html")) or path in pages: + response.headers["Cache-Control"] = "no-cache" + return response + + +# --- derived-store plumbing -------------------------------------------------- +# Every graded payload is cached in SQLite under (kind, cell, key) and validated +# by a token that encodes what it was computed from (see store.py). The freshness +# drivers stay where they were — climate.get_history tops up the archive tail +# hourly and get_recent_forecast refetches hourly — and the tokens are derived +# from what those return, so a cached payload expires exactly when its inputs +# change and never before. The same tokens double as ETags: a client sending +# If-None-Match gets an empty 304 without the payload even being loaded. +# The payloads themselves are assembled in views.py, shared with the offline +# migrate script. + +def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str: + """Deterministic weak ETag from a payload's identity + validity token. Weak + because the same content may be served under different encodings (gzip).""" + h = hashlib.sha1(f"{kind}:{cell_id}:{key}:{token}".encode()).hexdigest()[:20] + return f'W/"{h}"' + + +def _not_modified(request: Request, etag: str) -> bool: + """Does the client already hold this exact payload? Answerable from the token + alone — no payload load needed.""" + inm = request.headers.get("if-none-match") + if not inm: + return False + tags = {t.strip() for t in inm.split(",")} + return "*" in tags or etag in tags or etag.removeprefix("W/") in tags + + +def _encode(payload: dict) -> bytes: + """Compact JSON bytes, matching store.put_payload's encoding (allow_nan=False + mirrors Starlette — a NaN from grading should fail loudly, not reach a client).""" + return json.dumps(payload, default=str, allow_nan=False, separators=(",", ":")).encode() + + +def _json_response(body: bytes, etag: str | None = None) -> Response: + headers = {"ETag": etag} if etag else {} + return Response(content=body, media_type="application/json", headers=headers) + + +def _fetch_history(run, cell, recent_too=False, recent_phase="recent"): + """The shared fetch preamble for every data route: the archive record (and + optionally the hourly recent/forecast bundle) with upstream failures mapped + to clean HTTP errors and an empty record to a 404.""" + try: + with run.phase("history"): + history, cache_meta = climate.get_history(cell) + recent = None + if recent_too: + with run.phase(recent_phase): + recent = climate.get_recent_forecast(cell) + except Exception as e: # noqa: BLE001 + raise _weather_fetch_error(e) + if history.is_empty(): + raise HTTPException(status_code=404, detail="No historical data for this cell.") + return history, cache_meta, recent + + +def _cached_response(request, run, kind, cell, key, token, build, cache_placeless=True): + """The shared derived-store flow behind every per-view endpoint: + If-None-Match -> empty 304; a token-valid store row -> replay its exact + bytes; otherwise resolve the place label, build the payload, persist, serve. + + ``build(place) -> payload`` does any endpoint-specific audit tagging itself. + ``cache_placeless=False`` skips persisting a payload whose place label + failed to resolve (a transient reverse-geocode miss) — otherwise the + bare-coordinates fallback would stick for the life of the token.""" + cid = cell["id"] + etag = _etag_for(kind, cid, key, token) + if _not_modified(request, etag): + run.set(run_type="cache", not_modified=True) + return Response(status_code=304, headers={"ETag": etag}) + body = store.get_payload(kind, cid, key, token) + if body is not None: + run.set(run_type="cache", history_source="store") + return _json_response(body, etag) + with run.phase("reverse_geocode"): + place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) + payload = build(place) + return _json_response( + store.put_payload(kind, cid, key, token, payload, + cache=cache_placeless or place is not None), etag) + + + +# --- endpoints ---------------------------------------------------------------- + +def api_geocode(q: str = Query(..., min_length=1)): + try: + return {"results": climate.geocode(q)} + except Exception as e: # noqa: BLE001 - surface upstream failures to the client + raise HTTPException(status_code=502, detail=f"geocoding failed: {e}") + + +_SUGGEST_LIMIT = 5 + + +@functools.lru_cache(maxsize=1024) +def _suggest_upstream(q: str) -> tuple: + """Open-Meteo lookup for /suggest, memoized per query string — type-ahead + re-asks the same prefixes constantly (backspacing, retyping). Failures + raise and are not cached, so a transient upstream error doesn't stick.""" + return tuple(climate.geocode(q, count=_SUGGEST_LIMIT)) + + +def api_suggest(q: str = Query(..., min_length=1)): + """Type-ahead location suggestions: the top 5 places for a (possibly + typo'd) query prefix; `corrected` reports the respelling that produced the + results ("pest seattle" → "west seattle"), or null. The ranking/typo policy + lives in places.suggest.""" + try: + results, corrected = places.suggest(q, _suggest_upstream, _SUGGEST_LIMIT) + except Exception as e: # noqa: BLE001 - nothing at all to serve + raise HTTPException(status_code=502, detail=f"geocoding failed: {e}") + return {"results": results, "corrected": corrected} + + +def api_place( + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), +): + """Best-effort neighbourhood/city label for a point — the same string the view + endpoints expose as ``place`` (snapped to the grid cell, reverse-geocoded and + cached). Resolved on its own so the compare page can show a location's name as + soon as it's added, before its full series loads.""" + cell = grid.snap(lat, lon) + with audit.RunAudit(endpoint="place", lat=round(lat, 4), lon=round(lon, 4), + cell_id=cell["id"]) as run: + with run.phase("reverse_geocode"): + place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) + return {"place": place, + "cell": {"center_lat": cell["center_lat"], "center_lon": cell["center_lon"]}} + + +def api_grade( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), + date: str | None = Query(None, description="target date YYYY-MM-DD (default today)"), + days: int = Query(14, ge=1, le=60, description="days of history to grade before the target"), + after: int = Query(14, ge=0, le=14, + description="days to grade after the target (observed, or forecast when future; " + "the forecast reaches ~7 days out so future days cap there)"), +): + target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today() + cell = grid.snap(lat, lon) + + with audit.RunAudit( + endpoint="grade", + lat=round(lat, 4), + lon=round(lon, 4), + target_date=target.isoformat(), + recent_days=days, + cell_id=cell["id"], + ) as run: + history, cache_meta, recent = _fetch_history(run, cell, recent_too=True) + return _cached_response( + request, run, "grade", cell, + views.grade_key(target, days, after), views.recent_token(history, cell["id"]), + lambda place: views.build_grade(cell, target, days, history, recent, + cache_meta, place, run, after=after)) + + +def api_calendar( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), + start: str | None = Query(None, description="first date YYYY-MM-DD (overrides months)"), + end: str | None = Query(None, description="last date YYYY-MM-DD (default = latest available)"), + months: int = Query(24, ge=1, le=24, description="months back to grade when no start given"), +): + """Grade every day over a date range (default: last 2 years) for the calendar view. + + A custom [start, end] range is supported and capped at ~2 years per request; + the frontend splits longer spans into successive 2-year chunks. Grades against + the already-cached history record (which reaches to ~6 days ago), so no extra + fetch is needed. The graded payload is cached in the derived store keyed by + the clamped span and validated by the record's end date, so a repeat span is + a database read — across restarts — until new archive days arrive. Returns a + compact per-day shape (see grading.grade_range). New in API v2. + """ + cell = grid.snap(lat, lon) + + with audit.RunAudit( + endpoint="calendar", + lat=round(lat, 4), + lon=round(lon, 4), + recent_days=months * 31, # approximate span graded + cell_id=cell["id"], + ) as run: + history, cache_meta, _ = _fetch_history(run, cell) + start_ts, end_ts = views.cal_span(history, start, end, months) + + def build(place): + payload = views.build_calendar(cell, history, start_ts, end_ts, months, place, run) + full = not cache_meta.get("cached", False) + run.set(run_type="full" if full else "partial", + history_source="fetch" if full else "cache") + return payload + + # Compare loads several cells at once, so a transient reverse-geocode miss + # is most likely here; cache_placeless=False keeps that miss un-persisted. + return _cached_response(request, run, "calendar", cell, + views.calendar_key(start_ts, end_ts, months), + views.history_token(history), build, + cache_placeless=False) + + +def api_day( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), + date: str | None = Query(None, description="target date YYYY-MM-DD (default = latest available)"), +): + """Full percentile breakdown for a single day: the value at every tier boundary + in that day-of-year's ±7-day window, plus where the observed values land. + + Powers the single-day detail page. Grades against the cached history record; + for a date newer than the cache it pulls the recent window for the observation + (those payloads expire hourly — the recent bundle's own cadence — while fully + archived days stay valid until the record itself advances). New in API v2. + """ + cell = grid.snap(lat, lon) + + with audit.RunAudit( + endpoint="day", + lat=round(lat, 4), + lon=round(lon, 4), + cell_id=cell["id"], + ) as run: + history, cache_meta, _ = _fetch_history(run, cell) + last = history["date"].max() + target = datetime.date.fromisoformat(date[:10]) if date else last + run.set(target_date=target.isoformat()) + + def build(place): + payload = views.build_day(cell, history, target, place, run) + full = not cache_meta.get("cached", False) + run.set(run_type="full" if full else "partial", + history_source="fetch" if full else "cache") + return payload + + return _cached_response(request, run, "day", cell, + views.day_key(target), views.day_token(history, target), + build) + + +def api_score( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), +): + """Climate-drift score for a location: how far the last 6 years have moved + from the full 45-year baseline, per metric, percentile category and season, + rolled into per-metric and overall scores. + + Derived purely from the archive record, so it is cached until the record's + tail advances (the hourly top-up); the scoring-math version is baked into the + cache key so a math change invalidates only score rows. New in API v2. + """ + cell = grid.snap(lat, lon) + + with audit.RunAudit(endpoint="score", lat=round(lat, 4), lon=round(lon, 4), + cell_id=cell["id"]) as run: + history, cache_meta, _ = _fetch_history(run, cell) + + def build(place): + payload = views.build_score(cell, history, place, run) + full = not cache_meta.get("cached", False) + run.set(run_type="full" if full else "partial", + history_source="fetch" if full else "cache") + return payload + + return _cached_response(request, run, "score", cell, + views.score_key(), views.history_token(history), build) + + +def api_forecast( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), + days: int = Query(7, ge=1, le=14, description="how many forecast days ahead to grade"), +): + """Grade the next `days` of forecast weather against local climatology. + + Same shape as /grade (so the frontend renders it identically), but `recent` + holds the forward forecast — furthest-out day first. The forecast is fetched + fresh and cached only ~1 hour (see climate.get_recent_forecast) to track + updates; the graded payload's validity is tied to that fetch stamp, so it + expires exactly when a new forecast lands. New in API v2. + """ + cell = grid.snap(lat, lon) + + with audit.RunAudit(endpoint="forecast", lat=round(lat, 4), lon=round(lon, 4), + recent_days=days, cell_id=cell["id"]) as run: + history, _, fc = _fetch_history(run, cell, recent_too=True, recent_phase="forecast") + today = datetime.date.today() + + def build(place): + payload = views.build_forecast(cell, days, history, fc, today, place, run) + run.set(run_type="partial") + return payload + + return _cached_response(request, run, "forecast", cell, + views.forecast_key(today, days), + views.recent_token(history, cell["id"]), build) + + +def api_cell( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lon: float = Query(..., ge=-180, le=180), + prefetch: int = Query(0, ge=0, le=1, + description="1 = warm-only: never fetch weather upstream; 204 for a cold cell"), + neighbors: int = Query(0, ge=0, le=1, + description="1 = also warm the 8 surrounding cells in the background " + "(warm-only: cold neighbors are skipped, no upstream quota)"), +): + """One bundle carrying every view's payload for a cell, so the frontend warms + all views with a single request instead of four. + + Each slice is the EXACT payload its per-view endpoint returns — built by the + same builders and cached under the same derived-store keys/tokens — paired + with the etag that endpoint would emit. The client seeds its per-view cache + from the slices and later revalidates each view individually with + If-None-Match, so the bundle and the per-view endpoints stay one cache. + + prefetch=1 is the neighbor-warming mode with a hard guarantee: it never + spends weather-API quota. A cell with no cached archive answers 204 (no + body), and only the history-derived slices (calendar + latest-day detail) + are built — grading recent/forecast days needs the hourly upstream bundle. + Reverse geocoding makes at most one Nominatim call for a never-labeled cell; + the client staggers neighbor prefetches to respect that service. New in API v2. + """ + cell = grid.snap(lat, lon) + today = datetime.date.today() + + with audit.RunAudit(endpoint="cell", lat=round(lat, 4), lon=round(lon, 4), + cell_id=cell["id"], prefetch=bool(prefetch)) as run: + recent = None + if prefetch: + history = climate.load_cached_history(cell) + if history is None or history.is_empty(): + run.set(run_type="cold-skip") + return Response(status_code=204) + cache_meta = {"cached": True} + else: + history, cache_meta, recent = _fetch_history(run, cell, recent_too=True) + + cid = cell["id"] + last = history["date"].max() + + if neighbors: + _enqueue_neighbor_warming(cell) + + with run.phase("reverse_geocode"): + place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"]) + + def slice_for(kind: str, key: str, token: str, build) -> dict: + """The endpoint's derived row (or build + store it), paired with the + etag that endpoint would emit for the same request.""" + etag = _etag_for(kind, cid, key, token) + data = store.get_json(kind, cid, key, token) + if data is None: + data = build() + store.put_payload(kind, cid, key, token, data) + return {"etag": etag, "data": data} + + slices = {} + hist_token = views.history_token(history) + # Calendar: the default last-24-months span — what the calendar view (and + # the cross-view prefetch) requests first. + start_ts, end_ts = views.cal_span(history, None, None, 24) + slices["calendar"] = slice_for( + "calendar", views.calendar_key(start_ts, end_ts, 24), hist_token, + lambda: views.build_calendar(cell, history, start_ts, end_ts, 24, place, run)) + + if prefetch: + # Latest archived day: its observation comes from history alone, so + # it's buildable without the (skipped) recent bundle. + slices["day"] = slice_for( + "day", views.day_key(last), hist_token, + lambda: views.build_day(cell, history, last, place, run)) + else: + rf_token = views.recent_token(history, cid) + slices["grade"] = slice_for( + "grade", views.grade_key(today, 14, 14), rf_token, + lambda: views.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14)) + slices["forecast"] = slice_for( + "forecast", views.forecast_key(today, 7), rf_token, + lambda: views.build_forecast(cell, 7, history, recent, today, place, run)) + slices["day"] = slice_for( + "day", views.day_key(today), views.day_token(history, today), + lambda: views.build_day(cell, history, today, place, run, recent=recent)) + + # The bundle's identity is the combination of its slices' identities. + etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""), + "|".join(s["etag"] for s in slices.values())) + if _not_modified(request, etag): + run.set(run_type="cache", not_modified=True) + return Response(status_code=304, headers={"ETag": etag}) + run.set(run_type="bundle", slices=sorted(slices)) + payload = { + "api_version": "v2", + "cell": cell, + "place": place, + "today": today.isoformat(), + "slices": slices, + } + # Not stored as its own derived row — the slices already are. + return _json_response(_encode(payload), etag) + + +# --- ops metrics (gated) --------------------------------------------------- +def _metrics_allowed(request: Request) -> bool: + """Guard the ops metrics: a valid token allows from anywhere (for SSH tunnels); + with no token it's direct-loopback-only, and any forwarded header (i.e. proxied + via Caddy) is refused — so prod ops data never leaks through the public domain.""" + token = os.environ.get("THERMOGRAPH_METRICS_TOKEN", "").strip() + if token: + supplied = request.headers.get("x-metrics-token") or request.query_params.get("token") or "" + if hmac.compare_digest(supplied, token): + return True + client = request.client.host if request.client else None + forwarded = any(h in request.headers for h in + ("x-forwarded-for", "x-forwarded-host", "x-forwarded-proto")) + return client in ("127.0.0.1", "::1") and not forwarded + + +def api_metrics(request: Request): + """Live in-process traffic counters + worker facts for the ops dashboard.""" + if not _metrics_allowed(request): + raise HTTPException(status_code=404) # 404, not 403 — don't reveal the route + snap = metrics.snapshot() + snap["warm_queue_depth"] = _warm_queue.qsize() + snap["warm_seen"] = len(_WARM_SEEN) + snap["threads"] = threading.active_count() + snap["thread_names"] = sorted(t.name for t in threading.enumerate()) + return snap + + +async def api_event(request: Request) -> Response: + """Record one product event (a tap on "use my location", a digest signup, …). + + Deliberately minimal: the body carries only an allowlisted event name. The + referrer is read from this request's own Referer header — a client-supplied + one would be forgeable and buys nothing — and is reduced to a bare domain. + No cookies, no per-visitor id, no full URLs. + + Always answers 204, whether the event was counted, rejected by the allowlist, + or dropped by the rate limiter, so probing the endpoint reveals nothing. That + also suits navigator.sendBeacon, which ignores the response. + """ + name = "" + try: + body = await request.json() + if isinstance(body, dict): + name = str(body.get("event") or "") + except Exception: # noqa: BLE001 - a junk body is just a no-op event + pass + if name: + metrics.record_event( + name, + referer=request.headers.get("referer"), + host=request.headers.get("host"), + ip=_client_ip(request), + ) + return Response(status_code=204) + + +# --- API versioning -------------------------------------------------------- +# Non-backward-compatible additions land in a new version; every version stays +# mounted and served simultaneously. v1 is the original contract (grade/geocode); +# v2 adds the calendar. The unversioned /api/* paths are kept as aliases of v1 so +# existing clients keep working. +v1 = APIRouter(tags=["v1"]) +v1.add_api_route("/geocode", api_geocode, methods=["GET"]) +v1.add_api_route("/grade", api_grade, methods=["GET"]) + +v2 = APIRouter(tags=["v2"]) +v2.add_api_route("/geocode", api_geocode, methods=["GET"]) +v2.add_api_route("/suggest", api_suggest, methods=["GET"]) +v2.add_api_route("/place", api_place, methods=["GET"]) +v2.add_api_route("/grade", api_grade, methods=["GET"]) +v2.add_api_route("/calendar", api_calendar, methods=["GET"]) +v2.add_api_route("/day", api_day, methods=["GET"]) +v2.add_api_route("/score", api_score, methods=["GET"]) +v2.add_api_route("/forecast", api_forecast, methods=["GET"]) +v2.add_api_route("/cell", api_cell, methods=["GET"]) +v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False) +v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False) + +app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible) +app.include_router(v1, prefix=f"{BASE}/api/v1") +app.include_router(v2, prefix=f"{BASE}/api/v2") + + +# --- Discord slash commands (HTTP interactions) ---------------------------- +async def discord_interactions(request: Request) -> Response: + """Discord posts each slash-command interaction here. Verification runs over the + RAW body, so this must read bytes, not request.json().""" + raw = await request.body() + status, payload = discord_interactions_mod.handle( + raw, + request.headers.get("x-signature-ed25519", ""), + request.headers.get("x-signature-timestamp", ""), + ) + return Response(json.dumps(payload), status_code=status, media_type="application/json") + +app.add_api_route(f"{BASE}/discord/interactions", discord_interactions, + methods=["POST"], include_in_schema=False) + +# --- accounts (fastapi-users) ---------------------------------------------- +# Library-provided routers: /auth/login + /auth/logout (cookie session), +# /auth/register (signup), and /users/me (session check / profile). All under the +# same v2 prefix as the rest of the API, so the frontend calls them base-relative. +app.include_router( + users.fastapi_users.get_auth_router(users.auth_backend), + prefix=f"{BASE}/api/v2/auth", tags=["auth"], +) +app.include_router( + users.fastapi_users.get_register_router(UserRead, UserCreate), + prefix=f"{BASE}/api/v2/auth", tags=["auth"], +) +app.include_router( + users.fastapi_users.get_users_router(UserRead, UserUpdate), + prefix=f"{BASE}/api/v2/users", tags=["users"], +) +# Subscriptions + notifications (authed, user-scoped). +app.include_router(api_accounts.router, prefix=f"{BASE}/api/v2") +# Discord account linking (OAuth2, authed). +app.include_router(discord_link.router, prefix=f"{BASE}/api/v2") + + +# --- static frontend (served under BASE) ----------------------------------- +def _page(name): + """Route handler for one HTML page. Serves the file with its __ORIGIN__ + placeholders (the link-preview/Open Graph tags) filled in as the request's + scheme://host + BASE — preview crawlers (Discord, Slack, …) need absolute + URLs, and the host differs between LAN and prod. The scheme comes from + X-Forwarded-Proto when a reverse proxy (Caddy) fronts the plain-HTTP + uvicorn; the proxy passes the original Host header through untouched.""" + path = os.path.join(FRONTEND_DIR, name) + + def route(request: Request): + with open(path, encoding="utf-8") as f: + html = f.read() + proto = request.headers.get("x-forwarded-proto") or request.url.scheme + host = request.headers.get("host") or request.url.netloc + html = html.replace("__ORIGIN__", f"{proto}://{host}{BASE}") + # Search-engine verification tags (same source as the SEO pages), so + # the homepage Google/Bing verify against carries them too. + verify = content.head_verify_html() + if verify: + html = html.replace("", f"\n {verify}", 1) + etag = f'W/"{hashlib.sha1(html.encode()).hexdigest()[:20]}"' + if _not_modified(request, etag): + return Response(status_code=304, headers={"ETag": etag}) + return Response(html, media_type="text/html", headers={"ETag": etag}) + return route + + +# The un-slashed base redirects to BASE/ so the frontend's relative asset URLs +# resolve correctly. Under a sub-path this keeps the app scoped to BASE (never +# claiming "/", so the domain root stays free for another app via the proxy). When +# BASE is "" the app owns the root: "/" is already the index route below, so there +# is no bare base to redirect (and "" is not a valid route path). +# Pages also answer HEAD — link-preview crawlers probe with it before fetching. +if BASE: + app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False) +# NOTE: "/" is not here — the homepage is server-rendered by content.register() +# below, so its hero, records strip and city links reach crawlers and no-JS +# readers as real HTML. +app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False) +app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False) +app.add_api_route(f"{BASE}/score", _page("score.html"), methods=["GET", "HEAD"], include_in_schema=False) +app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False) +app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False) +app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False) + +# Monthly-digest signup (the footer form on every page). Answers the same generic +# message whatever the outcome, so it can't be probed for whether an address is +# already subscribed. +app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False) + +# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city / +# month / records / glossary / about / privacy) + robots.txt + sitemap.xml. +# Registered before the static mount so these explicit routes win. +content.register(app) + +# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset. +# Registered last so the explicit page routes above win. Mounts must start with +# "/", so at the root (BASE == "") mount at "/" — the earlier routes still win +# because Starlette matches in registration order. +app.mount(BASE or "/", StaticFiles(directory=FRONTEND_DIR), name="static") diff --git a/content.py b/web/content.py similarity index 99% rename from content.py rename to web/content.py index de414bc..34a22eb 100644 --- a/content.py +++ b/web/content.py @@ -20,18 +20,19 @@ from fastapi.responses import PlainTextResponse from jinja2 import Environment, FileSystemLoader, select_autoescape from markupsafe import Markup -import cities -import city_events -import climate -import grading -import grid -import homepage -from views import OBS_COLS +from data import cities +from data import city_events +from data import climate +from data import grading +from data import grid +from web import homepage +import paths +from web.views import OBS_COLS _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") BASE = f"/{_BASE}" if _BASE else "" -TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates") +TEMPLATES_DIR = paths.TEMPLATES_DIR _env = Environment( loader=FileSystemLoader(TEMPLATES_DIR), autoescape=select_autoescape(["html", "xml", "j2"]), @@ -359,8 +360,8 @@ def _content_lastmod() -> str: city list and this module (both change on a content/code rebuild, and the process restarts on deploy). Honest and stable, unlike a per-request today() that churns every fetch and trains crawlers to ignore lastmod entirely.""" - paths = [os.path.join(os.path.dirname(__file__), "cities.json"), __file__] - mtimes = [os.path.getmtime(p) for p in paths if os.path.exists(p)] + srcs = [paths.CITIES_JSON, __file__] + mtimes = [os.path.getmtime(p) for p in srcs if os.path.exists(p)] day = datetime.date.fromtimestamp(max(mtimes)) if mtimes else datetime.date.today() return day.isoformat() diff --git a/homepage.py b/web/homepage.py similarity index 98% rename from homepage.py rename to web/homepage.py index a38205f..91dc324 100644 --- a/homepage.py +++ b/web/homepage.py @@ -27,15 +27,16 @@ import time import polars as pl -import cities -import climate -import grading -import grid -from views import OBS_COLS +from data import cities +from data import climate +from data import grading +from data import grid +import paths +from web.views import OBS_COLS # data/ is the only writable path under the hardened systemd unit # (ReadWritePaths=/opt/thermograph/data …), so the feed lives there. -FEED_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "data", "homepage.json") +FEED_PATH = os.path.join(paths.DATA_DIR, "homepage.json") # How many cards the "Unusual right now" strip can show. RANK_LIMIT = 12 diff --git a/schemas.py b/web/schemas.py similarity index 99% rename from schemas.py rename to web/schemas.py index ebc0247..0898910 100644 --- a/schemas.py +++ b/web/schemas.py @@ -10,7 +10,7 @@ from typing import Literal from fastapi_users import schemas from pydantic import BaseModel, Field, field_validator -import grading +from data import grading # Canonical metric keys a subscription may watch (kept in lockstep with grading). ALLOWED_METRICS = tuple(grading.CLIMO_METRICS) diff --git a/views.py b/web/views.py similarity index 99% rename from views.py rename to web/views.py index c62ceb5..dfd94f5 100644 --- a/views.py +++ b/web/views.py @@ -13,9 +13,9 @@ import time import polars as pl -import climate -import grading -import scoring +from data import climate +from data import grading +from data import scoring # Observed values pulled from a daily record row for grading. Includes the # temperature-scale metrics (tmax/tmin/feels/wind/gust) plus precip; a column may