Jinemi/thermograph
4
0
Fork 0
Code Issues 2 Pull requests 3 Projects Releases Packages Wiki Activity Actions

Split the backend into domain packages (#217)

Browse source 
* Centralize filesystem paths in a single module

Add paths.py, which resolves the repo root once and derives the cache,
accounts DB, logs, templates, frontend and bundled-city-data locations
from it. Replace the 13 per-module `dirname(__file__)/..` anchors with
references to it, so a module's location no longer determines where the
app reads its data. Env overrides (accounts DB, VAPID, IndexNow) are
unchanged; every resolved path is byte-identical to before.

Groundwork for moving modules into packages without re-pointing paths.

Claude-Session: https://claude.ai/code/session_01XXxmNFy9cZ6Gh8Y9thZn62

* Split the backend into domain packages

Group the flat backend modules into packages that mirror their concerns:
  data/          climate, grading, scoring, grid, places, cities,
                 city_events, store
  web/           app, views, homepage, content, schemas
  notifications/ notify, digest, push, mailer, discord,
                 discord_interactions, discord_link
  accounts/      models, users, api_accounts, db
  core/          metrics, singleton, audit

Intra-project imports are rewritten to the package-qualified form. The
entry scripts (indexnow, warm_cities, migrate, gen_cities, gen_flavor)
and paths.py stay at the backend/ root, and backend/app.py becomes a
shim re-exporting web.app:app so the launch target stays `app:app` —
run.sh, the systemd units, and CI need no change.

Verified: full suite (318) passes, `uvicorn app:app` boots and serves
the home/SEO/static/API surfaces, and every root script imports clean.

Claude-Session: https://claude.ai/code/session_01XXxmNFy9cZ6Gh8Y9thZn62
This commit is contained in:
Emi Griffith 2026-07-19 22:31:03 -07:00 committed by GitHub
parent af57b89e4b
commit d17ac794fd
58 changed files with 1001 additions and 931 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
accounts/__init__.py Normal file
View file

@ -0,0 +1 @@
"""accounts package."""

14
api_accounts.py → accounts/api_accounts.py
View file

@ -12,12 +12,12 @@ from fastapi.concurrency import run_in_threadpool
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
import climate
import grid
import push
from db import get_async_session
from models import Notification, PushSubscription, Subscription
from schemas import (
from data import climate
from data import grid
from notifications import push
from accounts.db import get_async_session
from accounts.models import Notification, PushSubscription, Subscription
from web.schemas import (
NotificationList,
NotificationOut,
PushSubscriptionIn,
@ -26,7 +26,7 @@ from schemas import (
SubscriptionOut,
SubscriptionPatch,
)
from users import current_active_user
from accounts.users import current_active_user
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""

8
db.py → accounts/db.py
View file

@ -23,11 +23,11 @@ from sqlalchemy import create_engine, event
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase, sessionmaker
import paths
# Default to data/accounts.sqlite; THERMOGRAPH_ACCOUNTS_DB overrides it (tests point
# this at a throwaway temp file to stay hermetic).
DB_PATH = os.environ.get("THERMOGRAPH_ACCOUNTS_DB") or os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "data", "accounts.sqlite")
)
DB_PATH = os.environ.get("THERMOGRAPH_ACCOUNTS_DB") or os.path.join(paths.DATA_DIR, "accounts.sqlite")
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
@ -71,7 +71,7 @@ async def create_db_and_tables() -> None:
Imported for its side effect of registering the mapped classes on Base.metadata
before create_all runs.
"""
import models # noqa: F401 (registers tables on Base.metadata)
from accounts import models # noqa: F401 (registers tables on Base.metadata)
async with async_engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)

2
models.py → accounts/models.py
View file

@ -25,7 +25,7 @@ from sqlalchemy import (
)
from sqlalchemy.orm import Mapped, mapped_column
from db import Base
from accounts.db import Base
class User(SQLAlchemyBaseUserTableUUID, Base):

4
users.py → accounts/users.py
View file

@ -16,8 +16,8 @@ from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase
from fastapi_users_db_sqlalchemy.access_token import SQLAlchemyAccessTokenDatabase
from sqlalchemy.ext.asyncio import AsyncSession
from db import get_async_session
from models import AccessToken, User
from accounts.db import get_async_session
from accounts.models import AccessToken, User
BASE = "/" + os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
# Only used to sign reset/verification tokens — features that stay dormant until

785
app.py
View file

@ -1,782 +1,7 @@
"""Thermograph API — grade recent local weather against ~45 years of climatology."""
import contextlib
import datetime
import functools
import hashlib
import hmac
import json
import mimetypes
import os
import queue
import threading
import time
"""Entry-point shim.
from fastapi import APIRouter, FastAPI, HTTPException, Query, Request, Response
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
import api_accounts
import audit
import climate
import content
import digest
import discord_interactions as discord_interactions_mod
import discord_link
import db
import grid
import metrics
import notify
import places
import singleton
import store
import users
import views
from schemas import UserCreate, UserRead, UserUpdate
FRONTEND_DIR = os.path.join(os.path.dirname(__file__), "..", "frontend")
# Everything (pages, assets, API) is served under this base path so the app can
# live at https://<host>/thermograph/ behind a shared domain. Override with the
# THERMOGRAPH_BASE env var. The frontend uses base-relative URLs, so it follows
# this automatically without hardcoding the prefix.
#
# THERMOGRAPH_BASE=/ (or empty) means the app owns the whole domain: BASE becomes
# "" so every route sits at the domain root ("/", "/calendar", "/api/v2/…") with
# no prefix and no leading double slash. A non-empty value keeps the sub-path form
# ("/thermograph").
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""
# The PWA manifest is served as a static file; register its media type so
# StaticFiles labels it correctly (not in Python's default mimetypes map).
mimetypes.add_type("application/manifest+json", ".webmanifest")
def _weather_fetch_error(e) -> HTTPException:
"""A clean, retryable 503 when weather data is temporarily unavailable; the
raw error as a 502 otherwise. The fetch layer classifies its own failures
(climate.WeatherUnavailable carries the user-facing text) so no message
parsing happens here; the is_rate_limit fallback covers a raw upstream 429
from a fetcher that didn't classify it."""
if isinstance(e, climate.WeatherUnavailable):
return HTTPException(status_code=503, detail=str(e))
if climate.is_rate_limit(e):
return HTTPException(status_code=503, detail=climate.limit_message(False))
return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}")
# --- background neighbor warming ---------------------------------------------
# /cell?neighbors=1 asks the server to warm the 8 grid cells around the request
# (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's
# snapping math in JS). One worker drains the queue so warms never stack; the
# hard guarantee matches prefetch=1 — a cell with no cached archive is skipped,
# so no weather-API quota is ever spent, and reverse_geocode itself paces the
# at-most-one Nominatim call per never-labeled cell (~1/s policy).
_WARM_TTL = 3600.0 # don't re-enqueue a cell within the hour
_WARM_SEEN: dict[str, float] = {} # cell_id -> last enqueue time
_warm_queue: "queue.Queue[dict]" = queue.Queue()
def _warm_cell(cell: dict) -> None:
"""Materialize the history-derived slices for one cell — the same rows the
prefetch=1 bundle serves. Never fetches weather upstream."""
history = climate.load_cached_history(cell)
if history is None or history.is_empty():
return
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
token = views.history_token(history)
start_ts, end_ts = views.cal_span(history, None, None, 24)
cal_key = views.calendar_key(start_ts, end_ts, 24)
if store.get_payload("calendar", cell["id"], cal_key, token) is None:
store.put_payload("calendar", cell["id"], cal_key, token,
views.build_calendar(cell, history, start_ts, end_ts, 24, place))
last = history["date"].max()
day_key = views.day_key(last)
if store.get_payload("day", cell["id"], day_key, token) is None:
store.put_payload("day", cell["id"], day_key, token,
views.build_day(cell, history, last, place))
def _enqueue_neighbor_warming(cell: dict) -> None:
now = time.time()
for n in grid.neighbors(cell):
if now - _WARM_SEEN.get(n["id"], 0.0) < _WARM_TTL:
continue
_WARM_SEEN[n["id"]] = now
_warm_queue.put(n)
def _neighbor_warmer() -> None:
while True:
cell = _warm_queue.get()
try:
_warm_cell(cell)
except Exception: # noqa: BLE001 - warming is best-effort, never fatal
pass
finally:
_warm_queue.task_done()
@contextlib.asynccontextmanager
async def _lifespan(app):
# Warm the local place-name index (/suggest's typo tolerance) in the
# background; the app boots and serves fine without it — suggestions just
# fall back to the upstream geocoder until it's ready. A server-startup
# hook (not import time) so offline importers (tests, the migrate script's
# dependencies) don't kick off a GeoNames download. The neighbor warmer is
# also a server concern: enqueues before startup just wait in the queue.
# Create the account DB tables (users/sessions/subscriptions/notifications)
# before serving — a fast no-op once they exist.
await db.create_db_and_tables()
places.start_loading()
threading.Thread(target=_neighbor_warmer, name="neighbor-warmer", daemon=True).start()
# Background subscription evaluator. Its periodic sweep fetches upstream on a timer
# (not per request), so with multiple workers it must run in exactly one — elect a
# leader via a lockfile (THERMOGRAPH_SINGLETON_LOCK). Unset (single worker / dev /
# tests) => always the leader. The neighbor warmer above stays per-worker: each
# drains its own request-fed queue.
if singleton.claim(os.environ.get("THERMOGRAPH_SINGLETON_LOCK")):
notify.start()
yield
notify.stop()
app = FastAPI(title="Thermograph", version="0.2.0", lifespan=_lifespan)
# Compress every sizeable response (the 2-year calendar JSON shrinks ~6-8×).
# Applies to API JSON and static assets alike; tiny responses are left alone.
app.add_middleware(GZipMiddleware, minimum_size=1024)
def _client_ip(request) -> "str | None":
"""The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us
(Caddy on prod sets it), otherwise the direct peer address (LAN dev)."""
xff = request.headers.get("x-forwarded-for")
if xff:
return xff.split(",")[0].strip()
return request.client.host if request.client else None
@app.middleware("http")
async def revalidate_static(request, call_next):
"""Serve the frontend with no-cache (NOT no-store): browsers may keep a copy
but must revalidate it on every use. Starlette's FileResponse/StaticFiles
already emit ETag + Last-Modified, so an unchanged asset costs one conditional
request answered with an empty 304 instead of a full re-download — page-to-page
navigation stops re-transferring the JS/CSS/HTML while still picking up every
deploy immediately (no "my change isn't showing" bugs)."""
response = await call_next(request)
path = request.url.path
try:
cat = metrics.classify_inbound(path, BASE)
metrics.record_inbound(cat, response.status_code)
# Retain per-request client IPs for later analysis; skip static assets and the
# dashboard's own metrics polling to keep the log to real, meaningful traffic.
if cat not in ("static", "metrics", "event"):
audit.log_access({"ip": _client_ip(request), "method": request.method,
"path": path, "status": response.status_code, "cat": cat})
except Exception: # noqa: BLE001 - never let instrumentation break a response
pass
pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/score",
f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy")
if path.endswith((".js", ".css", ".html")) or path in pages:
response.headers["Cache-Control"] = "no-cache"
return response
# --- derived-store plumbing --------------------------------------------------
# Every graded payload is cached in SQLite under (kind, cell, key) and validated
# by a token that encodes what it was computed from (see store.py). The freshness
# drivers stay where they were — climate.get_history tops up the archive tail
# hourly and get_recent_forecast refetches hourly — and the tokens are derived
# from what those return, so a cached payload expires exactly when its inputs
# change and never before. The same tokens double as ETags: a client sending
# If-None-Match gets an empty 304 without the payload even being loaded.
# The payloads themselves are assembled in views.py, shared with the offline
# migrate script.
def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str:
"""Deterministic weak ETag from a payload's identity + validity token. Weak
because the same content may be served under different encodings (gzip)."""
h = hashlib.sha1(f"{kind}:{cell_id}:{key}:{token}".encode()).hexdigest()[:20]
return f'W/"{h}"'
def _not_modified(request: Request, etag: str) -> bool:
"""Does the client already hold this exact payload? Answerable from the token
alone — no payload load needed."""
inm = request.headers.get("if-none-match")
if not inm:
return False
tags = {t.strip() for t in inm.split(",")}
return "*" in tags or etag in tags or etag.removeprefix("W/") in tags
def _encode(payload: dict) -> bytes:
"""Compact JSON bytes, matching store.put_payload's encoding (allow_nan=False
mirrors Starlette — a NaN from grading should fail loudly, not reach a client)."""
return json.dumps(payload, default=str, allow_nan=False, separators=(",", ":")).encode()
def _json_response(body: bytes, etag: str | None = None) -> Response:
headers = {"ETag": etag} if etag else {}
return Response(content=body, media_type="application/json", headers=headers)
def _fetch_history(run, cell, recent_too=False, recent_phase="recent"):
"""The shared fetch preamble for every data route: the archive record (and
optionally the hourly recent/forecast bundle) with upstream failures mapped
to clean HTTP errors and an empty record to a 404."""
try:
with run.phase("history"):
history, cache_meta = climate.get_history(cell)
recent = None
if recent_too:
with run.phase(recent_phase):
recent = climate.get_recent_forecast(cell)
except Exception as e: # noqa: BLE001
raise _weather_fetch_error(e)
if history.is_empty():
raise HTTPException(status_code=404, detail="No historical data for this cell.")
return history, cache_meta, recent
def _cached_response(request, run, kind, cell, key, token, build, cache_placeless=True):
"""The shared derived-store flow behind every per-view endpoint:
If-None-Match -> empty 304; a token-valid store row -> replay its exact
bytes; otherwise resolve the place label, build the payload, persist, serve.
``build(place) -> payload`` does any endpoint-specific audit tagging itself.
``cache_placeless=False`` skips persisting a payload whose place label
failed to resolve (a transient reverse-geocode miss) — otherwise the
bare-coordinates fallback would stick for the life of the token."""
cid = cell["id"]
etag = _etag_for(kind, cid, key, token)
if _not_modified(request, etag):
run.set(run_type="cache", not_modified=True)
return Response(status_code=304, headers={"ETag": etag})
body = store.get_payload(kind, cid, key, token)
if body is not None:
run.set(run_type="cache", history_source="store")
return _json_response(body, etag)
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
payload = build(place)
return _json_response(
store.put_payload(kind, cid, key, token, payload,
cache=cache_placeless or place is not None), etag)
# --- endpoints ----------------------------------------------------------------
def api_geocode(q: str = Query(..., min_length=1)):
try:
return {"results": climate.geocode(q)}
except Exception as e: # noqa: BLE001 - surface upstream failures to the client
raise HTTPException(status_code=502, detail=f"geocoding failed: {e}")
_SUGGEST_LIMIT = 5
@functools.lru_cache(maxsize=1024)
def _suggest_upstream(q: str) -> tuple:
"""Open-Meteo lookup for /suggest, memoized per query string — type-ahead
re-asks the same prefixes constantly (backspacing, retyping). Failures
raise and are not cached, so a transient upstream error doesn't stick."""
return tuple(climate.geocode(q, count=_SUGGEST_LIMIT))
def api_suggest(q: str = Query(..., min_length=1)):
"""Type-ahead location suggestions: the top 5 places for a (possibly
typo'd) query prefix; `corrected` reports the respelling that produced the
results ("pest seattle" → "west seattle"), or null. The ranking/typo policy
lives in places.suggest."""
try:
results, corrected = places.suggest(q, _suggest_upstream, _SUGGEST_LIMIT)
except Exception as e: # noqa: BLE001 - nothing at all to serve
raise HTTPException(status_code=502, detail=f"geocoding failed: {e}")
return {"results": results, "corrected": corrected}
def api_place(
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
):
"""Best-effort neighbourhood/city label for a point — the same string the view
endpoints expose as ``place`` (snapped to the grid cell, reverse-geocoded and
cached). Resolved on its own so the compare page can show a location's name as
soon as it's added, before its full series loads."""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="place", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"]) as run:
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
return {"place": place,
"cell": {"center_lat": cell["center_lat"], "center_lon": cell["center_lon"]}}
def api_grade(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
date: str | None = Query(None, description="target date YYYY-MM-DD (default today)"),
days: int = Query(14, ge=1, le=60, description="days of history to grade before the target"),
after: int = Query(14, ge=0, le=14,
description="days to grade after the target (observed, or forecast when future; "
"the forecast reaches ~7 days out so future days cap there)"),
):
target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today()
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="grade",
lat=round(lat, 4),
lon=round(lon, 4),
target_date=target.isoformat(),
recent_days=days,
cell_id=cell["id"],
) as run:
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
return _cached_response(
request, run, "grade", cell,
views.grade_key(target, days, after), views.recent_token(history, cell["id"]),
lambda place: views.build_grade(cell, target, days, history, recent,
cache_meta, place, run, after=after))
def api_calendar(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
start: str | None = Query(None, description="first date YYYY-MM-DD (overrides months)"),
end: str | None = Query(None, description="last date YYYY-MM-DD (default = latest available)"),
months: int = Query(24, ge=1, le=24, description="months back to grade when no start given"),
):
"""Grade every day over a date range (default: last 2 years) for the calendar view.
A custom [start, end] range is supported and capped at ~2 years per request;
the frontend splits longer spans into successive 2-year chunks. Grades against
the already-cached history record (which reaches to ~6 days ago), so no extra
fetch is needed. The graded payload is cached in the derived store keyed by
the clamped span and validated by the record's end date, so a repeat span is
a database read — across restarts — until new archive days arrive. Returns a
compact per-day shape (see grading.grade_range). New in API v2.
The real application lives in ``web/app.py``. This re-export keeps the launch
target stable at ``app:app`` (run from ``backend/``), so the systemd units,
run.sh, and CI need no change after the move into packages.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="calendar",
lat=round(lat, 4),
lon=round(lon, 4),
recent_days=months * 31, # approximate span graded
cell_id=cell["id"],
) as run:
history, cache_meta, _ = _fetch_history(run, cell)
start_ts, end_ts = views.cal_span(history, start, end, months)
def build(place):
payload = views.build_calendar(cell, history, start_ts, end_ts, months, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
# Compare loads several cells at once, so a transient reverse-geocode miss
# is most likely here; cache_placeless=False keeps that miss un-persisted.
return _cached_response(request, run, "calendar", cell,
views.calendar_key(start_ts, end_ts, months),
views.history_token(history), build,
cache_placeless=False)
def api_day(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
date: str | None = Query(None, description="target date YYYY-MM-DD (default = latest available)"),
):
"""Full percentile breakdown for a single day: the value at every tier boundary
in that day-of-year's ±7-day window, plus where the observed values land.
Powers the single-day detail page. Grades against the cached history record;
for a date newer than the cache it pulls the recent window for the observation
(those payloads expire hourly — the recent bundle's own cadence — while fully
archived days stay valid until the record itself advances). New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="day",
lat=round(lat, 4),
lon=round(lon, 4),
cell_id=cell["id"],
) as run:
history, cache_meta, _ = _fetch_history(run, cell)
last = history["date"].max()
target = datetime.date.fromisoformat(date[:10]) if date else last
run.set(target_date=target.isoformat())
def build(place):
payload = views.build_day(cell, history, target, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
return _cached_response(request, run, "day", cell,
views.day_key(target), views.day_token(history, target),
build)
def api_score(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
):
"""Climate-drift score for a location: how far the last 6 years have moved
from the full 45-year baseline, per metric, percentile category and season,
rolled into per-metric and overall scores.
Derived purely from the archive record, so it is cached until the record's
tail advances (the hourly top-up); the scoring-math version is baked into the
cache key so a math change invalidates only score rows. New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="score", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"]) as run:
history, cache_meta, _ = _fetch_history(run, cell)
def build(place):
payload = views.build_score(cell, history, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
return _cached_response(request, run, "score", cell,
views.score_key(), views.history_token(history), build)
def api_forecast(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
days: int = Query(7, ge=1, le=14, description="how many forecast days ahead to grade"),
):
"""Grade the next `days` of forecast weather against local climatology.
Same shape as /grade (so the frontend renders it identically), but `recent`
holds the forward forecast — furthest-out day first. The forecast is fetched
fresh and cached only ~1 hour (see climate.get_recent_forecast) to track
updates; the graded payload's validity is tied to that fetch stamp, so it
expires exactly when a new forecast lands. New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="forecast", lat=round(lat, 4), lon=round(lon, 4),
recent_days=days, cell_id=cell["id"]) as run:
history, _, fc = _fetch_history(run, cell, recent_too=True, recent_phase="forecast")
today = datetime.date.today()
def build(place):
payload = views.build_forecast(cell, days, history, fc, today, place, run)
run.set(run_type="partial")
return payload
return _cached_response(request, run, "forecast", cell,
views.forecast_key(today, days),
views.recent_token(history, cell["id"]), build)
def api_cell(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
prefetch: int = Query(0, ge=0, le=1,
description="1 = warm-only: never fetch weather upstream; 204 for a cold cell"),
neighbors: int = Query(0, ge=0, le=1,
description="1 = also warm the 8 surrounding cells in the background "
"(warm-only: cold neighbors are skipped, no upstream quota)"),
):
"""One bundle carrying every view's payload for a cell, so the frontend warms
all views with a single request instead of four.
Each slice is the EXACT payload its per-view endpoint returns — built by the
same builders and cached under the same derived-store keys/tokens — paired
with the etag that endpoint would emit. The client seeds its per-view cache
from the slices and later revalidates each view individually with
If-None-Match, so the bundle and the per-view endpoints stay one cache.
prefetch=1 is the neighbor-warming mode with a hard guarantee: it never
spends weather-API quota. A cell with no cached archive answers 204 (no
body), and only the history-derived slices (calendar + latest-day detail)
are built — grading recent/forecast days needs the hourly upstream bundle.
Reverse geocoding makes at most one Nominatim call for a never-labeled cell;
the client staggers neighbor prefetches to respect that service. New in API v2.
"""
cell = grid.snap(lat, lon)
today = datetime.date.today()
with audit.RunAudit(endpoint="cell", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"], prefetch=bool(prefetch)) as run:
recent = None
if prefetch:
history = climate.load_cached_history(cell)
if history is None or history.is_empty():
run.set(run_type="cold-skip")
return Response(status_code=204)
cache_meta = {"cached": True}
else:
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
cid = cell["id"]
last = history["date"].max()
if neighbors:
_enqueue_neighbor_warming(cell)
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
def slice_for(kind: str, key: str, token: str, build) -> dict:
"""The endpoint's derived row (or build + store it), paired with the
etag that endpoint would emit for the same request."""
etag = _etag_for(kind, cid, key, token)
data = store.get_json(kind, cid, key, token)
if data is None:
data = build()
store.put_payload(kind, cid, key, token, data)
return {"etag": etag, "data": data}
slices = {}
hist_token = views.history_token(history)
# Calendar: the default last-24-months span — what the calendar view (and
# the cross-view prefetch) requests first.
start_ts, end_ts = views.cal_span(history, None, None, 24)
slices["calendar"] = slice_for(
"calendar", views.calendar_key(start_ts, end_ts, 24), hist_token,
lambda: views.build_calendar(cell, history, start_ts, end_ts, 24, place, run))
if prefetch:
# Latest archived day: its observation comes from history alone, so
# it's buildable without the (skipped) recent bundle.
slices["day"] = slice_for(
"day", views.day_key(last), hist_token,
lambda: views.build_day(cell, history, last, place, run))
else:
rf_token = views.recent_token(history, cid)
slices["grade"] = slice_for(
"grade", views.grade_key(today, 14, 14), rf_token,
lambda: views.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14))
slices["forecast"] = slice_for(
"forecast", views.forecast_key(today, 7), rf_token,
lambda: views.build_forecast(cell, 7, history, recent, today, place, run))
slices["day"] = slice_for(
"day", views.day_key(today), views.day_token(history, today),
lambda: views.build_day(cell, history, today, place, run, recent=recent))
# The bundle's identity is the combination of its slices' identities.
etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""),
"|".join(s["etag"] for s in slices.values()))
if _not_modified(request, etag):
run.set(run_type="cache", not_modified=True)
return Response(status_code=304, headers={"ETag": etag})
run.set(run_type="bundle", slices=sorted(slices))
payload = {
"api_version": "v2",
"cell": cell,
"place": place,
"today": today.isoformat(),
"slices": slices,
}
# Not stored as its own derived row — the slices already are.
return _json_response(_encode(payload), etag)
# --- ops metrics (gated) ---------------------------------------------------
def _metrics_allowed(request: Request) -> bool:
"""Guard the ops metrics: a valid token allows from anywhere (for SSH tunnels);
with no token it's direct-loopback-only, and any forwarded header (i.e. proxied
via Caddy) is refused — so prod ops data never leaks through the public domain."""
token = os.environ.get("THERMOGRAPH_METRICS_TOKEN", "").strip()
if token:
supplied = request.headers.get("x-metrics-token") or request.query_params.get("token") or ""
if hmac.compare_digest(supplied, token):
return True
client = request.client.host if request.client else None
forwarded = any(h in request.headers for h in
("x-forwarded-for", "x-forwarded-host", "x-forwarded-proto"))
return client in ("127.0.0.1", "::1") and not forwarded
def api_metrics(request: Request):
"""Live in-process traffic counters + worker facts for the ops dashboard."""
if not _metrics_allowed(request):
raise HTTPException(status_code=404) # 404, not 403 — don't reveal the route
snap = metrics.snapshot()
snap["warm_queue_depth"] = _warm_queue.qsize()
snap["warm_seen"] = len(_WARM_SEEN)
snap["threads"] = threading.active_count()
snap["thread_names"] = sorted(t.name for t in threading.enumerate())
return snap
async def api_event(request: Request) -> Response:
"""Record one product event (a tap on "use my location", a digest signup, …).
Deliberately minimal: the body carries only an allowlisted event name. The
referrer is read from this request's own Referer header — a client-supplied
one would be forgeable and buys nothing — and is reduced to a bare domain.
No cookies, no per-visitor id, no full URLs.
Always answers 204, whether the event was counted, rejected by the allowlist,
or dropped by the rate limiter, so probing the endpoint reveals nothing. That
also suits navigator.sendBeacon, which ignores the response.
"""
name = ""
try:
body = await request.json()
if isinstance(body, dict):
name = str(body.get("event") or "")
except Exception: # noqa: BLE001 - a junk body is just a no-op event
pass
if name:
metrics.record_event(
name,
referer=request.headers.get("referer"),
host=request.headers.get("host"),
ip=_client_ip(request),
)
return Response(status_code=204)
# --- API versioning --------------------------------------------------------
# Non-backward-compatible additions land in a new version; every version stays
# mounted and served simultaneously. v1 is the original contract (grade/geocode);
# v2 adds the calendar. The unversioned /api/* paths are kept as aliases of v1 so
# existing clients keep working.
v1 = APIRouter(tags=["v1"])
v1.add_api_route("/geocode", api_geocode, methods=["GET"])
v1.add_api_route("/grade", api_grade, methods=["GET"])
v2 = APIRouter(tags=["v2"])
v2.add_api_route("/geocode", api_geocode, methods=["GET"])
v2.add_api_route("/suggest", api_suggest, methods=["GET"])
v2.add_api_route("/place", api_place, methods=["GET"])
v2.add_api_route("/grade", api_grade, methods=["GET"])
v2.add_api_route("/calendar", api_calendar, methods=["GET"])
v2.add_api_route("/day", api_day, methods=["GET"])
v2.add_api_route("/score", api_score, methods=["GET"])
v2.add_api_route("/forecast", api_forecast, methods=["GET"])
v2.add_api_route("/cell", api_cell, methods=["GET"])
v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False)
v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False)
app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible)
app.include_router(v1, prefix=f"{BASE}/api/v1")
app.include_router(v2, prefix=f"{BASE}/api/v2")
# --- Discord slash commands (HTTP interactions) ----------------------------
async def discord_interactions(request: Request) -> Response:
"""Discord posts each slash-command interaction here. Verification runs over the
RAW body, so this must read bytes, not request.json()."""
raw = await request.body()
status, payload = discord_interactions_mod.handle(
raw,
request.headers.get("x-signature-ed25519", ""),
request.headers.get("x-signature-timestamp", ""),
)
return Response(json.dumps(payload), status_code=status, media_type="application/json")
app.add_api_route(f"{BASE}/discord/interactions", discord_interactions,
methods=["POST"], include_in_schema=False)
# --- accounts (fastapi-users) ----------------------------------------------
# Library-provided routers: /auth/login + /auth/logout (cookie session),
# /auth/register (signup), and /users/me (session check / profile). All under the
# same v2 prefix as the rest of the API, so the frontend calls them base-relative.
app.include_router(
users.fastapi_users.get_auth_router(users.auth_backend),
prefix=f"{BASE}/api/v2/auth", tags=["auth"],
)
app.include_router(
users.fastapi_users.get_register_router(UserRead, UserCreate),
prefix=f"{BASE}/api/v2/auth", tags=["auth"],
)
app.include_router(
users.fastapi_users.get_users_router(UserRead, UserUpdate),
prefix=f"{BASE}/api/v2/users", tags=["users"],
)
# Subscriptions + notifications (authed, user-scoped).
app.include_router(api_accounts.router, prefix=f"{BASE}/api/v2")
# Discord account linking (OAuth2, authed).
app.include_router(discord_link.router, prefix=f"{BASE}/api/v2")
# --- static frontend (served under BASE) -----------------------------------
def _page(name):
"""Route handler for one HTML page. Serves the file with its __ORIGIN__
placeholders (the link-preview/Open Graph tags) filled in as the request's
scheme://host + BASE — preview crawlers (Discord, Slack, …) need absolute
URLs, and the host differs between LAN and prod. The scheme comes from
X-Forwarded-Proto when a reverse proxy (Caddy) fronts the plain-HTTP
uvicorn; the proxy passes the original Host header through untouched."""
path = os.path.join(FRONTEND_DIR, name)
def route(request: Request):
with open(path, encoding="utf-8") as f:
html = f.read()
proto = request.headers.get("x-forwarded-proto") or request.url.scheme
host = request.headers.get("host") or request.url.netloc
html = html.replace("__ORIGIN__", f"{proto}://{host}{BASE}")
# Search-engine verification <meta> tags (same source as the SEO pages), so
# the homepage Google/Bing verify against carries them too.
verify = content.head_verify_html()
if verify:
html = html.replace("<head>", f"<head>\n {verify}", 1)
etag = f'W/"{hashlib.sha1(html.encode()).hexdigest()[:20]}"'
if _not_modified(request, etag):
return Response(status_code=304, headers={"ETag": etag})
return Response(html, media_type="text/html", headers={"ETag": etag})
return route
# The un-slashed base redirects to BASE/ so the frontend's relative asset URLs
# resolve correctly. Under a sub-path this keeps the app scoped to BASE (never
# claiming "/", so the domain root stays free for another app via the proxy). When
# BASE is "" the app owns the root: "/" is already the index route below, so there
# is no bare base to redirect (and "" is not a valid route path).
# Pages also answer HEAD — link-preview crawlers probe with it before fetching.
if BASE:
app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False)
# NOTE: "/" is not here — the homepage is server-rendered by content.register()
# below, so its hero, records strip and city links reach crawlers and no-JS
# readers as real HTML.
app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/score", _page("score.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False)
# Monthly-digest signup (the footer form on every page). Answers the same generic
# message whatever the outcome, so it can't be probed for whether an address is
# already subscribed.
app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False)
# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city /
# month / records / glossary / about / privacy) + robots.txt + sitemap.xml.
# Registered before the static mount so these explicit routes win.
content.register(app)
# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset.
# Registered last so the explicit page routes above win. Mounts must start with
# "/", so at the root (BASE == "") mount at "/" — the earlier routes still win
# because Starlette matches in registration order.
app.mount(BASE or "/", StaticFiles(directory=FRONTEND_DIR), name="static")
from web.app import app # noqa: F401

1
core/__init__.py Normal file
View file

@ -0,0 +1 @@
"""core package."""

4
audit.py → core/audit.py
View file

@ -23,7 +23,9 @@ import threading
import time
import uuid
LOG_ROOT = os.path.join(os.path.dirname(__file__), "..", "logs")
import paths
LOG_ROOT = paths.LOGS_DIR
AUDIT_DIR = os.path.join(LOG_ROOT, "audit")
ERROR_DIR = os.path.join(LOG_ROOT, "errors")
ACCESS_DIR = os.path.join(LOG_ROOT, "access")

0
metrics.py → core/metrics.py
View file

0
singleton.py → core/singleton.py
View file

1
data/__init__.py Normal file
View file

@ -0,0 +1 @@
"""data package."""

6
cities.py → data/cities.py
View file

@ -3,8 +3,10 @@ climate pages. Loaded once, lazily; regenerate the JSON with gen_cities.py."""
import json
import os
_PATH = os.path.join(os.path.dirname(__file__), "cities.json")
_FLAVOR_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json")
import paths
_PATH = paths.CITIES_JSON
_FLAVOR_PATH = paths.CITIES_FLAVOR_JSON
_CITIES: list[dict] | None = None
_BY_SLUG: dict[str, dict] | None = None
_FLAVOR: dict[str, dict] | None = None

0
city_events.py → data/city_events.py
View file

9
climate.py → data/climate.py
View file

@ -14,11 +14,12 @@ import httpx
import numpy as np
import polars as pl
import audit
import metrics
import store
from core import audit
from core import metrics
import paths
from data import store
CACHE_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "cache")
CACHE_DIR = os.path.join(paths.DATA_DIR, "cache")
MAX_ATTEMPTS = 3 # per upstream call, before giving up
START_DATE = "1980-01-01" # ERA5 reaches back to 1940; 1980 = 45 yrs, fast + robust
ARCHIVE_LATENCY_DAYS = 6 # ERA5 archive lags real time by a few days

0
grading.py → data/grading.py
View file

0
grid.py → data/grid.py
View file

5
places.py → data/places.py
View file

@ -28,9 +28,10 @@ import zipfile
import httpx
import audit
from core import audit
import paths
GEO_DIR = os.path.join(os.path.dirname(__file__), "..", "data", "geonames")
GEO_DIR = os.path.join(paths.DATA_DIR, "geonames")
GEONAMES_URL = "https://download.geonames.org/export/dump/"
# Which GeoNames cities dump to index. cities1000 (~170k places, population

2
scoring.py → data/scoring.py
View file

@ -18,7 +18,7 @@ payload (``baseline_overlaps_recent``).
import numpy as np
import polars as pl
import grading
from data import grading
RECENT_YEARS = 6
QS = (10, 25, 50, 75, 90) # the scored percentile categories

4
store.py → data/store.py
View file

@ -26,7 +26,9 @@ import threading
import time
import zlib
DB_PATH = os.path.join(os.path.dirname(__file__), "..", "data", "thermograph.sqlite")
import paths
DB_PATH = os.path.join(paths.DATA_DIR, "thermograph.sqlite")
_SCHEMA = """
CREATE TABLE IF NOT EXISTS derived (

5
gen_cities.py
View file

@ -15,9 +15,10 @@ import re
import sys
import unicodedata
import places
import paths
from data import places
OUT_PATH = os.path.join(os.path.dirname(__file__), "cities.json")
OUT_PATH = paths.CITIES_JSON
# GeoNames entry tuple layout (see places._load): the fields we keep.
_NAME, _ADMIN1, _COUNTRY, _CC, _LAT, _LON, _POP = 1, 2, 3, 4, 5, 6, 7

5
gen_flavor.py
View file

@ -20,9 +20,10 @@ from urllib.parse import quote
import httpx
import cities
from data import cities
import paths
OUT_PATH = os.path.join(os.path.dirname(__file__), "cities_flavor.json")
OUT_PATH = paths.CITIES_FLAVOR_JSON
UA = "ThermographBot/1.0 (https://thermograph.org; climate comparison tool)"
SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"

8
indexnow.py
View file

@ -25,9 +25,11 @@ from urllib.parse import urlparse
import httpx
import paths
log = logging.getLogger("thermograph.indexnow")
_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data"))
_DATA_DIR = paths.DATA_DIR
_KEY_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_KEY_FILE") or os.path.join(_DATA_DIR, "indexnow_key.txt")
_STATE_PATH = os.environ.get("THERMOGRAPH_INDEXNOW_STATE_FILE") or os.path.join(_DATA_DIR, "indexnow_state.txt")
_ENDPOINT = "https://api.indexnow.org/indexnow" # a shared endpoint; it fans out to all participants
@ -98,7 +100,7 @@ def submit(urls, host: str, key_location: str, scheme: str = "https", timeout: f
def submit_all(site_base_url: str, **kw) -> dict:
"""Submit every indexable page. ``site_base_url`` is the public origin + base
path, e.g. ``https://thermograph.org`` (root) or ``https://host/thermograph``."""
import content # lazy: avoids a content ↔ indexnow import cycle
from web import content # lazy: avoids a content ↔ indexnow import cycle
site = site_base_url.rstrip("/")
parsed = urlparse(site)
urls = [site + path for path in content.public_paths()]
@ -110,7 +112,7 @@ def url_signature() -> str:
"""A stable fingerprint of the indexable URL set — changes only when pages are
added or removed (e.g. a new city), not on code-only deploys. Used to skip
resubmitting an unchanged set (see the --if-changed CLI flag)."""
import content
from web import content
return hashlib.sha1("\n".join(content.public_paths()).encode()).hexdigest()

8
migrate.py
View file

@ -21,10 +21,10 @@ import os
import sys
import time
import climate
import grid
import store
import views
from data import climate
from data import grid
from data import store
from web import views
def migrate() -> int:

1
notifications/__init__.py Normal file
View file

@ -0,0 +1 @@
"""notifications package."""

8
digest.py → notifications/digest.py
View file

@ -16,10 +16,10 @@ from fastapi import Request, Response
from fastapi.responses import JSONResponse
from sqlalchemy import select
import mailer
import metrics
from db import async_session_maker
from models import PendingDigest
from notifications import mailer
from core import metrics
from accounts.db import async_session_maker
from accounts.models import PendingDigest
# Deliberately permissive: real-world addresses defeat clever patterns, and the
# confirmation email is the actual validator. This only rejects obvious junk.

2
discord.py → notifications/discord.py
View file

@ -17,7 +17,7 @@ import time
import httpx
import homepage
from web import homepage
# The webhook URL IS the credential — env only, never the repo. Unset => disabled.
WEBHOOK_URL = os.environ.get("THERMOGRAPH_DISCORD_WEBHOOK", "").strip()

6
discord_interactions.py → notifications/discord_interactions.py
View file

@ -21,9 +21,9 @@ import os
from nacl.exceptions import BadSignatureError
from nacl.signing import VerifyKey
import cities
import discord
import homepage
from data import cities
from notifications import discord
from web import homepage
# App's Ed25519 public key (Developer Portal -> General Information). Unset =>
# every request fails verification, which is the safe default for an unconfigured

6
discord_link.py → notifications/discord_link.py
View file

@ -29,9 +29,9 @@ from pydantic import BaseModel
from sqlalchemy import update
from sqlalchemy.ext.asyncio import AsyncSession
from db import get_async_session
from models import User
from users import SECRET, current_active_user
from accounts.db import get_async_session
from accounts.models import User
from accounts.users import SECRET, current_active_user
router = APIRouter(tags=["discord"])

0
mailer.py → notifications/mailer.py
View file

20
notify.py → notifications/notify.py
View file

@ -29,16 +29,16 @@ import time
import polars as pl
from sqlalchemy import delete, select
import climate
import discord
import grading
import grid
import homepage
import metrics
import push
from db import sync_session_maker
from models import AccessToken, Notification, PushSubscription, Subscription, User
from views import OBS_COLS
from data import climate
from notifications import discord
from data import grading
from data import grid
from web import homepage
from core import metrics
from notifications import push
from accounts.db import sync_session_maker
from accounts.models import AccessToken, Notification, PushSubscription, Subscription, User
from web.views import OBS_COLS
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""

5
push.py → notifications/push.py
View file

@ -25,7 +25,8 @@ import logging
import os
import threading
import audit
from core import audit
import paths
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import ec
@ -33,7 +34,7 @@ from pywebpush import WebPushException, webpush
log = logging.getLogger("thermograph.push")
_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data"))
_DATA_DIR = paths.DATA_DIR
_VAPID_PATH = os.environ.get("THERMOGRAPH_VAPID_FILE") or os.path.join(_DATA_DIR, "vapid.json")
# The VAPID "sub" claim — a contact the push service can reach about our traffic.
_CONTACT = os.environ.get("THERMOGRAPH_VAPID_CONTACT", "mailto:admin@thermograph.app")

42
paths.py Normal file
View file

@ -0,0 +1,42 @@
"""Canonical filesystem locations, resolved once from the repo root.
Every path the backend needs (the climate cache, the accounts DB, logs, the
Jinja templates, the frontend static dir, the bundled city data) is derived
here from the repository root rather than from each module's own ``__file__``.
That keeps the locations stable no matter how deep in the package tree a module
lives, so moving a module never silently re-points the cache or the accounts DB.
Callers that support an environment override (e.g. ``THERMOGRAPH_ACCOUNTS_DB``)
keep doing so; these are only the defaults.
"""
import os
_HERE = os.path.dirname(os.path.abspath(__file__))
def _repo_root() -> str:
"""Walk up from this file until a directory holds both ``backend/`` and
``frontend/`` — the repository root, in every checkout (dev, prod, worktree)."""
d = _HERE
while True:
if os.path.isdir(os.path.join(d, "backend")) and os.path.isdir(os.path.join(d, "frontend")):
return d
parent = os.path.dirname(d)
if parent == d:
raise RuntimeError(
"cannot locate the repo root (no ancestor has sibling backend/ and frontend/)"
)
d = parent
REPO_DIR = _repo_root()
BACKEND_DIR = os.path.join(REPO_DIR, "backend")
DATA_DIR = os.path.join(REPO_DIR, "data")
LOGS_DIR = os.path.join(REPO_DIR, "logs")
FRONTEND_DIR = os.path.join(REPO_DIR, "frontend")
TEMPLATES_DIR = os.path.join(BACKEND_DIR, "templates")
# Bundled reference data shipped alongside the code (generated by gen_cities /
# gen_flavor), not runtime state — hence under backend/, not data/.
CITIES_JSON = os.path.join(BACKEND_DIR, "cities.json")
CITIES_FLAVOR_JSON = os.path.join(BACKEND_DIR, "cities_flavor.json")

10
tests/accounts/test_api_accounts.py
View file

@ -4,8 +4,8 @@ THERMOGRAPH_ACCOUNTS_DB at a temp file) so nothing is written into the repo."""
import pytest
from fastapi.testclient import TestClient
import app as appmod
import db
from web import app as appmod
from accounts import db
V2 = "/thermograph/api/v2"
PW = "supersecret123"
@ -120,7 +120,7 @@ def test_push_requires_auth():
def test_push_subscribe_upsert_test_unsubscribe(monkeypatch):
import push
from notifications import push
sent = []
monkeypatch.setattr(push, "send", lambda info, payload: sent.append(info["endpoint"]) or "ok")
c = _client()
@ -142,7 +142,7 @@ def test_push_subscribe_upsert_test_unsubscribe(monkeypatch):
def test_push_test_prunes_gone_endpoint(monkeypatch):
import push
from notifications import push
monkeypatch.setattr(push, "send", lambda info, payload: "gone")
c = _client()
_login(c, "push-gone@example.com")
@ -156,7 +156,7 @@ def test_push_test_prunes_gone_endpoint(monkeypatch):
def test_push_test_reports_failed_delivery(monkeypatch):
# A rejected delivery (e.g. VAPID mismatch) must be reported, not silently dropped:
# the endpoint keeps returning 202 but with sent:0 / failed:N so the client can tell.
import push
from notifications import push
monkeypatch.setattr(push, "send", lambda info, payload: "error")
c = _client()
_login(c, "push-fail@example.com")

6
tests/conftest.py
View file

@ -16,7 +16,7 @@ import pytest
BACKEND = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BACKEND)
import places # noqa: E402
from data import places # noqa: E402
# app.py calls places.start_loading() at import; pretend it already ran so no
# background GeoNames download starts. Tests build their own tiny index.
@ -31,13 +31,13 @@ os.environ.setdefault("THERMOGRAPH_ENABLE_NOTIFIER", "0")
# Pin the IndexNow key so it isn't generated + written into the repo's data/ dir.
os.environ.setdefault("THERMOGRAPH_INDEXNOW_KEY", "test0000indexnow0000key000000000")
import store # noqa: E402
from data import store # noqa: E402
# Keep derived-store writes out of the repo's data/ folder. Individual store
# tests re-point this per test; everything else shares one throwaway DB.
store.DB_PATH = os.path.join(_TMP, "store.sqlite")
import audit # noqa: E402
from core import audit # noqa: E402
audit.AUDIT_DIR = os.path.join(_TMP, "logs", "audit")
audit.ERROR_DIR = os.path.join(_TMP, "logs", "errors")

2
tests/core/test_metrics.py
View file

@ -1,7 +1,7 @@
"""Unit tests for the in-process traffic counters (metrics.py)."""
import importlib
import metrics
from core import metrics
def _fresh():

2
tests/core/test_singleton.py
View file

@ -3,7 +3,7 @@ import fcntl
import importlib
import os
import singleton
from core import singleton
def _fresh():

2
tests/data/test_climate.py
View file

@ -5,7 +5,7 @@ import datetime
import polars as pl
import pytest
import climate
from data import climate
def _om_daily(n=3):

2
tests/data/test_grading.py
View file

@ -4,7 +4,7 @@ import numpy as np
import polars as pl
import pytest
import grading
from data import grading
# ---- empirical percentile ----------------------------------------------------

2
tests/data/test_grid.py
View file

@ -2,7 +2,7 @@ import math
import pytest
import grid
from data import grid
def test_snap_is_deterministic_and_contains_point():

2
tests/data/test_places.py
View file

@ -1,6 +1,6 @@
import pytest
import places
from data import places
def _entry(name, admin1, country, cc, lat, lon, pop):

2
tests/data/test_scoring.py
View file

@ -7,7 +7,7 @@ import numpy as np
import polars as pl
import pytest
import scoring
from data import scoring
ALL_METRICS = ("tmax", "tmin", "feels", "humid", "wetbulb", "wind", "gust", "precip")

14
tests/notifications/test_digest.py
View file

@ -3,12 +3,12 @@ import pytest
import sqlalchemy as sa
from fastapi.testclient import TestClient
import app as appmod
import db
import digest
import mailer
from db import async_session_maker
from models import PendingDigest
from web import app as appmod
from accounts import db
from notifications import digest
from notifications import mailer
from accounts.db import async_session_maker
from accounts.models import PendingDigest
B = "/thermograph"
@ -104,7 +104,7 @@ def test_signup_accepts_plain_form_post(client):
def test_signup_records_event(client):
import metrics
from core import metrics
before = metrics.snapshot()["events"].get("home.digest_signup", {})
before_n = sum(sum(days.values()) for days in before.values())
client.post(f"{B}/digest", json={"email": "counted@example.com"})

4
tests/notifications/test_discord.py
View file

@ -5,8 +5,8 @@ import time
import pytest
import discord
import notify
from notifications import discord
from notifications import notify
def _feed(ranked, generated_at=None, date=None):

10
tests/notifications/test_discord_dm.py
View file

@ -5,11 +5,11 @@ import types
import pytest
from fastapi.testclient import TestClient
import app as appmod
import db
import discord
import discord_link as dl
import notify
from web import app as appmod
from accounts import db
from notifications import discord
from notifications import discord_link as dl
from notifications import notify
V2 = "/thermograph/api/v2"
PW = "supersecret123"

6
tests/notifications/test_discord_interactions.py
View file

@ -8,9 +8,9 @@ import pytest
from fastapi.testclient import TestClient
from nacl.signing import SigningKey
import app as appmod
import discord_interactions as di
import homepage
from web import app as appmod
from notifications import discord_interactions as di
from web import homepage
B = "/thermograph"

6
tests/notifications/test_discord_link.py
View file

@ -3,9 +3,9 @@ mocked), unlink, and auth gating. Reuses the throwaway accounts DB from conftest
import pytest
from fastapi.testclient import TestClient
import app as appmod
import db
import discord_link as dl
from web import app as appmod
from accounts import db
from notifications import discord_link as dl
V2 = "/thermograph/api/v2"
PW = "supersecret123"

10
tests/notifications/test_notify.py
View file

@ -8,11 +8,11 @@ import uuid
import numpy as np
import polars as pl
import climate
import db
import notify
import push
from models import Notification, PushSubscription, Subscription, User
from data import climate
from accounts import db
from notifications import notify
from notifications import push
from accounts.models import Notification, PushSubscription, Subscription, User
from sqlalchemy import delete, func, select

14
tests/web/test_api.py
View file

@ -4,8 +4,8 @@ would catch wiring regressions (e.g. a handler calling a deleted helper)."""
import pytest
from fastapi.testclient import TestClient
import app as appmod
import climate
from web import app as appmod
from data import climate
@pytest.fixture
@ -194,7 +194,7 @@ def test_pages_serve_with_origin_filled_in(client):
def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch):
import app as appmod
from web import app as appmod
monkeypatch.setattr(appmod, "_WARM_SEEN", {})
# No lifespan in TestClient without a context manager, so no worker drains
# the queue — enqueued cells just accumulate for inspection.
@ -210,9 +210,9 @@ def test_cell_neighbors_flag_enqueues_warming(client, monkeypatch):
def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, history):
import app as appmod
import grid
import views
from web import app as appmod
from data import grid
from web import views
cell = grid.snap(-33.87, 151.21)
appmod._warm_cell(cell)
token = views.history_token(history)
@ -224,7 +224,7 @@ def test_warm_cell_materializes_history_slices(client, tmp_store, monkeypatch, h
def test_warm_cell_never_fetches_upstream(client, monkeypatch):
import app as appmod
from web import app as appmod
def boom(cell):
raise AssertionError("warming must never fetch weather upstream")
monkeypatch.setattr(climate, "get_history", boom)

20
tests/web/test_content.py
View file

@ -7,10 +7,10 @@ import re
import pytest
from fastapi.testclient import TestClient
import app as appmod
import cities
import climate
import grading
from web import app as appmod
from data import cities
from data import climate
from data import grading
# BASE defaults to /thermograph in tests (THERMOGRAPH_BASE unset).
B = "/thermograph"
@ -84,7 +84,7 @@ def test_city_404(client):
def test_curated_events_have_valid_slugs():
import city_events
from data import city_events
slugs = set(cities.all_slugs())
assert len(city_events.EVENTS) >= 20
assert all(s in slugs for s in city_events.EVENTS), \
@ -344,7 +344,7 @@ def test_wind_and_humidity_render_per_unit_system():
# The synthetic history carries only tmax/tmin/precip, so wind and humidity
# never reach a rendered page in tests — exercise the renderers directly
# rather than assert on a page that structurally cannot show them.
import content
from web import content
with content._unit_scope("F"):
assert content._wind_text(10) == "10 mph"
assert '<span class="wind" data-wind-mph="10.0">10 mph</span>' == content._wind(10)
@ -362,7 +362,7 @@ def test_wind_and_humidity_render_per_unit_system():
def test_precip_precision_differs_by_unit():
# Rainfall is reported in whole millimetres; a tenth of a mm is below what
# the source resolves. Inches keep two decimals.
import content
from web import content
with content._unit_scope("F"):
assert content._precip_text(0.04) == "0.04 in"
with content._unit_scope("C"):
@ -373,7 +373,7 @@ def test_precip_precision_differs_by_unit():
def test_measure_conversion_constants_match_the_frontend():
# Same drift risk as the country list: two hand-maintained copies.
import os
import content
from web import content
units_js = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
os.pardir, "frontend", "units.js")
with open(units_js, encoding="utf-8") as f:
@ -412,7 +412,7 @@ def test_f_country_list_matches_the_frontend():
# drifting apart, and drift means the server and the client disagree about
# what a first-time visitor should see.
import os
import content
from web import content
units_js = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
os.pardir, "frontend", "units.js")
with open(units_js, encoding="utf-8") as f:
@ -425,7 +425,7 @@ def test_celsius_rounding_matches_js():
# Python's round() goes to even, JS's Math.round goes up. Where they disagree
# the server prints one number and climate.js repaints another — a flicker for
# the visitor whose unit already matched.
import content
from web import content
assert content._round_half_up(0.5) == 1 # round(0.5) would give 0
assert content._round_half_up(1.5) == 2
assert content._round_half_up(2.5) == 3 # round(2.5) would give 2

10
tests/web/test_homepage.py
View file

@ -13,11 +13,11 @@ import time
import pytest
from fastapi.testclient import TestClient
import app as appmod
import cities
import climate
import content
import homepage
from web import app as appmod
from data import cities
from data import climate
from web import content
from web import homepage
B = "/thermograph"

10
tests/web/test_views.py
View file

@ -8,8 +8,8 @@ import sys
import polars as pl
import pytest
import climate
import views
from data import climate
from web import views
CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
@ -17,10 +17,10 @@ CELL = {"id": "1642_-4223", "center_lat": 47.6087, "center_lon": -122.29377}
def test_views_and_migrate_import_without_the_web_stack():
"""migrate.py must stay runnable offline: importing the payload layer may
not construct the FastAPI app or start the places-index download."""
code = ("import sys; import views, migrate; "
code = ("import sys; from web import views; import migrate; "
"assert 'fastapi' not in sys.modules, 'views/migrate pulled in FastAPI'; "
"assert 'app' not in sys.modules, 'views/migrate imported the web app'; "
"import places; assert places._load_started is False")
"assert 'web.app' not in sys.modules, 'views/migrate imported the web app'; "
"from data import places; assert places._load_started is False")
backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
subprocess.run([sys.executable, "-c", code], cwd=backend, check=True)

8
warm_cities.py
View file

@ -12,10 +12,10 @@ so this is an optimization, not a hard dependency.
import sys
import time
import cities
import climate
import grid
import homepage
from data import cities
from data import climate
from data import grid
from web import homepage
def main(limit: int | None = None, pace: float = 2.0) -> None:

1
web/__init__.py Normal file
View file

@ -0,0 +1 @@
"""web package."""

783
web/app.py Normal file
View file

@ -0,0 +1,783 @@
"""Thermograph API — grade recent local weather against ~45 years of climatology."""
import contextlib
import datetime
import functools
import hashlib
import hmac
import json
import mimetypes
import os
import queue
import threading
import time
from fastapi import APIRouter, FastAPI, HTTPException, Query, Request, Response
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
from accounts import api_accounts
from core import audit
from data import climate
from web import content
from notifications import digest
from notifications import discord_interactions as discord_interactions_mod
from notifications import discord_link
from accounts import db
from data import grid
from core import metrics
from notifications import notify
import paths
from data import places
from core import singleton
from data import store
from accounts import users
from web import views
from web.schemas import UserCreate, UserRead, UserUpdate
FRONTEND_DIR = paths.FRONTEND_DIR
# Everything (pages, assets, API) is served under this base path so the app can
# live at https://<host>/thermograph/ behind a shared domain. Override with the
# THERMOGRAPH_BASE env var. The frontend uses base-relative URLs, so it follows
# this automatically without hardcoding the prefix.
#
# THERMOGRAPH_BASE=/ (or empty) means the app owns the whole domain: BASE becomes
# "" so every route sits at the domain root ("/", "/calendar", "/api/v2/…") with
# no prefix and no leading double slash. A non-empty value keeps the sub-path form
# ("/thermograph").
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""
# The PWA manifest is served as a static file; register its media type so
# StaticFiles labels it correctly (not in Python's default mimetypes map).
mimetypes.add_type("application/manifest+json", ".webmanifest")
def _weather_fetch_error(e) -> HTTPException:
"""A clean, retryable 503 when weather data is temporarily unavailable; the
raw error as a 502 otherwise. The fetch layer classifies its own failures
(climate.WeatherUnavailable carries the user-facing text) so no message
parsing happens here; the is_rate_limit fallback covers a raw upstream 429
from a fetcher that didn't classify it."""
if isinstance(e, climate.WeatherUnavailable):
return HTTPException(status_code=503, detail=str(e))
if climate.is_rate_limit(e):
return HTTPException(status_code=503, detail=climate.limit_message(False))
return HTTPException(status_code=502, detail=f"weather data fetch failed: {e}")
# --- background neighbor warming ---------------------------------------------
# /cell?neighbors=1 asks the server to warm the 8 grid cells around the request
# (the frontend used to fire 8 staggered prefetch requests, mirroring grid.py's
# snapping math in JS). One worker drains the queue so warms never stack; the
# hard guarantee matches prefetch=1 — a cell with no cached archive is skipped,
# so no weather-API quota is ever spent, and reverse_geocode itself paces the
# at-most-one Nominatim call per never-labeled cell (~1/s policy).
_WARM_TTL = 3600.0 # don't re-enqueue a cell within the hour
_WARM_SEEN: dict[str, float] = {} # cell_id -> last enqueue time
_warm_queue: "queue.Queue[dict]" = queue.Queue()
def _warm_cell(cell: dict) -> None:
"""Materialize the history-derived slices for one cell — the same rows the
prefetch=1 bundle serves. Never fetches weather upstream."""
history = climate.load_cached_history(cell)
if history is None or history.is_empty():
return
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
token = views.history_token(history)
start_ts, end_ts = views.cal_span(history, None, None, 24)
cal_key = views.calendar_key(start_ts, end_ts, 24)
if store.get_payload("calendar", cell["id"], cal_key, token) is None:
store.put_payload("calendar", cell["id"], cal_key, token,
views.build_calendar(cell, history, start_ts, end_ts, 24, place))
last = history["date"].max()
day_key = views.day_key(last)
if store.get_payload("day", cell["id"], day_key, token) is None:
store.put_payload("day", cell["id"], day_key, token,
views.build_day(cell, history, last, place))
def _enqueue_neighbor_warming(cell: dict) -> None:
now = time.time()
for n in grid.neighbors(cell):
if now - _WARM_SEEN.get(n["id"], 0.0) < _WARM_TTL:
continue
_WARM_SEEN[n["id"]] = now
_warm_queue.put(n)
def _neighbor_warmer() -> None:
while True:
cell = _warm_queue.get()
try:
_warm_cell(cell)
except Exception: # noqa: BLE001 - warming is best-effort, never fatal
pass
finally:
_warm_queue.task_done()
@contextlib.asynccontextmanager
async def _lifespan(app):
# Warm the local place-name index (/suggest's typo tolerance) in the
# background; the app boots and serves fine without it — suggestions just
# fall back to the upstream geocoder until it's ready. A server-startup
# hook (not import time) so offline importers (tests, the migrate script's
# dependencies) don't kick off a GeoNames download. The neighbor warmer is
# also a server concern: enqueues before startup just wait in the queue.
# Create the account DB tables (users/sessions/subscriptions/notifications)
# before serving — a fast no-op once they exist.
await db.create_db_and_tables()
places.start_loading()
threading.Thread(target=_neighbor_warmer, name="neighbor-warmer", daemon=True).start()
# Background subscription evaluator. Its periodic sweep fetches upstream on a timer
# (not per request), so with multiple workers it must run in exactly one — elect a
# leader via a lockfile (THERMOGRAPH_SINGLETON_LOCK). Unset (single worker / dev /
# tests) => always the leader. The neighbor warmer above stays per-worker: each
# drains its own request-fed queue.
if singleton.claim(os.environ.get("THERMOGRAPH_SINGLETON_LOCK")):
notify.start()
yield
notify.stop()
app = FastAPI(title="Thermograph", version="0.2.0", lifespan=_lifespan)
# Compress every sizeable response (the 2-year calendar JSON shrinks ~6-8×).
# Applies to API JSON and static assets alike; tiny responses are left alone.
app.add_middleware(GZipMiddleware, minimum_size=1024)
def _client_ip(request) -> "str | None":
"""The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us
(Caddy on prod sets it), otherwise the direct peer address (LAN dev)."""
xff = request.headers.get("x-forwarded-for")
if xff:
return xff.split(",")[0].strip()
return request.client.host if request.client else None
@app.middleware("http")
async def revalidate_static(request, call_next):
"""Serve the frontend with no-cache (NOT no-store): browsers may keep a copy
but must revalidate it on every use. Starlette's FileResponse/StaticFiles
already emit ETag + Last-Modified, so an unchanged asset costs one conditional
request answered with an empty 304 instead of a full re-download — page-to-page
navigation stops re-transferring the JS/CSS/HTML while still picking up every
deploy immediately (no "my change isn't showing" bugs)."""
response = await call_next(request)
path = request.url.path
try:
cat = metrics.classify_inbound(path, BASE)
metrics.record_inbound(cat, response.status_code)
# Retain per-request client IPs for later analysis; skip static assets and the
# dashboard's own metrics polling to keep the log to real, meaningful traffic.
if cat not in ("static", "metrics", "event"):
audit.log_access({"ip": _client_ip(request), "method": request.method,
"path": path, "status": response.status_code, "cat": cat})
except Exception: # noqa: BLE001 - never let instrumentation break a response
pass
pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/score",
f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy")
if path.endswith((".js", ".css", ".html")) or path in pages:
response.headers["Cache-Control"] = "no-cache"
return response
# --- derived-store plumbing --------------------------------------------------
# Every graded payload is cached in SQLite under (kind, cell, key) and validated
# by a token that encodes what it was computed from (see store.py). The freshness
# drivers stay where they were — climate.get_history tops up the archive tail
# hourly and get_recent_forecast refetches hourly — and the tokens are derived
# from what those return, so a cached payload expires exactly when its inputs
# change and never before. The same tokens double as ETags: a client sending
# If-None-Match gets an empty 304 without the payload even being loaded.
# The payloads themselves are assembled in views.py, shared with the offline
# migrate script.
def _etag_for(kind: str, cell_id: str, key: str, token: str) -> str:
"""Deterministic weak ETag from a payload's identity + validity token. Weak
because the same content may be served under different encodings (gzip)."""
h = hashlib.sha1(f"{kind}:{cell_id}:{key}:{token}".encode()).hexdigest()[:20]
return f'W/"{h}"'
def _not_modified(request: Request, etag: str) -> bool:
"""Does the client already hold this exact payload? Answerable from the token
alone — no payload load needed."""
inm = request.headers.get("if-none-match")
if not inm:
return False
tags = {t.strip() for t in inm.split(",")}
return "*" in tags or etag in tags or etag.removeprefix("W/") in tags
def _encode(payload: dict) -> bytes:
"""Compact JSON bytes, matching store.put_payload's encoding (allow_nan=False
mirrors Starlette — a NaN from grading should fail loudly, not reach a client)."""
return json.dumps(payload, default=str, allow_nan=False, separators=(",", ":")).encode()
def _json_response(body: bytes, etag: str | None = None) -> Response:
headers = {"ETag": etag} if etag else {}
return Response(content=body, media_type="application/json", headers=headers)
def _fetch_history(run, cell, recent_too=False, recent_phase="recent"):
"""The shared fetch preamble for every data route: the archive record (and
optionally the hourly recent/forecast bundle) with upstream failures mapped
to clean HTTP errors and an empty record to a 404."""
try:
with run.phase("history"):
history, cache_meta = climate.get_history(cell)
recent = None
if recent_too:
with run.phase(recent_phase):
recent = climate.get_recent_forecast(cell)
except Exception as e: # noqa: BLE001
raise _weather_fetch_error(e)
if history.is_empty():
raise HTTPException(status_code=404, detail="No historical data for this cell.")
return history, cache_meta, recent
def _cached_response(request, run, kind, cell, key, token, build, cache_placeless=True):
"""The shared derived-store flow behind every per-view endpoint:
If-None-Match -> empty 304; a token-valid store row -> replay its exact
bytes; otherwise resolve the place label, build the payload, persist, serve.
``build(place) -> payload`` does any endpoint-specific audit tagging itself.
``cache_placeless=False`` skips persisting a payload whose place label
failed to resolve (a transient reverse-geocode miss) — otherwise the
bare-coordinates fallback would stick for the life of the token."""
cid = cell["id"]
etag = _etag_for(kind, cid, key, token)
if _not_modified(request, etag):
run.set(run_type="cache", not_modified=True)
return Response(status_code=304, headers={"ETag": etag})
body = store.get_payload(kind, cid, key, token)
if body is not None:
run.set(run_type="cache", history_source="store")
return _json_response(body, etag)
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
payload = build(place)
return _json_response(
store.put_payload(kind, cid, key, token, payload,
cache=cache_placeless or place is not None), etag)
# --- endpoints ----------------------------------------------------------------
def api_geocode(q: str = Query(..., min_length=1)):
try:
return {"results": climate.geocode(q)}
except Exception as e: # noqa: BLE001 - surface upstream failures to the client
raise HTTPException(status_code=502, detail=f"geocoding failed: {e}")
_SUGGEST_LIMIT = 5
@functools.lru_cache(maxsize=1024)
def _suggest_upstream(q: str) -> tuple:
"""Open-Meteo lookup for /suggest, memoized per query string — type-ahead
re-asks the same prefixes constantly (backspacing, retyping). Failures
raise and are not cached, so a transient upstream error doesn't stick."""
return tuple(climate.geocode(q, count=_SUGGEST_LIMIT))
def api_suggest(q: str = Query(..., min_length=1)):
"""Type-ahead location suggestions: the top 5 places for a (possibly
typo'd) query prefix; `corrected` reports the respelling that produced the
results ("pest seattle" → "west seattle"), or null. The ranking/typo policy
lives in places.suggest."""
try:
results, corrected = places.suggest(q, _suggest_upstream, _SUGGEST_LIMIT)
except Exception as e: # noqa: BLE001 - nothing at all to serve
raise HTTPException(status_code=502, detail=f"geocoding failed: {e}")
return {"results": results, "corrected": corrected}
def api_place(
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
):
"""Best-effort neighbourhood/city label for a point — the same string the view
endpoints expose as ``place`` (snapped to the grid cell, reverse-geocoded and
cached). Resolved on its own so the compare page can show a location's name as
soon as it's added, before its full series loads."""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="place", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"]) as run:
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
return {"place": place,
"cell": {"center_lat": cell["center_lat"], "center_lon": cell["center_lon"]}}
def api_grade(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
date: str | None = Query(None, description="target date YYYY-MM-DD (default today)"),
days: int = Query(14, ge=1, le=60, description="days of history to grade before the target"),
after: int = Query(14, ge=0, le=14,
description="days to grade after the target (observed, or forecast when future; "
"the forecast reaches ~7 days out so future days cap there)"),
):
target = datetime.date.fromisoformat(date[:10]) if date else datetime.date.today()
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="grade",
lat=round(lat, 4),
lon=round(lon, 4),
target_date=target.isoformat(),
recent_days=days,
cell_id=cell["id"],
) as run:
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
return _cached_response(
request, run, "grade", cell,
views.grade_key(target, days, after), views.recent_token(history, cell["id"]),
lambda place: views.build_grade(cell, target, days, history, recent,
cache_meta, place, run, after=after))
def api_calendar(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
start: str | None = Query(None, description="first date YYYY-MM-DD (overrides months)"),
end: str | None = Query(None, description="last date YYYY-MM-DD (default = latest available)"),
months: int = Query(24, ge=1, le=24, description="months back to grade when no start given"),
):
"""Grade every day over a date range (default: last 2 years) for the calendar view.
A custom [start, end] range is supported and capped at ~2 years per request;
the frontend splits longer spans into successive 2-year chunks. Grades against
the already-cached history record (which reaches to ~6 days ago), so no extra
fetch is needed. The graded payload is cached in the derived store keyed by
the clamped span and validated by the record's end date, so a repeat span is
a database read — across restarts — until new archive days arrive. Returns a
compact per-day shape (see grading.grade_range). New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="calendar",
lat=round(lat, 4),
lon=round(lon, 4),
recent_days=months * 31, # approximate span graded
cell_id=cell["id"],
) as run:
history, cache_meta, _ = _fetch_history(run, cell)
start_ts, end_ts = views.cal_span(history, start, end, months)
def build(place):
payload = views.build_calendar(cell, history, start_ts, end_ts, months, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
# Compare loads several cells at once, so a transient reverse-geocode miss
# is most likely here; cache_placeless=False keeps that miss un-persisted.
return _cached_response(request, run, "calendar", cell,
views.calendar_key(start_ts, end_ts, months),
views.history_token(history), build,
cache_placeless=False)
def api_day(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
date: str | None = Query(None, description="target date YYYY-MM-DD (default = latest available)"),
):
"""Full percentile breakdown for a single day: the value at every tier boundary
in that day-of-year's ±7-day window, plus where the observed values land.
Powers the single-day detail page. Grades against the cached history record;
for a date newer than the cache it pulls the recent window for the observation
(those payloads expire hourly — the recent bundle's own cadence — while fully
archived days stay valid until the record itself advances). New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(
endpoint="day",
lat=round(lat, 4),
lon=round(lon, 4),
cell_id=cell["id"],
) as run:
history, cache_meta, _ = _fetch_history(run, cell)
last = history["date"].max()
target = datetime.date.fromisoformat(date[:10]) if date else last
run.set(target_date=target.isoformat())
def build(place):
payload = views.build_day(cell, history, target, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
return _cached_response(request, run, "day", cell,
views.day_key(target), views.day_token(history, target),
build)
def api_score(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
):
"""Climate-drift score for a location: how far the last 6 years have moved
from the full 45-year baseline, per metric, percentile category and season,
rolled into per-metric and overall scores.
Derived purely from the archive record, so it is cached until the record's
tail advances (the hourly top-up); the scoring-math version is baked into the
cache key so a math change invalidates only score rows. New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="score", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"]) as run:
history, cache_meta, _ = _fetch_history(run, cell)
def build(place):
payload = views.build_score(cell, history, place, run)
full = not cache_meta.get("cached", False)
run.set(run_type="full" if full else "partial",
history_source="fetch" if full else "cache")
return payload
return _cached_response(request, run, "score", cell,
views.score_key(), views.history_token(history), build)
def api_forecast(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
days: int = Query(7, ge=1, le=14, description="how many forecast days ahead to grade"),
):
"""Grade the next `days` of forecast weather against local climatology.
Same shape as /grade (so the frontend renders it identically), but `recent`
holds the forward forecast — furthest-out day first. The forecast is fetched
fresh and cached only ~1 hour (see climate.get_recent_forecast) to track
updates; the graded payload's validity is tied to that fetch stamp, so it
expires exactly when a new forecast lands. New in API v2.
"""
cell = grid.snap(lat, lon)
with audit.RunAudit(endpoint="forecast", lat=round(lat, 4), lon=round(lon, 4),
recent_days=days, cell_id=cell["id"]) as run:
history, _, fc = _fetch_history(run, cell, recent_too=True, recent_phase="forecast")
today = datetime.date.today()
def build(place):
payload = views.build_forecast(cell, days, history, fc, today, place, run)
run.set(run_type="partial")
return payload
return _cached_response(request, run, "forecast", cell,
views.forecast_key(today, days),
views.recent_token(history, cell["id"]), build)
def api_cell(
request: Request,
lat: float = Query(..., ge=-90, le=90),
lon: float = Query(..., ge=-180, le=180),
prefetch: int = Query(0, ge=0, le=1,
description="1 = warm-only: never fetch weather upstream; 204 for a cold cell"),
neighbors: int = Query(0, ge=0, le=1,
description="1 = also warm the 8 surrounding cells in the background "
"(warm-only: cold neighbors are skipped, no upstream quota)"),
):
"""One bundle carrying every view's payload for a cell, so the frontend warms
all views with a single request instead of four.
Each slice is the EXACT payload its per-view endpoint returns — built by the
same builders and cached under the same derived-store keys/tokens — paired
with the etag that endpoint would emit. The client seeds its per-view cache
from the slices and later revalidates each view individually with
If-None-Match, so the bundle and the per-view endpoints stay one cache.
prefetch=1 is the neighbor-warming mode with a hard guarantee: it never
spends weather-API quota. A cell with no cached archive answers 204 (no
body), and only the history-derived slices (calendar + latest-day detail)
are built — grading recent/forecast days needs the hourly upstream bundle.
Reverse geocoding makes at most one Nominatim call for a never-labeled cell;
the client staggers neighbor prefetches to respect that service. New in API v2.
"""
cell = grid.snap(lat, lon)
today = datetime.date.today()
with audit.RunAudit(endpoint="cell", lat=round(lat, 4), lon=round(lon, 4),
cell_id=cell["id"], prefetch=bool(prefetch)) as run:
recent = None
if prefetch:
history = climate.load_cached_history(cell)
if history is None or history.is_empty():
run.set(run_type="cold-skip")
return Response(status_code=204)
cache_meta = {"cached": True}
else:
history, cache_meta, recent = _fetch_history(run, cell, recent_too=True)
cid = cell["id"]
last = history["date"].max()
if neighbors:
_enqueue_neighbor_warming(cell)
with run.phase("reverse_geocode"):
place = climate.reverse_geocode(cell["center_lat"], cell["center_lon"])
def slice_for(kind: str, key: str, token: str, build) -> dict:
"""The endpoint's derived row (or build + store it), paired with the
etag that endpoint would emit for the same request."""
etag = _etag_for(kind, cid, key, token)
data = store.get_json(kind, cid, key, token)
if data is None:
data = build()
store.put_payload(kind, cid, key, token, data)
return {"etag": etag, "data": data}
slices = {}
hist_token = views.history_token(history)
# Calendar: the default last-24-months span — what the calendar view (and
# the cross-view prefetch) requests first.
start_ts, end_ts = views.cal_span(history, None, None, 24)
slices["calendar"] = slice_for(
"calendar", views.calendar_key(start_ts, end_ts, 24), hist_token,
lambda: views.build_calendar(cell, history, start_ts, end_ts, 24, place, run))
if prefetch:
# Latest archived day: its observation comes from history alone, so
# it's buildable without the (skipped) recent bundle.
slices["day"] = slice_for(
"day", views.day_key(last), hist_token,
lambda: views.build_day(cell, history, last, place, run))
else:
rf_token = views.recent_token(history, cid)
slices["grade"] = slice_for(
"grade", views.grade_key(today, 14, 14), rf_token,
lambda: views.build_grade(cell, today, 14, history, recent, cache_meta, place, run, after=14))
slices["forecast"] = slice_for(
"forecast", views.forecast_key(today, 7), rf_token,
lambda: views.build_forecast(cell, 7, history, recent, today, place, run))
slices["day"] = slice_for(
"day", views.day_key(today), views.day_token(history, today),
lambda: views.build_day(cell, history, today, place, run, recent=recent))
# The bundle's identity is the combination of its slices' identities.
etag = _etag_for("cell", cid, "bundle" + (":p" if prefetch else ""),
"|".join(s["etag"] for s in slices.values()))
if _not_modified(request, etag):
run.set(run_type="cache", not_modified=True)
return Response(status_code=304, headers={"ETag": etag})
run.set(run_type="bundle", slices=sorted(slices))
payload = {
"api_version": "v2",
"cell": cell,
"place": place,
"today": today.isoformat(),
"slices": slices,
}
# Not stored as its own derived row — the slices already are.
return _json_response(_encode(payload), etag)
# --- ops metrics (gated) ---------------------------------------------------
def _metrics_allowed(request: Request) -> bool:
"""Guard the ops metrics: a valid token allows from anywhere (for SSH tunnels);
with no token it's direct-loopback-only, and any forwarded header (i.e. proxied
via Caddy) is refused — so prod ops data never leaks through the public domain."""
token = os.environ.get("THERMOGRAPH_METRICS_TOKEN", "").strip()
if token:
supplied = request.headers.get("x-metrics-token") or request.query_params.get("token") or ""
if hmac.compare_digest(supplied, token):
return True
client = request.client.host if request.client else None
forwarded = any(h in request.headers for h in
("x-forwarded-for", "x-forwarded-host", "x-forwarded-proto"))
return client in ("127.0.0.1", "::1") and not forwarded
def api_metrics(request: Request):
"""Live in-process traffic counters + worker facts for the ops dashboard."""
if not _metrics_allowed(request):
raise HTTPException(status_code=404) # 404, not 403 — don't reveal the route
snap = metrics.snapshot()
snap["warm_queue_depth"] = _warm_queue.qsize()
snap["warm_seen"] = len(_WARM_SEEN)
snap["threads"] = threading.active_count()
snap["thread_names"] = sorted(t.name for t in threading.enumerate())
return snap
async def api_event(request: Request) -> Response:
"""Record one product event (a tap on "use my location", a digest signup, …).
Deliberately minimal: the body carries only an allowlisted event name. The
referrer is read from this request's own Referer header — a client-supplied
one would be forgeable and buys nothing — and is reduced to a bare domain.
No cookies, no per-visitor id, no full URLs.
Always answers 204, whether the event was counted, rejected by the allowlist,
or dropped by the rate limiter, so probing the endpoint reveals nothing. That
also suits navigator.sendBeacon, which ignores the response.
"""
name = ""
try:
body = await request.json()
if isinstance(body, dict):
name = str(body.get("event") or "")
except Exception: # noqa: BLE001 - a junk body is just a no-op event
pass
if name:
metrics.record_event(
name,
referer=request.headers.get("referer"),
host=request.headers.get("host"),
ip=_client_ip(request),
)
return Response(status_code=204)
# --- API versioning --------------------------------------------------------
# Non-backward-compatible additions land in a new version; every version stays
# mounted and served simultaneously. v1 is the original contract (grade/geocode);
# v2 adds the calendar. The unversioned /api/* paths are kept as aliases of v1 so
# existing clients keep working.
v1 = APIRouter(tags=["v1"])
v1.add_api_route("/geocode", api_geocode, methods=["GET"])
v1.add_api_route("/grade", api_grade, methods=["GET"])
v2 = APIRouter(tags=["v2"])
v2.add_api_route("/geocode", api_geocode, methods=["GET"])
v2.add_api_route("/suggest", api_suggest, methods=["GET"])
v2.add_api_route("/place", api_place, methods=["GET"])
v2.add_api_route("/grade", api_grade, methods=["GET"])
v2.add_api_route("/calendar", api_calendar, methods=["GET"])
v2.add_api_route("/day", api_day, methods=["GET"])
v2.add_api_route("/score", api_score, methods=["GET"])
v2.add_api_route("/forecast", api_forecast, methods=["GET"])
v2.add_api_route("/cell", api_cell, methods=["GET"])
v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False)
v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False)
app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible)
app.include_router(v1, prefix=f"{BASE}/api/v1")
app.include_router(v2, prefix=f"{BASE}/api/v2")
# --- Discord slash commands (HTTP interactions) ----------------------------
async def discord_interactions(request: Request) -> Response:
"""Discord posts each slash-command interaction here. Verification runs over the
RAW body, so this must read bytes, not request.json()."""
raw = await request.body()
status, payload = discord_interactions_mod.handle(
raw,
request.headers.get("x-signature-ed25519", ""),
request.headers.get("x-signature-timestamp", ""),
)
return Response(json.dumps(payload), status_code=status, media_type="application/json")
app.add_api_route(f"{BASE}/discord/interactions", discord_interactions,
methods=["POST"], include_in_schema=False)
# --- accounts (fastapi-users) ----------------------------------------------
# Library-provided routers: /auth/login + /auth/logout (cookie session),
# /auth/register (signup), and /users/me (session check / profile). All under the
# same v2 prefix as the rest of the API, so the frontend calls them base-relative.
app.include_router(
users.fastapi_users.get_auth_router(users.auth_backend),
prefix=f"{BASE}/api/v2/auth", tags=["auth"],
)
app.include_router(
users.fastapi_users.get_register_router(UserRead, UserCreate),
prefix=f"{BASE}/api/v2/auth", tags=["auth"],
)
app.include_router(
users.fastapi_users.get_users_router(UserRead, UserUpdate),
prefix=f"{BASE}/api/v2/users", tags=["users"],
)
# Subscriptions + notifications (authed, user-scoped).
app.include_router(api_accounts.router, prefix=f"{BASE}/api/v2")
# Discord account linking (OAuth2, authed).
app.include_router(discord_link.router, prefix=f"{BASE}/api/v2")
# --- static frontend (served under BASE) -----------------------------------
def _page(name):
"""Route handler for one HTML page. Serves the file with its __ORIGIN__
placeholders (the link-preview/Open Graph tags) filled in as the request's
scheme://host + BASE — preview crawlers (Discord, Slack, …) need absolute
URLs, and the host differs between LAN and prod. The scheme comes from
X-Forwarded-Proto when a reverse proxy (Caddy) fronts the plain-HTTP
uvicorn; the proxy passes the original Host header through untouched."""
path = os.path.join(FRONTEND_DIR, name)
def route(request: Request):
with open(path, encoding="utf-8") as f:
html = f.read()
proto = request.headers.get("x-forwarded-proto") or request.url.scheme
host = request.headers.get("host") or request.url.netloc
html = html.replace("__ORIGIN__", f"{proto}://{host}{BASE}")
# Search-engine verification <meta> tags (same source as the SEO pages), so
# the homepage Google/Bing verify against carries them too.
verify = content.head_verify_html()
if verify:
html = html.replace("<head>", f"<head>\n {verify}", 1)
etag = f'W/"{hashlib.sha1(html.encode()).hexdigest()[:20]}"'
if _not_modified(request, etag):
return Response(status_code=304, headers={"ETag": etag})
return Response(html, media_type="text/html", headers={"ETag": etag})
return route
# The un-slashed base redirects to BASE/ so the frontend's relative asset URLs
# resolve correctly. Under a sub-path this keeps the app scoped to BASE (never
# claiming "/", so the domain root stays free for another app via the proxy). When
# BASE is "" the app owns the root: "/" is already the index route below, so there
# is no bare base to redirect (and "" is not a valid route path).
# Pages also answer HEAD — link-preview crawlers probe with it before fetching.
if BASE:
app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False)
# NOTE: "/" is not here — the homepage is server-rendered by content.register()
# below, so its hero, records strip and city links reach crawlers and no-JS
# readers as real HTML.
app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/score", _page("score.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False)
app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False)
# Monthly-digest signup (the footer form on every page). Answers the same generic
# message whatever the outcome, so it can't be probed for whether an address is
# already subscribed.
app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False)
# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city /
# month / records / glossary / about / privacy) + robots.txt + sitemap.xml.
# Registered before the static mount so these explicit routes win.
content.register(app)
# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset.
# Registered last so the explicit page routes above win. Mounts must start with
# "/", so at the root (BASE == "") mount at "/" — the earlier routes still win
# because Starlette matches in registration order.
app.mount(BASE or "/", StaticFiles(directory=FRONTEND_DIR), name="static")

21
content.py → web/content.py
View file

@ -20,18 +20,19 @@ from fastapi.responses import PlainTextResponse
from jinja2 import Environment, FileSystemLoader, select_autoescape
from markupsafe import Markup
import cities
import city_events
import climate
import grading
import grid
import homepage
from views import OBS_COLS
from data import cities
from data import city_events
from data import climate
from data import grading
from data import grid
from web import homepage
import paths
from web.views import OBS_COLS
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
BASE = f"/{_BASE}" if _BASE else ""
TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
TEMPLATES_DIR = paths.TEMPLATES_DIR
_env = Environment(
loader=FileSystemLoader(TEMPLATES_DIR),
autoescape=select_autoescape(["html", "xml", "j2"]),
@ -359,8 +360,8 @@ def _content_lastmod() -> str:
city list and this module (both change on a content/code rebuild, and the
process restarts on deploy). Honest and stable, unlike a per-request today()
that churns every fetch and trains crawlers to ignore lastmod entirely."""
paths = [os.path.join(os.path.dirname(__file__), "cities.json"), __file__]
mtimes = [os.path.getmtime(p) for p in paths if os.path.exists(p)]
srcs = [paths.CITIES_JSON, __file__]
mtimes = [os.path.getmtime(p) for p in srcs if os.path.exists(p)]
day = datetime.date.fromtimestamp(max(mtimes)) if mtimes else datetime.date.today()
return day.isoformat()

13
homepage.py → web/homepage.py
View file

@ -27,15 +27,16 @@ import time
import polars as pl
import cities
import climate
import grading
import grid
from views import OBS_COLS
from data import cities
from data import climate
from data import grading
from data import grid
import paths
from web.views import OBS_COLS
# data/ is the only writable path under the hardened systemd unit
# (ReadWritePaths=/opt/thermograph/data …), so the feed lives there.
FEED_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "data", "homepage.json")
FEED_PATH = os.path.join(paths.DATA_DIR, "homepage.json")
# How many cards the "Unusual right now" strip can show.
RANK_LIMIT = 12

2
schemas.py → web/schemas.py
View file

@ -10,7 +10,7 @@ from typing import Literal
from fastapi_users import schemas
from pydantic import BaseModel, Field, field_validator
import grading
from data import grading
# Canonical metric keys a subscription may watch (kept in lockstep with grading).
ALLOWED_METRICS = tuple(grading.CLIMO_METRICS)

6
views.py → web/views.py
View file

@ -13,9 +13,9 @@ import time
import polars as pl
import climate
import grading
import scoring
from data import climate
from data import grading
from data import scoring
# Observed values pulled from a daily record row for grading. Includes the
# temperature-scale metrics (tmax/tmin/feels/wind/gust) plus precip; a column may