Rebuild the homepage as a distribution landing; add an SMTP seam (#178)
The homepage was a bare tool: a find-bar and an empty panel reading "Find a location to begin." A visitor arriving from a search result or a shared link learned nothing about what the product does before deciding to leave. Rebuild it around the Weekly view, which is untouched: - Hero with the question as the h1, and a grade card showing a real graded example — the most unusual city we're currently tracking, either tail. The card's frame and text slots are server-rendered with reserved heights, so app.js re-pointing it at the visitor's own place shifts nothing. - "Unusual right now" strip, CSS scroll-snap, no JS carousel. A cold-tail city is force-included whenever one qualifies. - Stance line, how-it-works, explore cards, and 12 city chips linking into the ~1000-page /climate surface. - Monthly digest form, in the footer of every page. Serve / from Jinja instead of a static file with placeholder substitution, so crawlers and no-JS readers get the whole page as real HTML. home.html.j2 extends base.html.j2 and carries app.js's DOM contract over verbatim; the brand degrades to a <p> so the hero owns the sole h1. frontend/index.html is deleted rather than left behind the static mount, where it would keep serving indexable duplicate content. "Where is it most unusual right now" has no cheap answer at request time — percentiles live inside zlib-compressed payload blobs with no column to sort on. So homepage.py sweeps the warm cache and writes data/homepage.json, read by the template. The sweep is strictly cache-only (climate.load_cached_recent_forecast is new, the sibling of load_cached_history), so grading ~1000 cities costs zero upstream requests. It rides the notifier's timer behind an hourly guard rather than starting a second daemon, and also runs at the tail of warm_cities so a fresh deploy has a populated feed. Instrumentation: metrics gains a product-event counter keyed by (event, referrer domain, UTC day), behind an allowlist and a per-IP rate limit, fed by POST /api/v2/event. The referrer is taken from the request's own header, never from the client. The beacon gets its own inbound category that record_inbound ignores, so reporting an interaction doesn't also count as traffic. The dashboard grows an events block with per-referrer attribution. Email is scaffolded but sends nothing yet. mailer.py talks stdlib smtplib to a local Postfix null client on 127.0.0.1:25 (deploy/provision-mail.sh), so the choice between direct-to-MX and relaying through a provider stays a Postfix config change with no code change. The backend defaults to "console", which logs and sends nothing, so dev and tests exercise the whole signup path safely. Signups land in pending_digest unconfirmed; collecting the list shouldn't wait on delivery. Also adds /privacy, linked from the footer and kept out of the sitemap. The strip's classes are named unusual-* rather than record-*: .record-card is already the SEO records page's, and reusing it leaked layout rules onto /climate/<slug>/records.
This commit is contained in:
parent
858a5ab018
commit
3a66682322
16 changed files with 1517 additions and 16 deletions
51
app.py
51
app.py
|
|
@ -20,6 +20,7 @@ import api_accounts
|
||||||
import audit
|
import audit
|
||||||
import climate
|
import climate
|
||||||
import content
|
import content
|
||||||
|
import digest
|
||||||
import db
|
import db
|
||||||
import grid
|
import grid
|
||||||
import metrics
|
import metrics
|
||||||
|
|
@ -171,12 +172,13 @@ async def revalidate_static(request, call_next):
|
||||||
metrics.record_inbound(cat, response.status_code)
|
metrics.record_inbound(cat, response.status_code)
|
||||||
# Retain per-request client IPs for later analysis; skip static assets and the
|
# Retain per-request client IPs for later analysis; skip static assets and the
|
||||||
# dashboard's own metrics polling to keep the log to real, meaningful traffic.
|
# dashboard's own metrics polling to keep the log to real, meaningful traffic.
|
||||||
if cat not in ("static", "metrics"):
|
if cat not in ("static", "metrics", "event"):
|
||||||
audit.log_access({"ip": _client_ip(request), "method": request.method,
|
audit.log_access({"ip": _client_ip(request), "method": request.method,
|
||||||
"path": path, "status": response.status_code, "cat": cat})
|
"path": path, "status": response.status_code, "cat": cat})
|
||||||
except Exception: # noqa: BLE001 - never let instrumentation break a response
|
except Exception: # noqa: BLE001 - never let instrumentation break a response
|
||||||
pass
|
pass
|
||||||
pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts")
|
pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/compare",
|
||||||
|
f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy")
|
||||||
if path.endswith((".js", ".css", ".html")) or path in pages:
|
if path.endswith((".js", ".css", ".html")) or path in pages:
|
||||||
response.headers["Cache-Control"] = "no-cache"
|
response.headers["Cache-Control"] = "no-cache"
|
||||||
return response
|
return response
|
||||||
|
|
@ -592,6 +594,35 @@ def api_metrics(request: Request):
|
||||||
return snap
|
return snap
|
||||||
|
|
||||||
|
|
||||||
|
async def api_event(request: Request) -> Response:
|
||||||
|
"""Record one product event (a tap on "use my location", a digest signup, …).
|
||||||
|
|
||||||
|
Deliberately minimal: the body carries only an allowlisted event name. The
|
||||||
|
referrer is read from this request's own Referer header — a client-supplied
|
||||||
|
one would be forgeable and buys nothing — and is reduced to a bare domain.
|
||||||
|
No cookies, no per-visitor id, no full URLs.
|
||||||
|
|
||||||
|
Always answers 204, whether the event was counted, rejected by the allowlist,
|
||||||
|
or dropped by the rate limiter, so probing the endpoint reveals nothing. That
|
||||||
|
also suits navigator.sendBeacon, which ignores the response.
|
||||||
|
"""
|
||||||
|
name = ""
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
if isinstance(body, dict):
|
||||||
|
name = str(body.get("event") or "")
|
||||||
|
except Exception: # noqa: BLE001 - a junk body is just a no-op event
|
||||||
|
pass
|
||||||
|
if name:
|
||||||
|
metrics.record_event(
|
||||||
|
name,
|
||||||
|
referer=request.headers.get("referer"),
|
||||||
|
host=request.headers.get("host"),
|
||||||
|
ip=_client_ip(request),
|
||||||
|
)
|
||||||
|
return Response(status_code=204)
|
||||||
|
|
||||||
|
|
||||||
# --- API versioning --------------------------------------------------------
|
# --- API versioning --------------------------------------------------------
|
||||||
# Non-backward-compatible additions land in a new version; every version stays
|
# Non-backward-compatible additions land in a new version; every version stays
|
||||||
# mounted and served simultaneously. v1 is the original contract (grade/geocode);
|
# mounted and served simultaneously. v1 is the original contract (grade/geocode);
|
||||||
|
|
@ -611,6 +642,7 @@ v2.add_api_route("/day", api_day, methods=["GET"])
|
||||||
v2.add_api_route("/forecast", api_forecast, methods=["GET"])
|
v2.add_api_route("/forecast", api_forecast, methods=["GET"])
|
||||||
v2.add_api_route("/cell", api_cell, methods=["GET"])
|
v2.add_api_route("/cell", api_cell, methods=["GET"])
|
||||||
v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False)
|
v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False)
|
||||||
|
v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False)
|
||||||
|
|
||||||
app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible)
|
app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible)
|
||||||
app.include_router(v1, prefix=f"{BASE}/api/v1")
|
app.include_router(v1, prefix=f"{BASE}/api/v1")
|
||||||
|
|
@ -672,16 +704,23 @@ def _page(name):
|
||||||
# Pages also answer HEAD — link-preview crawlers probe with it before fetching.
|
# Pages also answer HEAD — link-preview crawlers probe with it before fetching.
|
||||||
if BASE:
|
if BASE:
|
||||||
app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/", _page("index.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
# NOTE: "/" is not here — the homepage is server-rendered by content.register()
|
||||||
|
# below, so its hero, records strip and city links reach crawlers and no-JS
|
||||||
|
# readers as real HTML.
|
||||||
app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
|
|
||||||
# Crawlable, server-rendered SEO pages (climate hub / per-city / month / records /
|
# Monthly-digest signup (the footer form on every page). Answers the same generic
|
||||||
# glossary / about) + robots.txt + sitemap.xml. Registered before the static mount
|
# message whatever the outcome, so it can't be probed for whether an address is
|
||||||
# so these explicit routes win.
|
# already subscribed.
|
||||||
|
app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False)
|
||||||
|
|
||||||
|
# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city /
|
||||||
|
# month / records / glossary / about / privacy) + robots.txt + sitemap.xml.
|
||||||
|
# Registered before the static mount so these explicit routes win.
|
||||||
content.register(app)
|
content.register(app)
|
||||||
|
|
||||||
# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset.
|
# Everything else under BASE (app.js, style.css, nav.js, …) is a static asset.
|
||||||
|
|
|
||||||
15
climate.py
15
climate.py
|
|
@ -587,6 +587,21 @@ def get_recent_forecast(cell: dict) -> pl.DataFrame:
|
||||||
return _derive_humidity(_load_recent_forecast(cell))
|
return _derive_humidity(_load_recent_forecast(cell))
|
||||||
|
|
||||||
|
|
||||||
|
def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None":
|
||||||
|
"""Recent+forecast from the parquet cache ONLY — never fetches upstream, and
|
||||||
|
unlike the loader below it does not care how stale the file is. The sibling of
|
||||||
|
load_cached_history, for the same reason: the homepage precompute sweeps every
|
||||||
|
cached city and must not spend a single upstream request doing it. None when
|
||||||
|
the cell has no cached recent/forecast record."""
|
||||||
|
path = _rf_cache_path(cell["id"])
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return _derive_humidity(_with_doy(_normalize_read(pl.read_parquet(path))))
|
||||||
|
except Exception: # noqa: BLE001 - a truncated/corrupt cache file is a miss, not a crash
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _load_recent_forecast(cell: dict) -> pl.DataFrame:
|
def _load_recent_forecast(cell: dict) -> pl.DataFrame:
|
||||||
"""Recent observations AND the forward forecast in ONE forecast-API call.
|
"""Recent observations AND the forward forecast in ONE forecast-API call.
|
||||||
|
|
||||||
|
|
|
||||||
97
content.py
97
content.py
|
|
@ -22,6 +22,7 @@ import city_events
|
||||||
import climate
|
import climate
|
||||||
import grading
|
import grading
|
||||||
import grid
|
import grid
|
||||||
|
import homepage
|
||||||
from views import OBS_COLS
|
from views import OBS_COLS
|
||||||
|
|
||||||
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
_BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/")
|
||||||
|
|
@ -131,9 +132,24 @@ def _range_bar(low_f, high_f) -> dict | None:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ordinal(n) -> str:
|
||||||
|
"""96 -> '96th'. Percentiles read as ordinals in prose ('in the 96th
|
||||||
|
percentile'), and the frontend's shared.js ord() does the same client-side."""
|
||||||
|
try:
|
||||||
|
i = int(round(float(n)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return "—"
|
||||||
|
if 10 <= i % 100 <= 20:
|
||||||
|
suffix = "th"
|
||||||
|
else:
|
||||||
|
suffix = {1: "st", 2: "nd", 3: "rd"}.get(i % 10, "th")
|
||||||
|
return f"{i}{suffix}"
|
||||||
|
|
||||||
|
|
||||||
_env.globals["temp_class"] = temp_class
|
_env.globals["temp_class"] = temp_class
|
||||||
_env.globals["temp"] = _temp
|
_env.globals["temp"] = _temp
|
||||||
_env.globals["temp_bare"] = _temp_bare
|
_env.globals["temp_bare"] = _temp_bare
|
||||||
|
_env.filters["ordinal"] = _ordinal
|
||||||
|
|
||||||
|
|
||||||
def _month_doy(month_idx: int) -> int:
|
def _month_doy(month_idx: int) -> int:
|
||||||
|
|
@ -731,6 +747,83 @@ def about_page(request: Request) -> Response:
|
||||||
return _respond_html(request, "about.html.j2", **ctx)
|
return _respond_html(request, "about.html.j2", **ctx)
|
||||||
|
|
||||||
|
|
||||||
|
def privacy_page(request: Request) -> Response:
|
||||||
|
ctx = {
|
||||||
|
"canonical_path": "/privacy",
|
||||||
|
"breadcrumb": [("Home", f"{BASE}/"), ("Privacy", None)],
|
||||||
|
"page_title": "Privacy — Thermograph",
|
||||||
|
"page_description": ("What Thermograph does and does not collect: no tracking, no ads, "
|
||||||
|
"no account required, and location that never leaves your browser."),
|
||||||
|
}
|
||||||
|
return _respond_html(request, "privacy.html.j2", **ctx)
|
||||||
|
|
||||||
|
|
||||||
|
# --- homepage -----------------------------------------------------------------
|
||||||
|
# The homepage is the Weekly tool plus the distribution surfaces around it. It is
|
||||||
|
# server-rendered like the SEO pages (rather than a static file with placeholder
|
||||||
|
# substitution) so a cold visitor with no JS still gets the headline, a real
|
||||||
|
# graded example, the records strip and the city links.
|
||||||
|
|
||||||
|
_HOME_JSONLD = {
|
||||||
|
"@context": "https://schema.org",
|
||||||
|
"@type": "WebApplication",
|
||||||
|
"name": "Thermograph",
|
||||||
|
"applicationCategory": "WeatherApplication",
|
||||||
|
"operatingSystem": "Web, iOS, Android",
|
||||||
|
"isAccessibleForFree": True,
|
||||||
|
"offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"},
|
||||||
|
"description": ("How unusual is your weather? Any day, anywhere on Earth — graded "
|
||||||
|
"against 45 years of that place's own history."),
|
||||||
|
}
|
||||||
|
|
||||||
|
# 12 city chips, chosen for geographic spread rather than raw population, so the
|
||||||
|
# strip reads as "anywhere on Earth" and seeds crawl paths across the hub.
|
||||||
|
HOME_CITY_SLUGS = (
|
||||||
|
"new-york-city-new-york-us", "london-england-gb", "tokyo-jp",
|
||||||
|
"sydney-new-south-wales-au", "sao-paulo-br", "lagos-ng",
|
||||||
|
"mumbai-maharashtra-in", "mexico-city-mx", "cairo-eg",
|
||||||
|
"toronto-ontario-ca", "berlin-state-of-berlin-de", "seattle-washington-us",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _home_cities() -> list[dict]:
|
||||||
|
"""The chip set, skipping any slug not in the routable city list so a
|
||||||
|
regenerated cities.json can never 404 a homepage link."""
|
||||||
|
out = []
|
||||||
|
for slug in HOME_CITY_SLUGS:
|
||||||
|
city = cities.get(slug)
|
||||||
|
if city:
|
||||||
|
out.append({"slug": slug, "name": city["name"]})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def home_page(request: Request) -> Response:
|
||||||
|
feed = homepage.load()
|
||||||
|
stale = bool(feed) and homepage.is_stale(feed)
|
||||||
|
unusual = None
|
||||||
|
ranked: list = []
|
||||||
|
if feed:
|
||||||
|
ranked = feed.get("ranked") or []
|
||||||
|
pick = (feed.get("picks") or {}).get("extreme")
|
||||||
|
if pick:
|
||||||
|
unusual = dict(pick, is_default=True)
|
||||||
|
|
||||||
|
ctx = {
|
||||||
|
"section": "home",
|
||||||
|
# The hero headline owns the page's sole h1, so the brand degrades to a <p>.
|
||||||
|
"brand_tag": "p",
|
||||||
|
# The tool needs the wide app column, not the 880px reading column.
|
||||||
|
"main_class": "",
|
||||||
|
"canonical_path": "/",
|
||||||
|
"unusual": unusual,
|
||||||
|
"stale": stale,
|
||||||
|
"ranked": ranked,
|
||||||
|
"cities": _home_cities(),
|
||||||
|
"jsonld_str": Markup(json.dumps({**_HOME_JSONLD, "url": f"{origin(request)}{BASE}/"})),
|
||||||
|
}
|
||||||
|
return _respond_html(request, "home.html.j2", **ctx)
|
||||||
|
|
||||||
|
|
||||||
# --- registration ------------------------------------------------------------
|
# --- registration ------------------------------------------------------------
|
||||||
def register(app) -> None:
|
def register(app) -> None:
|
||||||
"""Attach all content routes. Call from app.py BEFORE the StaticFiles mount."""
|
"""Attach all content routes. Call from app.py BEFORE the StaticFiles mount."""
|
||||||
|
|
@ -749,7 +842,11 @@ def register(app) -> None:
|
||||||
f"{BASE}/{_inkey}.txt", _indexnow_key_file,
|
f"{BASE}/{_inkey}.txt", _indexnow_key_file,
|
||||||
methods=["GET", "HEAD"], include_in_schema=False,
|
methods=["GET", "HEAD"], include_in_schema=False,
|
||||||
)
|
)
|
||||||
|
# The homepage. Registered here (not as a static file in app.py) so it is
|
||||||
|
# server-rendered from the same Jinja environment as the SEO pages.
|
||||||
|
app.add_api_route(f"{BASE}/", home_page, methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/about", about_page, methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/about", about_page, methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
|
app.add_api_route(f"{BASE}/privacy", privacy_page, methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/glossary", glossary_index, methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/glossary", glossary_index, methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
app.add_api_route(f"{BASE}/glossary/{{term}}", glossary_term, methods=["GET", "HEAD"], include_in_schema=False)
|
app.add_api_route(f"{BASE}/glossary/{{term}}", glossary_term, methods=["GET", "HEAD"], include_in_schema=False)
|
||||||
# Hub before /climate/{slug} so the literal path wins.
|
# Hub before /climate/{slug} so the literal path wins.
|
||||||
|
|
|
||||||
117
digest.py
Normal file
117
digest.py
Normal file
|
|
@ -0,0 +1,117 @@
|
||||||
|
"""Monthly-digest signup.
|
||||||
|
|
||||||
|
The form ships ahead of email delivery on purpose (see models.PendingDigest):
|
||||||
|
collecting the list is the slow part, so signups are stored now and confirmed
|
||||||
|
once SMTP is live. Nothing is mailed while mailer.enabled() is False.
|
||||||
|
|
||||||
|
Privacy: the address is used for the digest and nothing else, and the only other
|
||||||
|
thing recorded is the bare referrer domain for attribution.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from fastapi import Request, Response
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
import mailer
|
||||||
|
import metrics
|
||||||
|
from db import async_session_maker
|
||||||
|
from models import PendingDigest
|
||||||
|
|
||||||
|
# Deliberately permissive: real-world addresses defeat clever patterns, and the
|
||||||
|
# confirmation email is the actual validator. This only rejects obvious junk.
|
||||||
|
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s.]+(\.[^@\s.]+)+$")
|
||||||
|
|
||||||
|
# The same answer for every outcome — new row, duplicate, or throttled. Anything
|
||||||
|
# else would turn the endpoint into an oracle for "is this address subscribed?".
|
||||||
|
GENERIC_OK = "Check your inbox to confirm your address."
|
||||||
|
|
||||||
|
# Don't re-send a confirmation to an unconfirmed address more than this often.
|
||||||
|
RESEND_INTERVAL_S = 600
|
||||||
|
|
||||||
|
|
||||||
|
def _valid(email: str) -> bool:
|
||||||
|
return bool(email) and len(email) <= 320 and bool(_EMAIL_RE.match(email))
|
||||||
|
|
||||||
|
|
||||||
|
async def _store(email: str, place: str | None, source: str | None) -> None:
|
||||||
|
"""Upsert the signup. A repeat submit updates the existing row rather than
|
||||||
|
creating a second one — the unique constraint on email is the idempotency."""
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
existing = (
|
||||||
|
await session.execute(
|
||||||
|
select(PendingDigest).where(PendingDigest.email == email)
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
now = time.time()
|
||||||
|
if existing is None:
|
||||||
|
session.add(PendingDigest(
|
||||||
|
email=email, place_label=place, source=source, created_at=now,
|
||||||
|
))
|
||||||
|
else:
|
||||||
|
if place and not existing.place_label:
|
||||||
|
existing.place_label = place
|
||||||
|
# Re-subscribing after an unsubscribe re-opens the row.
|
||||||
|
existing.unsubscribed_at = None
|
||||||
|
if existing.confirmed_at is None:
|
||||||
|
if existing.last_sent_at and now - existing.last_sent_at < RESEND_INTERVAL_S:
|
||||||
|
await session.commit()
|
||||||
|
return
|
||||||
|
existing.last_sent_at = now
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
# Nothing is sent until a real mail backend is configured; the console
|
||||||
|
# backend logs it so the flow can be verified end-to-end without a server.
|
||||||
|
if mailer.enabled():
|
||||||
|
mailer.send(
|
||||||
|
email,
|
||||||
|
"Confirm your Thermograph digest",
|
||||||
|
"Thanks for signing up for the Thermograph monthly digest.\n\n"
|
||||||
|
"Confirmation links are not active yet — you're on the list, and "
|
||||||
|
"you'll get a confirmation as soon as they are.\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def signup(request: Request) -> Response:
|
||||||
|
"""POST {BASE}/digest — accepts a JSON body (the fetch path) or a plain form
|
||||||
|
post (the no-JS path). Always answers 200 with the same message."""
|
||||||
|
email = ""
|
||||||
|
place = None
|
||||||
|
try:
|
||||||
|
ctype = request.headers.get("content-type", "")
|
||||||
|
if ctype.startswith("application/json"):
|
||||||
|
body = await request.json()
|
||||||
|
if isinstance(body, dict):
|
||||||
|
email = str(body.get("email") or "").strip().lower()
|
||||||
|
place = (str(body.get("place")).strip() or None) if body.get("place") else None
|
||||||
|
else:
|
||||||
|
form = await request.form()
|
||||||
|
email = str(form.get("email") or "").strip().lower()
|
||||||
|
place = (str(form.get("place")).strip() or None) if form.get("place") else None
|
||||||
|
except Exception: # noqa: BLE001 - a malformed body is just an invalid signup
|
||||||
|
pass
|
||||||
|
|
||||||
|
if _valid(email):
|
||||||
|
source = metrics.normalize_referrer(
|
||||||
|
request.headers.get("referer"), request.headers.get("host")
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await _store(email, place, source)
|
||||||
|
except Exception: # noqa: BLE001 - never surface storage detail to the form
|
||||||
|
pass
|
||||||
|
metrics.record_event(
|
||||||
|
"home.digest_signup",
|
||||||
|
referer=request.headers.get("referer"),
|
||||||
|
host=request.headers.get("host"),
|
||||||
|
ip=request.client.host if request.client else None,
|
||||||
|
)
|
||||||
|
return JSONResponse({"ok": True, "message": GENERIC_OK})
|
||||||
|
|
||||||
|
return JSONResponse(
|
||||||
|
{"ok": False, "message": "That doesn't look like an email address."},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
222
homepage.py
Normal file
222
homepage.py
Normal file
|
|
@ -0,0 +1,222 @@
|
||||||
|
"""Homepage "unusual right now" feed.
|
||||||
|
|
||||||
|
The homepage wants to answer "where is the weather most unusual today?" across
|
||||||
|
every city we track. That question has no cheap answer at request time: the
|
||||||
|
derived store keeps graded payloads as zlib-compressed JSON blobs keyed by
|
||||||
|
(kind, cell, key), with no percentile column to sort on, so finding today's
|
||||||
|
extremes means grading every cached city from scratch.
|
||||||
|
|
||||||
|
So we precompute it. A sweep grades every city whose parquet cache is already
|
||||||
|
warm and writes the ranked result to data/homepage.json; the homepage template
|
||||||
|
reads that file. The sweep is deliberately **cache-only** — it calls
|
||||||
|
climate.load_cached_history and climate.load_cached_recent_forecast, never their
|
||||||
|
fetching counterparts, so a sweep over ~1000 cities costs zero upstream requests
|
||||||
|
and cannot burst the archive quota. Cities without a warm cache are skipped; the
|
||||||
|
deploy-time warm_cities.py run is what makes them eligible.
|
||||||
|
|
||||||
|
Runs from the notifier daemon (hourly guard) and at the tail of warm_cities, so
|
||||||
|
it refreshes both on a live server and right after a deploy.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
import cities
|
||||||
|
import climate
|
||||||
|
import grading
|
||||||
|
import grid
|
||||||
|
from views import OBS_COLS
|
||||||
|
|
||||||
|
# data/ is the only writable path under the hardened systemd unit
|
||||||
|
# (ReadWritePaths=/opt/thermograph/data …), so the feed lives there.
|
||||||
|
FEED_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "data", "homepage.json")
|
||||||
|
|
||||||
|
# How many cards the "Unusual right now" strip can show.
|
||||||
|
RANK_LIMIT = 12
|
||||||
|
|
||||||
|
# A city must be at least this far from the median to be worth calling unusual.
|
||||||
|
MIN_DEPARTURE = 15.0
|
||||||
|
|
||||||
|
# A cold-tail card is force-included when one is at or below this percentile,
|
||||||
|
# even if every warmer city outranks it. Both tails or it isn't honest.
|
||||||
|
COLD_TAIL_MAX_PCT = 10.0
|
||||||
|
|
||||||
|
# The feed is stale (label it "as of <date>" rather than claiming today) beyond this.
|
||||||
|
STALE_AFTER_S = 6 * 3600
|
||||||
|
|
||||||
|
# Ranking looks at the two metrics whose percentile people read as "how unusual
|
||||||
|
# was today" — the same pair grade_day's own `departure` score uses.
|
||||||
|
RANK_METRICS = ("tmax", "tmin")
|
||||||
|
|
||||||
|
_METRIC_LABEL = {"tmax": "High temp", "tmin": "Low temp"}
|
||||||
|
|
||||||
|
|
||||||
|
def _seasonal_window_label(d: datetime.date) -> str:
|
||||||
|
"""'mid-July' — the ±7-day seasonal window a percentile is measured against,
|
||||||
|
said the way a person would say it."""
|
||||||
|
part = "early" if d.day <= 10 else ("mid" if d.day <= 20 else "late")
|
||||||
|
return f"{part}-{d.strftime('%B')}"
|
||||||
|
|
||||||
|
|
||||||
|
def _grade_city(city: dict, today: datetime.date) -> dict | None:
|
||||||
|
"""Grade one city's latest observed day from cache alone. None when the city
|
||||||
|
has no warm cache, no observed row, or nothing gradeable."""
|
||||||
|
cell = grid.snap(city["lat"], city["lon"])
|
||||||
|
|
||||||
|
history = climate.load_cached_history(cell)
|
||||||
|
if history is None:
|
||||||
|
return None
|
||||||
|
recent = climate.load_cached_recent_forecast(cell)
|
||||||
|
if recent is None or recent.is_empty() or "date" not in recent.columns:
|
||||||
|
return None
|
||||||
|
|
||||||
|
observed = recent.filter(pl.col("date") <= today).sort("date")
|
||||||
|
if observed.is_empty():
|
||||||
|
return None
|
||||||
|
row = observed.row(observed.height - 1, named=True)
|
||||||
|
obs = {k: row[k] for k in OBS_COLS if k in row}
|
||||||
|
|
||||||
|
try:
|
||||||
|
graded = grading.grade_day(history, row["date"], obs)
|
||||||
|
except Exception: # noqa: BLE001 - one bad cell must not abort the sweep
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Pick the metric that strayed furthest from the median; that is the one the
|
||||||
|
# card should lead with.
|
||||||
|
best = None
|
||||||
|
for metric in RANK_METRICS:
|
||||||
|
g = graded.get(metric)
|
||||||
|
if not g or g.get("percentile") is None:
|
||||||
|
continue
|
||||||
|
departure = abs(g["percentile"] - 50)
|
||||||
|
if best is None or departure > best[0]:
|
||||||
|
best = (departure, metric, g)
|
||||||
|
if best is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
departure, metric, g = best
|
||||||
|
date = row["date"]
|
||||||
|
date_s = date.isoformat() if hasattr(date, "isoformat") else str(date)
|
||||||
|
return {
|
||||||
|
"slug": city["slug"],
|
||||||
|
"name": city["name"],
|
||||||
|
"display": cities.display_name(city),
|
||||||
|
"lat": round(city["lat"], 4),
|
||||||
|
"lon": round(city["lon"], 4),
|
||||||
|
"cell_id": cell["id"],
|
||||||
|
"date": date_s,
|
||||||
|
"metric": metric,
|
||||||
|
"metric_label": _METRIC_LABEL.get(metric, metric),
|
||||||
|
"window_label": _seasonal_window_label(
|
||||||
|
date if hasattr(date, "timetuple") else today
|
||||||
|
),
|
||||||
|
"value": g.get("value"),
|
||||||
|
"percentile": g.get("percentile"),
|
||||||
|
"grade": g.get("grade"),
|
||||||
|
# grading's css class is already the style.css token name minus the
|
||||||
|
# leading '--', so the template emits it with no mapping table.
|
||||||
|
"cls": g.get("class"),
|
||||||
|
"departure": round(departure, 1),
|
||||||
|
"tail": "cold" if g["percentile"] < 50 else "warm",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def build(limit: int | None = None) -> dict:
|
||||||
|
"""Sweep the warm cache and rank today's most unusual cities. Pure — does no
|
||||||
|
I/O beyond reading the parquet cache, and never touches the network."""
|
||||||
|
today = datetime.date.today()
|
||||||
|
graded: list[dict] = []
|
||||||
|
considered = 0
|
||||||
|
|
||||||
|
for city in cities.all_cities()[: limit or None]:
|
||||||
|
row = _grade_city(city, today)
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
considered += 1
|
||||||
|
if row["departure"] >= MIN_DEPARTURE:
|
||||||
|
graded.append(row)
|
||||||
|
|
||||||
|
graded.sort(key=lambda r: r["departure"], reverse=True)
|
||||||
|
ranked = graded[:RANK_LIMIT]
|
||||||
|
|
||||||
|
# Two-sided honesty: if any tracked city sits in the cold tail, one must
|
||||||
|
# appear in the strip even when warm anomalies dominate the ranking.
|
||||||
|
if not any(r["tail"] == "cold" for r in ranked):
|
||||||
|
cold = next(
|
||||||
|
(r for r in graded if r["tail"] == "cold"
|
||||||
|
and r["percentile"] <= COLD_TAIL_MAX_PCT),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if cold is not None:
|
||||||
|
ranked = ranked[: RANK_LIMIT - 1] + [cold]
|
||||||
|
|
||||||
|
picks = {}
|
||||||
|
if graded:
|
||||||
|
picks["extreme"] = graded[0]
|
||||||
|
cold = next((r for r in graded if r["tail"] == "cold"), None)
|
||||||
|
if cold is not None:
|
||||||
|
picks["cold"] = cold
|
||||||
|
|
||||||
|
return {
|
||||||
|
"generated_at": time.time(),
|
||||||
|
"date": today.isoformat(),
|
||||||
|
"considered": considered,
|
||||||
|
"picks": picks,
|
||||||
|
"ranked": ranked,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def refresh(limit: int | None = None) -> dict:
|
||||||
|
"""Rebuild the feed and write it atomically, so a concurrent reader never
|
||||||
|
sees a half-written file."""
|
||||||
|
feed = build(limit=limit)
|
||||||
|
path = os.path.abspath(FEED_PATH)
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp")
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(feed, f, separators=(",", ":"))
|
||||||
|
os.replace(tmp, path)
|
||||||
|
except BaseException:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
return feed
|
||||||
|
|
||||||
|
|
||||||
|
def load() -> dict | None:
|
||||||
|
"""Read the feed. Returns None when it is missing or unreadable — the
|
||||||
|
homepage renders its frame without live numbers rather than failing, so a
|
||||||
|
cold checkout with no feed still serves a complete page."""
|
||||||
|
try:
|
||||||
|
with open(os.path.abspath(FEED_PATH), encoding="utf-8") as f:
|
||||||
|
feed = json.load(f)
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return None
|
||||||
|
if not isinstance(feed, dict) or "ranked" not in feed:
|
||||||
|
return None
|
||||||
|
return feed
|
||||||
|
|
||||||
|
|
||||||
|
def is_stale(feed: dict) -> bool:
|
||||||
|
"""Older than STALE_AFTER_S, or built for a day that is no longer today."""
|
||||||
|
if feed.get("date") != datetime.date.today().isoformat():
|
||||||
|
return True
|
||||||
|
return (time.time() - float(feed.get("generated_at") or 0)) > STALE_AFTER_S
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
n = int(sys.argv[1]) if len(sys.argv) > 1 else None
|
||||||
|
out = refresh(limit=n)
|
||||||
|
print(f"graded {out['considered']} cached cities, "
|
||||||
|
f"{len(out['ranked'])} ranked, top: "
|
||||||
|
f"{out['ranked'][0]['display'] if out['ranked'] else '—'}")
|
||||||
100
mailer.py
Normal file
100
mailer.py
Normal file
|
|
@ -0,0 +1,100 @@
|
||||||
|
"""Outbound email, over stdlib smtplib only (no new dependency).
|
||||||
|
|
||||||
|
**The seam, not the policy.** The app always talks plain SMTP to whatever
|
||||||
|
``THERMOGRAPH_SMTP_HOST`` points at — in production, a Postfix null client
|
||||||
|
listening on 127.0.0.1:25 (see deploy/provision-mail.sh). Whether Postfix then
|
||||||
|
delivers straight to the recipient's MX or relays through a transactional
|
||||||
|
provider is a Postfix config decision that needs no change here. That is the
|
||||||
|
point of routing through a local MTA rather than talking to a provider's API:
|
||||||
|
delivery policy stays swappable, and a slow or failing upstream can't block a
|
||||||
|
request, because Postfix queues and retries on our behalf.
|
||||||
|
|
||||||
|
**Default-safe.** The backend defaults to ``console`` — it logs the message and
|
||||||
|
sends nothing — so LAN dev and the test suite can exercise the full signup path
|
||||||
|
without a mail server and without any risk of mailing a real person. Production
|
||||||
|
opts in with ``THERMOGRAPH_MAIL_BACKEND=smtp``, the same shape as
|
||||||
|
``THERMOGRAPH_COOKIE_SECURE``.
|
||||||
|
|
||||||
|
``send()`` never raises. Callers treat email like push notifications: a channel
|
||||||
|
that may fail without taking the request down with it.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import smtplib
|
||||||
|
import ssl
|
||||||
|
from email.message import EmailMessage
|
||||||
|
from email.utils import formataddr, parseaddr
|
||||||
|
|
||||||
|
log = logging.getLogger("thermograph.mail")
|
||||||
|
|
||||||
|
# console -> log the message, send nothing (default; dev + tests)
|
||||||
|
# smtp -> hand it to the configured SMTP host
|
||||||
|
# disabled -> drop silently (a kill switch that needs no config removal)
|
||||||
|
BACKEND = os.environ.get("THERMOGRAPH_MAIL_BACKEND", "console").strip().lower()
|
||||||
|
|
||||||
|
SMTP_HOST = os.environ.get("THERMOGRAPH_SMTP_HOST", "127.0.0.1").strip()
|
||||||
|
SMTP_PORT = int(os.environ.get("THERMOGRAPH_SMTP_PORT", "25") or 25)
|
||||||
|
SMTP_USER = os.environ.get("THERMOGRAPH_SMTP_USER", "").strip()
|
||||||
|
SMTP_PASSWORD = os.environ.get("THERMOGRAPH_SMTP_PASSWORD", "")
|
||||||
|
SMTP_STARTTLS = os.environ.get("THERMOGRAPH_SMTP_STARTTLS", "").strip() in ("1", "true", "yes")
|
||||||
|
SMTP_TIMEOUT = float(os.environ.get("THERMOGRAPH_SMTP_TIMEOUT", "10") or 10)
|
||||||
|
|
||||||
|
MAIL_FROM = os.environ.get("THERMOGRAPH_MAIL_FROM", "Thermograph <no-reply@thermograph.org>").strip()
|
||||||
|
MAIL_REPLY_TO = os.environ.get("THERMOGRAPH_MAIL_REPLY_TO", "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def enabled() -> bool:
|
||||||
|
"""Whether a send would actually leave the process."""
|
||||||
|
return BACKEND == "smtp"
|
||||||
|
|
||||||
|
|
||||||
|
def _build(to: str, subject: str, text: str, html: "str | None") -> EmailMessage:
|
||||||
|
msg = EmailMessage()
|
||||||
|
name, addr = parseaddr(MAIL_FROM)
|
||||||
|
msg["From"] = formataddr((name, addr)) if addr else MAIL_FROM
|
||||||
|
msg["To"] = to
|
||||||
|
msg["Subject"] = subject
|
||||||
|
if MAIL_REPLY_TO:
|
||||||
|
msg["Reply-To"] = MAIL_REPLY_TO
|
||||||
|
# Plain text is the body; HTML is the alternative, so text-only clients and
|
||||||
|
# spam filters both see real content.
|
||||||
|
msg.set_content(text)
|
||||||
|
if html:
|
||||||
|
msg.add_alternative(html, subtype="html")
|
||||||
|
return msg
|
||||||
|
|
||||||
|
|
||||||
|
def send(to: str, subject: str, text: str, html: "str | None" = None) -> bool:
|
||||||
|
"""Send one message. Returns whether it was handed off successfully.
|
||||||
|
|
||||||
|
Never raises: a mail failure must not break the request that triggered it.
|
||||||
|
"""
|
||||||
|
if not to or "@" not in to:
|
||||||
|
return False
|
||||||
|
if BACKEND == "disabled":
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
msg = _build(to, subject, text, html)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.exception("mail: could not build message")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if BACKEND != "smtp":
|
||||||
|
# console (the default): prove the flow end-to-end without sending.
|
||||||
|
log.info("mail[console] to=%s subject=%s\n%s", to, subject, text)
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=SMTP_TIMEOUT) as smtp:
|
||||||
|
if SMTP_STARTTLS:
|
||||||
|
smtp.starttls(context=ssl.create_default_context())
|
||||||
|
if SMTP_USER:
|
||||||
|
smtp.login(SMTP_USER, SMTP_PASSWORD)
|
||||||
|
smtp.send_message(msg)
|
||||||
|
return True
|
||||||
|
except Exception: # noqa: BLE001 - deliverability is never worth a 500
|
||||||
|
log.exception("mail: send failed to=%s host=%s:%s", to, SMTP_HOST, SMTP_PORT)
|
||||||
|
return False
|
||||||
201
metrics.py
201
metrics.py
|
|
@ -19,6 +19,7 @@ import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
# The ``phase=`` tag passed to ``climate._request`` maps 1:1 onto an external source.
|
# The ``phase=`` tag passed to ``climate._request`` maps 1:1 onto an external source.
|
||||||
# Keep this in sync with the phase names used at each call site in ``climate.py``.
|
# Keep this in sync with the phase names used at each call site in ``climate.py``.
|
||||||
|
|
@ -90,6 +91,10 @@ def classify_inbound(path: str, base: str = "") -> str:
|
||||||
return "accounts"
|
return "accounts"
|
||||||
if seg == "metrics":
|
if seg == "metrics":
|
||||||
return "metrics"
|
return "metrics"
|
||||||
|
# The event beacon reports traffic; it is not itself traffic. Its own
|
||||||
|
# category keeps record_inbound from double-counting every interaction.
|
||||||
|
if seg == "event":
|
||||||
|
return "event"
|
||||||
return f"api:{seg}" if seg else "api:other"
|
return f"api:{seg}" if seg else "api:other"
|
||||||
if p.endswith((".js", ".css", ".html", ".webmanifest", ".png", ".svg", ".ico",
|
if p.endswith((".js", ".css", ".html", ".webmanifest", ".png", ".svg", ".ico",
|
||||||
".json", ".woff", ".woff2", ".map", ".txt", ".xml")):
|
".json", ".woff", ".woff2", ".map", ".txt", ".xml")):
|
||||||
|
|
@ -97,13 +102,89 @@ def classify_inbound(path: str, base: str = "") -> str:
|
||||||
if p in ("/robots.txt", "/sitemap.xml"):
|
if p in ("/robots.txt", "/sitemap.xml"):
|
||||||
return "seo"
|
return "seo"
|
||||||
return "static"
|
return "static"
|
||||||
if p in ("", "/", "/calendar", "/day", "/compare", "/legend", "/alerts"):
|
if p in ("", "/", "/calendar", "/day", "/compare", "/legend", "/alerts", "/privacy"):
|
||||||
return "page"
|
return "page"
|
||||||
if p == "/about" or p.startswith("/climate") or p.startswith("/glossary"):
|
if p == "/about" or p.startswith("/climate") or p.startswith("/glossary"):
|
||||||
return "page"
|
return "page"
|
||||||
return "other"
|
return "other"
|
||||||
|
|
||||||
|
|
||||||
|
# --- product events ----------------------------------------------------------
|
||||||
|
# Distinct from inbound traffic: these count *intent* (tapped "use my location",
|
||||||
|
# submitted the digest form) rather than requests. Keyed by (event, referrer
|
||||||
|
# domain, UTC day) so a launch post's traffic can be told apart from search.
|
||||||
|
#
|
||||||
|
# Everything here is aggregate — no cookies, no per-visitor id, no full URLs.
|
||||||
|
|
||||||
|
EVENTS = frozenset({
|
||||||
|
"home.locate", "home.digest_signup", "home.records_click", "home.share",
|
||||||
|
})
|
||||||
|
# home.nav_{surface}: only these surfaces, so the key space stays bounded.
|
||||||
|
NAV_SURFACES = frozenset({
|
||||||
|
"calendar", "compare", "day", "alerts", "city", "cities", "records",
|
||||||
|
})
|
||||||
|
# Anything unrecognized collapses here, with no referrer dimension — visible
|
||||||
|
# enough to notice abuse, bounded enough to be harmless.
|
||||||
|
EVENT_OTHER = "event:other"
|
||||||
|
|
||||||
|
EVENT_DAYS = 7 # how many days of event history to keep
|
||||||
|
MAX_REFERRERS = 200 # distinct referrer domains tracked before overflow
|
||||||
|
REFERRER_OTHER = "other"
|
||||||
|
|
||||||
|
_REF_OK = set("abcdefghijklmnopqrstuvwxyz0123456789.-")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_event(name: str) -> str:
|
||||||
|
"""An allowlisted event name, or EVENT_OTHER. The allowlist is what keeps an
|
||||||
|
unauthenticated endpoint from being turned into unbounded storage."""
|
||||||
|
name = (name or "").strip()
|
||||||
|
if name in EVENTS:
|
||||||
|
return name
|
||||||
|
surface = name.partition("home.nav_")[2] if name.startswith("home.nav_") else ""
|
||||||
|
if surface and surface in NAV_SURFACES:
|
||||||
|
return name
|
||||||
|
return EVENT_OTHER
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_referrer(referer: "str | None", host: "str | None" = None) -> str:
|
||||||
|
"""The bare registrable-ish domain of a Referer header — never the full URL,
|
||||||
|
which can carry a search query or a private path. 'direct' when absent,
|
||||||
|
'self' for our own pages."""
|
||||||
|
if not referer:
|
||||||
|
return "direct"
|
||||||
|
try:
|
||||||
|
netloc = urllib.parse.urlsplit(referer).netloc
|
||||||
|
except ValueError:
|
||||||
|
return REFERRER_OTHER
|
||||||
|
domain = (netloc.rpartition("@")[2].partition(":")[0] or "").lower()
|
||||||
|
if domain.startswith("www."):
|
||||||
|
domain = domain[4:]
|
||||||
|
if host and domain == (host.partition(":")[0] or "").lower().removeprefix("www."):
|
||||||
|
return "self"
|
||||||
|
domain = "".join(ch for ch in domain if ch in _REF_OK)[:64]
|
||||||
|
return domain or REFERRER_OTHER
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_day() -> str:
|
||||||
|
return time.strftime("%Y-%m-%d", time.gmtime())
|
||||||
|
|
||||||
|
|
||||||
|
def _recent_days() -> "set[str]":
|
||||||
|
"""The EVENT_DAYS UTC days still worth keeping."""
|
||||||
|
now = time.time()
|
||||||
|
return {time.strftime("%Y-%m-%d", time.gmtime(now - d * 86400))
|
||||||
|
for d in range(EVENT_DAYS)}
|
||||||
|
|
||||||
|
|
||||||
|
def _nest_events(rows) -> dict:
|
||||||
|
"""Flat (event, referrer, day, count) rows -> {event: {referrer: {day: n}}}."""
|
||||||
|
out: "dict[str, dict[str, dict[str, int]]]" = {}
|
||||||
|
for key, count in rows:
|
||||||
|
event, referrer, day = key
|
||||||
|
out.setdefault(event, {}).setdefault(referrer, {})[day] = count
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _now_minute() -> int:
|
def _now_minute() -> int:
|
||||||
"""Current epoch-minute. Uses ``time.time()`` so tests can monkeypatch the clock."""
|
"""Current epoch-minute. Uses ``time.time()`` so tests can monkeypatch the clock."""
|
||||||
return int(time.time() // 60)
|
return int(time.time() // 60)
|
||||||
|
|
@ -129,6 +210,9 @@ class _MemStore:
|
||||||
# name -> {"ts": last-beat epoch, "interval_s": expected cadence}. Lets the
|
# name -> {"ts": last-beat epoch, "interval_s": expected cadence}. Lets the
|
||||||
# dashboard judge a background daemon by liveness, not per-worker thread presence.
|
# dashboard judge a background daemon by liveness, not per-worker thread presence.
|
||||||
self._heartbeats: "dict[str, dict[str, float]]" = {}
|
self._heartbeats: "dict[str, dict[str, float]]" = {}
|
||||||
|
# (event, referrer, day) -> count, pruned to EVENT_DAYS on write.
|
||||||
|
self._events: "dict[tuple[str, str, str], int]" = {}
|
||||||
|
self._recent_ev: "dict[int, dict[str, int]]" = {}
|
||||||
|
|
||||||
def _bump_recent(self, buckets, key) -> None:
|
def _bump_recent(self, buckets, key) -> None:
|
||||||
"""Count one event for ``key`` in the current-minute bucket (caller holds lock)."""
|
"""Count one event for ``key`` in the current-minute bucket (caller holds lock)."""
|
||||||
|
|
@ -173,6 +257,21 @@ class _MemStore:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._heartbeats[name] = {"ts": ts, "interval_s": interval_s}
|
self._heartbeats[name] = {"ts": ts, "interval_s": interval_s}
|
||||||
|
|
||||||
|
def record_event(self, event, referrer, day) -> None:
|
||||||
|
with self._lock:
|
||||||
|
# Cap distinct referrers so a spray of forged Referer headers can't
|
||||||
|
# grow the key space without bound.
|
||||||
|
if referrer not in ("direct", "self", REFERRER_OTHER):
|
||||||
|
seen = {r for _, r, _ in self._events}
|
||||||
|
if referrer not in seen and len(seen) >= MAX_REFERRERS:
|
||||||
|
referrer = REFERRER_OTHER
|
||||||
|
key = (event, referrer, day)
|
||||||
|
self._events[key] = self._events.get(key, 0) + 1
|
||||||
|
keep = _recent_days()
|
||||||
|
for k in [k for k in self._events if k[2] not in keep]:
|
||||||
|
del self._events[k]
|
||||||
|
self._bump_recent(self._recent_ev, event)
|
||||||
|
|
||||||
def read(self) -> dict:
|
def read(self) -> dict:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
inbound = {k: dict(v) for k, v in self._inbound.items()}
|
inbound = {k: dict(v) for k, v in self._inbound.items()}
|
||||||
|
|
@ -184,6 +283,8 @@ class _MemStore:
|
||||||
"inbound_recent": self._recent_totals(self._recent_in),
|
"inbound_recent": self._recent_totals(self._recent_in),
|
||||||
"outbound_recent": self._recent_totals(self._recent_out),
|
"outbound_recent": self._recent_totals(self._recent_out),
|
||||||
"heartbeats": {k: dict(v) for k, v in self._heartbeats.items()},
|
"heartbeats": {k: dict(v) for k, v in self._heartbeats.items()},
|
||||||
|
"events": _nest_events(self._events.items()),
|
||||||
|
"events_recent": self._recent_totals(self._recent_ev),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -212,6 +313,11 @@ class _SqliteStore:
|
||||||
minute INTEGER, category TEXT, count INTEGER, PRIMARY KEY(minute, category));
|
minute INTEGER, category TEXT, count INTEGER, PRIMARY KEY(minute, category));
|
||||||
CREATE TABLE IF NOT EXISTS outbound_minute(
|
CREATE TABLE IF NOT EXISTS outbound_minute(
|
||||||
minute INTEGER, source TEXT, count INTEGER, PRIMARY KEY(minute, source));
|
minute INTEGER, source TEXT, count INTEGER, PRIMARY KEY(minute, source));
|
||||||
|
CREATE TABLE IF NOT EXISTS event_cume(
|
||||||
|
event TEXT, referrer TEXT, day TEXT, count INTEGER,
|
||||||
|
PRIMARY KEY(event, referrer, day));
|
||||||
|
CREATE TABLE IF NOT EXISTS event_minute(
|
||||||
|
minute INTEGER, event TEXT, count INTEGER, PRIMARY KEY(minute, event));
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
# First worker to touch the DB stamps the shared start time; the rest inherit it.
|
# First worker to touch the DB stamps the shared start time; the rest inherit it.
|
||||||
|
|
@ -235,6 +341,12 @@ class _SqliteStore:
|
||||||
cutoff = minute - WINDOW_MINUTES
|
cutoff = minute - WINDOW_MINUTES
|
||||||
self._db.execute("DELETE FROM inbound_minute WHERE minute <= ?", (cutoff,))
|
self._db.execute("DELETE FROM inbound_minute WHERE minute <= ?", (cutoff,))
|
||||||
self._db.execute("DELETE FROM outbound_minute WHERE minute <= ?", (cutoff,))
|
self._db.execute("DELETE FROM outbound_minute WHERE minute <= ?", (cutoff,))
|
||||||
|
self._db.execute("DELETE FROM event_minute WHERE minute <= ?", (cutoff,))
|
||||||
|
# Event history is kept by day, not by minute.
|
||||||
|
self._db.execute(
|
||||||
|
"DELETE FROM event_cume WHERE day NOT IN (%s)"
|
||||||
|
% ",".join("?" * EVENT_DAYS), tuple(sorted(_recent_days())),
|
||||||
|
)
|
||||||
|
|
||||||
def record_inbound(self, category, bucket) -> None:
|
def record_inbound(self, category, bucket) -> None:
|
||||||
minute = _now_minute()
|
minute = _now_minute()
|
||||||
|
|
@ -266,6 +378,29 @@ class _SqliteStore:
|
||||||
)
|
)
|
||||||
self._prune(minute)
|
self._prune(minute)
|
||||||
|
|
||||||
|
def record_event(self, event, referrer, day) -> None:
|
||||||
|
minute = _now_minute()
|
||||||
|
with self._lock:
|
||||||
|
if referrer not in ("direct", "self", REFERRER_OTHER):
|
||||||
|
row = self._db.execute(
|
||||||
|
"SELECT COUNT(DISTINCT referrer) FROM event_cume").fetchone()
|
||||||
|
known = self._db.execute(
|
||||||
|
"SELECT 1 FROM event_cume WHERE referrer = ? LIMIT 1", (referrer,)
|
||||||
|
).fetchone()
|
||||||
|
if not known and row and row[0] >= MAX_REFERRERS:
|
||||||
|
referrer = REFERRER_OTHER
|
||||||
|
self._db.execute(
|
||||||
|
"INSERT INTO event_cume(event, referrer, day, count) VALUES(?, ?, ?, 1) "
|
||||||
|
"ON CONFLICT(event, referrer, day) DO UPDATE SET count = count + 1",
|
||||||
|
(event, referrer, day),
|
||||||
|
)
|
||||||
|
self._db.execute(
|
||||||
|
"INSERT INTO event_minute(minute, event, count) VALUES(?, ?, 1) "
|
||||||
|
"ON CONFLICT(minute, event) DO UPDATE SET count = count + 1",
|
||||||
|
(minute, event),
|
||||||
|
)
|
||||||
|
self._prune(minute)
|
||||||
|
|
||||||
def record_heartbeat(self, name, ts, interval_s) -> None:
|
def record_heartbeat(self, name, ts, interval_s) -> None:
|
||||||
# Reuse the meta table (two keys per daemon) so every worker's beats land in the
|
# Reuse the meta table (two keys per daemon) so every worker's beats land in the
|
||||||
# one shared file — the dashboard then sees the notifier's liveness whichever
|
# one shared file — the dashboard then sees the notifier's liveness whichever
|
||||||
|
|
@ -312,6 +447,14 @@ class _SqliteStore:
|
||||||
recent_out = dict(self._db.execute(
|
recent_out = dict(self._db.execute(
|
||||||
"SELECT source, SUM(count) FROM outbound_minute WHERE minute >= ? "
|
"SELECT source, SUM(count) FROM outbound_minute WHERE minute >= ? "
|
||||||
"GROUP BY source", (cutoff,)))
|
"GROUP BY source", (cutoff,)))
|
||||||
|
events = _nest_events(
|
||||||
|
((event, referrer, day), count)
|
||||||
|
for event, referrer, day, count in self._db.execute(
|
||||||
|
"SELECT event, referrer, day, count FROM event_cume")
|
||||||
|
)
|
||||||
|
recent_ev = dict(self._db.execute(
|
||||||
|
"SELECT event, SUM(count) FROM event_minute WHERE minute >= ? "
|
||||||
|
"GROUP BY event", (cutoff,)))
|
||||||
return {
|
return {
|
||||||
"started_at": self.started_at,
|
"started_at": self.started_at,
|
||||||
"inbound": inbound,
|
"inbound": inbound,
|
||||||
|
|
@ -319,6 +462,8 @@ class _SqliteStore:
|
||||||
"inbound_recent": recent_in,
|
"inbound_recent": recent_in,
|
||||||
"outbound_recent": recent_out,
|
"outbound_recent": recent_out,
|
||||||
"heartbeats": self._read_heartbeats(),
|
"heartbeats": self._read_heartbeats(),
|
||||||
|
"events": events,
|
||||||
|
"events_recent": recent_ev,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -342,7 +487,9 @@ def record_inbound(category: str, status) -> None:
|
||||||
"""Count one inbound request in ``category``, bucketed by status class."""
|
"""Count one inbound request in ``category``, bucketed by status class."""
|
||||||
# The metrics endpoint is polled by the dashboard itself — counting it would just
|
# The metrics endpoint is polled by the dashboard itself — counting it would just
|
||||||
# show the monitor watching itself, so it's excluded from traffic entirely.
|
# show the monitor watching itself, so it's excluded from traffic entirely.
|
||||||
if category == "metrics":
|
# The event beacon is likewise not traffic: it is the *record* of traffic, and
|
||||||
|
# counting it would double every interaction it reports.
|
||||||
|
if category in ("metrics", "event"):
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
_store.record_inbound(category, _status_bucket(status))
|
_store.record_inbound(category, _status_bucket(status))
|
||||||
|
|
@ -358,6 +505,44 @@ def record_outbound(phase: str, outcome: str) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# Per-IP token bucket for the unauthenticated event beacon. Bounded: the map is
|
||||||
|
# cleared whenever the minute rolls, so it holds at most one minute of clients.
|
||||||
|
_EVENT_RATE_PER_MIN = 30
|
||||||
|
_rate_lock = threading.Lock()
|
||||||
|
_rate_minute = -1
|
||||||
|
_rate_counts: "dict[str, int]" = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _rate_ok(ip: "str | None") -> bool:
|
||||||
|
global _rate_minute
|
||||||
|
key = ip or "?"
|
||||||
|
minute = _now_minute()
|
||||||
|
with _rate_lock:
|
||||||
|
if minute != _rate_minute:
|
||||||
|
_rate_minute = minute
|
||||||
|
_rate_counts.clear()
|
||||||
|
count = _rate_counts.get(key, 0) + 1
|
||||||
|
_rate_counts[key] = count
|
||||||
|
return count <= _EVENT_RATE_PER_MIN
|
||||||
|
|
||||||
|
|
||||||
|
def record_event(name: str, referer: "str | None" = None,
|
||||||
|
host: "str | None" = None, ip: "str | None" = None) -> None:
|
||||||
|
"""Count one product event. The name is allowlisted and the referrer is taken
|
||||||
|
from the request's own Referer header (never client-supplied, which would be
|
||||||
|
trivially forgeable). Best-effort and silent: the caller always answers 204,
|
||||||
|
so a rejected or rate-limited event tells a prober nothing."""
|
||||||
|
try:
|
||||||
|
if not _rate_ok(ip):
|
||||||
|
return
|
||||||
|
event = normalize_event(name)
|
||||||
|
# An unrecognized event carries no referrer dimension — one bucket total.
|
||||||
|
referrer = "direct" if event == EVENT_OTHER else normalize_referrer(referer, host)
|
||||||
|
_store.record_event(event, referrer, _utc_day())
|
||||||
|
except Exception: # noqa: BLE001 - instrumentation must never break a request
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def record_heartbeat(name: str, interval_s: float) -> None:
|
def record_heartbeat(name: str, interval_s: float) -> None:
|
||||||
"""Stamp ``name``'s liveness. ``interval_s`` is the daemon's expected beat cadence, so
|
"""Stamp ``name``'s liveness. ``interval_s`` is the daemon's expected beat cadence, so
|
||||||
a reader can tell fresh from stale without knowing the schedule. Best-effort."""
|
a reader can tell fresh from stale without knowing the schedule. Best-effort."""
|
||||||
|
|
@ -373,7 +558,8 @@ def snapshot() -> dict:
|
||||||
data = _store.read()
|
data = _store.read()
|
||||||
except Exception: # noqa: BLE001 - a locked/broken store must not 500 the dashboard
|
except Exception: # noqa: BLE001 - a locked/broken store must not 500 the dashboard
|
||||||
data = {"started_at": time.time(), "inbound": {}, "outbound": {},
|
data = {"started_at": time.time(), "inbound": {}, "outbound": {},
|
||||||
"inbound_recent": {}, "outbound_recent": {}, "heartbeats": {}}
|
"inbound_recent": {}, "outbound_recent": {}, "heartbeats": {},
|
||||||
|
"events": {}, "events_recent": {}}
|
||||||
inbound = data["inbound"]
|
inbound = data["inbound"]
|
||||||
outbound = data["outbound"]
|
outbound = data["outbound"]
|
||||||
started_at = data["started_at"]
|
started_at = data["started_at"]
|
||||||
|
|
@ -398,4 +584,13 @@ def snapshot() -> dict:
|
||||||
"inbound_recent": data["inbound_recent"],
|
"inbound_recent": data["inbound_recent"],
|
||||||
"outbound_recent": data["outbound_recent"],
|
"outbound_recent": data["outbound_recent"],
|
||||||
"heartbeats": heartbeats,
|
"heartbeats": heartbeats,
|
||||||
|
# Product events: {event: {referrer_domain: {day: count}}}, aggregate only.
|
||||||
|
"events": data.get("events") or {},
|
||||||
|
"events_recent": data.get("events_recent") or {},
|
||||||
|
"events_total": sum(
|
||||||
|
n
|
||||||
|
for refs in (data.get("events") or {}).values()
|
||||||
|
for days in refs.values()
|
||||||
|
for n in days.values()
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
39
models.py
39
models.py
|
|
@ -141,3 +141,42 @@ class PushSubscription(Base):
|
||||||
UniqueConstraint("endpoint", name="uq_push_endpoint"),
|
UniqueConstraint("endpoint", name="uq_push_endpoint"),
|
||||||
Index("idx_push_user", "user_id"),
|
Index("idx_push_user", "user_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PendingDigest(Base):
|
||||||
|
"""A monthly-digest signup, collected before email delivery is wired up.
|
||||||
|
|
||||||
|
The digest form ships ahead of SMTP on purpose: building the list is the
|
||||||
|
slow part, and making people wait for the mailer would throw away every
|
||||||
|
signup in the meantime. Rows land here unconfirmed; once delivery is live, a
|
||||||
|
confirmation pass mails each address and stamps ``confirmed_at``.
|
||||||
|
|
||||||
|
Deliberately NOT tied to ``user`` — signing up for the digest must not
|
||||||
|
require an account, and most subscribers won't have one.
|
||||||
|
"""
|
||||||
|
__tablename__ = "pending_digest"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
# 320 = the practical maximum length of an email address (64 local + @ + 255 domain).
|
||||||
|
email: Mapped[str] = mapped_column(String(320), nullable=False)
|
||||||
|
# The place the digest should cover. Optional: an address with no place is
|
||||||
|
# still a real signup, and the place can be asked for at confirmation time.
|
||||||
|
place_label: Mapped[str | None] = mapped_column(String(200), nullable=True)
|
||||||
|
lat: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
lon: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
cell_id: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||||
|
# Where the signup came from (bare referrer domain), for attribution only.
|
||||||
|
source: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[float] = mapped_column(Float, nullable=False, default=time.time)
|
||||||
|
# Set when the address is verified. The double opt-in token is stored as a
|
||||||
|
# sha256 hash, never in the clear, so a leaked database can't confirm addresses.
|
||||||
|
token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
confirmed_at: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
last_sent_at: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
unsubscribed_at: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# One row per address: a re-submit updates in place rather than
|
||||||
|
# duplicating, which is what makes the form idempotent.
|
||||||
|
UniqueConstraint("email", name="uq_pending_digest_email"),
|
||||||
|
)
|
||||||
|
|
|
||||||
22
notify.py
22
notify.py
|
|
@ -32,6 +32,7 @@ from sqlalchemy import delete, select
|
||||||
import climate
|
import climate
|
||||||
import grading
|
import grading
|
||||||
import grid
|
import grid
|
||||||
|
import homepage
|
||||||
import metrics
|
import metrics
|
||||||
import push
|
import push
|
||||||
from db import sync_session_maker
|
from db import sync_session_maker
|
||||||
|
|
@ -291,6 +292,26 @@ def run_pass() -> int:
|
||||||
return created
|
return created
|
||||||
|
|
||||||
|
|
||||||
|
# The homepage's "unusual right now" feed rides along on this loop rather than
|
||||||
|
# running its own thread: this one already wakes on a timer and already carries
|
||||||
|
# the single-leader invariant a second daemon would have to re-establish. The
|
||||||
|
# sweep is cache-only, so it costs no upstream quota.
|
||||||
|
HOMEPAGE_REFRESH_INTERVAL = 3600.0
|
||||||
|
_last_homepage_refresh = 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _maybe_refresh_homepage() -> None:
|
||||||
|
global _last_homepage_refresh
|
||||||
|
now = time.time()
|
||||||
|
if now - _last_homepage_refresh < HOMEPAGE_REFRESH_INTERVAL:
|
||||||
|
return
|
||||||
|
_last_homepage_refresh = now
|
||||||
|
try:
|
||||||
|
homepage.refresh()
|
||||||
|
except Exception: # noqa: BLE001 - the feed is optional; the homepage renders without it
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def run_loop():
|
def run_loop():
|
||||||
# Beat once at startup so the dashboard shows the notifier alive immediately after a
|
# Beat once at startup so the dashboard shows the notifier alive immediately after a
|
||||||
# (re)start, without waiting a full INTERVAL for the first pass. Then beat every wake:
|
# (re)start, without waiting a full INTERVAL for the first pass. Then beat every wake:
|
||||||
|
|
@ -304,6 +325,7 @@ def run_loop():
|
||||||
run_pass()
|
run_pass()
|
||||||
except Exception: # noqa: BLE001 - a bad pass must never kill the loop
|
except Exception: # noqa: BLE001 - a bad pass must never kill the loop
|
||||||
pass
|
pass
|
||||||
|
_maybe_refresh_homepage()
|
||||||
|
|
||||||
|
|
||||||
def start():
|
def start():
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@
|
||||||
<link rel="icon" href="{{ base }}/favicon-16.png" type="image/png" sizes="16x16" />
|
<link rel="icon" href="{{ base }}/favicon-16.png" type="image/png" sizes="16x16" />
|
||||||
<link rel="apple-touch-icon" href="{{ base }}/apple-touch-icon.png" />
|
<link rel="apple-touch-icon" href="{{ base }}/apple-touch-icon.png" />
|
||||||
<link rel="manifest" href="{{ base }}/manifest.webmanifest" />
|
<link rel="manifest" href="{{ base }}/manifest.webmanifest" />
|
||||||
|
{% block head_extra %}{% endblock %}
|
||||||
<link rel="stylesheet" href="{{ base }}/style.css" />
|
<link rel="stylesheet" href="{{ base }}/style.css" />
|
||||||
{% block jsonld %}{% endblock %}
|
{% block jsonld %}{% endblock %}
|
||||||
</head>
|
</head>
|
||||||
|
|
@ -31,17 +32,25 @@
|
||||||
<header>
|
<header>
|
||||||
<div class="brand">
|
<div class="brand">
|
||||||
<div>
|
<div>
|
||||||
<h1 class="site-name"><a href="{{ base }}/"><span class="logo"><svg viewBox="0 0 512 512" width="28" height="28" aria-hidden="true"><rect x="32" y="32" width="448" height="448" rx="96" fill="#10141B" stroke="#fff" stroke-opacity=".08" stroke-width="4"/><rect x="64" y="170" width="384" height="48" rx="8" fill="#27131C"/><rect x="64" y="222" width="384" height="48" rx="8" fill="#38322E"/><rect x="64" y="274" width="384" height="48" rx="8" fill="#1A2D27"/><rect x="64" y="326" width="384" height="48" rx="8" fill="#26333F"/><rect x="64" y="378" width="384" height="48" rx="8" fill="#18293A"/><line x1="64" y1="132" x2="448" y2="132" stroke="#46586E" stroke-width="8" stroke-dasharray="26 20"/><path d="M64 410H126V386H160" fill="none" stroke="var(--cold)" stroke-width="34"/><path d="M160 403V352H244V368H282" fill="none" stroke="var(--cool)" stroke-width="34"/><path d="M282 385V320H312" fill="none" stroke="var(--normal)" stroke-width="34"/><path d="M312 337V248H338" fill="none" stroke="var(--warm)" stroke-width="34"/><path d="M338 265V196H366" fill="none" stroke="var(--hot)" stroke-width="34"/><path d="M366 213V132H396" fill="none" stroke="var(--very-hot)" stroke-width="34"/><rect x="382" y="100" width="64" height="64" rx="12" fill="var(--rec-hot)" stroke="#10141B" stroke-width="12"/></svg></span>Thermograph</a></h1>
|
{# The brand is the page's h1 everywhere EXCEPT the homepage, where the
|
||||||
|
hero headline owns the sole h1 and the brand degrades to a <p>. The
|
||||||
|
.brand h1, .brand .site-name selector pair in style.css keeps the
|
||||||
|
lockup looking identical either way. #}
|
||||||
|
<{{ brand_tag|default('h1') }} class="site-name"><a href="{{ base }}/"><span class="logo"><svg viewBox="0 0 512 512" width="28" height="28" aria-hidden="true"><rect x="32" y="32" width="448" height="448" rx="96" fill="#10141B" stroke="#fff" stroke-opacity=".08" stroke-width="4"/><rect x="64" y="170" width="384" height="48" rx="8" fill="#27131C"/><rect x="64" y="222" width="384" height="48" rx="8" fill="#38322E"/><rect x="64" y="274" width="384" height="48" rx="8" fill="#1A2D27"/><rect x="64" y="326" width="384" height="48" rx="8" fill="#26333F"/><rect x="64" y="378" width="384" height="48" rx="8" fill="#18293A"/><line x1="64" y1="132" x2="448" y2="132" stroke="#46586E" stroke-width="8" stroke-dasharray="26 20"/><path d="M64 410H126V386H160" fill="none" stroke="var(--cold)" stroke-width="34"/><path d="M160 403V352H244V368H282" fill="none" stroke="var(--cool)" stroke-width="34"/><path d="M282 385V320H312" fill="none" stroke="var(--normal)" stroke-width="34"/><path d="M312 337V248H338" fill="none" stroke="var(--warm)" stroke-width="34"/><path d="M338 265V196H366" fill="none" stroke="var(--hot)" stroke-width="34"/><path d="M366 213V132H396" fill="none" stroke="var(--very-hot)" stroke-width="34"/><rect x="382" y="100" width="64" height="64" rx="12" fill="var(--rec-hot)" stroke="#10141B" stroke-width="12"/></svg></span>Thermograph</a></{{ brand_tag|default('h1') }}>
|
||||||
<p class="tag">How unusual is the weather? Graded against ~45 years of local climate history.</p>
|
<p class="tag">How unusual is the weather? Graded against ~45 years of local climate history.</p>
|
||||||
</div>
|
</div>
|
||||||
<details class="nav-menu">
|
<details class="nav-menu">
|
||||||
<summary class="nav-toggle" aria-label="Menu"><svg viewBox="0 0 24 24" width="22" height="22" aria-hidden="true" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 7h16M4 12h16M4 17h16"/></svg></summary>
|
<summary class="nav-toggle" aria-label="Menu"><svg viewBox="0 0 24 24" width="22" height="22" aria-hidden="true" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 7h16M4 12h16M4 17h16"/></svg></summary>
|
||||||
<div class="nav-panel">
|
<div class="nav-panel">
|
||||||
|
{# data-view marks the links nav.js rewrites to carry the current
|
||||||
|
location hash, so moving between views keeps the place you picked.
|
||||||
|
Climate and Alerts deliberately opt out (they aren't cell-scoped).
|
||||||
|
Inert on the SEO pages, which never load nav.js. #}
|
||||||
<nav class="view-nav" aria-label="Views">
|
<nav class="view-nav" aria-label="Views">
|
||||||
<a href="{{ base }}/">Weekly</a>
|
<a href="{{ base }}/" data-view="map"{% if section == 'home' %} class="active"{% endif %}>Weekly</a>
|
||||||
<a href="{{ base }}/calendar">Calendar</a>
|
<a href="{{ base }}/calendar" data-view="calendar">Calendar</a>
|
||||||
<a href="{{ base }}/day">Day Detail</a>
|
<a href="{{ base }}/day" data-view="day">Day Detail</a>
|
||||||
<a href="{{ base }}/compare">Compare</a>
|
<a href="{{ base }}/compare" data-view="compare">Compare</a>
|
||||||
<a href="{{ base }}/climate"{% if section == 'climate' %} class="active"{% endif %}>Climate</a>
|
<a href="{{ base }}/climate"{% if section == 'climate' %} class="active"{% endif %}>Climate</a>
|
||||||
<a href="{{ base }}/alerts">Alerts</a>
|
<a href="{{ base }}/alerts">Alerts</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
|
@ -50,24 +59,42 @@
|
||||||
</div>
|
</div>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<main class="content">
|
<main{% if main_class|default('content') %} class="{{ main_class|default('content') }}"{% endif %}>
|
||||||
{% block content %}{% endblock %}
|
{% block content %}{% endblock %}
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<footer class="site-footer">
|
<footer class="site-footer">
|
||||||
|
{# The compact digest signup is the site-wide footer pattern, not a
|
||||||
|
homepage-only surface — every page is a possible entry point. #}
|
||||||
|
<form class="digest-form digest-compact" method="post" action="{{ base }}/digest"
|
||||||
|
data-digest aria-labelledby="footer-digest-label">
|
||||||
|
<label id="footer-digest-label" for="footer-digest-email">Your city's weather, graded — monthly</label>
|
||||||
|
<div class="digest-row">
|
||||||
|
<input id="footer-digest-email" name="email" type="email" required
|
||||||
|
autocomplete="email" placeholder="you@example.com" />
|
||||||
|
<button type="submit">Get the digest</button>
|
||||||
|
</div>
|
||||||
|
<p class="digest-note muted">No spam, no sharing your address, unsubscribe anytime.</p>
|
||||||
|
<p class="digest-status" role="status" aria-live="polite" hidden></p>
|
||||||
|
</form>
|
||||||
<nav aria-label="Footer">
|
<nav aria-label="Footer">
|
||||||
<a href="{{ base }}/climate">City climates</a>
|
<a href="{{ base }}/climate">City climates</a>
|
||||||
<a href="{{ base }}/glossary">Weather glossary</a>
|
<a href="{{ base }}/glossary">Weather glossary</a>
|
||||||
<a href="{{ base }}/about">About & methodology</a>
|
<a href="{{ base }}/about">About & methodology</a>
|
||||||
|
<a href="{{ base }}/privacy">Privacy</a>
|
||||||
<a href="{{ base }}/">Open the tool</a>
|
<a href="{{ base }}/">Open the tool</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
<p class="muted">Free. No ads. No tracking. No account needed. Built by one person in the open.</p>
|
||||||
<p class="muted">Thermograph grades weather by its percentile against ~45 years of local
|
<p class="muted">Thermograph grades weather by its percentile against ~45 years of local
|
||||||
climate history. Climate data via Open-Meteo (ERA5 reanalysis).</p>
|
climate history. Data: ERA5 via Open-Meteo · NASA POWER · MET Norway.</p>
|
||||||
</footer>
|
</footer>
|
||||||
|
{% block body_scripts %}
|
||||||
<!-- Header parity with the interactive pages: the °F/°C toggle, account + bell,
|
<!-- Header parity with the interactive pages: the °F/°C toggle, account + bell,
|
||||||
and the client-side temperature converter. All self-contained modules. -->
|
and the client-side temperature converter. All self-contained modules. -->
|
||||||
<script type="module" src="{{ base }}/units.js"></script>
|
<script type="module" src="{{ base }}/units.js"></script>
|
||||||
<script type="module" src="{{ base }}/account.js"></script>
|
<script type="module" src="{{ base }}/account.js"></script>
|
||||||
<script type="module" src="{{ base }}/climate.js"></script>
|
<script type="module" src="{{ base }}/climate.js"></script>
|
||||||
|
{% endblock %}
|
||||||
|
<script type="module" src="{{ base }}/digest.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
|
||||||
179
templates/home.html.j2
Normal file
179
templates/home.html.j2
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
{% extends "base.html.j2" %}
|
||||||
|
{# The Weekly view — the product itself — with the distribution surfaces built
|
||||||
|
around it. Everything structural is server-rendered so crawlers and no-JS
|
||||||
|
readers get a complete page; app.js hydrates the live numbers in place.
|
||||||
|
|
||||||
|
The interactive controls below (#find-btn, #date-input, #panel, #results, …)
|
||||||
|
are app.js's DOM contract, carried over verbatim from the old static
|
||||||
|
frontend/index.html. Do not rename these ids.
|
||||||
|
|
||||||
|
section / brand_tag / main_class come from the route's render context
|
||||||
|
(content.home_page), not a child-template {% set %} — the SEO pages already
|
||||||
|
pass `section` that way. #}
|
||||||
|
|
||||||
|
{% block title %}Thermograph — how unusual is your weather?{% endblock %}
|
||||||
|
{% block description %}See how today's weather compares to decades of local climate history for any place on Earth — daily high, low, feels-like, humidity, wind and rain graded by percentile.{% endblock %}
|
||||||
|
{% block canonical_path %}/{% endblock %}
|
||||||
|
|
||||||
|
{% block head_extra %}
|
||||||
|
<link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" />
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block jsonld %}
|
||||||
|
<script type="application/ld+json">
|
||||||
|
{{ jsonld_str }}
|
||||||
|
</script>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
{# --- Hero ------------------------------------------------------------
|
||||||
|
The grade card is the LCP element: its frame is server-rendered and its
|
||||||
|
slots are sized here, so hydrating the live numbers causes no layout
|
||||||
|
shift. Band colour is always paired with the band word — colour is never
|
||||||
|
the only signal. #}
|
||||||
|
<section id="hero" class="hero">
|
||||||
|
<div class="hero-copy">
|
||||||
|
<h1>How unusual is your weather?</h1>
|
||||||
|
<p class="hero-sub">Any day, anywhere on Earth — graded against 45 years of that place's own history.</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="hero-card">
|
||||||
|
{# --cat carries the band colour, the same convention .normal-card uses,
|
||||||
|
so the band token stays the single source of truth. #}
|
||||||
|
<div class="grade-card" id="hero-grade" aria-live="polite"
|
||||||
|
{% if unusual %}style="--cat: var(--{{ unusual.cls }})"{% endif %}>
|
||||||
|
<p class="grade-where" id="hero-where">
|
||||||
|
{%- if unusual -%}
|
||||||
|
Today in {{ unusual.display }}
|
||||||
|
{%- else -%}
|
||||||
|
Today
|
||||||
|
{%- endif -%}
|
||||||
|
</p>
|
||||||
|
<p class="grade-band" id="hero-band">{{ unusual.grade if unusual else '—' }}</p>
|
||||||
|
<p class="grade-detail" id="hero-detail">
|
||||||
|
{%- if unusual -%}
|
||||||
|
{{ unusual.metric_label }} in the {{ unusual.percentile|ordinal }} percentile for {{ unusual.window_label }}
|
||||||
|
{%- else -%}
|
||||||
|
Pick a place to see how unusual its weather is today.
|
||||||
|
{%- endif -%}
|
||||||
|
</p>
|
||||||
|
{% if unusual and unusual.is_default %}
|
||||||
|
<p class="grade-why muted">the most unusual weather we're tracking{% if stale %} · as of {{ unusual.date }}{% endif %}</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<div class="hero-actions">
|
||||||
|
<button type="button" id="hero-locate" class="btn-primary">Use my location</button>
|
||||||
|
<button type="button" id="hero-pick" class="btn-ghost">Pick on the map</button>
|
||||||
|
</div>
|
||||||
|
<p class="hero-jump"><a href="#panel">See your week ↓</a></p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section class="controls">
|
||||||
|
<div class="find-bar">
|
||||||
|
<button type="button" id="find-btn" class="find-btn"></button>
|
||||||
|
<span id="loc-label" class="loc-label" hidden></span>
|
||||||
|
</div>
|
||||||
|
<div class="date-row">
|
||||||
|
<label>Target date
|
||||||
|
<input id="date-input" type="date" />
|
||||||
|
</label>
|
||||||
|
<button type="button" id="today-btn" class="today-chip" title="Reset the target date to today">Today</button>
|
||||||
|
<span class="hint"><a href="{{ base }}/legend">What do the grades mean? →</a></span>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section id="panel" class="panel panel-full">
|
||||||
|
<div class="placeholder" id="placeholder">
|
||||||
|
<p>Find a location to begin.</p>
|
||||||
|
<p class="muted">Thermograph fetches decades of daily highs, lows, and precipitation
|
||||||
|
for your ~4 sq mi cell, builds a ±7-day seasonal distribution for each
|
||||||
|
day of the year, and grades recent weather by where it falls on that distribution.</p>
|
||||||
|
</div>
|
||||||
|
<div id="results" hidden></div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{# Sits outside #panel on purpose: app.js rewrites results.innerHTML
|
||||||
|
wholesale on every grade, so anything inside it would be destroyed. #}
|
||||||
|
<p class="stance-line muted">
|
||||||
|
Free. No ads. No tracking. No account needed.
|
||||||
|
<a href="{{ base }}/about">How we work →</a>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{# --- Unusual right now ----------------------------------------------
|
||||||
|
Scroll-snap row, no JS carousel. Both tails are represented whenever a
|
||||||
|
cold-tail city qualifies — two-sided honesty is product identity. #}
|
||||||
|
{% if ranked %}
|
||||||
|
<section class="unusual-strip" aria-labelledby="records-title">
|
||||||
|
<h2 id="records-title">Unusual right now</h2>
|
||||||
|
<ul class="unusual-row">
|
||||||
|
{% for r in ranked %}
|
||||||
|
<li class="unusual-card" style="--cat: var(--{{ r.cls }})">
|
||||||
|
<a href="{{ base }}/day#lat={{ r.lat }}&lon={{ r.lon }}"
|
||||||
|
data-event="home.records_click">
|
||||||
|
<span class="unusual-city">{{ r.display }}</span>
|
||||||
|
<span class="unusual-pct">{{ r.percentile|ordinal }}</span>
|
||||||
|
<span class="unusual-band">{{ r.grade }}</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
<p class="unusual-more"><a href="{{ base }}/climate">See all records →</a></p>
|
||||||
|
</section>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{# --- How it works ---------------------------------------------------- #}
|
||||||
|
<section class="how-it-works" aria-labelledby="how-title">
|
||||||
|
<h2 id="how-title">How it works</h2>
|
||||||
|
<ol class="how-steps">
|
||||||
|
<li>We keep 45 years of climate history (ERA5 reanalysis) for your exact ~2-mile grid square.</li>
|
||||||
|
<li>Today gets compared to the same two weeks of the year, every year back to 1980.</li>
|
||||||
|
<li>The result is a percentile and a plain-language grade — Near Record cold to Near Record hot.</li>
|
||||||
|
</ol>
|
||||||
|
<p class="muted">
|
||||||
|
<a href="{{ base }}/legend">Full methodology →</a>
|
||||||
|
· Data: ERA5 via Open-Meteo · NASA POWER · MET Norway
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{# --- Explore --------------------------------------------------------- #}
|
||||||
|
<section class="explore" aria-labelledby="explore-title">
|
||||||
|
<h2 id="explore-title">Explore</h2>
|
||||||
|
<div class="explore-cards">
|
||||||
|
<a class="explore-card" href="{{ base }}/calendar" data-event="home.nav_calendar">
|
||||||
|
<span class="explore-name">Calendar</span>
|
||||||
|
<span class="explore-desc">Two years of your days, each colored by how unusual it was.</span>
|
||||||
|
</a>
|
||||||
|
<a class="explore-card" href="{{ base }}/compare" data-event="home.nav_compare">
|
||||||
|
<span class="explore-name">Compare</span>
|
||||||
|
<span class="explore-desc">Rank any places by comfort for a trip or a move.</span>
|
||||||
|
</a>
|
||||||
|
<a class="explore-card" href="{{ base }}/day" data-event="home.nav_day">
|
||||||
|
<span class="explore-name">Day</span>
|
||||||
|
<span class="explore-desc">Any date since 1980, anywhere: what was normal, what happened.</span>
|
||||||
|
</a>
|
||||||
|
<a class="explore-card" href="{{ base }}/alerts" data-event="home.nav_alerts">
|
||||||
|
<span class="explore-name">Alerts</span>
|
||||||
|
<span class="explore-desc">Free percentile alerts: get pinged when your weather goes statistically weird.</span>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{# --- City hub links --------------------------------------------------
|
||||||
|
Real HTML links funnelling homepage authority into the ~1000-page
|
||||||
|
/climate surface. #}
|
||||||
|
<section class="city-hub" aria-labelledby="cities-title">
|
||||||
|
<h2 id="cities-title">Climate context for 1,000 cities</h2>
|
||||||
|
<ul class="city-chips">
|
||||||
|
{% for c in cities %}
|
||||||
|
<li><a href="{{ base }}/climate/{{ c.slug }}" data-event="home.nav_city">{{ c.name }}</a></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
<p><a href="{{ base }}/climate" data-event="home.nav_cities">Browse all cities →</a></p>
|
||||||
|
</section>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block body_scripts %}
|
||||||
|
<script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>
|
||||||
|
<script type="module" src="{{ base }}/app.js"></script>
|
||||||
|
{% endblock %}
|
||||||
39
templates/privacy.html.j2
Normal file
39
templates/privacy.html.j2
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
{% extends "base.html.j2" %}
|
||||||
|
{% block title %}{{ page_title }}{% endblock %}
|
||||||
|
{% block description %}{{ page_description }}{% endblock %}
|
||||||
|
{% block canonical_path %}{{ canonical_path }}{% endblock %}
|
||||||
|
{% block content %}
|
||||||
|
<nav class="breadcrumb" aria-label="Breadcrumb">
|
||||||
|
{% for label, href in breadcrumb %}{% if href %}<a href="{{ href }}">{{ label }}</a>{% else %}<span>{{ label }}</span>{% endif %}{% if not loop.last %} <span class="sep">›</span> {% endif %}{% endfor %}
|
||||||
|
</nav>
|
||||||
|
<article class="climate-page">
|
||||||
|
<h1>Privacy</h1>
|
||||||
|
<p class="lede">Thermograph is free, has no ads, and needs no account. It is built so that
|
||||||
|
there is very little about you to collect in the first place.</p>
|
||||||
|
|
||||||
|
<h2>Your location</h2>
|
||||||
|
<p>Thermograph never looks up your location from your IP address. The only way the site
|
||||||
|
learns where you are is if you press <b>Use my location</b>, which asks your browser for
|
||||||
|
permission — you can decline, and picking a place on the map works just as well.</p>
|
||||||
|
<p>Whichever way you choose a place, the coordinates are sent to the server only as part of
|
||||||
|
the ordinary request for that grid square's climate data, and are not logged beyond the
|
||||||
|
normal web-server request handling every site does. The place you picked is remembered in
|
||||||
|
your own browser's local storage, not on the server, so clearing your browser data
|
||||||
|
forgets it.</p>
|
||||||
|
|
||||||
|
<h2>Analytics</h2>
|
||||||
|
<p>There is no analytics library, no cookies for tracking, and no per-visitor identifier.
|
||||||
|
The server keeps simple aggregate counts — how many people opened a page or tapped a
|
||||||
|
button, and which site referred them — with nothing tying any of it to an individual.</p>
|
||||||
|
|
||||||
|
<h2>Email</h2>
|
||||||
|
<p>If you sign up for the monthly digest, your address is used for that digest and nothing
|
||||||
|
else. It is never sold or shared, and every message can unsubscribe you.</p>
|
||||||
|
|
||||||
|
<h2>Accounts</h2>
|
||||||
|
<p>An account is optional and only exists to hold weather alerts you have asked for. It
|
||||||
|
stores your email address and the places you are watching.</p>
|
||||||
|
|
||||||
|
<p class="muted">Questions: see <a href="{{ base }}/about">About & methodology</a>.</p>
|
||||||
|
</article>
|
||||||
|
{% endblock %}
|
||||||
120
tests/test_digest.py
Normal file
120
tests/test_digest.py
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
"""Tests for the digest signup and the mailer seam."""
|
||||||
|
import pytest
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import app as appmod
|
||||||
|
import db
|
||||||
|
import digest
|
||||||
|
import mailer
|
||||||
|
from db import async_session_maker
|
||||||
|
from models import PendingDigest
|
||||||
|
|
||||||
|
B = "/thermograph"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def _tables():
|
||||||
|
# TestClient(app) (no context manager) skips the lifespan, so create the
|
||||||
|
# account tables directly against the temp DB.
|
||||||
|
db.Base.metadata.create_all(db.sync_engine)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client():
|
||||||
|
return TestClient(appmod.app)
|
||||||
|
|
||||||
|
|
||||||
|
def _rows(client, email):
|
||||||
|
"""Count stored rows for an address, via the running event loop."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
async def go():
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
return (await session.execute(
|
||||||
|
sa.select(sa.func.count()).select_from(PendingDigest)
|
||||||
|
.where(PendingDigest.email == email)
|
||||||
|
)).scalar_one()
|
||||||
|
return asyncio.run(go())
|
||||||
|
|
||||||
|
|
||||||
|
# --- mailer ------------------------------------------------------------------
|
||||||
|
def test_console_backend_sends_nothing(monkeypatch):
|
||||||
|
"""The default backend must never open a socket — this is what makes it safe
|
||||||
|
to run the full signup path in dev and in tests."""
|
||||||
|
monkeypatch.setattr(mailer, "BACKEND", "console")
|
||||||
|
|
||||||
|
def boom(*a, **k):
|
||||||
|
raise AssertionError("console backend must not connect to SMTP")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mailer.smtplib, "SMTP", boom)
|
||||||
|
assert mailer.send("a@example.com", "s", "t") is True
|
||||||
|
assert mailer.enabled() is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_send_swallows_connection_failure(monkeypatch):
|
||||||
|
"""A dead mail server must not raise into the request that triggered it."""
|
||||||
|
monkeypatch.setattr(mailer, "BACKEND", "smtp")
|
||||||
|
|
||||||
|
def boom(*a, **k):
|
||||||
|
raise OSError("connection refused")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mailer.smtplib, "SMTP", boom)
|
||||||
|
assert mailer.send("a@example.com", "s", "t") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_backend_drops(monkeypatch):
|
||||||
|
monkeypatch.setattr(mailer, "BACKEND", "disabled")
|
||||||
|
assert mailer.send("a@example.com", "s", "t") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_recipient_rejected():
|
||||||
|
assert mailer.send("", "s", "t") is False
|
||||||
|
assert mailer.send("not-an-address", "s", "t") is False
|
||||||
|
|
||||||
|
|
||||||
|
# --- signup ------------------------------------------------------------------
|
||||||
|
def test_signup_stores_and_confirms(client):
|
||||||
|
r = client.post(f"{B}/digest", json={"email": "reader@example.com"})
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert r.json()["message"] == digest.GENERIC_OK
|
||||||
|
assert _rows(client, "reader@example.com") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_signup_is_idempotent_and_case_insensitive(client):
|
||||||
|
first = client.post(f"{B}/digest", json={"email": "Dup@Example.COM"})
|
||||||
|
second = client.post(f"{B}/digest", json={"email": "dup@example.com"})
|
||||||
|
# Same answer either way: the endpoint must not reveal whether an address is
|
||||||
|
# already on the list.
|
||||||
|
assert first.json()["message"] == second.json()["message"] == digest.GENERIC_OK
|
||||||
|
assert _rows(client, "dup@example.com") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_signup_rejects_junk(client):
|
||||||
|
for bad in ["", "nope", "a@b", "@example.com", "x@", "a b@example.com"]:
|
||||||
|
r = client.post(f"{B}/digest", json={"email": bad})
|
||||||
|
assert r.status_code == 400, bad
|
||||||
|
|
||||||
|
|
||||||
|
def test_signup_accepts_plain_form_post(client):
|
||||||
|
"""The no-JS path: a real <form method=post> submission."""
|
||||||
|
r = client.post(f"{B}/digest", data={"email": "nojs@example.com"})
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert _rows(client, "nojs@example.com") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_signup_records_event(client):
|
||||||
|
import metrics
|
||||||
|
before = metrics.snapshot()["events"].get("home.digest_signup", {})
|
||||||
|
before_n = sum(sum(days.values()) for days in before.values())
|
||||||
|
client.post(f"{B}/digest", json={"email": "counted@example.com"})
|
||||||
|
after = metrics.snapshot()["events"].get("home.digest_signup", {})
|
||||||
|
assert sum(sum(days.values()) for days in after.values()) == before_n + 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_form_is_on_every_page(client):
|
||||||
|
"""The digest form is the site-wide footer pattern, not homepage-only."""
|
||||||
|
for path in ["/", "/about", "/climate", "/privacy", "/glossary"]:
|
||||||
|
html = client.get(f"{B}{path}").text
|
||||||
|
assert "data-digest" in html, path
|
||||||
|
assert 'action="/thermograph/digest"' in html, path
|
||||||
202
tests/test_homepage.py
Normal file
202
tests/test_homepage.py
Normal file
|
|
@ -0,0 +1,202 @@
|
||||||
|
"""Tests for the homepage: the cache-only "unusual right now" precompute, the
|
||||||
|
server-rendered surfaces, and the digest signup.
|
||||||
|
|
||||||
|
The precompute must never touch the network — a sweep over ~1000 cities that
|
||||||
|
fetched would burst the archive quota — so the tests here assert that the
|
||||||
|
fetching climate helpers are not called at all.
|
||||||
|
"""
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import app as appmod
|
||||||
|
import cities
|
||||||
|
import climate
|
||||||
|
import content
|
||||||
|
import homepage
|
||||||
|
|
||||||
|
B = "/thermograph"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client():
|
||||||
|
return TestClient(appmod.app)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def no_feed(monkeypatch):
|
||||||
|
"""The homepage with no precomputed feed at all."""
|
||||||
|
monkeypatch.setattr(homepage, "load", lambda: None)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _pick(name, pct, cls, grade, tail, slug=None):
|
||||||
|
return {
|
||||||
|
"slug": slug or (name.lower().replace(" ", "-") + "-xx"),
|
||||||
|
"name": name, "display": f"{name}, Somewhere",
|
||||||
|
"lat": 1.0, "lon": 2.0, "cell_id": "1_2",
|
||||||
|
"date": datetime.date.today().isoformat(),
|
||||||
|
"metric": "tmax", "metric_label": "High temp", "window_label": "mid-July",
|
||||||
|
"value": 99.0, "percentile": pct, "grade": grade, "cls": cls,
|
||||||
|
"departure": abs(pct - 50), "tail": tail,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- the precompute ----------------------------------------------------------
|
||||||
|
def test_build_never_fetches_upstream(monkeypatch):
|
||||||
|
"""The sweep uses the cache-only readers, never the fetching ones."""
|
||||||
|
def boom(*a, **k):
|
||||||
|
raise AssertionError("the homepage sweep must not fetch upstream")
|
||||||
|
|
||||||
|
monkeypatch.setattr(climate, "get_history", boom)
|
||||||
|
monkeypatch.setattr(climate, "get_recent_forecast", boom)
|
||||||
|
feed = homepage.build(limit=20)
|
||||||
|
assert set(feed) >= {"generated_at", "date", "considered", "picks", "ranked"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_grades_cached_cities(monkeypatch, history, recent):
|
||||||
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone())
|
||||||
|
monkeypatch.setattr(climate, "load_cached_recent_forecast", lambda cell: recent.clone())
|
||||||
|
feed = homepage.build(limit=5)
|
||||||
|
assert feed["considered"] == 5
|
||||||
|
for row in feed["ranked"]:
|
||||||
|
assert row["percentile"] is not None
|
||||||
|
assert row["cls"] and not row["cls"].startswith("--") # token name, no prefix
|
||||||
|
assert row["tail"] in ("warm", "cold")
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_cell_cache_is_skipped_not_fatal(monkeypatch):
|
||||||
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
||||||
|
monkeypatch.setattr(climate, "load_cached_recent_forecast", lambda cell: None)
|
||||||
|
feed = homepage.build(limit=10)
|
||||||
|
assert feed["considered"] == 0 and feed["ranked"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_survives_missing_and_corrupt_file(monkeypatch, tmp_path):
|
||||||
|
monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "nope.json"))
|
||||||
|
assert homepage.load() is None
|
||||||
|
|
||||||
|
bad = tmp_path / "homepage.json"
|
||||||
|
bad.write_text("{not json")
|
||||||
|
monkeypatch.setattr(homepage, "FEED_PATH", str(bad))
|
||||||
|
assert homepage.load() is None
|
||||||
|
|
||||||
|
bad.write_text('{"unexpected": true}')
|
||||||
|
assert homepage.load() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_writes_atomically(monkeypatch, tmp_path):
|
||||||
|
monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "homepage.json"))
|
||||||
|
monkeypatch.setattr(climate, "load_cached_history", lambda cell: None)
|
||||||
|
homepage.refresh(limit=3)
|
||||||
|
written = json.loads((tmp_path / "homepage.json").read_text())
|
||||||
|
assert written["ranked"] == []
|
||||||
|
# No temp files left behind.
|
||||||
|
assert [p.name for p in tmp_path.iterdir()] == ["homepage.json"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_staleness(monkeypatch):
|
||||||
|
today = datetime.date.today().isoformat()
|
||||||
|
assert not homepage.is_stale({"date": today, "generated_at": time.time()})
|
||||||
|
assert homepage.is_stale({"date": today, "generated_at": time.time() - 7 * 3600})
|
||||||
|
assert homepage.is_stale({"date": "1999-01-01", "generated_at": time.time()})
|
||||||
|
|
||||||
|
|
||||||
|
# --- the rendered page -------------------------------------------------------
|
||||||
|
def test_homepage_is_complete_without_js(client, no_feed):
|
||||||
|
"""A crawler (or a reader with JS off) gets the whole page as real HTML."""
|
||||||
|
r = client.get(f"{B}/")
|
||||||
|
assert r.status_code == 200
|
||||||
|
html = r.text
|
||||||
|
assert "How unusual is your weather?" in html
|
||||||
|
assert "How it works" in html
|
||||||
|
assert "Climate context for 1,000 cities" in html
|
||||||
|
assert "Free. No ads. No tracking." in html
|
||||||
|
# The interactive tool's DOM contract survives the move to Jinja.
|
||||||
|
for dom_id in ('id="find-btn"', 'id="date-input"', 'id="panel"',
|
||||||
|
'id="placeholder"', 'id="results"'):
|
||||||
|
assert dom_id in html
|
||||||
|
# Exactly one h1: the hero headline. The brand degrades to a <p>.
|
||||||
|
assert html.count("<h1") == 1
|
||||||
|
assert 'p class="site-name"' in html
|
||||||
|
|
||||||
|
|
||||||
|
def test_city_chips_all_resolve(client, no_feed):
|
||||||
|
"""Every homepage chip must be a real, routable city — a stale slug would
|
||||||
|
render an empty chip list and leak a 404 link."""
|
||||||
|
resolved = content._home_cities()
|
||||||
|
assert len(resolved) == len(content.HOME_CITY_SLUGS)
|
||||||
|
html = client.get(f"{B}/").text
|
||||||
|
for city in resolved:
|
||||||
|
assert f"/climate/{city['slug']}" in html
|
||||||
|
assert cities.get(city["slug"]) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_records_strip_shows_both_tails(client, monkeypatch):
|
||||||
|
"""Two-sided honesty: a qualifying cold-tail city appears alongside the heat."""
|
||||||
|
feed = {
|
||||||
|
"generated_at": time.time(), "date": datetime.date.today().isoformat(),
|
||||||
|
"considered": 800,
|
||||||
|
"picks": {"extreme": _pick("Phoenix", 99.4, "rec-hot", "Near Record", "warm")},
|
||||||
|
"ranked": [_pick("Phoenix", 99.4, "rec-hot", "Near Record", "warm"),
|
||||||
|
_pick("Ushuaia", 2.1, "very-cold", "Very Low", "cold")],
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(homepage, "load", lambda: feed)
|
||||||
|
html = client.get(f"{B}/").text
|
||||||
|
assert "Unusual right now" in html
|
||||||
|
assert "Phoenix" in html and "Ushuaia" in html
|
||||||
|
# Band colour comes from the shared tokens, and the band word is always
|
||||||
|
# present too — colour is never the only signal.
|
||||||
|
assert "var(--rec-hot)" in html and "var(--very-cold)" in html
|
||||||
|
assert "Near Record" in html and "Very Low" in html
|
||||||
|
|
||||||
|
|
||||||
|
def test_hero_uses_most_unusual_default(client, monkeypatch):
|
||||||
|
feed = {
|
||||||
|
"generated_at": time.time(), "date": datetime.date.today().isoformat(),
|
||||||
|
"considered": 800,
|
||||||
|
"picks": {"extreme": _pick("Phoenix", 96.0, "very-hot", "Very High", "warm")},
|
||||||
|
"ranked": [],
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(homepage, "load", lambda: feed)
|
||||||
|
html = client.get(f"{B}/").text
|
||||||
|
assert "Today in Phoenix, Somewhere" in html
|
||||||
|
assert "96th percentile" in html
|
||||||
|
assert "the most unusual weather we're tracking" in html
|
||||||
|
|
||||||
|
|
||||||
|
def test_homepage_renders_without_feed(client, no_feed):
|
||||||
|
"""A cold checkout with no precomputed feed still serves a complete page."""
|
||||||
|
html = client.get(f"{B}/").text
|
||||||
|
assert "How unusual is your weather?" in html
|
||||||
|
assert "Unusual right now" not in html # strip is omitted, not broken
|
||||||
|
assert 'id="hero-grade"' in html # the frame is still there
|
||||||
|
|
||||||
|
|
||||||
|
def test_homepage_etag_revalidates(client, no_feed):
|
||||||
|
r = client.get(f"{B}/")
|
||||||
|
etag = r.headers["etag"]
|
||||||
|
again = client.get(f"{B}/", headers={"If-None-Match": etag})
|
||||||
|
assert again.status_code == 304
|
||||||
|
|
||||||
|
|
||||||
|
def test_old_static_index_is_gone(client):
|
||||||
|
"""index.html must not linger behind the static mount as indexable duplicate
|
||||||
|
content now that / is server-rendered."""
|
||||||
|
assert client.get(f"{B}/index.html").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_privacy_page(client):
|
||||||
|
r = client.get(f"{B}/privacy")
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert "no analytics library" in r.text.lower() or "no analytics" in r.text.lower()
|
||||||
|
assert "Use my location" in r.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_privacy_is_linked_but_not_in_sitemap(client):
|
||||||
|
assert f"{B}/privacy" in client.get(f"{B}/").text
|
||||||
|
# It's a policy page, not a crawl target we want ranked.
|
||||||
|
assert "/privacy" not in client.get(f"{B}/sitemap.xml").text
|
||||||
|
|
@ -239,3 +239,80 @@ def test_heartbeat_survives_store_read_shape(tmp_path):
|
||||||
mem = m._MemStore()
|
mem = m._MemStore()
|
||||||
mem.record_heartbeat("subscription-notifier", 10.0, 900)
|
mem.record_heartbeat("subscription-notifier", 10.0, 900)
|
||||||
assert mem.read()["heartbeats"]["subscription-notifier"] == {"ts": 10.0, "interval_s": 900}
|
assert mem.read()["heartbeats"]["subscription-notifier"] == {"ts": 10.0, "interval_s": 900}
|
||||||
|
|
||||||
|
|
||||||
|
# --- product events -----------------------------------------------------------
|
||||||
|
# These count intent (a tap, a signup), not requests, and are keyed by
|
||||||
|
# (event, referrer domain, UTC day). The endpoint is unauthenticated, so the
|
||||||
|
# allowlist and the caps are load-bearing, not decoration.
|
||||||
|
|
||||||
|
def test_event_allowlist():
|
||||||
|
m = _fresh()
|
||||||
|
assert m.normalize_event("home.locate") == "home.locate"
|
||||||
|
assert m.normalize_event("home.nav_calendar") == "home.nav_calendar"
|
||||||
|
# Unknown names collapse into one bucket rather than growing the key space.
|
||||||
|
assert m.normalize_event("home.nav_not_a_surface") == m.EVENT_OTHER
|
||||||
|
assert m.normalize_event("totally.made.up") == m.EVENT_OTHER
|
||||||
|
assert m.normalize_event("") == m.EVENT_OTHER
|
||||||
|
assert m.normalize_event(None) == m.EVENT_OTHER
|
||||||
|
|
||||||
|
|
||||||
|
def test_referrer_normalization():
|
||||||
|
m = _fresh()
|
||||||
|
# Only the bare domain is kept — never the path or query, which can carry a
|
||||||
|
# search term or a private URL.
|
||||||
|
assert m.normalize_referrer("https://news.ycombinator.com/item?id=1") == "news.ycombinator.com"
|
||||||
|
assert m.normalize_referrer("https://www.reddit.com/r/weather/") == "reddit.com"
|
||||||
|
assert m.normalize_referrer(None) == "direct"
|
||||||
|
assert m.normalize_referrer("") == "direct"
|
||||||
|
assert m.normalize_referrer("https://thermograph.org/x", host="thermograph.org") == "self"
|
||||||
|
assert m.normalize_referrer("https://www.thermograph.org/x", host="thermograph.org") == "self"
|
||||||
|
# Junk is sanitized, never stored raw.
|
||||||
|
assert " " not in m.normalize_referrer("https://ev il<>.com/")
|
||||||
|
assert len(m.normalize_referrer("https://" + "a" * 200 + ".com/")) <= 64
|
||||||
|
|
||||||
|
|
||||||
|
def test_events_are_not_counted_as_inbound_traffic():
|
||||||
|
"""The beacon reports traffic; counting it would double every interaction."""
|
||||||
|
m = _fresh()
|
||||||
|
assert m.classify_inbound("/thermograph/api/v2/event", "/thermograph") == "event"
|
||||||
|
m.record_inbound("event", 204)
|
||||||
|
assert m.snapshot()["inbound_total"] == 0
|
||||||
|
assert "event" not in m.snapshot()["inbound"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_event_keys_by_event_referrer_day():
|
||||||
|
m = _fresh()
|
||||||
|
for _ in range(3):
|
||||||
|
m.record_event("home.locate", referer="https://news.ycombinator.com/x")
|
||||||
|
m.record_event("home.share", referer=None)
|
||||||
|
events = m.snapshot()["events"]
|
||||||
|
day = m._utc_day()
|
||||||
|
assert events["home.locate"]["news.ycombinator.com"][day] == 3
|
||||||
|
assert events["home.share"]["direct"][day] == 1
|
||||||
|
assert m.snapshot()["events_total"] == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_event_carries_no_referrer_dimension():
|
||||||
|
m = _fresh()
|
||||||
|
m.record_event("junk.event", referer="https://spam.example/")
|
||||||
|
events = m.snapshot()["events"]
|
||||||
|
assert list(events[m.EVENT_OTHER]) == ["direct"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_referrer_cardinality_is_capped():
|
||||||
|
"""A spray of forged Referer headers must not grow the key space forever."""
|
||||||
|
m = _fresh()
|
||||||
|
m._EVENT_RATE_PER_MIN = 10_000
|
||||||
|
for i in range(m.MAX_REFERRERS + 50):
|
||||||
|
m.record_event("home.locate", referer=f"https://ref{i}.example/")
|
||||||
|
refs = m.snapshot()["events"]["home.locate"]
|
||||||
|
assert len(refs) <= m.MAX_REFERRERS + 1 # +1 for the overflow bucket
|
||||||
|
assert m.REFERRER_OTHER in refs
|
||||||
|
|
||||||
|
|
||||||
|
def test_event_rate_limit_per_ip():
|
||||||
|
m = _fresh()
|
||||||
|
for _ in range(m._EVENT_RATE_PER_MIN + 25):
|
||||||
|
m.record_event("home.locate", ip="203.0.113.9")
|
||||||
|
assert m.snapshot()["events_total"] == m._EVENT_RATE_PER_MIN
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ import time
|
||||||
import cities
|
import cities
|
||||||
import climate
|
import climate
|
||||||
import grid
|
import grid
|
||||||
|
import homepage
|
||||||
|
|
||||||
|
|
||||||
def main(limit: int | None = None, pace: float = 2.0) -> None:
|
def main(limit: int | None = None, pace: float = 2.0) -> None:
|
||||||
|
|
@ -40,6 +41,16 @@ def main(limit: int | None = None, pace: float = 2.0) -> None:
|
||||||
time.sleep(pace)
|
time.sleep(pace)
|
||||||
print(f"done: fetched={fetched} skipped(cached)={skipped} failed={failed}")
|
print(f"done: fetched={fetched} skipped(cached)={skipped} failed={failed}")
|
||||||
|
|
||||||
|
# Rebuild the homepage's "unusual right now" feed from whatever is now warm.
|
||||||
|
# Runs here as well as on the notifier loop so a fresh deploy has a populated
|
||||||
|
# feed immediately, and so it still refreshes when the notifier is disabled.
|
||||||
|
try:
|
||||||
|
feed = homepage.refresh()
|
||||||
|
print(f"homepage feed: {feed['considered']} cities graded, "
|
||||||
|
f"{len(feed['ranked'])} ranked")
|
||||||
|
except Exception as e: # noqa: BLE001 - the homepage renders without the feed
|
||||||
|
print(f"homepage feed: FAILED {e}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
args = sys.argv[1:]
|
args = sys.argv[1:]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue