diff --git a/app.py b/app.py index 2f2315c..8775b03 100644 --- a/app.py +++ b/app.py @@ -20,6 +20,7 @@ import api_accounts import audit import climate import content +import digest import db import grid import metrics @@ -171,12 +172,13 @@ async def revalidate_static(request, call_next): metrics.record_inbound(cat, response.status_code) # Retain per-request client IPs for later analysis; skip static assets and the # dashboard's own metrics polling to keep the log to real, meaningful traffic. - if cat not in ("static", "metrics"): + if cat not in ("static", "metrics", "event"): audit.log_access({"ip": _client_ip(request), "method": request.method, "path": path, "status": response.status_code, "cat": cat}) except Exception: # noqa: BLE001 - never let instrumentation break a response pass - pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/compare", f"{BASE}/legend", f"{BASE}/alerts") + pages = (BASE, f"{BASE}/", f"{BASE}/calendar", f"{BASE}/day", f"{BASE}/compare", + f"{BASE}/legend", f"{BASE}/alerts", f"{BASE}/privacy") if path.endswith((".js", ".css", ".html")) or path in pages: response.headers["Cache-Control"] = "no-cache" return response @@ -592,6 +594,35 @@ def api_metrics(request: Request): return snap +async def api_event(request: Request) -> Response: + """Record one product event (a tap on "use my location", a digest signup, …). + + Deliberately minimal: the body carries only an allowlisted event name. The + referrer is read from this request's own Referer header — a client-supplied + one would be forgeable and buys nothing — and is reduced to a bare domain. + No cookies, no per-visitor id, no full URLs. + + Always answers 204, whether the event was counted, rejected by the allowlist, + or dropped by the rate limiter, so probing the endpoint reveals nothing. That + also suits navigator.sendBeacon, which ignores the response. + """ + name = "" + try: + body = await request.json() + if isinstance(body, dict): + name = str(body.get("event") or "") + except Exception: # noqa: BLE001 - a junk body is just a no-op event + pass + if name: + metrics.record_event( + name, + referer=request.headers.get("referer"), + host=request.headers.get("host"), + ip=_client_ip(request), + ) + return Response(status_code=204) + + # --- API versioning -------------------------------------------------------- # Non-backward-compatible additions land in a new version; every version stays # mounted and served simultaneously. v1 is the original contract (grade/geocode); @@ -611,6 +642,7 @@ v2.add_api_route("/day", api_day, methods=["GET"]) v2.add_api_route("/forecast", api_forecast, methods=["GET"]) v2.add_api_route("/cell", api_cell, methods=["GET"]) v2.add_api_route("/metrics", api_metrics, methods=["GET"], include_in_schema=False) +v2.add_api_route("/event", api_event, methods=["POST"], include_in_schema=False) app.include_router(v1, prefix=f"{BASE}/api") # legacy unversioned == v1 (backward compatible) app.include_router(v1, prefix=f"{BASE}/api/v1") @@ -672,16 +704,23 @@ def _page(name): # Pages also answer HEAD — link-preview crawlers probe with it before fetching. if BASE: app.add_api_route(BASE, lambda: RedirectResponse(url=f"{BASE}/"), methods=["GET", "HEAD"], include_in_schema=False) -app.add_api_route(f"{BASE}/", _page("index.html"), methods=["GET", "HEAD"], include_in_schema=False) +# NOTE: "/" is not here — the homepage is server-rendered by content.register() +# below, so its hero, records strip and city links reach crawlers and no-JS +# readers as real HTML. app.add_api_route(f"{BASE}/calendar", _page("calendar.html"), methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/day", _page("day.html"), methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/compare", _page("compare.html"), methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/legend", _page("legend.html"), methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/alerts", _page("subscriptions.html"), methods=["GET", "HEAD"], include_in_schema=False) -# Crawlable, server-rendered SEO pages (climate hub / per-city / month / records / -# glossary / about) + robots.txt + sitemap.xml. Registered before the static mount -# so these explicit routes win. +# Monthly-digest signup (the footer form on every page). Answers the same generic +# message whatever the outcome, so it can't be probed for whether an address is +# already subscribed. +app.add_api_route(f"{BASE}/digest", digest.signup, methods=["POST"], include_in_schema=False) + +# Crawlable, server-rendered SEO pages (the homepage, climate hub / per-city / +# month / records / glossary / about / privacy) + robots.txt + sitemap.xml. +# Registered before the static mount so these explicit routes win. content.register(app) # Everything else under BASE (app.js, style.css, nav.js, …) is a static asset. diff --git a/climate.py b/climate.py index d2415a1..088a1a8 100644 --- a/climate.py +++ b/climate.py @@ -587,6 +587,21 @@ def get_recent_forecast(cell: dict) -> pl.DataFrame: return _derive_humidity(_load_recent_forecast(cell)) +def load_cached_recent_forecast(cell: dict) -> "pl.DataFrame | None": + """Recent+forecast from the parquet cache ONLY — never fetches upstream, and + unlike the loader below it does not care how stale the file is. The sibling of + load_cached_history, for the same reason: the homepage precompute sweeps every + cached city and must not spend a single upstream request doing it. None when + the cell has no cached recent/forecast record.""" + path = _rf_cache_path(cell["id"]) + if not os.path.exists(path): + return None + try: + return _derive_humidity(_with_doy(_normalize_read(pl.read_parquet(path)))) + except Exception: # noqa: BLE001 - a truncated/corrupt cache file is a miss, not a crash + return None + + def _load_recent_forecast(cell: dict) -> pl.DataFrame: """Recent observations AND the forward forecast in ONE forecast-API call. diff --git a/content.py b/content.py index 4bb1c04..36338d2 100644 --- a/content.py +++ b/content.py @@ -22,6 +22,7 @@ import city_events import climate import grading import grid +import homepage from views import OBS_COLS _BASE = os.environ.get("THERMOGRAPH_BASE", "/thermograph").strip("/") @@ -131,9 +132,24 @@ def _range_bar(low_f, high_f) -> dict | None: } +def _ordinal(n) -> str: + """96 -> '96th'. Percentiles read as ordinals in prose ('in the 96th + percentile'), and the frontend's shared.js ord() does the same client-side.""" + try: + i = int(round(float(n))) + except (TypeError, ValueError): + return "—" + if 10 <= i % 100 <= 20: + suffix = "th" + else: + suffix = {1: "st", 2: "nd", 3: "rd"}.get(i % 10, "th") + return f"{i}{suffix}" + + _env.globals["temp_class"] = temp_class _env.globals["temp"] = _temp _env.globals["temp_bare"] = _temp_bare +_env.filters["ordinal"] = _ordinal def _month_doy(month_idx: int) -> int: @@ -731,6 +747,83 @@ def about_page(request: Request) -> Response: return _respond_html(request, "about.html.j2", **ctx) +def privacy_page(request: Request) -> Response: + ctx = { + "canonical_path": "/privacy", + "breadcrumb": [("Home", f"{BASE}/"), ("Privacy", None)], + "page_title": "Privacy — Thermograph", + "page_description": ("What Thermograph does and does not collect: no tracking, no ads, " + "no account required, and location that never leaves your browser."), + } + return _respond_html(request, "privacy.html.j2", **ctx) + + +# --- homepage ----------------------------------------------------------------- +# The homepage is the Weekly tool plus the distribution surfaces around it. It is +# server-rendered like the SEO pages (rather than a static file with placeholder +# substitution) so a cold visitor with no JS still gets the headline, a real +# graded example, the records strip and the city links. + +_HOME_JSONLD = { + "@context": "https://schema.org", + "@type": "WebApplication", + "name": "Thermograph", + "applicationCategory": "WeatherApplication", + "operatingSystem": "Web, iOS, Android", + "isAccessibleForFree": True, + "offers": {"@type": "Offer", "price": "0", "priceCurrency": "USD"}, + "description": ("How unusual is your weather? Any day, anywhere on Earth — graded " + "against 45 years of that place's own history."), +} + +# 12 city chips, chosen for geographic spread rather than raw population, so the +# strip reads as "anywhere on Earth" and seeds crawl paths across the hub. +HOME_CITY_SLUGS = ( + "new-york-city-new-york-us", "london-england-gb", "tokyo-jp", + "sydney-new-south-wales-au", "sao-paulo-br", "lagos-ng", + "mumbai-maharashtra-in", "mexico-city-mx", "cairo-eg", + "toronto-ontario-ca", "berlin-state-of-berlin-de", "seattle-washington-us", +) + + +def _home_cities() -> list[dict]: + """The chip set, skipping any slug not in the routable city list so a + regenerated cities.json can never 404 a homepage link.""" + out = [] + for slug in HOME_CITY_SLUGS: + city = cities.get(slug) + if city: + out.append({"slug": slug, "name": city["name"]}) + return out + + +def home_page(request: Request) -> Response: + feed = homepage.load() + stale = bool(feed) and homepage.is_stale(feed) + unusual = None + ranked: list = [] + if feed: + ranked = feed.get("ranked") or [] + pick = (feed.get("picks") or {}).get("extreme") + if pick: + unusual = dict(pick, is_default=True) + + ctx = { + "section": "home", + # The hero headline owns the page's sole h1, so the brand degrades to a

. + "brand_tag": "p", + # The tool needs the wide app column, not the 880px reading column. + "main_class": "", + "canonical_path": "/", + "unusual": unusual, + "stale": stale, + "ranked": ranked, + "cities": _home_cities(), + "jsonld_str": Markup(json.dumps({**_HOME_JSONLD, "url": f"{origin(request)}{BASE}/"})), + } + return _respond_html(request, "home.html.j2", **ctx) + + # --- registration ------------------------------------------------------------ def register(app) -> None: """Attach all content routes. Call from app.py BEFORE the StaticFiles mount.""" @@ -749,7 +842,11 @@ def register(app) -> None: f"{BASE}/{_inkey}.txt", _indexnow_key_file, methods=["GET", "HEAD"], include_in_schema=False, ) + # The homepage. Registered here (not as a static file in app.py) so it is + # server-rendered from the same Jinja environment as the SEO pages. + app.add_api_route(f"{BASE}/", home_page, methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/about", about_page, methods=["GET", "HEAD"], include_in_schema=False) + app.add_api_route(f"{BASE}/privacy", privacy_page, methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/glossary", glossary_index, methods=["GET", "HEAD"], include_in_schema=False) app.add_api_route(f"{BASE}/glossary/{{term}}", glossary_term, methods=["GET", "HEAD"], include_in_schema=False) # Hub before /climate/{slug} so the literal path wins. diff --git a/digest.py b/digest.py new file mode 100644 index 0000000..a889ab9 --- /dev/null +++ b/digest.py @@ -0,0 +1,117 @@ +"""Monthly-digest signup. + +The form ships ahead of email delivery on purpose (see models.PendingDigest): +collecting the list is the slow part, so signups are stored now and confirmed +once SMTP is live. Nothing is mailed while mailer.enabled() is False. + +Privacy: the address is used for the digest and nothing else, and the only other +thing recorded is the bare referrer domain for attribution. +""" +from __future__ import annotations + +import re +import time + +from fastapi import Request, Response +from fastapi.responses import JSONResponse +from sqlalchemy import select + +import mailer +import metrics +from db import async_session_maker +from models import PendingDigest + +# Deliberately permissive: real-world addresses defeat clever patterns, and the +# confirmation email is the actual validator. This only rejects obvious junk. +_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s.]+(\.[^@\s.]+)+$") + +# The same answer for every outcome — new row, duplicate, or throttled. Anything +# else would turn the endpoint into an oracle for "is this address subscribed?". +GENERIC_OK = "Check your inbox to confirm your address." + +# Don't re-send a confirmation to an unconfirmed address more than this often. +RESEND_INTERVAL_S = 600 + + +def _valid(email: str) -> bool: + return bool(email) and len(email) <= 320 and bool(_EMAIL_RE.match(email)) + + +async def _store(email: str, place: str | None, source: str | None) -> None: + """Upsert the signup. A repeat submit updates the existing row rather than + creating a second one — the unique constraint on email is the idempotency.""" + async with async_session_maker() as session: + existing = ( + await session.execute( + select(PendingDigest).where(PendingDigest.email == email) + ) + ).scalar_one_or_none() + + now = time.time() + if existing is None: + session.add(PendingDigest( + email=email, place_label=place, source=source, created_at=now, + )) + else: + if place and not existing.place_label: + existing.place_label = place + # Re-subscribing after an unsubscribe re-opens the row. + existing.unsubscribed_at = None + if existing.confirmed_at is None: + if existing.last_sent_at and now - existing.last_sent_at < RESEND_INTERVAL_S: + await session.commit() + return + existing.last_sent_at = now + await session.commit() + + # Nothing is sent until a real mail backend is configured; the console + # backend logs it so the flow can be verified end-to-end without a server. + if mailer.enabled(): + mailer.send( + email, + "Confirm your Thermograph digest", + "Thanks for signing up for the Thermograph monthly digest.\n\n" + "Confirmation links are not active yet — you're on the list, and " + "you'll get a confirmation as soon as they are.\n", + ) + + +async def signup(request: Request) -> Response: + """POST {BASE}/digest — accepts a JSON body (the fetch path) or a plain form + post (the no-JS path). Always answers 200 with the same message.""" + email = "" + place = None + try: + ctype = request.headers.get("content-type", "") + if ctype.startswith("application/json"): + body = await request.json() + if isinstance(body, dict): + email = str(body.get("email") or "").strip().lower() + place = (str(body.get("place")).strip() or None) if body.get("place") else None + else: + form = await request.form() + email = str(form.get("email") or "").strip().lower() + place = (str(form.get("place")).strip() or None) if form.get("place") else None + except Exception: # noqa: BLE001 - a malformed body is just an invalid signup + pass + + if _valid(email): + source = metrics.normalize_referrer( + request.headers.get("referer"), request.headers.get("host") + ) + try: + await _store(email, place, source) + except Exception: # noqa: BLE001 - never surface storage detail to the form + pass + metrics.record_event( + "home.digest_signup", + referer=request.headers.get("referer"), + host=request.headers.get("host"), + ip=request.client.host if request.client else None, + ) + return JSONResponse({"ok": True, "message": GENERIC_OK}) + + return JSONResponse( + {"ok": False, "message": "That doesn't look like an email address."}, + status_code=400, + ) diff --git a/homepage.py b/homepage.py new file mode 100644 index 0000000..ce0ba25 --- /dev/null +++ b/homepage.py @@ -0,0 +1,222 @@ +"""Homepage "unusual right now" feed. + +The homepage wants to answer "where is the weather most unusual today?" across +every city we track. That question has no cheap answer at request time: the +derived store keeps graded payloads as zlib-compressed JSON blobs keyed by +(kind, cell, key), with no percentile column to sort on, so finding today's +extremes means grading every cached city from scratch. + +So we precompute it. A sweep grades every city whose parquet cache is already +warm and writes the ranked result to data/homepage.json; the homepage template +reads that file. The sweep is deliberately **cache-only** — it calls +climate.load_cached_history and climate.load_cached_recent_forecast, never their +fetching counterparts, so a sweep over ~1000 cities costs zero upstream requests +and cannot burst the archive quota. Cities without a warm cache are skipped; the +deploy-time warm_cities.py run is what makes them eligible. + +Runs from the notifier daemon (hourly guard) and at the tail of warm_cities, so +it refreshes both on a live server and right after a deploy. +""" +from __future__ import annotations + +import datetime +import json +import os +import tempfile +import time + +import polars as pl + +import cities +import climate +import grading +import grid +from views import OBS_COLS + +# data/ is the only writable path under the hardened systemd unit +# (ReadWritePaths=/opt/thermograph/data …), so the feed lives there. +FEED_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "data", "homepage.json") + +# How many cards the "Unusual right now" strip can show. +RANK_LIMIT = 12 + +# A city must be at least this far from the median to be worth calling unusual. +MIN_DEPARTURE = 15.0 + +# A cold-tail card is force-included when one is at or below this percentile, +# even if every warmer city outranks it. Both tails or it isn't honest. +COLD_TAIL_MAX_PCT = 10.0 + +# The feed is stale (label it "as of " rather than claiming today) beyond this. +STALE_AFTER_S = 6 * 3600 + +# Ranking looks at the two metrics whose percentile people read as "how unusual +# was today" — the same pair grade_day's own `departure` score uses. +RANK_METRICS = ("tmax", "tmin") + +_METRIC_LABEL = {"tmax": "High temp", "tmin": "Low temp"} + + +def _seasonal_window_label(d: datetime.date) -> str: + """'mid-July' — the ±7-day seasonal window a percentile is measured against, + said the way a person would say it.""" + part = "early" if d.day <= 10 else ("mid" if d.day <= 20 else "late") + return f"{part}-{d.strftime('%B')}" + + +def _grade_city(city: dict, today: datetime.date) -> dict | None: + """Grade one city's latest observed day from cache alone. None when the city + has no warm cache, no observed row, or nothing gradeable.""" + cell = grid.snap(city["lat"], city["lon"]) + + history = climate.load_cached_history(cell) + if history is None: + return None + recent = climate.load_cached_recent_forecast(cell) + if recent is None or recent.is_empty() or "date" not in recent.columns: + return None + + observed = recent.filter(pl.col("date") <= today).sort("date") + if observed.is_empty(): + return None + row = observed.row(observed.height - 1, named=True) + obs = {k: row[k] for k in OBS_COLS if k in row} + + try: + graded = grading.grade_day(history, row["date"], obs) + except Exception: # noqa: BLE001 - one bad cell must not abort the sweep + return None + + # Pick the metric that strayed furthest from the median; that is the one the + # card should lead with. + best = None + for metric in RANK_METRICS: + g = graded.get(metric) + if not g or g.get("percentile") is None: + continue + departure = abs(g["percentile"] - 50) + if best is None or departure > best[0]: + best = (departure, metric, g) + if best is None: + return None + + departure, metric, g = best + date = row["date"] + date_s = date.isoformat() if hasattr(date, "isoformat") else str(date) + return { + "slug": city["slug"], + "name": city["name"], + "display": cities.display_name(city), + "lat": round(city["lat"], 4), + "lon": round(city["lon"], 4), + "cell_id": cell["id"], + "date": date_s, + "metric": metric, + "metric_label": _METRIC_LABEL.get(metric, metric), + "window_label": _seasonal_window_label( + date if hasattr(date, "timetuple") else today + ), + "value": g.get("value"), + "percentile": g.get("percentile"), + "grade": g.get("grade"), + # grading's css class is already the style.css token name minus the + # leading '--', so the template emits it with no mapping table. + "cls": g.get("class"), + "departure": round(departure, 1), + "tail": "cold" if g["percentile"] < 50 else "warm", + } + + +def build(limit: int | None = None) -> dict: + """Sweep the warm cache and rank today's most unusual cities. Pure — does no + I/O beyond reading the parquet cache, and never touches the network.""" + today = datetime.date.today() + graded: list[dict] = [] + considered = 0 + + for city in cities.all_cities()[: limit or None]: + row = _grade_city(city, today) + if row is None: + continue + considered += 1 + if row["departure"] >= MIN_DEPARTURE: + graded.append(row) + + graded.sort(key=lambda r: r["departure"], reverse=True) + ranked = graded[:RANK_LIMIT] + + # Two-sided honesty: if any tracked city sits in the cold tail, one must + # appear in the strip even when warm anomalies dominate the ranking. + if not any(r["tail"] == "cold" for r in ranked): + cold = next( + (r for r in graded if r["tail"] == "cold" + and r["percentile"] <= COLD_TAIL_MAX_PCT), + None, + ) + if cold is not None: + ranked = ranked[: RANK_LIMIT - 1] + [cold] + + picks = {} + if graded: + picks["extreme"] = graded[0] + cold = next((r for r in graded if r["tail"] == "cold"), None) + if cold is not None: + picks["cold"] = cold + + return { + "generated_at": time.time(), + "date": today.isoformat(), + "considered": considered, + "picks": picks, + "ranked": ranked, + } + + +def refresh(limit: int | None = None) -> dict: + """Rebuild the feed and write it atomically, so a concurrent reader never + sees a half-written file.""" + feed = build(limit=limit) + path = os.path.abspath(FEED_PATH) + os.makedirs(os.path.dirname(path), exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(feed, f, separators=(",", ":")) + os.replace(tmp, path) + except BaseException: + try: + os.unlink(tmp) + except OSError: + pass + raise + return feed + + +def load() -> dict | None: + """Read the feed. Returns None when it is missing or unreadable — the + homepage renders its frame without live numbers rather than failing, so a + cold checkout with no feed still serves a complete page.""" + try: + with open(os.path.abspath(FEED_PATH), encoding="utf-8") as f: + feed = json.load(f) + except (OSError, ValueError): + return None + if not isinstance(feed, dict) or "ranked" not in feed: + return None + return feed + + +def is_stale(feed: dict) -> bool: + """Older than STALE_AFTER_S, or built for a day that is no longer today.""" + if feed.get("date") != datetime.date.today().isoformat(): + return True + return (time.time() - float(feed.get("generated_at") or 0)) > STALE_AFTER_S + + +if __name__ == "__main__": + import sys + n = int(sys.argv[1]) if len(sys.argv) > 1 else None + out = refresh(limit=n) + print(f"graded {out['considered']} cached cities, " + f"{len(out['ranked'])} ranked, top: " + f"{out['ranked'][0]['display'] if out['ranked'] else '—'}") diff --git a/mailer.py b/mailer.py new file mode 100644 index 0000000..f95e646 --- /dev/null +++ b/mailer.py @@ -0,0 +1,100 @@ +"""Outbound email, over stdlib smtplib only (no new dependency). + +**The seam, not the policy.** The app always talks plain SMTP to whatever +``THERMOGRAPH_SMTP_HOST`` points at — in production, a Postfix null client +listening on 127.0.0.1:25 (see deploy/provision-mail.sh). Whether Postfix then +delivers straight to the recipient's MX or relays through a transactional +provider is a Postfix config decision that needs no change here. That is the +point of routing through a local MTA rather than talking to a provider's API: +delivery policy stays swappable, and a slow or failing upstream can't block a +request, because Postfix queues and retries on our behalf. + +**Default-safe.** The backend defaults to ``console`` — it logs the message and +sends nothing — so LAN dev and the test suite can exercise the full signup path +without a mail server and without any risk of mailing a real person. Production +opts in with ``THERMOGRAPH_MAIL_BACKEND=smtp``, the same shape as +``THERMOGRAPH_COOKIE_SECURE``. + +``send()`` never raises. Callers treat email like push notifications: a channel +that may fail without taking the request down with it. +""" +from __future__ import annotations + +import logging +import os +import smtplib +import ssl +from email.message import EmailMessage +from email.utils import formataddr, parseaddr + +log = logging.getLogger("thermograph.mail") + +# console -> log the message, send nothing (default; dev + tests) +# smtp -> hand it to the configured SMTP host +# disabled -> drop silently (a kill switch that needs no config removal) +BACKEND = os.environ.get("THERMOGRAPH_MAIL_BACKEND", "console").strip().lower() + +SMTP_HOST = os.environ.get("THERMOGRAPH_SMTP_HOST", "127.0.0.1").strip() +SMTP_PORT = int(os.environ.get("THERMOGRAPH_SMTP_PORT", "25") or 25) +SMTP_USER = os.environ.get("THERMOGRAPH_SMTP_USER", "").strip() +SMTP_PASSWORD = os.environ.get("THERMOGRAPH_SMTP_PASSWORD", "") +SMTP_STARTTLS = os.environ.get("THERMOGRAPH_SMTP_STARTTLS", "").strip() in ("1", "true", "yes") +SMTP_TIMEOUT = float(os.environ.get("THERMOGRAPH_SMTP_TIMEOUT", "10") or 10) + +MAIL_FROM = os.environ.get("THERMOGRAPH_MAIL_FROM", "Thermograph ").strip() +MAIL_REPLY_TO = os.environ.get("THERMOGRAPH_MAIL_REPLY_TO", "").strip() + + +def enabled() -> bool: + """Whether a send would actually leave the process.""" + return BACKEND == "smtp" + + +def _build(to: str, subject: str, text: str, html: "str | None") -> EmailMessage: + msg = EmailMessage() + name, addr = parseaddr(MAIL_FROM) + msg["From"] = formataddr((name, addr)) if addr else MAIL_FROM + msg["To"] = to + msg["Subject"] = subject + if MAIL_REPLY_TO: + msg["Reply-To"] = MAIL_REPLY_TO + # Plain text is the body; HTML is the alternative, so text-only clients and + # spam filters both see real content. + msg.set_content(text) + if html: + msg.add_alternative(html, subtype="html") + return msg + + +def send(to: str, subject: str, text: str, html: "str | None" = None) -> bool: + """Send one message. Returns whether it was handed off successfully. + + Never raises: a mail failure must not break the request that triggered it. + """ + if not to or "@" not in to: + return False + if BACKEND == "disabled": + return False + + try: + msg = _build(to, subject, text, html) + except Exception: # noqa: BLE001 + log.exception("mail: could not build message") + return False + + if BACKEND != "smtp": + # console (the default): prove the flow end-to-end without sending. + log.info("mail[console] to=%s subject=%s\n%s", to, subject, text) + return True + + try: + with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=SMTP_TIMEOUT) as smtp: + if SMTP_STARTTLS: + smtp.starttls(context=ssl.create_default_context()) + if SMTP_USER: + smtp.login(SMTP_USER, SMTP_PASSWORD) + smtp.send_message(msg) + return True + except Exception: # noqa: BLE001 - deliverability is never worth a 500 + log.exception("mail: send failed to=%s host=%s:%s", to, SMTP_HOST, SMTP_PORT) + return False diff --git a/metrics.py b/metrics.py index d59ed95..0c681ab 100644 --- a/metrics.py +++ b/metrics.py @@ -19,6 +19,7 @@ import os import sqlite3 import threading import time +import urllib.parse # The ``phase=`` tag passed to ``climate._request`` maps 1:1 onto an external source. # Keep this in sync with the phase names used at each call site in ``climate.py``. @@ -90,6 +91,10 @@ def classify_inbound(path: str, base: str = "") -> str: return "accounts" if seg == "metrics": return "metrics" + # The event beacon reports traffic; it is not itself traffic. Its own + # category keeps record_inbound from double-counting every interaction. + if seg == "event": + return "event" return f"api:{seg}" if seg else "api:other" if p.endswith((".js", ".css", ".html", ".webmanifest", ".png", ".svg", ".ico", ".json", ".woff", ".woff2", ".map", ".txt", ".xml")): @@ -97,13 +102,89 @@ def classify_inbound(path: str, base: str = "") -> str: if p in ("/robots.txt", "/sitemap.xml"): return "seo" return "static" - if p in ("", "/", "/calendar", "/day", "/compare", "/legend", "/alerts"): + if p in ("", "/", "/calendar", "/day", "/compare", "/legend", "/alerts", "/privacy"): return "page" if p == "/about" or p.startswith("/climate") or p.startswith("/glossary"): return "page" return "other" +# --- product events ---------------------------------------------------------- +# Distinct from inbound traffic: these count *intent* (tapped "use my location", +# submitted the digest form) rather than requests. Keyed by (event, referrer +# domain, UTC day) so a launch post's traffic can be told apart from search. +# +# Everything here is aggregate — no cookies, no per-visitor id, no full URLs. + +EVENTS = frozenset({ + "home.locate", "home.digest_signup", "home.records_click", "home.share", +}) +# home.nav_{surface}: only these surfaces, so the key space stays bounded. +NAV_SURFACES = frozenset({ + "calendar", "compare", "day", "alerts", "city", "cities", "records", +}) +# Anything unrecognized collapses here, with no referrer dimension — visible +# enough to notice abuse, bounded enough to be harmless. +EVENT_OTHER = "event:other" + +EVENT_DAYS = 7 # how many days of event history to keep +MAX_REFERRERS = 200 # distinct referrer domains tracked before overflow +REFERRER_OTHER = "other" + +_REF_OK = set("abcdefghijklmnopqrstuvwxyz0123456789.-") + + +def normalize_event(name: str) -> str: + """An allowlisted event name, or EVENT_OTHER. The allowlist is what keeps an + unauthenticated endpoint from being turned into unbounded storage.""" + name = (name or "").strip() + if name in EVENTS: + return name + surface = name.partition("home.nav_")[2] if name.startswith("home.nav_") else "" + if surface and surface in NAV_SURFACES: + return name + return EVENT_OTHER + + +def normalize_referrer(referer: "str | None", host: "str | None" = None) -> str: + """The bare registrable-ish domain of a Referer header — never the full URL, + which can carry a search query or a private path. 'direct' when absent, + 'self' for our own pages.""" + if not referer: + return "direct" + try: + netloc = urllib.parse.urlsplit(referer).netloc + except ValueError: + return REFERRER_OTHER + domain = (netloc.rpartition("@")[2].partition(":")[0] or "").lower() + if domain.startswith("www."): + domain = domain[4:] + if host and domain == (host.partition(":")[0] or "").lower().removeprefix("www."): + return "self" + domain = "".join(ch for ch in domain if ch in _REF_OK)[:64] + return domain or REFERRER_OTHER + + +def _utc_day() -> str: + return time.strftime("%Y-%m-%d", time.gmtime()) + + +def _recent_days() -> "set[str]": + """The EVENT_DAYS UTC days still worth keeping.""" + now = time.time() + return {time.strftime("%Y-%m-%d", time.gmtime(now - d * 86400)) + for d in range(EVENT_DAYS)} + + +def _nest_events(rows) -> dict: + """Flat (event, referrer, day, count) rows -> {event: {referrer: {day: n}}}.""" + out: "dict[str, dict[str, dict[str, int]]]" = {} + for key, count in rows: + event, referrer, day = key + out.setdefault(event, {}).setdefault(referrer, {})[day] = count + return out + + def _now_minute() -> int: """Current epoch-minute. Uses ``time.time()`` so tests can monkeypatch the clock.""" return int(time.time() // 60) @@ -129,6 +210,9 @@ class _MemStore: # name -> {"ts": last-beat epoch, "interval_s": expected cadence}. Lets the # dashboard judge a background daemon by liveness, not per-worker thread presence. self._heartbeats: "dict[str, dict[str, float]]" = {} + # (event, referrer, day) -> count, pruned to EVENT_DAYS on write. + self._events: "dict[tuple[str, str, str], int]" = {} + self._recent_ev: "dict[int, dict[str, int]]" = {} def _bump_recent(self, buckets, key) -> None: """Count one event for ``key`` in the current-minute bucket (caller holds lock).""" @@ -173,6 +257,21 @@ class _MemStore: with self._lock: self._heartbeats[name] = {"ts": ts, "interval_s": interval_s} + def record_event(self, event, referrer, day) -> None: + with self._lock: + # Cap distinct referrers so a spray of forged Referer headers can't + # grow the key space without bound. + if referrer not in ("direct", "self", REFERRER_OTHER): + seen = {r for _, r, _ in self._events} + if referrer not in seen and len(seen) >= MAX_REFERRERS: + referrer = REFERRER_OTHER + key = (event, referrer, day) + self._events[key] = self._events.get(key, 0) + 1 + keep = _recent_days() + for k in [k for k in self._events if k[2] not in keep]: + del self._events[k] + self._bump_recent(self._recent_ev, event) + def read(self) -> dict: with self._lock: inbound = {k: dict(v) for k, v in self._inbound.items()} @@ -184,6 +283,8 @@ class _MemStore: "inbound_recent": self._recent_totals(self._recent_in), "outbound_recent": self._recent_totals(self._recent_out), "heartbeats": {k: dict(v) for k, v in self._heartbeats.items()}, + "events": _nest_events(self._events.items()), + "events_recent": self._recent_totals(self._recent_ev), } @@ -212,6 +313,11 @@ class _SqliteStore: minute INTEGER, category TEXT, count INTEGER, PRIMARY KEY(minute, category)); CREATE TABLE IF NOT EXISTS outbound_minute( minute INTEGER, source TEXT, count INTEGER, PRIMARY KEY(minute, source)); + CREATE TABLE IF NOT EXISTS event_cume( + event TEXT, referrer TEXT, day TEXT, count INTEGER, + PRIMARY KEY(event, referrer, day)); + CREATE TABLE IF NOT EXISTS event_minute( + minute INTEGER, event TEXT, count INTEGER, PRIMARY KEY(minute, event)); """ ) # First worker to touch the DB stamps the shared start time; the rest inherit it. @@ -235,6 +341,12 @@ class _SqliteStore: cutoff = minute - WINDOW_MINUTES self._db.execute("DELETE FROM inbound_minute WHERE minute <= ?", (cutoff,)) self._db.execute("DELETE FROM outbound_minute WHERE minute <= ?", (cutoff,)) + self._db.execute("DELETE FROM event_minute WHERE minute <= ?", (cutoff,)) + # Event history is kept by day, not by minute. + self._db.execute( + "DELETE FROM event_cume WHERE day NOT IN (%s)" + % ",".join("?" * EVENT_DAYS), tuple(sorted(_recent_days())), + ) def record_inbound(self, category, bucket) -> None: minute = _now_minute() @@ -266,6 +378,29 @@ class _SqliteStore: ) self._prune(minute) + def record_event(self, event, referrer, day) -> None: + minute = _now_minute() + with self._lock: + if referrer not in ("direct", "self", REFERRER_OTHER): + row = self._db.execute( + "SELECT COUNT(DISTINCT referrer) FROM event_cume").fetchone() + known = self._db.execute( + "SELECT 1 FROM event_cume WHERE referrer = ? LIMIT 1", (referrer,) + ).fetchone() + if not known and row and row[0] >= MAX_REFERRERS: + referrer = REFERRER_OTHER + self._db.execute( + "INSERT INTO event_cume(event, referrer, day, count) VALUES(?, ?, ?, 1) " + "ON CONFLICT(event, referrer, day) DO UPDATE SET count = count + 1", + (event, referrer, day), + ) + self._db.execute( + "INSERT INTO event_minute(minute, event, count) VALUES(?, ?, 1) " + "ON CONFLICT(minute, event) DO UPDATE SET count = count + 1", + (minute, event), + ) + self._prune(minute) + def record_heartbeat(self, name, ts, interval_s) -> None: # Reuse the meta table (two keys per daemon) so every worker's beats land in the # one shared file — the dashboard then sees the notifier's liveness whichever @@ -312,6 +447,14 @@ class _SqliteStore: recent_out = dict(self._db.execute( "SELECT source, SUM(count) FROM outbound_minute WHERE minute >= ? " "GROUP BY source", (cutoff,))) + events = _nest_events( + ((event, referrer, day), count) + for event, referrer, day, count in self._db.execute( + "SELECT event, referrer, day, count FROM event_cume") + ) + recent_ev = dict(self._db.execute( + "SELECT event, SUM(count) FROM event_minute WHERE minute >= ? " + "GROUP BY event", (cutoff,))) return { "started_at": self.started_at, "inbound": inbound, @@ -319,6 +462,8 @@ class _SqliteStore: "inbound_recent": recent_in, "outbound_recent": recent_out, "heartbeats": self._read_heartbeats(), + "events": events, + "events_recent": recent_ev, } @@ -342,7 +487,9 @@ def record_inbound(category: str, status) -> None: """Count one inbound request in ``category``, bucketed by status class.""" # The metrics endpoint is polled by the dashboard itself — counting it would just # show the monitor watching itself, so it's excluded from traffic entirely. - if category == "metrics": + # The event beacon is likewise not traffic: it is the *record* of traffic, and + # counting it would double every interaction it reports. + if category in ("metrics", "event"): return try: _store.record_inbound(category, _status_bucket(status)) @@ -358,6 +505,44 @@ def record_outbound(phase: str, outcome: str) -> None: pass +# Per-IP token bucket for the unauthenticated event beacon. Bounded: the map is +# cleared whenever the minute rolls, so it holds at most one minute of clients. +_EVENT_RATE_PER_MIN = 30 +_rate_lock = threading.Lock() +_rate_minute = -1 +_rate_counts: "dict[str, int]" = {} + + +def _rate_ok(ip: "str | None") -> bool: + global _rate_minute + key = ip or "?" + minute = _now_minute() + with _rate_lock: + if minute != _rate_minute: + _rate_minute = minute + _rate_counts.clear() + count = _rate_counts.get(key, 0) + 1 + _rate_counts[key] = count + return count <= _EVENT_RATE_PER_MIN + + +def record_event(name: str, referer: "str | None" = None, + host: "str | None" = None, ip: "str | None" = None) -> None: + """Count one product event. The name is allowlisted and the referrer is taken + from the request's own Referer header (never client-supplied, which would be + trivially forgeable). Best-effort and silent: the caller always answers 204, + so a rejected or rate-limited event tells a prober nothing.""" + try: + if not _rate_ok(ip): + return + event = normalize_event(name) + # An unrecognized event carries no referrer dimension — one bucket total. + referrer = "direct" if event == EVENT_OTHER else normalize_referrer(referer, host) + _store.record_event(event, referrer, _utc_day()) + except Exception: # noqa: BLE001 - instrumentation must never break a request + pass + + def record_heartbeat(name: str, interval_s: float) -> None: """Stamp ``name``'s liveness. ``interval_s`` is the daemon's expected beat cadence, so a reader can tell fresh from stale without knowing the schedule. Best-effort.""" @@ -373,7 +558,8 @@ def snapshot() -> dict: data = _store.read() except Exception: # noqa: BLE001 - a locked/broken store must not 500 the dashboard data = {"started_at": time.time(), "inbound": {}, "outbound": {}, - "inbound_recent": {}, "outbound_recent": {}, "heartbeats": {}} + "inbound_recent": {}, "outbound_recent": {}, "heartbeats": {}, + "events": {}, "events_recent": {}} inbound = data["inbound"] outbound = data["outbound"] started_at = data["started_at"] @@ -398,4 +584,13 @@ def snapshot() -> dict: "inbound_recent": data["inbound_recent"], "outbound_recent": data["outbound_recent"], "heartbeats": heartbeats, + # Product events: {event: {referrer_domain: {day: count}}}, aggregate only. + "events": data.get("events") or {}, + "events_recent": data.get("events_recent") or {}, + "events_total": sum( + n + for refs in (data.get("events") or {}).values() + for days in refs.values() + for n in days.values() + ), } diff --git a/models.py b/models.py index a805803..6168356 100644 --- a/models.py +++ b/models.py @@ -141,3 +141,42 @@ class PushSubscription(Base): UniqueConstraint("endpoint", name="uq_push_endpoint"), Index("idx_push_user", "user_id"), ) + + +class PendingDigest(Base): + """A monthly-digest signup, collected before email delivery is wired up. + + The digest form ships ahead of SMTP on purpose: building the list is the + slow part, and making people wait for the mailer would throw away every + signup in the meantime. Rows land here unconfirmed; once delivery is live, a + confirmation pass mails each address and stamps ``confirmed_at``. + + Deliberately NOT tied to ``user`` — signing up for the digest must not + require an account, and most subscribers won't have one. + """ + __tablename__ = "pending_digest" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + # 320 = the practical maximum length of an email address (64 local + @ + 255 domain). + email: Mapped[str] = mapped_column(String(320), nullable=False) + # The place the digest should cover. Optional: an address with no place is + # still a real signup, and the place can be asked for at confirmation time. + place_label: Mapped[str | None] = mapped_column(String(200), nullable=True) + lat: Mapped[float | None] = mapped_column(Float, nullable=True) + lon: Mapped[float | None] = mapped_column(Float, nullable=True) + cell_id: Mapped[str | None] = mapped_column(String(32), nullable=True) + # Where the signup came from (bare referrer domain), for attribution only. + source: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[float] = mapped_column(Float, nullable=False, default=time.time) + # Set when the address is verified. The double opt-in token is stored as a + # sha256 hash, never in the clear, so a leaked database can't confirm addresses. + token_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + confirmed_at: Mapped[float | None] = mapped_column(Float, nullable=True) + last_sent_at: Mapped[float | None] = mapped_column(Float, nullable=True) + unsubscribed_at: Mapped[float | None] = mapped_column(Float, nullable=True) + + __table_args__ = ( + # One row per address: a re-submit updates in place rather than + # duplicating, which is what makes the form idempotent. + UniqueConstraint("email", name="uq_pending_digest_email"), + ) diff --git a/notify.py b/notify.py index 12efda2..78bdae1 100644 --- a/notify.py +++ b/notify.py @@ -32,6 +32,7 @@ from sqlalchemy import delete, select import climate import grading import grid +import homepage import metrics import push from db import sync_session_maker @@ -291,6 +292,26 @@ def run_pass() -> int: return created +# The homepage's "unusual right now" feed rides along on this loop rather than +# running its own thread: this one already wakes on a timer and already carries +# the single-leader invariant a second daemon would have to re-establish. The +# sweep is cache-only, so it costs no upstream quota. +HOMEPAGE_REFRESH_INTERVAL = 3600.0 +_last_homepage_refresh = 0.0 + + +def _maybe_refresh_homepage() -> None: + global _last_homepage_refresh + now = time.time() + if now - _last_homepage_refresh < HOMEPAGE_REFRESH_INTERVAL: + return + _last_homepage_refresh = now + try: + homepage.refresh() + except Exception: # noqa: BLE001 - the feed is optional; the homepage renders without it + pass + + def run_loop(): # Beat once at startup so the dashboard shows the notifier alive immediately after a # (re)start, without waiting a full INTERVAL for the first pass. Then beat every wake: @@ -304,6 +325,7 @@ def run_loop(): run_pass() except Exception: # noqa: BLE001 - a bad pass must never kill the loop pass + _maybe_refresh_homepage() def start(): diff --git a/templates/base.html.j2 b/templates/base.html.j2 index 47c059f..53da82b 100644 --- a/templates/base.html.j2 +++ b/templates/base.html.j2 @@ -24,6 +24,7 @@ + {% block head_extra %}{% endblock %} {% block jsonld %}{% endblock %} @@ -31,17 +32,25 @@

-

Thermograph

+ {# The brand is the page's h1 everywhere EXCEPT the homepage, where the + hero headline owns the sole h1 and the brand degrades to a

. The + .brand h1, .brand .site-name selector pair in style.css keeps the + lockup looking identical either way. #} + <{{ brand_tag|default('h1') }} class="site-name">Thermograph

How unusual is the weather? Graded against ~45 years of local climate history.

+ {# data-view marks the links nav.js rewrites to carry the current + location hash, so moving between views keeps the place you picked. + Climate and Alerts deliberately opt out (they aren't cell-scoped). + Inert on the SEO pages, which never load nav.js. #} @@ -50,24 +59,42 @@
-
+ {% block content %}{% endblock %}
+ {% block body_scripts %} + {% endblock %} + diff --git a/templates/home.html.j2 b/templates/home.html.j2 new file mode 100644 index 0000000..b42018e --- /dev/null +++ b/templates/home.html.j2 @@ -0,0 +1,179 @@ +{% extends "base.html.j2" %} +{# The Weekly view — the product itself — with the distribution surfaces built + around it. Everything structural is server-rendered so crawlers and no-JS + readers get a complete page; app.js hydrates the live numbers in place. + + The interactive controls below (#find-btn, #date-input, #panel, #results, …) + are app.js's DOM contract, carried over verbatim from the old static + frontend/index.html. Do not rename these ids. + + section / brand_tag / main_class come from the route's render context + (content.home_page), not a child-template {% set %} — the SEO pages already + pass `section` that way. #} + +{% block title %}Thermograph — how unusual is your weather?{% endblock %} +{% block description %}See how today's weather compares to decades of local climate history for any place on Earth — daily high, low, feels-like, humidity, wind and rain graded by percentile.{% endblock %} +{% block canonical_path %}/{% endblock %} + +{% block head_extra %} + +{% endblock %} + +{% block jsonld %} + +{% endblock %} + +{% block content %} + {# --- Hero ------------------------------------------------------------ + The grade card is the LCP element: its frame is server-rendered and its + slots are sized here, so hydrating the live numbers causes no layout + shift. Band colour is always paired with the band word — colour is never + the only signal. #} +
+
+

How unusual is your weather?

+

Any day, anywhere on Earth — graded against 45 years of that place's own history.

+
+ +
+ {# --cat carries the band colour, the same convention .normal-card uses, + so the band token stays the single source of truth. #} +
+

+ {%- if unusual -%} + Today in {{ unusual.display }} + {%- else -%} + Today + {%- endif -%} +

+

{{ unusual.grade if unusual else '—' }}

+

+ {%- if unusual -%} + {{ unusual.metric_label }} in the {{ unusual.percentile|ordinal }} percentile for {{ unusual.window_label }} + {%- else -%} + Pick a place to see how unusual its weather is today. + {%- endif -%} +

+ {% if unusual and unusual.is_default %} +

the most unusual weather we're tracking{% if stale %} · as of {{ unusual.date }}{% endif %}

+ {% endif %} +
+
+ + +
+

See your week ↓

+
+
+ +
+
+ + +
+
+ + + What do the grades mean? → +
+
+ +
+
+

Find a location to begin.

+

Thermograph fetches decades of daily highs, lows, and precipitation + for your ~4 sq mi cell, builds a ±7-day seasonal distribution for each + day of the year, and grades recent weather by where it falls on that distribution.

+
+ +
+ + {# Sits outside #panel on purpose: app.js rewrites results.innerHTML + wholesale on every grade, so anything inside it would be destroyed. #} +

+ Free. No ads. No tracking. No account needed. + How we work → +

+ + {# --- Unusual right now ---------------------------------------------- + Scroll-snap row, no JS carousel. Both tails are represented whenever a + cold-tail city qualifies — two-sided honesty is product identity. #} + {% if ranked %} +
+

Unusual right now

+ +

See all records →

+
+ {% endif %} + + {# --- How it works ---------------------------------------------------- #} +
+

How it works

+
    +
  1. We keep 45 years of climate history (ERA5 reanalysis) for your exact ~2-mile grid square.
    2. +
  2. Today gets compared to the same two weeks of the year, every year back to 1980.
    3. +
  3. The result is a percentile and a plain-language grade — Near Record cold to Near Record hot.
    4. +
+

+ Full methodology → + · Data: ERA5 via Open-Meteo · NASA POWER · MET Norway +

+
+ + {# --- Explore --------------------------------------------------------- #} +
+

Explore

+
+ + Calendar + Two years of your days, each colored by how unusual it was. + + + Compare + Rank any places by comfort for a trip or a move. + + + Day + Any date since 1980, anywhere: what was normal, what happened. + + + Alerts + Free percentile alerts: get pinged when your weather goes statistically weird. + +
+
+ + {# --- City hub links -------------------------------------------------- + Real HTML links funnelling homepage authority into the ~1000-page + /climate surface. #} +
+

Climate context for 1,000 cities

+ +

Browse all cities →

+
+{% endblock %} + +{% block body_scripts %} + + +{% endblock %} diff --git a/templates/privacy.html.j2 b/templates/privacy.html.j2 new file mode 100644 index 0000000..b0e4036 --- /dev/null +++ b/templates/privacy.html.j2 @@ -0,0 +1,39 @@ +{% extends "base.html.j2" %} +{% block title %}{{ page_title }}{% endblock %} +{% block description %}{{ page_description }}{% endblock %} +{% block canonical_path %}{{ canonical_path }}{% endblock %} +{% block content %} + +
+

Privacy

+

Thermograph is free, has no ads, and needs no account. It is built so that + there is very little about you to collect in the first place.

+ +

Your location

+

Thermograph never looks up your location from your IP address. The only way the site + learns where you are is if you press Use my location, which asks your browser for + permission — you can decline, and picking a place on the map works just as well.

+

Whichever way you choose a place, the coordinates are sent to the server only as part of + the ordinary request for that grid square's climate data, and are not logged beyond the + normal web-server request handling every site does. The place you picked is remembered in + your own browser's local storage, not on the server, so clearing your browser data + forgets it.

+ +

Analytics

+

There is no analytics library, no cookies for tracking, and no per-visitor identifier. + The server keeps simple aggregate counts — how many people opened a page or tapped a + button, and which site referred them — with nothing tying any of it to an individual.

+ +

Email

+

If you sign up for the monthly digest, your address is used for that digest and nothing + else. It is never sold or shared, and every message can unsubscribe you.

+ +

Accounts

+

An account is optional and only exists to hold weather alerts you have asked for. It + stores your email address and the places you are watching.

+ +

Questions: see About & methodology.

+
+{% endblock %} diff --git a/tests/test_digest.py b/tests/test_digest.py new file mode 100644 index 0000000..7b1d7ff --- /dev/null +++ b/tests/test_digest.py @@ -0,0 +1,120 @@ +"""Tests for the digest signup and the mailer seam.""" +import pytest +import sqlalchemy as sa +from fastapi.testclient import TestClient + +import app as appmod +import db +import digest +import mailer +from db import async_session_maker +from models import PendingDigest + +B = "/thermograph" + + +@pytest.fixture(scope="module", autouse=True) +def _tables(): + # TestClient(app) (no context manager) skips the lifespan, so create the + # account tables directly against the temp DB. + db.Base.metadata.create_all(db.sync_engine) + + +@pytest.fixture +def client(): + return TestClient(appmod.app) + + +def _rows(client, email): + """Count stored rows for an address, via the running event loop.""" + import asyncio + + async def go(): + async with async_session_maker() as session: + return (await session.execute( + sa.select(sa.func.count()).select_from(PendingDigest) + .where(PendingDigest.email == email) + )).scalar_one() + return asyncio.run(go()) + + +# --- mailer ------------------------------------------------------------------ +def test_console_backend_sends_nothing(monkeypatch): + """The default backend must never open a socket — this is what makes it safe + to run the full signup path in dev and in tests.""" + monkeypatch.setattr(mailer, "BACKEND", "console") + + def boom(*a, **k): + raise AssertionError("console backend must not connect to SMTP") + + monkeypatch.setattr(mailer.smtplib, "SMTP", boom) + assert mailer.send("a@example.com", "s", "t") is True + assert mailer.enabled() is False + + +def test_send_swallows_connection_failure(monkeypatch): + """A dead mail server must not raise into the request that triggered it.""" + monkeypatch.setattr(mailer, "BACKEND", "smtp") + + def boom(*a, **k): + raise OSError("connection refused") + + monkeypatch.setattr(mailer.smtplib, "SMTP", boom) + assert mailer.send("a@example.com", "s", "t") is False + + +def test_disabled_backend_drops(monkeypatch): + monkeypatch.setattr(mailer, "BACKEND", "disabled") + assert mailer.send("a@example.com", "s", "t") is False + + +def test_invalid_recipient_rejected(): + assert mailer.send("", "s", "t") is False + assert mailer.send("not-an-address", "s", "t") is False + + +# --- signup ------------------------------------------------------------------ +def test_signup_stores_and_confirms(client): + r = client.post(f"{B}/digest", json={"email": "reader@example.com"}) + assert r.status_code == 200 + assert r.json()["message"] == digest.GENERIC_OK + assert _rows(client, "reader@example.com") == 1 + + +def test_signup_is_idempotent_and_case_insensitive(client): + first = client.post(f"{B}/digest", json={"email": "Dup@Example.COM"}) + second = client.post(f"{B}/digest", json={"email": "dup@example.com"}) + # Same answer either way: the endpoint must not reveal whether an address is + # already on the list. + assert first.json()["message"] == second.json()["message"] == digest.GENERIC_OK + assert _rows(client, "dup@example.com") == 1 + + +def test_signup_rejects_junk(client): + for bad in ["", "nope", "a@b", "@example.com", "x@", "a b@example.com"]: + r = client.post(f"{B}/digest", json={"email": bad}) + assert r.status_code == 400, bad + + +def test_signup_accepts_plain_form_post(client): + """The no-JS path: a real
submission.""" + r = client.post(f"{B}/digest", data={"email": "nojs@example.com"}) + assert r.status_code == 200 + assert _rows(client, "nojs@example.com") == 1 + + +def test_signup_records_event(client): + import metrics + before = metrics.snapshot()["events"].get("home.digest_signup", {}) + before_n = sum(sum(days.values()) for days in before.values()) + client.post(f"{B}/digest", json={"email": "counted@example.com"}) + after = metrics.snapshot()["events"].get("home.digest_signup", {}) + assert sum(sum(days.values()) for days in after.values()) == before_n + 1 + + +def test_form_is_on_every_page(client): + """The digest form is the site-wide footer pattern, not homepage-only.""" + for path in ["/", "/about", "/climate", "/privacy", "/glossary"]: + html = client.get(f"{B}{path}").text + assert "data-digest" in html, path + assert 'action="/thermograph/digest"' in html, path diff --git a/tests/test_homepage.py b/tests/test_homepage.py new file mode 100644 index 0000000..30e07ff --- /dev/null +++ b/tests/test_homepage.py @@ -0,0 +1,202 @@ +"""Tests for the homepage: the cache-only "unusual right now" precompute, the +server-rendered surfaces, and the digest signup. + +The precompute must never touch the network — a sweep over ~1000 cities that +fetched would burst the archive quota — so the tests here assert that the +fetching climate helpers are not called at all. +""" +import datetime +import json +import time + +import pytest +from fastapi.testclient import TestClient + +import app as appmod +import cities +import climate +import content +import homepage + +B = "/thermograph" + + +@pytest.fixture +def client(): + return TestClient(appmod.app) + + +@pytest.fixture +def no_feed(monkeypatch): + """The homepage with no precomputed feed at all.""" + monkeypatch.setattr(homepage, "load", lambda: None) + return None + + +def _pick(name, pct, cls, grade, tail, slug=None): + return { + "slug": slug or (name.lower().replace(" ", "-") + "-xx"), + "name": name, "display": f"{name}, Somewhere", + "lat": 1.0, "lon": 2.0, "cell_id": "1_2", + "date": datetime.date.today().isoformat(), + "metric": "tmax", "metric_label": "High temp", "window_label": "mid-July", + "value": 99.0, "percentile": pct, "grade": grade, "cls": cls, + "departure": abs(pct - 50), "tail": tail, + } + + +# --- the precompute ---------------------------------------------------------- +def test_build_never_fetches_upstream(monkeypatch): + """The sweep uses the cache-only readers, never the fetching ones.""" + def boom(*a, **k): + raise AssertionError("the homepage sweep must not fetch upstream") + + monkeypatch.setattr(climate, "get_history", boom) + monkeypatch.setattr(climate, "get_recent_forecast", boom) + feed = homepage.build(limit=20) + assert set(feed) >= {"generated_at", "date", "considered", "picks", "ranked"} + + +def test_build_grades_cached_cities(monkeypatch, history, recent): + monkeypatch.setattr(climate, "load_cached_history", lambda cell: history.clone()) + monkeypatch.setattr(climate, "load_cached_recent_forecast", lambda cell: recent.clone()) + feed = homepage.build(limit=5) + assert feed["considered"] == 5 + for row in feed["ranked"]: + assert row["percentile"] is not None + assert row["cls"] and not row["cls"].startswith("--") # token name, no prefix + assert row["tail"] in ("warm", "cold") + + +def test_missing_cell_cache_is_skipped_not_fatal(monkeypatch): + monkeypatch.setattr(climate, "load_cached_history", lambda cell: None) + monkeypatch.setattr(climate, "load_cached_recent_forecast", lambda cell: None) + feed = homepage.build(limit=10) + assert feed["considered"] == 0 and feed["ranked"] == [] + + +def test_load_survives_missing_and_corrupt_file(monkeypatch, tmp_path): + monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "nope.json")) + assert homepage.load() is None + + bad = tmp_path / "homepage.json" + bad.write_text("{not json") + monkeypatch.setattr(homepage, "FEED_PATH", str(bad)) + assert homepage.load() is None + + bad.write_text('{"unexpected": true}') + assert homepage.load() is None + + +def test_refresh_writes_atomically(monkeypatch, tmp_path): + monkeypatch.setattr(homepage, "FEED_PATH", str(tmp_path / "homepage.json")) + monkeypatch.setattr(climate, "load_cached_history", lambda cell: None) + homepage.refresh(limit=3) + written = json.loads((tmp_path / "homepage.json").read_text()) + assert written["ranked"] == [] + # No temp files left behind. + assert [p.name for p in tmp_path.iterdir()] == ["homepage.json"] + + +def test_staleness(monkeypatch): + today = datetime.date.today().isoformat() + assert not homepage.is_stale({"date": today, "generated_at": time.time()}) + assert homepage.is_stale({"date": today, "generated_at": time.time() - 7 * 3600}) + assert homepage.is_stale({"date": "1999-01-01", "generated_at": time.time()}) + + +# --- the rendered page ------------------------------------------------------- +def test_homepage_is_complete_without_js(client, no_feed): + """A crawler (or a reader with JS off) gets the whole page as real HTML.""" + r = client.get(f"{B}/") + assert r.status_code == 200 + html = r.text + assert "How unusual is your weather?" in html + assert "How it works" in html + assert "Climate context for 1,000 cities" in html + assert "Free. No ads. No tracking." in html + # The interactive tool's DOM contract survives the move to Jinja. + for dom_id in ('id="find-btn"', 'id="date-input"', 'id="panel"', + 'id="placeholder"', 'id="results"'): + assert dom_id in html + # Exactly one h1: the hero headline. The brand degrades to a

. + assert html.count(".com/") + assert len(m.normalize_referrer("https://" + "a" * 200 + ".com/")) <= 64 + + +def test_events_are_not_counted_as_inbound_traffic(): + """The beacon reports traffic; counting it would double every interaction.""" + m = _fresh() + assert m.classify_inbound("/thermograph/api/v2/event", "/thermograph") == "event" + m.record_inbound("event", 204) + assert m.snapshot()["inbound_total"] == 0 + assert "event" not in m.snapshot()["inbound"] + + +def test_record_event_keys_by_event_referrer_day(): + m = _fresh() + for _ in range(3): + m.record_event("home.locate", referer="https://news.ycombinator.com/x") + m.record_event("home.share", referer=None) + events = m.snapshot()["events"] + day = m._utc_day() + assert events["home.locate"]["news.ycombinator.com"][day] == 3 + assert events["home.share"]["direct"][day] == 1 + assert m.snapshot()["events_total"] == 4 + + +def test_unknown_event_carries_no_referrer_dimension(): + m = _fresh() + m.record_event("junk.event", referer="https://spam.example/") + events = m.snapshot()["events"] + assert list(events[m.EVENT_OTHER]) == ["direct"] + + +def test_referrer_cardinality_is_capped(): + """A spray of forged Referer headers must not grow the key space forever.""" + m = _fresh() + m._EVENT_RATE_PER_MIN = 10_000 + for i in range(m.MAX_REFERRERS + 50): + m.record_event("home.locate", referer=f"https://ref{i}.example/") + refs = m.snapshot()["events"]["home.locate"] + assert len(refs) <= m.MAX_REFERRERS + 1 # +1 for the overflow bucket + assert m.REFERRER_OTHER in refs + + +def test_event_rate_limit_per_ip(): + m = _fresh() + for _ in range(m._EVENT_RATE_PER_MIN + 25): + m.record_event("home.locate", ip="203.0.113.9") + assert m.snapshot()["events_total"] == m._EVENT_RATE_PER_MIN diff --git a/warm_cities.py b/warm_cities.py index 3813f1c..77a428c 100644 --- a/warm_cities.py +++ b/warm_cities.py @@ -15,6 +15,7 @@ import time import cities import climate import grid +import homepage def main(limit: int | None = None, pace: float = 2.0) -> None: @@ -40,6 +41,16 @@ def main(limit: int | None = None, pace: float = 2.0) -> None: time.sleep(pace) print(f"done: fetched={fetched} skipped(cached)={skipped} failed={failed}") + # Rebuild the homepage's "unusual right now" feed from whatever is now warm. + # Runs here as well as on the notifier loop so a fresh deploy has a populated + # feed immediately, and so it still refreshes when the notifier is disabled. + try: + feed = homepage.refresh() + print(f"homepage feed: {feed['considered']} cities graded, " + f"{len(feed['ranked'])} ranked") + except Exception as e: # noqa: BLE001 - the homepage renders without the feed + print(f"homepage feed: FAILED {e}") + if __name__ == "__main__": args = sys.argv[1:]