"""Product-event instrumentation: what people actually *do* in the UI. This is the durable, dimensioned tier that sits above ``core/metrics.py``'s since-start counters. metrics.py answers "is traffic flowing right now"; this module answers "did anyone ever get a location, and did search work" — over weeks, across deploys. **Feature-flagged OFF.** Nothing here records anything unless ``THERMOGRAPH_EVENTS=1``. With the flag unset the beacon still answers 204 and this module is inert, so shipping it is a no-op. Design constraints, in priority order: 1. **No identifier, ever.** No cookie, no localStorage id, no fingerprint, no user id — not even when the request carries a logged-in session. The unit of record is an *hour bucket + a tuple of allowlisted enum values*, never a person and never a row-per-interaction. That is a structural guarantee: there is no column an identifier could be written into. 2. **No free text, no coordinates.** Every field is drawn from a fixed allowlist (see ``SCHEMA``). An unrecognised name or value collapses to a bounded "other" bucket rather than being stored. So the endpoint cannot be turned into arbitrary storage, and cannot accidentally capture a search query, a URL, or a lat/lon. 3. **No IP.** The caller's IP is used only as an in-memory rate-limit key for the current minute (see ``metrics._rate_ok``) and is never passed in here. 4. **Never breaks a request.** Every entry point swallows its own errors. Storage: an hourly aggregate upsert into ``ui_event_hourly`` (a TimescaleDB hypertable on Postgres — see ``alembic/versions/0003_ui_events.py``), plus one JSONL line per event into the activity stream so Loki/Grafana can show the last 24h live. Neither carries anything person-linked. Off Postgres (dev, tests, offline tooling) the aggregate sink degrades to a no-op and only the JSONL line is written — the same dialect split ``data/store.py`` and ``core/metrics.py`` already use. """ import datetime import os import threading from core import audit # --- feature flag ------------------------------------------------------------- # Default OFF everywhere. Turn on per-environment (dev first, then beta, then # prod) once the privacy copy in frontend_ssr/templates/privacy.html.j2 has been # updated to match what is actually recorded. ENABLED = os.environ.get("THERMOGRAPH_EVENTS", "").strip().lower() in ("1", "true", "yes", "on") DATABASE_URL = os.environ.get("THERMOGRAPH_DATABASE_URL", "").strip() IS_POSTGRES = DATABASE_URL.startswith("postgresql") def _libpq_dsn(url: str) -> str: """A plain libpq URL psycopg accepts (mirrors climate_store/metrics).""" return url.replace("+asyncpg", "").replace("+psycopg", "") _PG_DSN = _libpq_dsn(DATABASE_URL) if IS_POSTGRES else "" # --- schema ------------------------------------------------------------------- # Seven events. Every one answers a product question that cannot be answered from # the request log alone; anything the request log *can* answer (page views, API # usage, error rates) is deliberately NOT duplicated here. # # Three dimension slots are stored — ``view``, ``prop``, ``value`` — plus a # referrer domain on ``view.open`` only (acquisition), so the key space stays # small enough to eyeball. Each event declares which slots it uses and the exact # allowlist for each. VIEWS = frozenset({ "home", "calendar", "day", "compare", "score", "alerts", "legend", "city", "cities", "records", "month", "glossary", "about", "privacy", }) #: Where a location came from. The activation question: do visitors ever get #: past "no place picked", and by which route? PICK_METHODS = frozenset({"geolocate", "map", "search", "restore", "link"}) #: Did the location search actually work? ``hit`` = results shown, ``typo`` = #: results only after respelling (places.suggest's ``corrected``), ``miss`` = #: nothing found. The query string itself is never sent or stored. SEARCH_OUTCOMES = frozenset({"hit", "typo", "miss"}) #: In-view controls worth knowing about. Anything not listed is dropped, so a #: new control has to be added here deliberately (and reviewed for privacy). CONTROLS = frozenset({ "chart_metric", # value: the metric key "unit", # value: F | C "date", # value: today | past "range", # value: the span bucket "season", # value: season key or "custom" "totals", # calendar totals mode "dist_metric", # compare distribution metric "comfort", # compare comfort slicer moved "compare_add", # value: the location-count bucket after the add "compare_remove", "filter_sheet", # mobile filter sheet opened "day_step", # day view prev/next }) #: Bounded value vocabulary, shared across controls. A value outside this set is #: stored as "other" — never verbatim. VALUES = frozenset({ # chart / distribution metrics "tmax", "tmin", "feels", "humid", "wind", "gust", "precip", "dry", # units "F", "C", # dates "today", "past", "future", # spans / counts (bucketed client-side, never a raw number) "1", "2", "3", "4", "5plus", "1y", "2y", "5y", "10y", "max", # seasons "winter", "spring", "summer", "fall", "custom", "all", # generic "on", "off", "other", }) SHARE_KINDS = frozenset({"link", "png", "permalink"}) #: The alerts funnel. ``ok`` records whether the step succeeded, so a high #: start/low create ratio is visible without any per-person tracking. ALERT_STEPS = frozenset({"open", "start", "create", "push_on", "push_off", "test"}) #: Dead ends — the highest-signal events here. A spike in any of these is a bug #: report nobody filed. DEADENDS = frozenset({ "grade_error", # /api/grade returned an error the user saw "no_data", # a cell with no usable history "geo_denied", # browser geolocation refused / timed out "search_miss", # search returned nothing (paired with place.search=miss) "not_found", # a 404 page was rendered "offline", # the fetch failed with no network }) OK = frozenset({"y", "n"}) #: name -> {slot: allowlist}. A slot absent from an event is always stored "". SCHEMA: "dict[str, dict[str, frozenset]]" = { # A view was opened (page load or SPA view). ``prop`` is how it was reached. "view.open": {"view": VIEWS, "prop": frozenset({"direct", "nav", "link", "restore"})}, # A location was chosen. NEVER carries coordinates — only the route taken. "place.pick": {"view": VIEWS, "prop": PICK_METHODS}, # A location search completed. Recorded server-side from api_suggest as well # as client-side, so searches from every surface are covered. "place.search": {"prop": SEARCH_OUTCOMES}, # An in-view control was used. "view.control": {"view": VIEWS, "prop": CONTROLS, "value": VALUES}, # Share / export. "share": {"view": VIEWS, "prop": SHARE_KINDS}, # The alerts funnel. "alert": {"prop": ALERT_STEPS, "value": OK}, # A dead end the visitor actually hit. "deadend": {"view": VIEWS, "prop": DEADENDS}, } #: Anything unrecognised collapses here with every dimension blank — visible #: enough to notice abuse, bounded enough to be harmless. EVENT_OTHER = "other" VALUE_OTHER = "other" SLOTS = ("view", "prop", "value") #: Max events per batched beacon payload. The client batches bursts (chart-metric #: taps, slider drags) rather than firing a request each. MAX_BATCH = 20 #: Max JSON body the beacon will read, in bytes. Must comfortably exceed #: MAX_BATCH full-sized events (~90 bytes each) or the byte cap, not the batch #: cap, becomes the binding limit and a legitimate full batch is silently #: dropped. Still far too small for the endpoint to be worth abusing as storage. MAX_BODY_BYTES = 4096 def normalize(name: str, props: "dict | None" = None) -> "tuple[str, dict[str, str]]": """An allowlisted ``(event, {view, prop, value})`` pair. The allowlist is the whole security model of an unauthenticated write endpoint: it is what stops the beacon from becoming unbounded storage, a spam sink, or an accidental PII channel. Unknown event -> ``EVENT_OTHER`` with blank dimensions; unknown value in a known slot -> ``VALUE_OTHER``; a slot the event does not declare -> dropped entirely. """ name = (name or "").strip() spec = SCHEMA.get(name) if spec is None: return EVENT_OTHER, {s: "" for s in SLOTS} props = props if isinstance(props, dict) else {} out: "dict[str, str]" = {} for slot in SLOTS: allowed = spec.get(slot) if allowed is None: out[slot] = "" continue raw = props.get(slot) raw = raw.strip() if isinstance(raw, str) else "" out[slot] = raw if raw in allowed else (VALUE_OTHER if raw else "") return name, out # --- durable aggregate sink --------------------------------------------------- # One row per (hour, event, view, prop, value, referrer). No row is ever written # per interaction, so the table cannot hold a sequence of one person's actions # even in principle. _local = threading.local() def _conn(): """Thread-local autocommit psycopg connection, or None when Postgres is off or unreachable. Mirrors data/climate_store.py.""" if not IS_POSTGRES: return None conn = getattr(_local, "conn", None) if conn is not None and not conn.closed: return conn try: import psycopg # local import: only the Postgres path needs it _local.conn = psycopg.connect(_PG_DSN, autocommit=True) except Exception: # noqa: BLE001 - degrade to "no durable sink", never raise _local.conn = None return _local.conn def _hour_bucket() -> datetime.datetime: now = datetime.datetime.now(datetime.timezone.utc) return now.replace(minute=0, second=0, microsecond=0) _UPSERT = """ INSERT INTO ui_event_hourly (bucket, event, view, prop, value, referrer, count) VALUES (%s, %s, %s, %s, %s, %s, 1) ON CONFLICT (bucket, event, view, prop, value, referrer) DO UPDATE SET count = ui_event_hourly.count + 1 """ def _upsert(event: str, dims: "dict[str, str]", referrer: str) -> None: conn = _conn() if conn is None: return try: conn.execute(_UPSERT, (_hour_bucket(), event, dims["view"], dims["prop"], dims["value"], referrer)) except Exception: # noqa: BLE001 - a broken connection retires itself below try: conn.close() except Exception: # noqa: BLE001 pass _local.conn = None # --- public entry point ------------------------------------------------------- def record(name: str, props: "dict | None" = None, referer: "str | None" = None, host: "str | None" = None) -> None: """Record one product event. Best-effort, silent, and a no-op unless the feature flag is on. ``referer``/``host`` come from the request's own headers (a client-supplied referrer would be forgeable and buys nothing) and are reduced to a bare domain by ``metrics.normalize_referrer`` — never a full URL, which can carry a search query or a private path. The referrer dimension is kept only for ``view.open``; every other event stores "" there, which is what keeps the key space small. """ if not ENABLED: return try: # Import here, not at module scope: metrics imports cleanly on its own and # this keeps the dependency one-directional at import time. from core import metrics event, dims = normalize(name, props) referrer = metrics.normalize_referrer(referer, host) if event == "view.open" else "" _upsert(event, dims, referrer) # Second sink: one structured line per event into the activity stream, so # the Loki/Grafana stack can show the last 24h live without waiting on an # hourly rollup. Carries the same allowlisted enums and nothing else. audit.log_activity("ui.event", {"event": event, **dims, "referrer": referrer}) except Exception: # noqa: BLE001 - instrumentation must never break a request pass