thermograph/backend/core/events.py

"""Product-event instrumentation: what people actually *do* in the UI.

This is the durable, dimensioned tier that sits above ``core/metrics.py``'s
since-start counters. metrics.py answers "is traffic flowing right now"; this
module answers "did anyone ever get a location, and did search work" — over
weeks, across deploys.

**Feature-flagged OFF.** Nothing here records anything unless
``THERMOGRAPH_EVENTS=1``. With the flag unset the beacon still answers 204 and
this module is inert, so shipping it is a no-op.

Design constraints, in priority order:

1. **No identifier, ever.** No cookie, no localStorage id, no fingerprint, no
   user id — not even when the request carries a logged-in session. The unit of
   record is an *hour bucket + a tuple of allowlisted enum values*, never a
   person and never a row-per-interaction. That is a structural guarantee: there
   is no column an identifier could be written into.
2. **No free text, no coordinates.** Every field is drawn from a fixed
   allowlist (see ``SCHEMA``). An unrecognised name or value collapses to a
   bounded "other" bucket rather than being stored. So the endpoint cannot be
   turned into arbitrary storage, and cannot accidentally capture a search
   query, a URL, or a lat/lon.
3. **No IP.** The caller's IP is used only as an in-memory rate-limit key for
   the current minute (see ``metrics._rate_ok``) and is never passed in here.
4. **Never breaks a request.** Every entry point swallows its own errors.

Storage: an hourly aggregate upsert into ``ui_event_hourly`` (a TimescaleDB
hypertable on Postgres — see ``alembic/versions/0003_ui_events.py``), plus one
JSONL line per event into the activity stream so Loki/Grafana can show the last
24h live. Neither carries anything person-linked. Off Postgres (dev, tests,
offline tooling) the aggregate sink degrades to a no-op and only the JSONL line
is written — the same dialect split ``data/store.py`` and ``core/metrics.py``
already use.
"""
import datetime
import os
import threading

from core import audit

# --- feature flag -------------------------------------------------------------
# Default OFF everywhere. Turn on per-environment (dev first, then beta, then
# prod) once the privacy copy in frontend_ssr/templates/privacy.html.j2 has been
# updated to match what is actually recorded.
ENABLED = os.environ.get("THERMOGRAPH_EVENTS", "").strip().lower() in ("1", "true", "yes", "on")

DATABASE_URL = os.environ.get("THERMOGRAPH_DATABASE_URL", "").strip()
IS_POSTGRES = DATABASE_URL.startswith("postgresql")


def _libpq_dsn(url: str) -> str:
    """A plain libpq URL psycopg accepts (mirrors climate_store/metrics)."""
    return url.replace("+asyncpg", "").replace("+psycopg", "")


_PG_DSN = _libpq_dsn(DATABASE_URL) if IS_POSTGRES else ""

# --- schema -------------------------------------------------------------------
# Seven events. Every one answers a product question that cannot be answered from
# the request log alone; anything the request log *can* answer (page views, API
# usage, error rates) is deliberately NOT duplicated here.
#
# Three dimension slots are stored — ``view``, ``prop``, ``value`` — plus a
# referrer domain on ``view.open`` only (acquisition), so the key space stays
# small enough to eyeball. Each event declares which slots it uses and the exact
# allowlist for each.

VIEWS = frozenset({
    "home", "calendar", "day", "compare", "score", "alerts", "legend",
    "city", "cities", "records", "month", "glossary", "about", "privacy",
})

#: Where a location came from. The activation question: do visitors ever get
#: past "no place picked", and by which route?
PICK_METHODS = frozenset({"geolocate", "map", "search", "restore", "link"})

#: Did the location search actually work? ``hit`` = results shown, ``typo`` =
#: results only after respelling (places.suggest's ``corrected``), ``miss`` =
#: nothing found. The query string itself is never sent or stored.
SEARCH_OUTCOMES = frozenset({"hit", "typo", "miss"})

#: In-view controls worth knowing about. Anything not listed is dropped, so a
#: new control has to be added here deliberately (and reviewed for privacy).
CONTROLS = frozenset({
    "chart_metric",     # value: the metric key
    "unit",             # value: F | C
    "date",             # value: today | past
    "range",            # value: the span bucket
    "season",           # value: season key or "custom"
    "totals",           # calendar totals mode
    "dist_metric",      # compare distribution metric
    "comfort",          # compare comfort slicer moved
    "compare_add",      # value: the location-count bucket after the add
    "compare_remove",
    "filter_sheet",     # mobile filter sheet opened
    "day_step",         # day view prev/next
})

#: Bounded value vocabulary, shared across controls. A value outside this set is
#: stored as "other" — never verbatim.
VALUES = frozenset({
    # chart / distribution metrics
    "tmax", "tmin", "feels", "humid", "wind", "gust", "precip", "dry",
    # units
    "F", "C",
    # dates
    "today", "past", "future",
    # spans / counts (bucketed client-side, never a raw number)
    "1", "2", "3", "4", "5plus",
    "1y", "2y", "5y", "10y", "max",
    # seasons
    "winter", "spring", "summer", "fall", "custom", "all",
    # generic
    "on", "off", "other",
})

SHARE_KINDS = frozenset({"link", "png", "permalink"})

#: The alerts funnel. ``ok`` records whether the step succeeded, so a high
#: start/low create ratio is visible without any per-person tracking.
ALERT_STEPS = frozenset({"open", "start", "create", "push_on", "push_off", "test"})

#: Dead ends — the highest-signal events here. A spike in any of these is a bug
#: report nobody filed.
DEADENDS = frozenset({
    "grade_error",   # /api/grade returned an error the user saw
    "no_data",       # a cell with no usable history
    "geo_denied",    # browser geolocation refused / timed out
    "search_miss",   # search returned nothing (paired with place.search=miss)
    "not_found",     # a 404 page was rendered
    "offline",       # the fetch failed with no network
})

OK = frozenset({"y", "n"})

#: name -> {slot: allowlist}. A slot absent from an event is always stored "".
SCHEMA: "dict[str, dict[str, frozenset]]" = {
    # A view was opened (page load or SPA view). ``prop`` is how it was reached.
    "view.open":    {"view": VIEWS, "prop": frozenset({"direct", "nav", "link", "restore"})},
    # A location was chosen. NEVER carries coordinates — only the route taken.
    "place.pick":   {"view": VIEWS, "prop": PICK_METHODS},
    # A location search completed. Recorded server-side from api_suggest as well
    # as client-side, so searches from every surface are covered.
    "place.search": {"prop": SEARCH_OUTCOMES},
    # An in-view control was used.
    "view.control": {"view": VIEWS, "prop": CONTROLS, "value": VALUES},
    # Share / export.
    "share":        {"view": VIEWS, "prop": SHARE_KINDS},
    # The alerts funnel.
    "alert":        {"prop": ALERT_STEPS, "value": OK},
    # A dead end the visitor actually hit.
    "deadend":      {"view": VIEWS, "prop": DEADENDS},
}

#: Anything unrecognised collapses here with every dimension blank — visible
#: enough to notice abuse, bounded enough to be harmless.
EVENT_OTHER = "other"
VALUE_OTHER = "other"

SLOTS = ("view", "prop", "value")

#: Max events per batched beacon payload. The client batches bursts (chart-metric
#: taps, slider drags) rather than firing a request each.
MAX_BATCH = 20

#: Max JSON body the beacon will read, in bytes. Must comfortably exceed
#: MAX_BATCH full-sized events (~90 bytes each) or the byte cap, not the batch
#: cap, becomes the binding limit and a legitimate full batch is silently
#: dropped. Still far too small for the endpoint to be worth abusing as storage.
MAX_BODY_BYTES = 4096


def normalize(name: str, props: "dict | None" = None) -> "tuple[str, dict[str, str]]":
    """An allowlisted ``(event, {view, prop, value})`` pair.

    The allowlist is the whole security model of an unauthenticated write
    endpoint: it is what stops the beacon from becoming unbounded storage, a
    spam sink, or an accidental PII channel. Unknown event -> ``EVENT_OTHER``
    with blank dimensions; unknown value in a known slot -> ``VALUE_OTHER``;
    a slot the event does not declare -> dropped entirely.
    """
    name = (name or "").strip()
    spec = SCHEMA.get(name)
    if spec is None:
        return EVENT_OTHER, {s: "" for s in SLOTS}
    props = props if isinstance(props, dict) else {}
    out: "dict[str, str]" = {}
    for slot in SLOTS:
        allowed = spec.get(slot)
        if allowed is None:
            out[slot] = ""
            continue
        raw = props.get(slot)
        raw = raw.strip() if isinstance(raw, str) else ""
        out[slot] = raw if raw in allowed else (VALUE_OTHER if raw else "")
    return name, out


# --- durable aggregate sink ---------------------------------------------------
# One row per (hour, event, view, prop, value, referrer). No row is ever written
# per interaction, so the table cannot hold a sequence of one person's actions
# even in principle.

_local = threading.local()


def _conn():
    """Thread-local autocommit psycopg connection, or None when Postgres is off
    or unreachable. Mirrors data/climate_store.py."""
    if not IS_POSTGRES:
        return None
    conn = getattr(_local, "conn", None)
    if conn is not None and not conn.closed:
        return conn
    try:
        import psycopg  # local import: only the Postgres path needs it
        _local.conn = psycopg.connect(_PG_DSN, autocommit=True)
    except Exception:  # noqa: BLE001 - degrade to "no durable sink", never raise
        _local.conn = None
    return _local.conn


def _hour_bucket() -> datetime.datetime:
    now = datetime.datetime.now(datetime.timezone.utc)
    return now.replace(minute=0, second=0, microsecond=0)


_UPSERT = """
    INSERT INTO ui_event_hourly (bucket, event, view, prop, value, referrer, count)
    VALUES (%s, %s, %s, %s, %s, %s, 1)
    ON CONFLICT (bucket, event, view, prop, value, referrer)
    DO UPDATE SET count = ui_event_hourly.count + 1
"""


def _upsert(event: str, dims: "dict[str, str]", referrer: str) -> None:
    conn = _conn()
    if conn is None:
        return
    try:
        conn.execute(_UPSERT, (_hour_bucket(), event, dims["view"], dims["prop"],
                               dims["value"], referrer))
    except Exception:  # noqa: BLE001 - a broken connection retires itself below
        try:
            conn.close()
        except Exception:  # noqa: BLE001
            pass
        _local.conn = None


# --- public entry point -------------------------------------------------------

def record(name: str, props: "dict | None" = None,
           referer: "str | None" = None, host: "str | None" = None) -> None:
    """Record one product event. Best-effort, silent, and a no-op unless the
    feature flag is on.

    ``referer``/``host`` come from the request's own headers (a client-supplied
    referrer would be forgeable and buys nothing) and are reduced to a bare
    domain by ``metrics.normalize_referrer`` — never a full URL, which can carry
    a search query or a private path. The referrer dimension is kept only for
    ``view.open``; every other event stores "" there, which is what keeps the
    key space small.
    """
    if not ENABLED:
        return
    try:
        # Import here, not at module scope: metrics imports cleanly on its own and
        # this keeps the dependency one-directional at import time.
        from core import metrics
        event, dims = normalize(name, props)
        referrer = metrics.normalize_referrer(referer, host) if event == "view.open" else ""
        _upsert(event, dims, referrer)
        # Second sink: one structured line per event into the activity stream, so
        # the Loki/Grafana stack can show the last 24h live without waiting on an
        # hourly rollup. Carries the same allowlisted enums and nothing else.
        audit.log_activity("ui.event", {"event": event, **dims, "referrer": referrer})
    except Exception:  # noqa: BLE001 - instrumentation must never break a request
        pass