UI product-event instrumentation (design + flagged-off prototype)
Extends the existing /api/v2/event beacon into a small, typed, durable event
schema so the interactive views can answer product questions the request log
cannot: whether a visitor ever gets a location, whether search finds anything,
which controls and views earn their maintenance, and which dead ends people
actually hit.
Seven events with three enum dimension slots, stored as hourly aggregates in a
TimescaleDB hypertable plus one JSONL line per event for the 30-day Loki view.
No row per interaction and no column an identifier could go in: no cookie, no
session id, no user id, no IP, no coordinates, no free text, no URLs. The IP is
a per-minute rate-limit key and nothing else.
Abuse resistance for a public endpoint on a 60%-crawler site: closed allowlist
(unknown names collapse to one bucket), 4 KB streaming body cap, per-IP ceiling
raised to 120/min (30 was sized for four coarse events and would have silently
truncated a batched stream), soft Sec-Fetch-Site first-party check, uniform 204.
Ships inert -- THERMOGRAPH_EVENTS gates both the recorder and the flag stamp on
<html> that makes the client send anything, and is unset everywhere. See
UI-EVENTS.md for the schema, the rejected transport/storage alternatives, and
the privacy decisions that must be settled before the flag is turned on.
2026-07-23 22:49:55 +00:00
|
|
|
"""Route-level tests for the public product-event beacon (POST /api/v2/event).
|
|
|
|
|
|
|
|
|
|
The endpoint is unauthenticated and reachable by anything on the internet — on a
|
|
|
|
|
site where ~60% of inbound traffic is crawlers and scanners — so these tests are
|
|
|
|
|
mostly about what it REFUSES to do: grow storage, echo anything back, or accept
|
|
|
|
|
a payload big enough to be worth abusing.
|
|
|
|
|
"""
|
|
|
|
|
import json
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
|
|
|
|
from core import events
|
|
|
|
|
from web import app as appmod
|
|
|
|
|
|
|
|
|
|
EVENT_URL = "/thermograph/api/v2/event"
|
|
|
|
|
# Browsers set this on fetch/sendBeacon; a bare scanner POST does not.
|
|
|
|
|
FIRST_PARTY = {"sec-fetch-site": "same-origin"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
|
|
|
def _fresh_rate_limiter():
|
|
|
|
|
"""TestClient always presents the same client IP, so without this one test's
|
|
|
|
|
beacon traffic silently rate-limits the next one's."""
|
|
|
|
|
from core import metrics
|
|
|
|
|
metrics.reset_rate_limiter()
|
|
|
|
|
yield
|
|
|
|
|
metrics.reset_rate_limiter()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
|
def client():
|
|
|
|
|
return TestClient(appmod.app)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
|
def recorded(monkeypatch):
|
|
|
|
|
"""Capture what reaches the recorder, with the flag forced on."""
|
|
|
|
|
seen = []
|
|
|
|
|
monkeypatch.setattr(events, "ENABLED", True)
|
|
|
|
|
monkeypatch.setattr(events, "_upsert", lambda *a: None)
|
|
|
|
|
monkeypatch.setattr(events.audit, "log_activity", lambda tag, rec: seen.append(rec))
|
|
|
|
|
return seen
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_beacon_always_answers_204_and_returns_no_body(client, recorded):
|
|
|
|
|
"""Counted, rejected, or rate-limited must be indistinguishable to a prober."""
|
|
|
|
|
for body in ({"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
{"event": "not-a-real-event"},
|
|
|
|
|
{"nonsense": True},
|
|
|
|
|
[1, 2, 3],
|
|
|
|
|
"plain string"):
|
|
|
|
|
r = client.post(EVENT_URL, content=json.dumps(body), headers=FIRST_PARTY)
|
|
|
|
|
assert r.status_code == 204
|
|
|
|
|
assert r.content == b""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_valid_event_is_recorded_with_allowlisted_dimensions(client, recorded):
|
|
|
|
|
client.post(EVENT_URL, json={"event": "view.control",
|
|
|
|
|
"props": {"view": "compare", "prop": "unit", "value": "C"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
assert recorded == [{"event": "view.control", "view": "compare", "prop": "unit",
|
|
|
|
|
"value": "C", "referrer": ""}]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_batched_payload_is_accepted_and_capped(client, recorded):
|
|
|
|
|
items = [{"event": "share", "props": {"prop": "link"}}] * (events.MAX_BATCH + 5)
|
|
|
|
|
client.post(EVENT_URL, json={"events": items}, headers=FIRST_PARTY)
|
|
|
|
|
assert len(recorded) == events.MAX_BATCH
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_full_batch_of_real_events_fits_under_the_byte_cap(client, recorded):
|
|
|
|
|
"""Regression: if MAX_BODY_BYTES ever drops below a full MAX_BATCH payload,
|
|
|
|
|
the byte cap silently eats legitimate batches instead of abusive ones."""
|
|
|
|
|
items = [{"event": "view.control",
|
|
|
|
|
"props": {"view": "compare", "prop": "chart_metric", "value": "precip"}}
|
|
|
|
|
] * events.MAX_BATCH
|
|
|
|
|
assert len(json.dumps({"events": items})) < events.MAX_BODY_BYTES
|
|
|
|
|
client.post(EVENT_URL, json={"events": items}, headers=FIRST_PARTY)
|
|
|
|
|
assert len(recorded) == events.MAX_BATCH
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_oversized_body_is_dropped_without_being_parsed(client, recorded):
|
|
|
|
|
"""The endpoint must not become storage: a body past the cap is never read
|
|
|
|
|
into a record at all."""
|
|
|
|
|
fat = {"event": "view.open", "props": {"view": "home"},
|
|
|
|
|
"junk": "x" * (events.MAX_BODY_BYTES * 4)}
|
|
|
|
|
r = client.post(EVENT_URL, content=json.dumps(fat), headers=FIRST_PARTY)
|
|
|
|
|
assert r.status_code == 204
|
|
|
|
|
assert recorded == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_free_text_and_coordinates_never_survive(client, recorded):
|
|
|
|
|
"""Anything a client puts in a dimension slot is allowlisted or bucketed —
|
|
|
|
|
a leaked query string or lat/lon cannot reach a sink."""
|
|
|
|
|
client.post(EVENT_URL, json={"event": "place.pick",
|
|
|
|
|
"props": {"view": "home", "prop": "47.6062,-122.3321"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
client.post(EVENT_URL, json={"event": "view.control",
|
|
|
|
|
"props": {"view": "home", "prop": "chart_metric",
|
|
|
|
|
"value": "user@example.com"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
stored = {v for rec in recorded for v in rec.values()}
|
|
|
|
|
assert not any("47.6" in s or "@" in s for s in stored)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cross_site_post_is_dropped(client, recorded):
|
|
|
|
|
r = client.post(EVENT_URL, json={"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
headers={"sec-fetch-site": "cross-site"})
|
|
|
|
|
assert r.status_code == 204
|
|
|
|
|
assert recorded == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_flag_off_records_nothing(client, monkeypatch):
|
|
|
|
|
"""The whole feature ships inert: with THERMOGRAPH_EVENTS unset the beacon
|
|
|
|
|
still answers, and nothing reaches the durable sink."""
|
|
|
|
|
seen = []
|
|
|
|
|
monkeypatch.setattr(events, "ENABLED", False)
|
|
|
|
|
monkeypatch.setattr(events, "_upsert", lambda *a: seen.append(a))
|
|
|
|
|
monkeypatch.setattr(events.audit, "log_activity", lambda tag, rec: seen.append(rec))
|
|
|
|
|
r = client.post(EVENT_URL, json={"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
assert r.status_code == 204
|
|
|
|
|
assert seen == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_legacy_single_event_shape_still_counts(client, monkeypatch):
|
|
|
|
|
"""The four home.* aggregate counters already in production keep working —
|
|
|
|
|
they are unflagged and predate the schema."""
|
|
|
|
|
from core import metrics
|
|
|
|
|
seen = []
|
|
|
|
|
monkeypatch.setattr(metrics, "_store", type("S", (), {
|
|
|
|
|
"record_event": lambda self, *a: seen.append(a)})())
|
|
|
|
|
client.post(EVENT_URL, json={"event": "home.locate"}, headers=FIRST_PARTY)
|
|
|
|
|
assert seen and seen[0][0] == "home.locate"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_beacon_is_not_counted_as_inbound_traffic(client):
|
|
|
|
|
"""It reports traffic; counting it would double every interaction."""
|
|
|
|
|
from core import metrics
|
|
|
|
|
assert metrics.classify_inbound(EVENT_URL, "/thermograph") == "event"
|
UI events v2: consented session tier, failed-search capture, privacy rewrite
Three operator decisions change v1's premises, so the design is now two tiers
with separate flags:
Tier A (THERMOGRAPH_EVENTS) is v1 unchanged -- anonymous hourly aggregates, no
identifier, nothing stored on the device, no consent needed, runs for everyone.
It stays the primary signal precisely so a poor consent rate cannot take the
numbers down with it, and so Tier B's rates can be calibrated against it to
measure the consent bias rather than ignoring it.
Tier B (THERMOGRAPH_EVENT_SESSIONS) adds an ephemeral per-tab id -- 16 random
bytes in sessionStorage, rotating on a 30-minute idle and a 2-hour absolute cap,
capped at 200 events, never linked to another session, device, or to the account
(api_event does not read the auth cookie and there is no user column). Rows land
in a 30-day hypertable with minute-granularity timestamps and an in-session
sequence number instead of precise clock times. Storing an identifier engages
ePrivacy Art. 5(3) and analytics is not strictly necessary, so it is gated on
opt-in consent: an equal-weight banner that mints nothing before "Allow",
one-click withdrawal in every footer that drops the live id immediately, and
GPC/DNT treated as a refusal already given.
THERMOGRAPH_SEARCH_MISS captures zero-result search text server-side in
api_suggest only -- normalised, rejected outright on any personal-data smell,
stored as a per-day count, and pruned below a three-person floor after a week.
The privacy page is rewritten rather than deferred: "no per-visitor identifier"
becomes false the moment Tier B ships. Tests assert the load-bearing promises so
the copy and the code cannot drift apart silently.
Raw-IP truncation is NOT implemented here (the logging pipeline owns it) but
UI-EVENTS.md states what the app side must do, and records that Caddy's default
JSON log already stores full request URIs -- so every search query is in Loki
with the client IP today, which the search-miss mitigations depend on fixing.
UI-EVENTS.md carries the v1-to-v2 diff, the consent reasoning including why
legitimate interest is not available, and an explicit argument that the session
identifier is the wrong trade at this traffic volume.
2026-07-23 23:11:06 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# --- Tier B over the wire -------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
|
def session_rows(monkeypatch, recorded):
|
|
|
|
|
rows = []
|
|
|
|
|
monkeypatch.setattr(events, "SESSIONS_ENABLED", True)
|
|
|
|
|
monkeypatch.setattr(events, "_insert_session", lambda *a: rows.append(a))
|
|
|
|
|
return rows
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_session_and_seq_travel_with_a_batch(client, session_rows):
|
|
|
|
|
client.post(EVENT_URL, json={
|
|
|
|
|
"session": "D" * 22,
|
|
|
|
|
"events": [{"event": "view.open", "props": {"view": "home"}, "seq": 1},
|
|
|
|
|
{"event": "place.pick", "props": {"view": "home", "prop": "map"}, "seq": 2}],
|
|
|
|
|
}, headers=FIRST_PARTY)
|
|
|
|
|
assert [(r[0], r[1], r[2]) for r in session_rows] == [
|
|
|
|
|
("D" * 22, 1, "view.open"), ("D" * 22, 2, "place.pick")]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_forged_session_field_cannot_smuggle_text(client, session_rows, recorded):
|
|
|
|
|
"""The session field is the only client-controlled string that reaches a
|
|
|
|
|
sink, so a malformed one must lose its payload entirely — while the
|
|
|
|
|
anonymous aggregate still records."""
|
|
|
|
|
for forged in ("'; DROP TABLE ui_event_session; --", "user@example.com",
|
|
|
|
|
"47.6062,-122.3321", "x" * 500):
|
|
|
|
|
client.post(EVENT_URL, json={"session": forged, "seq": 1,
|
|
|
|
|
"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
assert session_rows == []
|
|
|
|
|
assert len(recorded) == 4 # Tier A recorded every one of them
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_sessions_flag_off_records_only_the_aggregate(client, monkeypatch, recorded):
|
|
|
|
|
rows = []
|
|
|
|
|
monkeypatch.setattr(events, "SESSIONS_ENABLED", False)
|
|
|
|
|
monkeypatch.setattr(events, "_insert_session", lambda *a: rows.append(a))
|
|
|
|
|
client.post(EVENT_URL, json={"session": "E" * 22, "seq": 1,
|
|
|
|
|
"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
assert rows == [] and len(recorded) == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_beacon_never_reads_the_account_session_cookie(client, session_rows, recorded):
|
|
|
|
|
"""Even a signed-in visitor's events stay unlinked to the account: the route
|
|
|
|
|
does not consult the auth cookie, so there is nothing to join on."""
|
|
|
|
|
client.cookies.set("thermographauth", "pretend-jwt-value")
|
|
|
|
|
client.post(EVENT_URL, json={"event": "view.open", "props": {"view": "home"}},
|
|
|
|
|
headers=FIRST_PARTY)
|
|
|
|
|
assert "pretend-jwt" not in str(recorded) + str(session_rows)
|
|
|
|
|
assert recorded and all("user" not in k for rec in recorded for k in rec)
|