diff --git a/core/audit.py b/core/audit.py index 53835e9..8af0502 100644 --- a/core/audit.py +++ b/core/audit.py @@ -29,6 +29,7 @@ LOG_ROOT = paths.LOGS_DIR AUDIT_DIR = os.path.join(LOG_ROOT, "audit") ERROR_DIR = os.path.join(LOG_ROOT, "errors") ACCESS_DIR = os.path.join(LOG_ROOT, "access") +HEARTBEAT_DIR = os.path.join(LOG_ROOT, "heartbeat") _lock = threading.Lock() @@ -75,6 +76,20 @@ def log_access(record: dict) -> None: _append(os.path.join(ACCESS_DIR, f"access-{_today()}.jsonl"), {"ts": _now_iso(), **record}) +def log_heartbeat(daemon: str, interval_s: float) -> None: + """Emit one liveness beat for a background daemon (e.g. the subscription + notifier) to the heartbeat log stream. This is the log-shipping counterpart to + ``metrics.record_heartbeat``: the observability stack tails these JSONL files, so + a beat here becomes a queryable ``tag="heartbeat"`` signal in Loki/Grafana — the + dashboard can tell a live daemon from a dead one by whether beats keep arriving + within ``interval_s``, without polling the in-process metrics endpoint. Best-effort.""" + _append( + os.path.join(HEARTBEAT_DIR, f"heartbeat-{_today()}.jsonl"), + {"ts": _now_iso(), "level": "info", "tag": "heartbeat", + "daemon": daemon, "interval_s": interval_s}, + ) + + class RunAudit: """Times a graded run and writes one audit line when the ``with`` block exits. diff --git a/notifications/notify.py b/notifications/notify.py index 128c64b..e00955e 100644 --- a/notifications/notify.py +++ b/notifications/notify.py @@ -35,6 +35,7 @@ from data import climate from notifications import discord from data import grading from data import grid +from core import audit from core import metrics from notifications import push from accounts.db import sync_session_maker @@ -356,6 +357,7 @@ def run_loop(): # healthy => a fresh beat at most INTERVAL apart, which is how the dashboard tells this # single-leader daemon apart from a genuinely dead one (per-worker thread checks can't). metrics.record_heartbeat("subscription-notifier", INTERVAL) + audit.log_heartbeat("subscription-notifier", INTERVAL) # Wait first, then run — gives the app a moment to finish booting. The whole # body is guarded: this is a daemon thread, so ANY escaping exception (a bad # pass, a homepage refresh, a Discord post) would silently kill it for the @@ -363,6 +365,9 @@ def run_loop(): # iteration instead; the next wake retries and keeps the heartbeat fresh. while not _STOP.wait(INTERVAL): metrics.record_heartbeat("subscription-notifier", INTERVAL) + # Log-shipped twin of the in-process metric above, so the observability + # stack (Loki/Grafana) can show notifier liveness straight from the logs. + audit.log_heartbeat("subscription-notifier", INTERVAL) try: run_pass() _maybe_refresh_homepage() diff --git a/tests/core/test_audit.py b/tests/core/test_audit.py new file mode 100644 index 0000000..3072993 --- /dev/null +++ b/tests/core/test_audit.py @@ -0,0 +1,28 @@ +"""Tests for the file-based audit/heartbeat logging (core/audit.py).""" +import json + +from core import audit + + +def test_log_heartbeat_writes_a_queryable_record(tmp_path, monkeypatch): + # The observability stack (Alloy → Loki) tails these JSONL files and lifts + # `level`/`tag` into labels, so a heartbeat must carry exactly those fields for + # the dashboard to query notifier liveness as {job="app-json", tag="heartbeat"}. + monkeypatch.setattr(audit, "HEARTBEAT_DIR", str(tmp_path)) + audit.log_heartbeat("subscription-notifier", 900) + + files = list(tmp_path.glob("heartbeat-*.jsonl")) + assert len(files) == 1 + rec = json.loads(files[0].read_text().strip()) + assert rec["tag"] == "heartbeat" + assert rec["level"] == "info" + assert rec["daemon"] == "subscription-notifier" + assert rec["interval_s"] == 900 + assert "ts" in rec + + +def test_log_heartbeat_never_raises(monkeypatch): + # Best-effort, like the rest of audit: an unwritable dir must not bubble up into + # the notifier loop (which is why the loop can rely on calling it unguarded). + monkeypatch.setattr(audit, "HEARTBEAT_DIR", "/proc/nonexistent/cannot-write") + audit.log_heartbeat("subscription-notifier", 900) # no exception