Emit a notifier heartbeat to the log stream for the dashboard (#12)

core/audit.py

@@ -29,6 +29,7 @@ LOG_ROOT = paths.LOGS_DIR
 AUDIT_DIR = os.path.join(LOG_ROOT, "audit")
 ERROR_DIR = os.path.join(LOG_ROOT, "errors")
 ACCESS_DIR = os.path.join(LOG_ROOT, "access")
+HEARTBEAT_DIR = os.path.join(LOG_ROOT, "heartbeat")
 
 _lock = threading.Lock()
 
@@ -75,6 +76,20 @@ def log_access(record: dict) -> None:
     _append(os.path.join(ACCESS_DIR, f"access-{_today()}.jsonl"), {"ts": _now_iso(), **record})
 
 
+def log_heartbeat(daemon: str, interval_s: float) -> None:
+    """Emit one liveness beat for a background daemon (e.g. the subscription
+    notifier) to the heartbeat log stream. This is the log-shipping counterpart to
+    ``metrics.record_heartbeat``: the observability stack tails these JSONL files, so
+    a beat here becomes a queryable ``tag="heartbeat"`` signal in Loki/Grafana — the
+    dashboard can tell a live daemon from a dead one by whether beats keep arriving
+    within ``interval_s``, without polling the in-process metrics endpoint. Best-effort."""
+    _append(
+        os.path.join(HEARTBEAT_DIR, f"heartbeat-{_today()}.jsonl"),
+        {"ts": _now_iso(), "level": "info", "tag": "heartbeat",
+         "daemon": daemon, "interval_s": interval_s},
+    )
+
+
 class RunAudit:
     """Times a graded run and writes one audit line when the ``with`` block exits.
 

notifications/notify.py

@@ -35,6 +35,7 @@ from data import climate
 from notifications import discord
 from data import grading
 from data import grid
+from core import audit
 from core import metrics
 from notifications import push
 from accounts.db import sync_session_maker
@@ -356,6 +357,7 @@ def run_loop():
     # healthy => a fresh beat at most INTERVAL apart, which is how the dashboard tells this
     # single-leader daemon apart from a genuinely dead one (per-worker thread checks can't).
     metrics.record_heartbeat("subscription-notifier", INTERVAL)
+    audit.log_heartbeat("subscription-notifier", INTERVAL)
     # Wait first, then run — gives the app a moment to finish booting. The whole
     # body is guarded: this is a daemon thread, so ANY escaping exception (a bad
     # pass, a homepage refresh, a Discord post) would silently kill it for the
@@ -363,6 +365,9 @@ def run_loop():
     # iteration instead; the next wake retries and keeps the heartbeat fresh.
     while not _STOP.wait(INTERVAL):
         metrics.record_heartbeat("subscription-notifier", INTERVAL)
+        # Log-shipped twin of the in-process metric above, so the observability
+        # stack (Loki/Grafana) can show notifier liveness straight from the logs.
+        audit.log_heartbeat("subscription-notifier", INTERVAL)
         try:
             run_pass()
             _maybe_refresh_homepage()

tests/core/test_audit.py

@@ -0,0 +1,28 @@
+"""Tests for the file-based audit/heartbeat logging (core/audit.py)."""
+import json
+
+from core import audit
+
+
+def test_log_heartbeat_writes_a_queryable_record(tmp_path, monkeypatch):
+    # The observability stack (Alloy → Loki) tails these JSONL files and lifts
+    # `level`/`tag` into labels, so a heartbeat must carry exactly those fields for
+    # the dashboard to query notifier liveness as {job="app-json", tag="heartbeat"}.
+    monkeypatch.setattr(audit, "HEARTBEAT_DIR", str(tmp_path))
+    audit.log_heartbeat("subscription-notifier", 900)
+
+    files = list(tmp_path.glob("heartbeat-*.jsonl"))
+    assert len(files) == 1
+    rec = json.loads(files[0].read_text().strip())
+    assert rec["tag"] == "heartbeat"
+    assert rec["level"] == "info"
+    assert rec["daemon"] == "subscription-notifier"
+    assert rec["interval_s"] == 900
+    assert "ts" in rec
+
+
+def test_log_heartbeat_never_raises(monkeypatch):
+    # Best-effort, like the rest of audit: an unwritable dir must not bubble up into
+    # the notifier loop (which is why the loop can rely on calling it unguarded).
+    monkeypatch.setattr(audit, "HEARTBEAT_DIR", "/proc/nonexistent/cannot-write")
+    audit.log_heartbeat("subscription-notifier", 900)  # no exception