diff --git a/README.md b/README.md index 53c3878..c27b427 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ curl -s 'http://10.10.0.2:3100/loki/api/v1/label/host/values' # -> {"data":["b `Thermograph — Fleet Logs` (auto-provisioned), with a `Node` selector: log volume by service, app error rate by tag, upstream 429 count, Caddy 5xx -count, a notifier-liveness log panel, recent app errors, and a live all-container +count, a per-node notifier-liveness indicator (from the heartbeat log), recent app errors, and a live all-container tail. Edit it in Grafana and re-export the JSON here to version a change. ## Notes / follow-ups @@ -118,6 +118,6 @@ tail. Edit it in Grafana and re-export the JSON here to version a change. - **Loki is mesh-only** by design; only Grafana (with auth) is public. - Alloy reads the Docker socket read-only to discover containers. - Retention is 30 days on beta's disk — bump `loki/config.yml` if you want more. -- Possible next step: have the app emit a periodic `notifier heartbeat` log line - so notifier liveness is a first-class log-derived signal, not just container - chatter. +- The app emits a `tag=heartbeat` line for the subscription notifier every + ~15m; the dashboard's "Notifier alive?" panel turns red if a node misses it. + Extend the same `audit.log_heartbeat()` to any future background daemon. diff --git a/grafana/dashboards/thermograph-logs.json b/grafana/dashboards/thermograph-logs.json index 8587568..c417c8a 100644 --- a/grafana/dashboards/thermograph-logs.json +++ b/grafana/dashboards/thermograph-logs.json @@ -87,16 +87,25 @@ ] }, { - "type": "logs", - "title": "Subscription notifier", - "description": "Everything the app logs mentioning the notifier — its liveness at a glance.", + "type": "stat", + "title": "Notifier alive? (heartbeats / 20m, per node)", + "description": "The subscription notifier logs a heartbeat every ~15m (tag=heartbeat). 0 beats in the last 20m on a node means its notifier is down.", "gridPos": { "h": 4, "w": 12, "x": 12, "y": 8 }, "datasource": { "type": "loki", "uid": "loki" }, - "options": { "showTime": true, "wrapLogMessage": true, "dedupStrategy": "exact", "sortOrder": "Descending" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, + "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN" } } } ] + } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "textMode": "value_and_name", "graphMode": "none" }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, - "expr": "{job=\"docker\", service=\"app\", host=~\"$host\"} |~ \"(?i)notifier\"" + "expr": "sum by (host) (count_over_time({job=\"app-json\", tag=\"heartbeat\", host=~\"$host\"} [20m]))", + "legendFormat": "{{host}}", + "instant": true } ] },