{ "uid": "tg-app-activity", "title": "Thermograph \u2014 App Activity & Delivery", "tags": [ "thermograph", "activity", "notifier", "auth" ], "schemaVersion": 39, "editable": true, "refresh": "1m", "time": { "from": "now-24h", "to": "now" }, "timezone": "browser", "templating": { "list": [ { "name": "host", "type": "query", "datasource": { "type": "loki", "uid": "loki" }, "query": { "label": "host", "stream": "", "type": 1 }, "definition": "label_values(host)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".+", "current": { "text": "All", "value": "$__all" }, "sort": 1 } ] }, "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "name": "Annotations & Alerts", "type": "dashboard" } ] }, "panels": [ { "id": 1, "type": "row", "title": "Notifier throughput", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } }, { "id": 2, "type": "stat", "title": "Alerts fired", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notif.emit\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 3, "type": "stat", "title": "Sweeps run", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notify.pass\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 4, "type": "stat", "title": "Registrations", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"auth.register\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 5, "type": "stat", "title": "Logins", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 12, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"auth.login\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 6, "type": "stat", "title": "Email failures", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 16, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"mail.send\" | json | ok=\"false\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 7, "type": "stat", "title": "Discord DM failures", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 20, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notif.discord\" | json | ok=\"false\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 8, "type": "timeseries", "title": "Notifier sweep \u2014 subscriptions evaluated vs alerts created", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "max_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notify.pass\" | json | unwrap subs_evaluated [$__auto])", "refId": "A", "queryType": "range", "legendFormat": "subs evaluated" }, { "datasource": { "type": "loki", "uid": "loki" }, "expr": "max_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notify.pass\" | json | unwrap created [$__auto])", "refId": "B", "queryType": "range", "legendFormat": "alerts created" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "type": "timeseries", "title": "Notifier sweep duration (ms)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "max_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notify.pass\" | json | unwrap duration_ms [$__auto])", "refId": "A", "queryType": "range", "legendFormat": "duration" } ], "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 10, "type": "timeseries", "title": "Alerts fired \u2014 by metric", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (metric) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notif.emit\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{metric}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 11, "type": "timeseries", "title": "Alerts fired \u2014 by kind & direction", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (kind, direction) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notif.emit\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{kind}}/{{direction}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 12, "type": "row", "title": "Signup funnel & account lifecycle", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 } }, { "id": 13, "type": "timeseries", "title": "Auth funnel \u2014 register \u2192 verify \u2192 login", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (tag) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=~\"auth.+\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{tag}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 14, "type": "timeseries", "title": "Account lifecycle \u2014 subscriptions & push devices", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (tag) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=~\"sub.+|push.+\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{tag}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 15, "type": "row", "title": "Delivery health", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 } }, { "id": 16, "type": "timeseries", "title": "Email sends \u2014 ok vs fail", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (ok) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"mail.send\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "ok={{ok}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 17, "type": "timeseries", "title": "Discord DMs \u2014 ok vs fail", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (ok) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"notif.discord\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "ok={{ok}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 18, "type": "timeseries", "title": "Email sends by backend", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (backend) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"mail.send\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{backend}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 19, "type": "row", "title": "Unhandled 500s & slow requests", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 } }, { "id": 20, "type": "stat", "title": "Unhandled 500s (1h)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 41 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} | tag=\"http.error\" [1h]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 5 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 21, "type": "stat", "title": "Slow requests >2s (1h)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 6, "x": 6, "y": 41 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"http.slow\" [1h]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 20 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto" } }, { "id": 22, "type": "timeseries", "title": "500s by route (category)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} | tag=\"http.error\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 23, "type": "timeseries", "title": "Slow requests (>2s) by route (category)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"http.slow\" | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 2, "fillOpacity": 60, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 24, "type": "logs", "title": "Recent unhandled 500s (exception + path)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} | tag=\"http.error\"", "refId": "A", "queryType": "range" } ], "options": { "showTime": true, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" } }, { "id": 25, "type": "logs", "title": "Recent slow requests (>2s)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=\"http.slow\"", "refId": "A", "queryType": "range" } ], "options": { "showTime": true, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" } }, { "id": 26, "type": "row", "title": "Deploys & lifecycle", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 } }, { "id": 27, "type": "logs", "title": "Process starts / stops (version, role)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 66 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{job=\"app-json\", host=~\"$host\", filename=~\".+/activity/.+\"} | tag=~\"app.start|app.stop\"", "refId": "A", "queryType": "range" } ], "options": { "showTime": true, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" } } ] }