{ "uid": "tg-traffic-domains", "title": "Thermograph \u2014 Traffic & Domains", "tags": [ "thermograph", "traffic", "domains" ], "schemaVersion": 39, "editable": true, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "templating": { "list": [ { "name": "host", "type": "query", "datasource": { "type": "loki", "uid": "loki" }, "query": { "label": "host", "stream": "", "type": 1 }, "definition": "label_values(host)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".+", "current": { "text": "All", "value": "$__all" }, "sort": 1 } ] }, "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "name": "Annotations & Alerts", "type": "dashboard" } ] }, "panels": [ { "id": 1, "type": "row", "title": "Overview", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 } }, { "id": 2, "type": "stat", "title": "App requests", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 3, "type": "stat", "title": "HTTP errors (4xx/5xx)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | status=~\"4..|5..\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 25 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 4, "type": "stat", "title": "Upstream failures", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 20 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 5, "type": "stat", "title": "Rate-limited (429)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 12, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} | json | status=\"429\" [$__range]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 6, "type": "stat", "title": "Notifier beats / 20m", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 16, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (host) (count_over_time({job=\"app-json\", host=~\"$host\"} | tag=\"heartbeat\" [20m]))", "refId": "A", "queryType": "range", "legendFormat": "{{host}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 7, "type": "stat", "title": "Caddy 5xx / 1h", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 5, "w": 4, "x": 20, "y": 1 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"caddy\", host=~\"$host\"} | json | status>=500 [1h]))", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "color": { "mode": "thresholds", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 8, "type": "piechart", "title": "Traffic mix by domain (category)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json [$__range]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {} }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "value", "percent" ] }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "values": true }, "displayLabels": [ "percent" ] } }, { "id": 9, "type": "timeseries", "title": "Requests/sec by domain", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (rate({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 10, "type": "row", "title": "Weather API \u2014 core product", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 } }, { "id": 11, "type": "timeseries", "title": "Weather endpoints \u2014 req/s", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (rate({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | cat=~\"api:.+\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 12, "type": "timeseries", "title": "Grade run latency (ms)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "quantile_over_time(0.95, {job=\"app-json\", host=~\"$host\", filename=~\".+/audit/.+\"} | json | unwrap duration_ms [$__auto]) by ()", "refId": "A", "queryType": "range", "legendFormat": "p95" }, { "datasource": { "type": "loki", "uid": "loki" }, "expr": "quantile_over_time(0.50, {job=\"app-json\", host=~\"$host\", filename=~\".+/audit/.+\"} | json | unwrap duration_ms [$__auto]) by ()", "refId": "B", "queryType": "range", "legendFormat": "p50" } ], "fieldConfig": { "defaults": { "unit": "ms", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 0, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 13, "type": "table", "title": "Top endpoints (by volume)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "topk(12, sum by (cat) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | cat=~\"api:.+\" [$__range])))", "refId": "A", "queryType": "instant", "legendFormat": "{{cat}}", "instant": true, "format": "table" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": {} }, "overrides": [] }, "options": { "showHeader": true, "cellHeight": "sm", "sortBy": [ { "displayName": "Value", "desc": true } ] } }, { "id": 14, "type": "stat", "title": "Cache-served runs (partial %)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 24 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/audit/.+\"} | json | run_type=\"partial\" [$__range])) / sum(count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/audit/.+\"} | json | run_type=~\".+\" [$__range])) * 100", "refId": "A", "queryType": "range" } ], "fieldConfig": { "defaults": { "unit": "percent", "custom": {}, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "color": { "mode": "fixed", "fixedColor": "text" } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "textMode": "auto", "orientation": "auto" } }, { "id": 15, "type": "timeseries", "title": "Fresh (full) vs cached (partial) runs", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 24 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (run_type) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/audit/.+\"} | json | run_type=~\".+\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{run_type}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 16, "type": "row", "title": "Accounts, Auth & Content", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 } }, { "id": 17, "type": "timeseries", "title": "Auth & Accounts \u2014 req/s", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (rate({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | cat=~\"auth|accounts\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 18, "type": "timeseries", "title": "Pages & SEO \u2014 req/s", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (cat) (rate({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | cat=~\"page|seo\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{cat}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 19, "type": "row", "title": "Upstream fetches & Reliability", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 } }, { "id": 20, "type": "timeseries", "title": "Upstream failures by phase", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (phase) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"} [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{phase}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 21, "type": "timeseries", "title": "HTTP errors by status", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (status) (count_over_time({job=\"app-json\", host=~\"$host\", filename=~\".+/access/.+\"} | json | status=~\"4..|5..\" [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{status}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 22, "type": "logs", "title": "Recent errors & retries (app)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 50 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{job=\"app-json\", host=~\"$host\", filename=~\".+/errors/.+\"}", "refId": "A", "queryType": "range" } ], "options": { "showTime": true, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" } }, { "id": 23, "type": "row", "title": "By site (Caddy) & Fleet", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 } }, { "id": 24, "type": "timeseries", "title": "Requests by site (Host header)", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (request_host) (rate({job=\"caddy\", host=~\"$host\"} | json [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{request_host}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 25, "type": "timeseries", "title": "Log volume by service", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "sum by (service) (rate({job=\"docker\", host=~\"$host\"} [$__auto]))", "refId": "A", "queryType": "range", "legendFormat": "{{service}}" } ], "fieldConfig": { "defaults": { "unit": "logs", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 18, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 26, "type": "logs", "title": "Live tail \u2014 all containers", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 68 }, "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{job=\"docker\", host=~\"$host\"}", "refId": "A", "queryType": "range" } ], "options": { "showTime": true, "wrapLogMessage": true, "prettifyLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none", "sortOrder": "Descending" } } ] }