thermograph/observability/grafana/dashboards/thermograph-logs.json

{
  "annotations": { "list": [] },
  "editable": true,
  "graphTooltip": 1,
  "title": "Thermograph — Fleet Logs",
  "uid": "thermograph-logs",
  "tags": ["thermograph"],
  "timezone": "",
  "schemaVersion": 39,
  "refresh": "30s",
  "time": { "from": "now-6h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "host",
        "type": "custom",
        "label": "Node",
        "query": "prod,beta,dev",
        "current": { "text": "All", "value": "$__all" },
        "includeAll": true,
        "allValue": ".+",
        "multi": true
      }
    ]
  },
  "panels": [
    {
      "type": "timeseries",
      "title": "Log volume by service",
      "description": "Lines/sec from every container, split by compose service (app, db, forgejo…).",
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "datasource": { "type": "loki", "uid": "loki" },
      "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" } } } },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "sum by (service) (count_over_time({job=\"docker\", host=~\"$host\"} [$__auto]))",
          "legendFormat": "{{service}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "App errors/sec (structured logs)",
      "description": "count_over_time of the app's errors-*.jsonl lines, by tag.",
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "datasource": { "type": "loki", "uid": "loki" },
      "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 60 }, "color": { "mode": "palette-classic" } } },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "sum by (tag) (count_over_time({job=\"app-json\", source=\"app\", host=~\"$host\"} | level=~\"error|warn\" [$__auto]))",
          "legendFormat": "{{tag}}"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Upstream rate-limits (429) — last 1h",
      "description": "Open-Meteo / upstream 429s pulled from the app's error logs.",
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 },
      "datasource": { "type": "loki", "uid": "loki" },
      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 50 } ] } } },
      "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "sum (count_over_time({job=\"app-json\", host=~\"$host\"} | status=\"429\" [1h]))",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Caddy 5xx — last 1h",
      "description": "Server errors seen by the reverse proxy across all sites.",
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 },
      "datasource": { "type": "loki", "uid": "loki" },
      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } },
      "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "sum (count_over_time({job=\"caddy\", host=~\"$host\"} | json | status>=500 [1h]))",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Notifier alive? (heartbeats / 20m, per node)",
      "description": "The subscription notifier logs a heartbeat every ~15m (tag=heartbeat). 0 beats in the last 20m on a node means its notifier is down.",
      "gridPos": { "h": 4, "w": 12, "x": 12, "y": 8 },
      "datasource": { "type": "loki", "uid": "loki" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] },
          "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN" } } } ]
        }
      },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "textMode": "value_and_name", "graphMode": "none" },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "sum by (host) (count_over_time({job=\"app-json\", tag=\"heartbeat\", host=~\"$host\"} [20m]))",
          "legendFormat": "{{host}}",
          "instant": true
        }
      ]
    },
    {
      "type": "logs",
      "title": "Recent app errors",
      "description": "The app's errors-*.jsonl, newest first.",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 },
      "datasource": { "type": "loki", "uid": "loki" },
      "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "{job=\"app-json\", host=~\"$host\"} | level=~\"error|warn\""
        }
      ]
    },
    {
      "type": "logs",
      "title": "Live tail — all containers",
      "description": "Raw stdout/stderr from every container on the selected node(s).",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 },
      "datasource": { "type": "loki", "uid": "loki" },
      "options": { "showTime": true, "wrapLogMessage": false, "sortOrder": "Descending", "enableLogDetails": true },
      "targets": [
        {
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "{job=\"docker\", host=~\"$host\"}"
        }
      ]
    }
  ]
}