Subtree-merge thermograph-observability (origin/main) into observability/

git-subtree-dir: observability git-subtree-mainline: ae1d9bb534 git-subtree-split: 19e74af9ca
2026-07-22 22:01:11 -07:00 · 2026-07-22 22:01:11 -07:00 · f2fd8f6835
commit f2fd8f6835
parent ae1d9bb534 19e74af9ca
15 changed files with 3401 additions and 0 deletions
--- a/observability/.env.example
+++ b/observability/.env.example
@ -0,0 +1,20 @@
+# Copy to .env on the monitoring host (beta) before `docker compose up`.
+# .env is gitignored — never commit real secrets.
+
+# The public hostname beta's Caddy serves Grafana at (see caddy-grafana.conf).
+GRAFANA_DOMAIN=dashboard.thermograph.org
+
+# Break-glass local admin (Google SSO below is the primary login). Keep this
+# strong; it's the fallback if OAuth ever breaks.
+GF_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD=change-me-to-something-strong
+
+# --- Google SSO ------------------------------------------------------------------
+# Create a Google Cloud "OAuth 2.0 Client ID" (type: Web application) and set:
+#   Authorized redirect URI:  https://dashboard.thermograph.org/login/google
+# Then paste the client id/secret here and flip OAUTH_ENABLED=true.
+# Login is locked to pre-provisioned accounts (allow_sign_up=false) — provision
+# your email as an admin (see README) so only you can sign in.
+OAUTH_ENABLED=false
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
--- a/observability/.forgejo/workflows/validate.yml
+++ b/observability/.forgejo/workflows/validate.yml
@ -0,0 +1,66 @@
+name: Validate observability stack
+
+# This repo deploys by hand (`docker compose up -d` on beta + the Alloy agent on
+# each node) with no build step, so nothing caught a malformed compose file, a
+# broken dashboard JSON, or an unparseable config until it failed live on the
+# host. This is that missing guard: it parses every YAML/JSON artifact that ships
+# to a node. It does NOT deploy, and deliberately needs no docker CLI (the
+# node:20-bookworm job image doesn't ship one) — a YAML parse catches the real
+# breakage (bad indent / unparseable file) without it.
+#
+# Not validated here: the Alloy config (alloy/config.alloy, an HCL-like format,
+# needs the `alloy` binary) and docker-compose *schema* (unknown-key checks, which
+# would need the docker CLI). Add either if the churn warrants the extra tooling.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  validate:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Dashboards are valid JSON
+        run: |
+          set -euo pipefail
+          python3 - <<'PY'
+          import json, pathlib, sys
+          bad = False
+          files = sorted(pathlib.Path("grafana/dashboards").glob("*.json"))
+          if not files:
+              print("no dashboards found"); sys.exit(1)
+          for f in files:
+              try:
+                  json.loads(f.read_text()); print("OK", f)
+              except Exception as e:  # noqa: BLE001
+                  print("INVALID JSON", f, "-", e); bad = True
+          sys.exit(1 if bad else 0)
+          PY
+
+      - name: YAML artifacts parse (compose + loki + grafana provisioning)
+        run: |
+          set -euo pipefail
+          apt-get update -qq && apt-get install -y -qq python3-yaml >/dev/null
+          python3 - <<'PY'
+          import pathlib, sys
+          import yaml
+          bad = False
+          targets = [
+              pathlib.Path("docker-compose.yml"),
+              pathlib.Path("alloy/docker-compose.agent.yml"),
+              pathlib.Path("loki/config.yml"),
+              *sorted(pathlib.Path("grafana/provisioning").rglob("*.yml")),
+          ]
+          for f in targets:
+              if not f.exists():
+                  print("MISSING", f); bad = True; continue
+              try:
+                  yaml.safe_load(f.read_text()); print("OK", f)
+              except Exception as e:  # noqa: BLE001
+                  print("INVALID YAML", f, "-", e); bad = True
+          sys.exit(1 if bad else 0)
+          PY
--- a/observability/.gitignore
+++ b/observability/.gitignore
@ -0,0 +1,3 @@
+.env
+alloy/.env
+*.log
--- a/observability/CLAUDE.md
+++ b/observability/CLAUDE.md
@ -0,0 +1,41 @@
+# thermograph-observability — agent instructions
+
+The **logging/metrics stack** for the Thermograph fleet: Loki + Grafana on beta,
+with a Grafana Alloy agent on every node (prod, beta, dev) shipping container and
+app logs over the WireGuard mesh. Grafana is fronted by beta's Caddy at
+`dashboard.thermograph.org` (Google SSO, pre-provisioned users only).
+
+This is **operational infra config, not application code** — there is no build.
+It deploys by hand: `docker compose up -d` on beta for Loki+Grafana, and the
+Alloy agent (`alloy/docker-compose.agent.yml`) on each node. See the README for
+the full per-node procedure.
+
+## Layout
+
+- `docker-compose.yml` — the Loki + Grafana stack (runs on beta).
+- `loki/config.yml` — Loki config (mesh-only, filesystem storage).
+- `grafana/provisioning/` — datasource (Loki) + dashboard provider, auto-loaded
+  at startup. `grafana/dashboards/*.json` — the dashboards themselves.
+- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node log
+  shipper. The node name comes from `ALLOY_NODE` (set per host).
+- `caddy-grafana.conf` — the Caddy vhost for `dashboard.thermograph.org` (lives
+  in beta's Caddy config, kept here for reference).
+- `.env.example` — copy to `.env` on beta before `docker compose up`. `.env` is
+  gitignored; never commit real OAuth secrets or the admin password.
+
+## Conventions
+
+- **Every artifact that ships to a node is CI-validated** (`.forgejo/workflows/validate.yml`):
+  both compose files parse, all dashboard JSON is valid, and the Loki/provisioning
+  YAML parses. Keep new dashboards as valid JSON and new config as valid YAML or
+  CI fails. The Alloy config is not yet CI-validated (needs the `alloy` binary).
+- The public hostname is **`dashboard.thermograph.org`** everywhere — not
+  `grafana.thermograph.org`. Match it in any new comment/config.
+- Secrets (OAuth client id/secret, admin password) live only in the host `.env`,
+  never in the repo.
+
+## Branching
+
+Single `main` branch, no protection. Open a PR against `main`; the validator
+gates it. Deploying the change to beta / the nodes is a separate manual step
+(this repo has no deploy automation — a known gap).
--- a/observability/README.md
+++ b/observability/README.md
@ -0,0 +1,123 @@
+# Thermograph — observability
+
+Fleet-wide log aggregation for [Thermograph](https://thermograph.org): a central
+**Grafana + Loki** stack, fed by a **Grafana Alloy** agent on every node, all
+over the private WireGuard mesh. This replaces the old in-repo
+`scripts/dashboard.py` (an SSH-tailed, single-host, metrics-endpoint view) with a
+persistent, queryable, fleet-wide log store and UI.
+
+```
+  prod (10.10.0.1)          beta (10.10.0.2)            desktop / LAN dev (10.10.0.3)
+  ┌────────────┐            ┌──────────────────┐        ┌────────────┐
+  │ Alloy agent│──wg0──┐    │ Loki  ◀── Alloy  │   ┌──wg0│ Alloy agent│
+  └────────────┘       └───▶│ Grafana (Caddy)  │◀──┘    └────────────┘
+   docker+caddy+app         └──────────────────┘         docker+caddy+app
+     logs                    central store + UI            logs
+```
+
+- **Loki** stores logs (filesystem, 30-day retention). Listens on the mesh IP
+  `10.10.0.2:3100` — never public.
+- **Grafana** is the UI, fronted by beta's Caddy at `dashboard.thermograph.org`,
+  login via **Google SSO** (with a break-glass local admin); Loki is not exposed.
+- **Alloy** runs on each node and ships three sources to Loki: every Docker
+  container's stdout/stderr, Caddy's host access logs, and the app's structured
+  JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase`
+  become labels). Every line is tagged with its node (`host = prod|beta|dev`).
+
+## Layout
+
+| Path | What |
+|------|------|
+| `docker-compose.yml` | the central Loki + Grafana stack (runs on beta) |
+| `loki/config.yml` | Loki config (filesystem, retention) |
+| `grafana/provisioning/` | auto-wired Loki datasource + dashboard provider |
+| `grafana/dashboards/thermograph-logs.json` | the fleet-logs dashboard |
+| `alloy/config.alloy` | the per-node collector config |
+| `alloy/docker-compose.agent.yml` | runs the Alloy agent on a node |
+| `caddy-grafana.conf` | beta Caddy site block for the Grafana UI |
+
+## Deploy
+
+### 1. Central stack (on beta)
+
+```bash
+# on beta (75.119.132.91):
+git clone <this repo> thermograph-observability && cd thermograph-observability
+cp .env.example .env      # set GF_SECURITY_ADMIN_PASSWORD
+docker compose up -d      # Loki on 10.10.0.2:3100 + 127.0.0.1:3100; Grafana on 127.0.0.1:3000
+```
+
+Expose the Grafana UI: point `dashboard.thermograph.org` at beta, append
+`caddy-grafana.conf` to `/etc/caddy/Caddyfile`, `systemctl reload caddy`.
+
+### Google SSO (Grafana + Forgejo)
+
+Both the dashboard and Forgejo log in with Google. Create **one** Google Cloud
+OAuth 2.0 Client (type: *Web application*) and register both redirect URIs:
+
+- Grafana: `https://dashboard.thermograph.org/login/google`
+- Forgejo: `https://git.thermograph.org/user/oauth2/google/callback`
+
+**Grafana:** put the client id/secret in `.env`, set `OAUTH_ENABLED=true`, and
+`docker compose up -d`. Login is locked to pre-provisioned accounts
+(`allow_sign_up=false`), so provision your email as an admin once:
+
+```bash
+# on beta, after the stack is up (uses the break-glass admin creds from .env):
+curl -s -u admin:"$GF_SECURITY_ADMIN_PASSWORD" -H 'Content-Type: application/json' \
+  -X POST http://127.0.0.1:3000/api/admin/users \
+  -d '{"name":"you","login":"you@gmail.com","email":"you@gmail.com","password":"'"$(openssl rand -base64 24)"'"}'
+# then make that user a Grafana admin (org role Admin + server admin) via the UI
+# or /api/org/users; the Google login matches on email.
+```
+
+**Forgejo:** add the Google auth source (once, as a Forgejo admin):
+
+```bash
+docker exec -u git <forgejo-container> forgejo admin auth add-oauth \
+  --name google --provider openidConnect \
+  --key "<CLIENT_ID>" --secret "<CLIENT_SECRET>" \
+  --auto-discover-url https://accounts.google.com/.well-known/openid-configuration
+```
+
+### 2. Alloy agent (on every node — prod, beta, desktop)
+
+```bash
+cd thermograph-observability/alloy
+# Every node (beta included) pushes to Loki over the mesh IP — a containerized
+# agent can't reach the host's 127.0.0.1, but it can reach beta's wg0 address:
+#   beta:    LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push   ALLOY_NODE=beta
+#   prod:    LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push   ALLOY_NODE=prod
+#   desktop: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push   ALLOY_NODE=dev
+ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \
+  docker compose -f docker-compose.agent.yml up -d
+```
+
+On the LAN dev node the app's log volume is `thermograph-dev_applogs` (not
+`thermograph_applogs`) — edit the two references in `docker-compose.agent.yml`
+there. The mesh must already be up (see the main repo's `deploy/swarm/` /
+`INFRA.md`); Swarm is **not** required — the agents talk plain HTTP over wg0.
+
+## Verify
+
+```bash
+# From any mesh node, confirm Loki is receiving from every host:
+curl -s 'http://10.10.0.2:3100/loki/api/v1/label/host/values'   # -> {"data":["beta","dev","prod"]}
+# Then open Grafana → the "Thermograph — Fleet Logs" dashboard.
+```
+
+## The dashboard
+
+`Thermograph — Fleet Logs` (auto-provisioned), with a `Node` selector:
+log volume by service, app error rate by tag, upstream 429 count, Caddy 5xx
+count, a per-node notifier-liveness indicator (from the heartbeat log), recent app errors, and a live all-container
+tail. Edit it in Grafana and re-export the JSON here to version a change.
+
+## Notes / follow-ups
+
+- **Loki is mesh-only** by design; only Grafana (with auth) is public.
+- Alloy reads the Docker socket read-only to discover containers.
+- Retention is 30 days on beta's disk — bump `loki/config.yml` if you want more.
+- The app emits a `tag=heartbeat` line for the subscription notifier every
+  ~15m; the dashboard's "Notifier alive?" panel turns red if a node misses it.
+  Extend the same `audit.log_heartbeat()` to any future background daemon.
--- a/observability/alloy/config.alloy
+++ b/observability/alloy/config.alloy
@ -0,0 +1,96 @@
+// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN
+// dev). It gathers three sources and ships them to the central Loki on beta:
+//
+//   1. Every Docker container's stdout/stderr (app, db, and on beta forgejo)
+//      via the Docker socket.
+//   2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs
+//      on the host, not in a container, so its logs aren't in Docker.
+//   3. The app's structured JSON logs (errors/access/audit *.jsonl) from the
+//      `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels.
+//
+// Per-node settings come from the environment (see docker-compose.agent.yml):
+//   ALLOY_NODE  — this node's name label (prod | beta | dev)
+//   LOKI_URL    — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0)
+
+livedebugging { enabled = false }
+
+// --- 1. All Docker container logs ------------------------------------------------
+discovery.docker "containers" {
+	host = "unix:///var/run/docker.sock"
+}
+
+// Turn Docker metadata into tidy labels: `container` (short name) and `service`
+// (the compose service, e.g. app/db). Drop Alloy's own container to avoid a loop.
+discovery.relabel "containers" {
+	targets = discovery.docker.containers.targets
+
+	rule {
+		source_labels = ["__meta_docker_container_name"]
+		regex         = "/(.*)"
+		target_label  = "container"
+	}
+	rule {
+		source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+		target_label  = "service"
+	}
+	rule {
+		source_labels = ["container"]
+		regex         = ".*alloy.*"
+		action        = "drop"
+	}
+}
+
+loki.source.docker "containers" {
+	host          = "unix:///var/run/docker.sock"
+	targets       = discovery.relabel.containers.output
+	forward_to    = [loki.write.central.receiver]
+	relabel_rules = discovery.relabel.containers.rules
+	labels        = { job = "docker" }
+}
+
+// --- 2. Caddy host access logs ---------------------------------------------------
+local.file_match "caddy" {
+	path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }]
+}
+
+loki.source.file "caddy" {
+	targets    = local.file_match.caddy.targets
+	forward_to = [loki.write.central.receiver]
+}
+
+// --- 3. App structured JSON logs (errors / access / audit) -----------------------
+// Mounted read-only from the app's `applogs` volume at /applogs (see the agent
+// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable.
+local.file_match "app_jsonl" {
+	path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }]
+}
+
+loki.source.file "app_jsonl" {
+	targets    = local.file_match.app_jsonl.targets
+	forward_to = [loki.process.app_jsonl.receiver]
+}
+
+loki.process "app_jsonl" {
+	forward_to = [loki.write.central.receiver]
+
+	stage.json {
+		expressions = { level = "level", tag = "tag", phase = "phase", status = "status" }
+	}
+	// A record with no explicit level: an error-folder line is an error, else info.
+	stage.static_labels {
+		values = { source = "app" }
+	}
+	stage.labels {
+		values = { level = "", tag = "", phase = "" }
+	}
+}
+
+// --- Ship to the central Loki over the WireGuard mesh ----------------------------
+loki.write "central" {
+	endpoint {
+		url = sys.env("LOKI_URL")
+	}
+	// Every line from this node is stamped with its node name, so one Grafana
+	// view can slice prod vs beta vs dev.
+	external_labels = { host = sys.env("ALLOY_NODE") }
+}
--- a/observability/alloy/docker-compose.agent.yml
+++ b/observability/alloy/docker-compose.agent.yml
@ -0,0 +1,48 @@
+# The Alloy log-collector agent. Runs on EACH node (prod, beta, LAN dev) — one
+# per node — reading that node's Docker containers, Caddy logs, and the app's
+# JSON logs, and shipping them to the central Loki on beta over the mesh.
+#
+# Per-node config comes from the environment (an .env file next to this, or
+# exported before `up`):
+#   ALLOY_NODE   this node's label:  prod | beta | dev
+#   LOKI_URL     push endpoint. All nodes push over the mesh:
+#                http://10.10.0.2:3100/loki/api/v1/push
+#
+# Deploy on a node:
+#   ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \
+#     docker compose -f docker-compose.agent.yml up -d
+#
+# The app's JSON logs are read from its `applogs` Docker volume, named
+# `<project>_applogs`. prod and beta run the app under the compose project
+# `thermograph`, so the external volume below is `thermograph_applogs`. The LAN
+# dev stack uses the project `thermograph-dev` — on that node, change the two
+# `thermograph_applogs` references below to `thermograph-dev_applogs`.
+
+services:
+  alloy:
+    image: grafana/alloy:v1.9.1
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.listen-addr=0.0.0.0:12345
+    environment:
+      ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (prod|beta|dev)}
+      LOKI_URL: ${LOKI_URL:?set LOKI_URL}
+    volumes:
+      - ./config.alloy:/etc/alloy/config.alloy:ro
+      # Read-only Docker socket: discover + tail every container's stdout/stderr.
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      # Caddy's host access logs (runs on the host, not a container).
+      - /var/log/caddy:/var/log/caddy:ro
+      # The app's structured JSON logs, straight off its named volume.
+      - thermograph_applogs:/applogs:ro
+      - alloy_data:/var/lib/alloy/data
+    restart: unless-stopped
+
+volumes:
+  # The app's existing log volume (created by the app's own stack). external:true
+  # means compose references it, never creates or deletes it.
+  thermograph_applogs:
+    external: true
+  alloy_data: {}
--- a/observability/caddy-grafana.conf
+++ b/observability/caddy-grafana.conf
@ -0,0 +1,15 @@
+# Append to beta's /etc/caddy/Caddyfile (beta is the monitoring host), then
+# `systemctl reload caddy`. Grafana publishes to 127.0.0.1:3000; this fronts it
+# with TLS, same pattern as beta's other site blocks. Login is Google SSO (with a
+# break-glass local admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied
+# here and stays mesh-only.
+#
+# dashboard.thermograph.org's A record must already point at beta (75.119.132.91).
+
+dashboard.thermograph.org {
+	encode zstd gzip
+	reverse_proxy 127.0.0.1:3000
+	log {
+		output file /var/log/caddy/dashboard.log
+	}
+}
--- a/observability/docker-compose.yml
+++ b/observability/docker-compose.yml
@ -0,0 +1,72 @@
+# The central observability stack: Loki (log store) + Grafana (UI). Runs on ONE
+# node — the monitoring host, `beta` (75.119.132.91 / mesh 10.10.0.2) — because
+# beta is an always-on VPS with a public Caddy already fronting it. Every node's
+# Alloy agent (see alloy/) ships logs here; agents on prod and the desktop reach
+# Loki over the WireGuard mesh, so Loki must listen on the mesh interface.
+#
+# Deploy (on beta):   docker compose up -d
+# Grafana is proxied by beta's Caddy at dashboard.thermograph.org (see README);
+# Loki is NOT public — it binds the mesh IP only, reachable to agents over wg0.
+
+services:
+  loki:
+    image: grafana/loki:3.5.1
+    command: -config.file=/etc/loki/config.yml
+    volumes:
+      - ./loki/config.yml:/etc/loki/config.yml:ro
+      - loki_data:/loki
+    ports:
+      # Mesh-only: agents on prod (10.10.0.1) and desktop (10.10.0.3) push here
+      # over wg0. Never published on the public interface.
+      - "10.10.0.2:3100:3100"
+      # Also localhost, so the beta-local Alloy agent and curl checks can reach it.
+      - "127.0.0.1:3100:3100"
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.6.1
+    depends_on: [loki]
+    environment:
+      # Primary login is Google SSO (below). The admin user is kept as a
+      # break-glass local fallback; its password comes from the host env
+      # (.env — see .env.example), never baked into the compose file.
+      GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:?set GF_SECURITY_ADMIN_PASSWORD}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      # Served behind Caddy at this external URL (sub-path-safe cookie/redirects).
+      GF_SERVER_ROOT_URL: "https://${GRAFANA_DOMAIN:-dashboard.thermograph.org}/"
+      # --- Google SSO (OIDC) -----------------------------------------------------
+      # Enabled once GOOGLE_CLIENT_ID/SECRET are set in .env and OAUTH_ENABLED=true.
+      # allow_sign_up is FALSE: a Google login only succeeds if a Grafana user with
+      # that email already exists — so this dashboard is locked to pre-provisioned
+      # accounts (see README), not "any Google user". Redirect URI to register in
+      # Google Cloud: https://dashboard.thermograph.org/login/google
+      GF_AUTH_GOOGLE_ENABLED: ${OAUTH_ENABLED:-false}
+      GF_AUTH_GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-}
+      GF_AUTH_GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-}
+      GF_AUTH_GOOGLE_SCOPES: "openid email profile"
+      GF_AUTH_GOOGLE_AUTH_URL: "https://accounts.google.com/o/oauth2/v2/auth"
+      GF_AUTH_GOOGLE_TOKEN_URL: "https://oauth2.googleapis.com/token"
+      GF_AUTH_GOOGLE_API_URL: "https://openidconnect.googleapis.com/v1/userinfo"
+      GF_AUTH_GOOGLE_ALLOW_SIGN_UP: "false"
+      # Match a Google login to a PRE-PROVISIONED user by email (the admin created
+      # via the API has no prior Google-auth linkage). Without this, Grafana 9.3+
+      # won't look up by email and instead tries to auto-create the user — which
+      # allow_sign_up=false then rejects ("signup is disabled"). Safe here: Google
+      # verifies email ownership and it's the only OAuth provider, so there's no
+      # cross-provider takeover vector the "insecure" name warns about.
+      GF_AUTH_OAUTH_ALLOW_INSECURE_EMAIL_LOOKUP: "true"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - grafana_data:/var/lib/grafana
+    ports:
+      # Host-local; beta's Caddy reverse-proxies to it. Not public directly.
+      - "127.0.0.1:3000:3000"
+    restart: unless-stopped
+
+volumes:
+  loki_data: {}
+  grafana_data: {}
--- a/observability/grafana/dashboards/thermograph-activity.json
+++ b/observability/grafana/dashboards/thermograph-activity.json
--- a/observability/grafana/dashboards/thermograph-logs.json
+++ b/observability/grafana/dashboards/thermograph-logs.json
@ -0,0 +1,141 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "graphTooltip": 1,
+  "title": "Thermograph — Fleet Logs",
+  "uid": "thermograph-logs",
+  "tags": ["thermograph"],
+  "timezone": "",
+  "schemaVersion": 39,
+  "refresh": "30s",
+  "time": { "from": "now-6h", "to": "now" },
+  "templating": {
+    "list": [
+      {
+        "name": "host",
+        "type": "custom",
+        "label": "Node",
+        "query": "prod,beta,dev",
+        "current": { "text": "All", "value": "$__all" },
+        "includeAll": true,
+        "allValue": ".+",
+        "multi": true
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "timeseries",
+      "title": "Log volume by service",
+      "description": "Lines/sec from every container, split by compose service (app, db, forgejo…).",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" } } } },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "sum by (service) (count_over_time({job=\"docker\", host=~\"$host\"} [$__auto]))",
+          "legendFormat": "{{service}}"
+        }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "App errors/sec (structured logs)",
+      "description": "count_over_time of the app's errors-*.jsonl lines, by tag.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 60 }, "color": { "mode": "palette-classic" } } },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "sum by (tag) (count_over_time({job=\"app-json\", source=\"app\", host=~\"$host\"} | level=~\"error|warn\" [$__auto]))",
+          "legendFormat": "{{tag}}"
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Upstream rate-limits (429) — last 1h",
+      "description": "Open-Meteo / upstream 429s pulled from the app's error logs.",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 50 } ] } } },
+      "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "sum (count_over_time({job=\"app-json\", host=~\"$host\"} | status=\"429\" [1h]))",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Caddy 5xx — last 1h",
+      "description": "Server errors seen by the reverse proxy across all sites.",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } },
+      "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "sum (count_over_time({job=\"caddy\", host=~\"$host\"} | json | status>=500 [1h]))",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "stat",
+      "title": "Notifier alive? (heartbeats / 20m, per node)",
+      "description": "The subscription notifier logs a heartbeat every ~15m (tag=heartbeat). 0 beats in the last 20m on a node means its notifier is down.",
+      "gridPos": { "h": 4, "w": 12, "x": 12, "y": 8 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] },
+          "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN" } } } ]
+        }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "textMode": "value_and_name", "graphMode": "none" },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "sum by (host) (count_over_time({job=\"app-json\", tag=\"heartbeat\", host=~\"$host\"} [20m]))",
+          "legendFormat": "{{host}}",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "type": "logs",
+      "title": "Recent app errors",
+      "description": "The app's errors-*.jsonl, newest first.",
+      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "{job=\"app-json\", host=~\"$host\"} | level=~\"error|warn\""
+        }
+      ]
+    },
+    {
+      "type": "logs",
+      "title": "Live tail — all containers",
+      "description": "Raw stdout/stderr from every container on the selected node(s).",
+      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "loki", "uid": "loki" },
+      "options": { "showTime": true, "wrapLogMessage": false, "sortOrder": "Descending", "enableLogDetails": true },
+      "targets": [
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "expr": "{job=\"docker\", host=~\"$host\"}"
+        }
+      ]
+    }
+  ]
+}
--- a/observability/grafana/dashboards/thermograph-traffic.json
+++ b/observability/grafana/dashboards/thermograph-traffic.json
--- a/observability/grafana/provisioning/dashboards/provider.yml
+++ b/observability/grafana/provisioning/dashboards/provider.yml
@ -0,0 +1,16 @@
+# Load any dashboard JSON dropped in /var/lib/grafana/dashboards (mounted from
+# ./grafana/dashboards) at startup, so the fleet overview is version-controlled
+# in this repo rather than hand-built in the UI.
+apiVersion: 1
+
+providers:
+  - name: thermograph
+    orgId: 1
+    folder: Thermograph
+    type: file
+    disableDeletion: false
+    editable: true
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false
--- a/observability/grafana/provisioning/datasources/loki.yml
+++ b/observability/grafana/provisioning/datasources/loki.yml
@ -0,0 +1,13 @@
+# Auto-provision the Loki datasource so Grafana comes up already wired to it —
+# no click-ops. Loki runs in the same compose stack, reachable by service name.
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki                 # stable UID so the provisioned dashboards can bind to it
+    isDefault: true
+    jsonData:
+      maxLines: 2000
--- a/observability/loki/config.yml
+++ b/observability/loki/config.yml
@ -0,0 +1,51 @@
+# Loki — the single log store for the whole Thermograph fleet (prod, beta, LAN
+# dev). Every node's Alloy agent pushes here over the WireGuard mesh; Grafana
+# reads from here. Single-binary, filesystem-backed: no object storage, no
+# clustering — right for a three-node hobby fleet, and everything lives on one
+# volume you can back up or blow away.
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: warn
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  # A hobby fleet's volume is tiny; keep 30 days and cap ingestion generously.
+  retention_period: 720h
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  max_query_series: 5000
+  allow_structured_metadata: true
+  volume_enabled: true
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+
+# No analytics phone-home from a private box.
+analytics:
+  reporting_enabled: false