Emi Griffith
46edb0bca0
A central Grafana + Loki stack (on beta) fed by a Grafana Alloy agent on every node, replacing the old SSH-tailed single-host scripts/dashboard.py. - docker-compose.yml: central Loki (mesh-only :3100) + Grafana (Caddy-fronted) - loki/config.yml: single-binary, filesystem storage, 30-day retention - alloy/config.alloy + docker-compose.agent.yml: per-node collector — every container's stdout/stderr via the Docker socket, Caddy host logs, and the app's structured JSON logs (errors/access/audit), each line tagged by node - grafana/: auto-provisioned Loki datasource + a fleet-logs dashboard (volume by service, error rate, upstream 429s, Caddy 5xx, notifier liveness, live tail), with a per-node selector - caddy-grafana.conf, README, .env.example
51 lines
2.2 KiB
YAML
51 lines
2.2 KiB
YAML
# The central observability stack: Loki (log store) + Grafana (UI). Runs on ONE
|
|
# node — the monitoring host, `beta` (75.119.132.91 / mesh 10.10.0.2) — because
|
|
# beta is an always-on VPS with a public Caddy already fronting it. Every node's
|
|
# Alloy agent (see alloy/) ships logs here; agents on prod and the desktop reach
|
|
# Loki over the WireGuard mesh, so Loki must listen on the mesh interface.
|
|
#
|
|
# Deploy (on beta): docker compose up -d
|
|
# Grafana is proxied by beta's Caddy at grafana.thermograph.org (see README);
|
|
# Loki is NOT public — it binds the mesh IP only, reachable to agents over wg0.
|
|
|
|
services:
|
|
loki:
|
|
image: grafana/loki:3.5.1
|
|
command: -config.file=/etc/loki/config.yml
|
|
volumes:
|
|
- ./loki/config.yml:/etc/loki/config.yml:ro
|
|
- loki_data:/loki
|
|
ports:
|
|
# Mesh-only: agents on prod (10.10.0.1) and desktop (10.10.0.3) push here
|
|
# over wg0. Never published on the public interface.
|
|
- "10.10.0.2:3100:3100"
|
|
# Also localhost, so the beta-local Alloy agent and curl checks can reach it.
|
|
- "127.0.0.1:3100:3100"
|
|
restart: unless-stopped
|
|
|
|
grafana:
|
|
image: grafana/grafana:11.6.1
|
|
depends_on: [loki]
|
|
environment:
|
|
# Grafana owns its own auth. Admin password is injected from the host env
|
|
# (set GF_SECURITY_ADMIN_PASSWORD before `up`, or in an .env file — see
|
|
# .env.example); never bake a credential into the compose file.
|
|
GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER:-admin}
|
|
GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:?set GF_SECURITY_ADMIN_PASSWORD}
|
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
|
# Served behind Caddy at this external URL (sub-path-safe cookie/redirects).
|
|
GF_SERVER_ROOT_URL: "https://${GRAFANA_DOMAIN:-grafana.thermograph.org}/"
|
|
volumes:
|
|
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
|
- grafana_data:/var/lib/grafana
|
|
ports:
|
|
# Host-local; beta's Caddy reverse-proxies to it. Not public directly.
|
|
- "127.0.0.1:3000:3000"
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
loki_data: {}
|
|
grafana_data: {}
|