Subtree-merge thermograph-observability (origin/main) into observability/
git-subtree-dir: observability git-subtree-mainline:ae1d9bb534git-subtree-split:19e74af9ca
This commit is contained in:
Emi Griffith
commit
f2fd8f6835
15 changed files with 3401 additions and 0 deletions
20
observability/.env.example
Normal file
20
observability/.env.example
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Copy to .env on the monitoring host (beta) before `docker compose up`.
|
||||||
|
# .env is gitignored — never commit real secrets.
|
||||||
|
|
||||||
|
# The public hostname beta's Caddy serves Grafana at (see caddy-grafana.conf).
|
||||||
|
GRAFANA_DOMAIN=dashboard.thermograph.org
|
||||||
|
|
||||||
|
# Break-glass local admin (Google SSO below is the primary login). Keep this
|
||||||
|
# strong; it's the fallback if OAuth ever breaks.
|
||||||
|
GF_ADMIN_USER=admin
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD=change-me-to-something-strong
|
||||||
|
|
||||||
|
# --- Google SSO ------------------------------------------------------------------
|
||||||
|
# Create a Google Cloud "OAuth 2.0 Client ID" (type: Web application) and set:
|
||||||
|
# Authorized redirect URI: https://dashboard.thermograph.org/login/google
|
||||||
|
# Then paste the client id/secret here and flip OAUTH_ENABLED=true.
|
||||||
|
# Login is locked to pre-provisioned accounts (allow_sign_up=false) — provision
|
||||||
|
# your email as an admin (see README) so only you can sign in.
|
||||||
|
OAUTH_ENABLED=false
|
||||||
|
GOOGLE_CLIENT_ID=
|
||||||
|
GOOGLE_CLIENT_SECRET=
|
||||||
66
observability/.forgejo/workflows/validate.yml
Normal file
66
observability/.forgejo/workflows/validate.yml
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
name: Validate observability stack
|
||||||
|
|
||||||
|
# This repo deploys by hand (`docker compose up -d` on beta + the Alloy agent on
|
||||||
|
# each node) with no build step, so nothing caught a malformed compose file, a
|
||||||
|
# broken dashboard JSON, or an unparseable config until it failed live on the
|
||||||
|
# host. This is that missing guard: it parses every YAML/JSON artifact that ships
|
||||||
|
# to a node. It does NOT deploy, and deliberately needs no docker CLI (the
|
||||||
|
# node:20-bookworm job image doesn't ship one) — a YAML parse catches the real
|
||||||
|
# breakage (bad indent / unparseable file) without it.
|
||||||
|
#
|
||||||
|
# Not validated here: the Alloy config (alloy/config.alloy, an HCL-like format,
|
||||||
|
# needs the `alloy` binary) and docker-compose *schema* (unknown-key checks, which
|
||||||
|
# would need the docker CLI). Add either if the churn warrants the extra tooling.
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
branches: [main]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
validate:
|
||||||
|
runs-on: docker
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Dashboards are valid JSON
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
python3 - <<'PY'
|
||||||
|
import json, pathlib, sys
|
||||||
|
bad = False
|
||||||
|
files = sorted(pathlib.Path("grafana/dashboards").glob("*.json"))
|
||||||
|
if not files:
|
||||||
|
print("no dashboards found"); sys.exit(1)
|
||||||
|
for f in files:
|
||||||
|
try:
|
||||||
|
json.loads(f.read_text()); print("OK", f)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
print("INVALID JSON", f, "-", e); bad = True
|
||||||
|
sys.exit(1 if bad else 0)
|
||||||
|
PY
|
||||||
|
|
||||||
|
- name: YAML artifacts parse (compose + loki + grafana provisioning)
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
apt-get update -qq && apt-get install -y -qq python3-yaml >/dev/null
|
||||||
|
python3 - <<'PY'
|
||||||
|
import pathlib, sys
|
||||||
|
import yaml
|
||||||
|
bad = False
|
||||||
|
targets = [
|
||||||
|
pathlib.Path("docker-compose.yml"),
|
||||||
|
pathlib.Path("alloy/docker-compose.agent.yml"),
|
||||||
|
pathlib.Path("loki/config.yml"),
|
||||||
|
*sorted(pathlib.Path("grafana/provisioning").rglob("*.yml")),
|
||||||
|
]
|
||||||
|
for f in targets:
|
||||||
|
if not f.exists():
|
||||||
|
print("MISSING", f); bad = True; continue
|
||||||
|
try:
|
||||||
|
yaml.safe_load(f.read_text()); print("OK", f)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
print("INVALID YAML", f, "-", e); bad = True
|
||||||
|
sys.exit(1 if bad else 0)
|
||||||
|
PY
|
||||||
3
observability/.gitignore
vendored
Normal file
3
observability/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
.env
|
||||||
|
alloy/.env
|
||||||
|
*.log
|
||||||
41
observability/CLAUDE.md
Normal file
41
observability/CLAUDE.md
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
# thermograph-observability — agent instructions
|
||||||
|
|
||||||
|
The **logging/metrics stack** for the Thermograph fleet: Loki + Grafana on beta,
|
||||||
|
with a Grafana Alloy agent on every node (prod, beta, dev) shipping container and
|
||||||
|
app logs over the WireGuard mesh. Grafana is fronted by beta's Caddy at
|
||||||
|
`dashboard.thermograph.org` (Google SSO, pre-provisioned users only).
|
||||||
|
|
||||||
|
This is **operational infra config, not application code** — there is no build.
|
||||||
|
It deploys by hand: `docker compose up -d` on beta for Loki+Grafana, and the
|
||||||
|
Alloy agent (`alloy/docker-compose.agent.yml`) on each node. See the README for
|
||||||
|
the full per-node procedure.
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
- `docker-compose.yml` — the Loki + Grafana stack (runs on beta).
|
||||||
|
- `loki/config.yml` — Loki config (mesh-only, filesystem storage).
|
||||||
|
- `grafana/provisioning/` — datasource (Loki) + dashboard provider, auto-loaded
|
||||||
|
at startup. `grafana/dashboards/*.json` — the dashboards themselves.
|
||||||
|
- `alloy/config.alloy` + `alloy/docker-compose.agent.yml` — the per-node log
|
||||||
|
shipper. The node name comes from `ALLOY_NODE` (set per host).
|
||||||
|
- `caddy-grafana.conf` — the Caddy vhost for `dashboard.thermograph.org` (lives
|
||||||
|
in beta's Caddy config, kept here for reference).
|
||||||
|
- `.env.example` — copy to `.env` on beta before `docker compose up`. `.env` is
|
||||||
|
gitignored; never commit real OAuth secrets or the admin password.
|
||||||
|
|
||||||
|
## Conventions
|
||||||
|
|
||||||
|
- **Every artifact that ships to a node is CI-validated** (`.forgejo/workflows/validate.yml`):
|
||||||
|
both compose files parse, all dashboard JSON is valid, and the Loki/provisioning
|
||||||
|
YAML parses. Keep new dashboards as valid JSON and new config as valid YAML or
|
||||||
|
CI fails. The Alloy config is not yet CI-validated (needs the `alloy` binary).
|
||||||
|
- The public hostname is **`dashboard.thermograph.org`** everywhere — not
|
||||||
|
`grafana.thermograph.org`. Match it in any new comment/config.
|
||||||
|
- Secrets (OAuth client id/secret, admin password) live only in the host `.env`,
|
||||||
|
never in the repo.
|
||||||
|
|
||||||
|
## Branching
|
||||||
|
|
||||||
|
Single `main` branch, no protection. Open a PR against `main`; the validator
|
||||||
|
gates it. Deploying the change to beta / the nodes is a separate manual step
|
||||||
|
(this repo has no deploy automation — a known gap).
|
||||||
123
observability/README.md
Normal file
123
observability/README.md
Normal file
|
|
@ -0,0 +1,123 @@
|
||||||
|
# Thermograph — observability
|
||||||
|
|
||||||
|
Fleet-wide log aggregation for [Thermograph](https://thermograph.org): a central
|
||||||
|
**Grafana + Loki** stack, fed by a **Grafana Alloy** agent on every node, all
|
||||||
|
over the private WireGuard mesh. This replaces the old in-repo
|
||||||
|
`scripts/dashboard.py` (an SSH-tailed, single-host, metrics-endpoint view) with a
|
||||||
|
persistent, queryable, fleet-wide log store and UI.
|
||||||
|
|
||||||
|
```
|
||||||
|
prod (10.10.0.1) beta (10.10.0.2) desktop / LAN dev (10.10.0.3)
|
||||||
|
┌────────────┐ ┌──────────────────┐ ┌────────────┐
|
||||||
|
│ Alloy agent│──wg0──┐ │ Loki ◀── Alloy │ ┌──wg0│ Alloy agent│
|
||||||
|
└────────────┘ └───▶│ Grafana (Caddy) │◀──┘ └────────────┘
|
||||||
|
docker+caddy+app └──────────────────┘ docker+caddy+app
|
||||||
|
logs central store + UI logs
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Loki** stores logs (filesystem, 30-day retention). Listens on the mesh IP
|
||||||
|
`10.10.0.2:3100` — never public.
|
||||||
|
- **Grafana** is the UI, fronted by beta's Caddy at `dashboard.thermograph.org`,
|
||||||
|
login via **Google SSO** (with a break-glass local admin); Loki is not exposed.
|
||||||
|
- **Alloy** runs on each node and ships three sources to Loki: every Docker
|
||||||
|
container's stdout/stderr, Caddy's host access logs, and the app's structured
|
||||||
|
JSON logs (`errors`/`access`/`audit` `*.jsonl`, parsed so `level`/`tag`/`phase`
|
||||||
|
become labels). Every line is tagged with its node (`host = prod|beta|dev`).
|
||||||
|
|
||||||
|
## Layout
|
||||||
|
|
||||||
|
| Path | What |
|
||||||
|
|------|------|
|
||||||
|
| `docker-compose.yml` | the central Loki + Grafana stack (runs on beta) |
|
||||||
|
| `loki/config.yml` | Loki config (filesystem, retention) |
|
||||||
|
| `grafana/provisioning/` | auto-wired Loki datasource + dashboard provider |
|
||||||
|
| `grafana/dashboards/thermograph-logs.json` | the fleet-logs dashboard |
|
||||||
|
| `alloy/config.alloy` | the per-node collector config |
|
||||||
|
| `alloy/docker-compose.agent.yml` | runs the Alloy agent on a node |
|
||||||
|
| `caddy-grafana.conf` | beta Caddy site block for the Grafana UI |
|
||||||
|
|
||||||
|
## Deploy
|
||||||
|
|
||||||
|
### 1. Central stack (on beta)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# on beta (75.119.132.91):
|
||||||
|
git clone <this repo> thermograph-observability && cd thermograph-observability
|
||||||
|
cp .env.example .env # set GF_SECURITY_ADMIN_PASSWORD
|
||||||
|
docker compose up -d # Loki on 10.10.0.2:3100 + 127.0.0.1:3100; Grafana on 127.0.0.1:3000
|
||||||
|
```
|
||||||
|
|
||||||
|
Expose the Grafana UI: point `dashboard.thermograph.org` at beta, append
|
||||||
|
`caddy-grafana.conf` to `/etc/caddy/Caddyfile`, `systemctl reload caddy`.
|
||||||
|
|
||||||
|
### Google SSO (Grafana + Forgejo)
|
||||||
|
|
||||||
|
Both the dashboard and Forgejo log in with Google. Create **one** Google Cloud
|
||||||
|
OAuth 2.0 Client (type: *Web application*) and register both redirect URIs:
|
||||||
|
|
||||||
|
- Grafana: `https://dashboard.thermograph.org/login/google`
|
||||||
|
- Forgejo: `https://git.thermograph.org/user/oauth2/google/callback`
|
||||||
|
|
||||||
|
**Grafana:** put the client id/secret in `.env`, set `OAUTH_ENABLED=true`, and
|
||||||
|
`docker compose up -d`. Login is locked to pre-provisioned accounts
|
||||||
|
(`allow_sign_up=false`), so provision your email as an admin once:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# on beta, after the stack is up (uses the break-glass admin creds from .env):
|
||||||
|
curl -s -u admin:"$GF_SECURITY_ADMIN_PASSWORD" -H 'Content-Type: application/json' \
|
||||||
|
-X POST http://127.0.0.1:3000/api/admin/users \
|
||||||
|
-d '{"name":"you","login":"you@gmail.com","email":"you@gmail.com","password":"'"$(openssl rand -base64 24)"'"}'
|
||||||
|
# then make that user a Grafana admin (org role Admin + server admin) via the UI
|
||||||
|
# or /api/org/users; the Google login matches on email.
|
||||||
|
```
|
||||||
|
|
||||||
|
**Forgejo:** add the Google auth source (once, as a Forgejo admin):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -u git <forgejo-container> forgejo admin auth add-oauth \
|
||||||
|
--name google --provider openidConnect \
|
||||||
|
--key "<CLIENT_ID>" --secret "<CLIENT_SECRET>" \
|
||||||
|
--auto-discover-url https://accounts.google.com/.well-known/openid-configuration
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Alloy agent (on every node — prod, beta, desktop)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd thermograph-observability/alloy
|
||||||
|
# Every node (beta included) pushes to Loki over the mesh IP — a containerized
|
||||||
|
# agent can't reach the host's 127.0.0.1, but it can reach beta's wg0 address:
|
||||||
|
# beta: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=beta
|
||||||
|
# prod: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=prod
|
||||||
|
# desktop: LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push ALLOY_NODE=dev
|
||||||
|
ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \
|
||||||
|
docker compose -f docker-compose.agent.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
On the LAN dev node the app's log volume is `thermograph-dev_applogs` (not
|
||||||
|
`thermograph_applogs`) — edit the two references in `docker-compose.agent.yml`
|
||||||
|
there. The mesh must already be up (see the main repo's `deploy/swarm/` /
|
||||||
|
`INFRA.md`); Swarm is **not** required — the agents talk plain HTTP over wg0.
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# From any mesh node, confirm Loki is receiving from every host:
|
||||||
|
curl -s 'http://10.10.0.2:3100/loki/api/v1/label/host/values' # -> {"data":["beta","dev","prod"]}
|
||||||
|
# Then open Grafana → the "Thermograph — Fleet Logs" dashboard.
|
||||||
|
```
|
||||||
|
|
||||||
|
## The dashboard
|
||||||
|
|
||||||
|
`Thermograph — Fleet Logs` (auto-provisioned), with a `Node` selector:
|
||||||
|
log volume by service, app error rate by tag, upstream 429 count, Caddy 5xx
|
||||||
|
count, a per-node notifier-liveness indicator (from the heartbeat log), recent app errors, and a live all-container
|
||||||
|
tail. Edit it in Grafana and re-export the JSON here to version a change.
|
||||||
|
|
||||||
|
## Notes / follow-ups
|
||||||
|
|
||||||
|
- **Loki is mesh-only** by design; only Grafana (with auth) is public.
|
||||||
|
- Alloy reads the Docker socket read-only to discover containers.
|
||||||
|
- Retention is 30 days on beta's disk — bump `loki/config.yml` if you want more.
|
||||||
|
- The app emits a `tag=heartbeat` line for the subscription notifier every
|
||||||
|
~15m; the dashboard's "Notifier alive?" panel turns red if a node misses it.
|
||||||
|
Extend the same `audit.log_heartbeat()` to any future background daemon.
|
||||||
96
observability/alloy/config.alloy
Normal file
96
observability/alloy/config.alloy
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN
|
||||||
|
// dev). It gathers three sources and ships them to the central Loki on beta:
|
||||||
|
//
|
||||||
|
// 1. Every Docker container's stdout/stderr (app, db, and on beta forgejo)
|
||||||
|
// via the Docker socket.
|
||||||
|
// 2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs
|
||||||
|
// on the host, not in a container, so its logs aren't in Docker.
|
||||||
|
// 3. The app's structured JSON logs (errors/access/audit *.jsonl) from the
|
||||||
|
// `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels.
|
||||||
|
//
|
||||||
|
// Per-node settings come from the environment (see docker-compose.agent.yml):
|
||||||
|
// ALLOY_NODE — this node's name label (prod | beta | dev)
|
||||||
|
// LOKI_URL — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0)
|
||||||
|
|
||||||
|
livedebugging { enabled = false }
|
||||||
|
|
||||||
|
// --- 1. All Docker container logs ------------------------------------------------
|
||||||
|
discovery.docker "containers" {
|
||||||
|
host = "unix:///var/run/docker.sock"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Turn Docker metadata into tidy labels: `container` (short name) and `service`
|
||||||
|
// (the compose service, e.g. app/db). Drop Alloy's own container to avoid a loop.
|
||||||
|
discovery.relabel "containers" {
|
||||||
|
targets = discovery.docker.containers.targets
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_docker_container_name"]
|
||||||
|
regex = "/(.*)"
|
||||||
|
target_label = "container"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
||||||
|
target_label = "service"
|
||||||
|
}
|
||||||
|
rule {
|
||||||
|
source_labels = ["container"]
|
||||||
|
regex = ".*alloy.*"
|
||||||
|
action = "drop"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.docker "containers" {
|
||||||
|
host = "unix:///var/run/docker.sock"
|
||||||
|
targets = discovery.relabel.containers.output
|
||||||
|
forward_to = [loki.write.central.receiver]
|
||||||
|
relabel_rules = discovery.relabel.containers.rules
|
||||||
|
labels = { job = "docker" }
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 2. Caddy host access logs ---------------------------------------------------
|
||||||
|
local.file_match "caddy" {
|
||||||
|
path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.file "caddy" {
|
||||||
|
targets = local.file_match.caddy.targets
|
||||||
|
forward_to = [loki.write.central.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- 3. App structured JSON logs (errors / access / audit) -----------------------
|
||||||
|
// Mounted read-only from the app's `applogs` volume at /applogs (see the agent
|
||||||
|
// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable.
|
||||||
|
local.file_match "app_jsonl" {
|
||||||
|
path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.file "app_jsonl" {
|
||||||
|
targets = local.file_match.app_jsonl.targets
|
||||||
|
forward_to = [loki.process.app_jsonl.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.process "app_jsonl" {
|
||||||
|
forward_to = [loki.write.central.receiver]
|
||||||
|
|
||||||
|
stage.json {
|
||||||
|
expressions = { level = "level", tag = "tag", phase = "phase", status = "status" }
|
||||||
|
}
|
||||||
|
// A record with no explicit level: an error-folder line is an error, else info.
|
||||||
|
stage.static_labels {
|
||||||
|
values = { source = "app" }
|
||||||
|
}
|
||||||
|
stage.labels {
|
||||||
|
values = { level = "", tag = "", phase = "" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Ship to the central Loki over the WireGuard mesh ----------------------------
|
||||||
|
loki.write "central" {
|
||||||
|
endpoint {
|
||||||
|
url = sys.env("LOKI_URL")
|
||||||
|
}
|
||||||
|
// Every line from this node is stamped with its node name, so one Grafana
|
||||||
|
// view can slice prod vs beta vs dev.
|
||||||
|
external_labels = { host = sys.env("ALLOY_NODE") }
|
||||||
|
}
|
||||||
48
observability/alloy/docker-compose.agent.yml
Normal file
48
observability/alloy/docker-compose.agent.yml
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
# The Alloy log-collector agent. Runs on EACH node (prod, beta, LAN dev) — one
|
||||||
|
# per node — reading that node's Docker containers, Caddy logs, and the app's
|
||||||
|
# JSON logs, and shipping them to the central Loki on beta over the mesh.
|
||||||
|
#
|
||||||
|
# Per-node config comes from the environment (an .env file next to this, or
|
||||||
|
# exported before `up`):
|
||||||
|
# ALLOY_NODE this node's label: prod | beta | dev
|
||||||
|
# LOKI_URL push endpoint. All nodes push over the mesh:
|
||||||
|
# http://10.10.0.2:3100/loki/api/v1/push
|
||||||
|
#
|
||||||
|
# Deploy on a node:
|
||||||
|
# ALLOY_NODE=prod LOKI_URL=http://10.10.0.2:3100/loki/api/v1/push \
|
||||||
|
# docker compose -f docker-compose.agent.yml up -d
|
||||||
|
#
|
||||||
|
# The app's JSON logs are read from its `applogs` Docker volume, named
|
||||||
|
# `<project>_applogs`. prod and beta run the app under the compose project
|
||||||
|
# `thermograph`, so the external volume below is `thermograph_applogs`. The LAN
|
||||||
|
# dev stack uses the project `thermograph-dev` — on that node, change the two
|
||||||
|
# `thermograph_applogs` references below to `thermograph-dev_applogs`.
|
||||||
|
|
||||||
|
services:
|
||||||
|
alloy:
|
||||||
|
image: grafana/alloy:v1.9.1
|
||||||
|
command:
|
||||||
|
- run
|
||||||
|
- /etc/alloy/config.alloy
|
||||||
|
- --storage.path=/var/lib/alloy/data
|
||||||
|
- --server.http.listen-addr=0.0.0.0:12345
|
||||||
|
environment:
|
||||||
|
ALLOY_NODE: ${ALLOY_NODE:?set ALLOY_NODE (prod|beta|dev)}
|
||||||
|
LOKI_URL: ${LOKI_URL:?set LOKI_URL}
|
||||||
|
volumes:
|
||||||
|
- ./config.alloy:/etc/alloy/config.alloy:ro
|
||||||
|
# Read-only Docker socket: discover + tail every container's stdout/stderr.
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
# Caddy's host access logs (runs on the host, not a container).
|
||||||
|
- /var/log/caddy:/var/log/caddy:ro
|
||||||
|
# The app's structured JSON logs, straight off its named volume.
|
||||||
|
- thermograph_applogs:/applogs:ro
|
||||||
|
- alloy_data:/var/lib/alloy/data
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
# The app's existing log volume (created by the app's own stack). external:true
|
||||||
|
# means compose references it, never creates or deletes it.
|
||||||
|
thermograph_applogs:
|
||||||
|
external: true
|
||||||
|
alloy_data: {}
|
||||||
15
observability/caddy-grafana.conf
Normal file
15
observability/caddy-grafana.conf
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
# Append to beta's /etc/caddy/Caddyfile (beta is the monitoring host), then
|
||||||
|
# `systemctl reload caddy`. Grafana publishes to 127.0.0.1:3000; this fronts it
|
||||||
|
# with TLS, same pattern as beta's other site blocks. Login is Google SSO (with a
|
||||||
|
# break-glass local admin); Loki (the log store, on 10.10.0.2:3100) is NOT proxied
|
||||||
|
# here and stays mesh-only.
|
||||||
|
#
|
||||||
|
# dashboard.thermograph.org's A record must already point at beta (75.119.132.91).
|
||||||
|
|
||||||
|
dashboard.thermograph.org {
|
||||||
|
encode zstd gzip
|
||||||
|
reverse_proxy 127.0.0.1:3000
|
||||||
|
log {
|
||||||
|
output file /var/log/caddy/dashboard.log
|
||||||
|
}
|
||||||
|
}
|
||||||
72
observability/docker-compose.yml
Normal file
72
observability/docker-compose.yml
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
# The central observability stack: Loki (log store) + Grafana (UI). Runs on ONE
|
||||||
|
# node — the monitoring host, `beta` (75.119.132.91 / mesh 10.10.0.2) — because
|
||||||
|
# beta is an always-on VPS with a public Caddy already fronting it. Every node's
|
||||||
|
# Alloy agent (see alloy/) ships logs here; agents on prod and the desktop reach
|
||||||
|
# Loki over the WireGuard mesh, so Loki must listen on the mesh interface.
|
||||||
|
#
|
||||||
|
# Deploy (on beta): docker compose up -d
|
||||||
|
# Grafana is proxied by beta's Caddy at dashboard.thermograph.org (see README);
|
||||||
|
# Loki is NOT public — it binds the mesh IP only, reachable to agents over wg0.
|
||||||
|
|
||||||
|
services:
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.5.1
|
||||||
|
command: -config.file=/etc/loki/config.yml
|
||||||
|
volumes:
|
||||||
|
- ./loki/config.yml:/etc/loki/config.yml:ro
|
||||||
|
- loki_data:/loki
|
||||||
|
ports:
|
||||||
|
# Mesh-only: agents on prod (10.10.0.1) and desktop (10.10.0.3) push here
|
||||||
|
# over wg0. Never published on the public interface.
|
||||||
|
- "10.10.0.2:3100:3100"
|
||||||
|
# Also localhost, so the beta-local Alloy agent and curl checks can reach it.
|
||||||
|
- "127.0.0.1:3100:3100"
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:11.6.1
|
||||||
|
depends_on: [loki]
|
||||||
|
environment:
|
||||||
|
# Primary login is Google SSO (below). The admin user is kept as a
|
||||||
|
# break-glass local fallback; its password comes from the host env
|
||||||
|
# (.env — see .env.example), never baked into the compose file.
|
||||||
|
GF_SECURITY_ADMIN_USER: ${GF_ADMIN_USER:-admin}
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:?set GF_SECURITY_ADMIN_PASSWORD}
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||||
|
# Served behind Caddy at this external URL (sub-path-safe cookie/redirects).
|
||||||
|
GF_SERVER_ROOT_URL: "https://${GRAFANA_DOMAIN:-dashboard.thermograph.org}/"
|
||||||
|
# --- Google SSO (OIDC) -----------------------------------------------------
|
||||||
|
# Enabled once GOOGLE_CLIENT_ID/SECRET are set in .env and OAUTH_ENABLED=true.
|
||||||
|
# allow_sign_up is FALSE: a Google login only succeeds if a Grafana user with
|
||||||
|
# that email already exists — so this dashboard is locked to pre-provisioned
|
||||||
|
# accounts (see README), not "any Google user". Redirect URI to register in
|
||||||
|
# Google Cloud: https://dashboard.thermograph.org/login/google
|
||||||
|
GF_AUTH_GOOGLE_ENABLED: ${OAUTH_ENABLED:-false}
|
||||||
|
GF_AUTH_GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID:-}
|
||||||
|
GF_AUTH_GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET:-}
|
||||||
|
GF_AUTH_GOOGLE_SCOPES: "openid email profile"
|
||||||
|
GF_AUTH_GOOGLE_AUTH_URL: "https://accounts.google.com/o/oauth2/v2/auth"
|
||||||
|
GF_AUTH_GOOGLE_TOKEN_URL: "https://oauth2.googleapis.com/token"
|
||||||
|
GF_AUTH_GOOGLE_API_URL: "https://openidconnect.googleapis.com/v1/userinfo"
|
||||||
|
GF_AUTH_GOOGLE_ALLOW_SIGN_UP: "false"
|
||||||
|
# Match a Google login to a PRE-PROVISIONED user by email (the admin created
|
||||||
|
# via the API has no prior Google-auth linkage). Without this, Grafana 9.3+
|
||||||
|
# won't look up by email and instead tries to auto-create the user — which
|
||||||
|
# allow_sign_up=false then rejects ("signup is disabled"). Safe here: Google
|
||||||
|
# verifies email ownership and it's the only OAuth provider, so there's no
|
||||||
|
# cross-provider takeover vector the "insecure" name warns about.
|
||||||
|
GF_AUTH_OAUTH_ALLOW_INSECURE_EMAIL_LOOKUP: "true"
|
||||||
|
volumes:
|
||||||
|
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
ports:
|
||||||
|
# Host-local; beta's Caddy reverse-proxies to it. Not public directly.
|
||||||
|
- "127.0.0.1:3000:3000"
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
loki_data: {}
|
||||||
|
grafana_data: {}
|
||||||
1368
observability/grafana/dashboards/thermograph-activity.json
Normal file
1368
observability/grafana/dashboards/thermograph-activity.json
Normal file
File diff suppressed because it is too large
Load diff
141
observability/grafana/dashboards/thermograph-logs.json
Normal file
141
observability/grafana/dashboards/thermograph-logs.json
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
{
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"editable": true,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"title": "Thermograph — Fleet Logs",
|
||||||
|
"uid": "thermograph-logs",
|
||||||
|
"tags": ["thermograph"],
|
||||||
|
"timezone": "",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"refresh": "30s",
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "host",
|
||||||
|
"type": "custom",
|
||||||
|
"label": "Node",
|
||||||
|
"query": "prod,beta,dev",
|
||||||
|
"current": { "text": "All", "value": "$__all" },
|
||||||
|
"includeAll": true,
|
||||||
|
"allValue": ".+",
|
||||||
|
"multi": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Log volume by service",
|
||||||
|
"description": "Lines/sec from every container, split by compose service (app, db, forgejo…).",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" } } } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "sum by (service) (count_over_time({job=\"docker\", host=~\"$host\"} [$__auto]))",
|
||||||
|
"legendFormat": "{{service}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "App errors/sec (structured logs)",
|
||||||
|
"description": "count_over_time of the app's errors-*.jsonl lines, by tag.",
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 60 }, "color": { "mode": "palette-classic" } } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "sum by (tag) (count_over_time({job=\"app-json\", source=\"app\", host=~\"$host\"} | level=~\"error|warn\" [$__auto]))",
|
||||||
|
"legendFormat": "{{tag}}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Upstream rate-limits (429) — last 1h",
|
||||||
|
"description": "Open-Meteo / upstream 429s pulled from the app's error logs.",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 8 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 50 } ] } } },
|
||||||
|
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "sum (count_over_time({job=\"app-json\", host=~\"$host\"} | status=\"429\" [1h]))",
|
||||||
|
"instant": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Caddy 5xx — last 1h",
|
||||||
|
"description": "Server errors seen by the reverse proxy across all sites.",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 8 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } },
|
||||||
|
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "sum (count_over_time({job=\"caddy\", host=~\"$host\"} | json | status>=500 [1h]))",
|
||||||
|
"instant": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Notifier alive? (heartbeats / 20m, per node)",
|
||||||
|
"description": "The subscription notifier logs a heartbeat every ~15m (tag=heartbeat). 0 beats in the last 20m on a node means its notifier is down.",
|
||||||
|
"gridPos": { "h": 4, "w": 12, "x": 12, "y": 8 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] },
|
||||||
|
"mappings": [ { "type": "value", "options": { "0": { "text": "DOWN" } } } ]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "textMode": "value_and_name", "graphMode": "none" },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "sum by (host) (count_over_time({job=\"app-json\", tag=\"heartbeat\", host=~\"$host\"} [20m]))",
|
||||||
|
"legendFormat": "{{host}}",
|
||||||
|
"instant": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Recent app errors",
|
||||||
|
"description": "The app's errors-*.jsonl, newest first.",
|
||||||
|
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "{job=\"app-json\", host=~\"$host\"} | level=~\"error|warn\""
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "logs",
|
||||||
|
"title": "Live tail — all containers",
|
||||||
|
"description": "Raw stdout/stderr from every container on the selected node(s).",
|
||||||
|
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 12 },
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"options": { "showTime": true, "wrapLogMessage": false, "sortOrder": "Descending", "enableLogDetails": true },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"expr": "{job=\"docker\", host=~\"$host\"}"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
1328
observability/grafana/dashboards/thermograph-traffic.json
Normal file
1328
observability/grafana/dashboards/thermograph-traffic.json
Normal file
File diff suppressed because it is too large
Load diff
16
observability/grafana/provisioning/dashboards/provider.yml
Normal file
16
observability/grafana/provisioning/dashboards/provider.yml
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Load any dashboard JSON dropped in /var/lib/grafana/dashboards (mounted from
|
||||||
|
# ./grafana/dashboards) at startup, so the fleet overview is version-controlled
|
||||||
|
# in this repo rather than hand-built in the UI.
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: thermograph
|
||||||
|
orgId: 1
|
||||||
|
folder: Thermograph
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
editable: true
|
||||||
|
updateIntervalSeconds: 30
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
|
foldersFromFilesStructure: false
|
||||||
13
observability/grafana/provisioning/datasources/loki.yml
Normal file
13
observability/grafana/provisioning/datasources/loki.yml
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
# Auto-provision the Loki datasource so Grafana comes up already wired to it —
|
||||||
|
# no click-ops. Loki runs in the same compose stack, reachable by service name.
|
||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
uid: loki # stable UID so the provisioned dashboards can bind to it
|
||||||
|
isDefault: true
|
||||||
|
jsonData:
|
||||||
|
maxLines: 2000
|
||||||
51
observability/loki/config.yml
Normal file
51
observability/loki/config.yml
Normal file
|
|
@ -0,0 +1,51 @@
|
||||||
|
# Loki — the single log store for the whole Thermograph fleet (prod, beta, LAN
|
||||||
|
# dev). Every node's Alloy agent pushes here over the WireGuard mesh; Grafana
|
||||||
|
# reads from here. Single-binary, filesystem-backed: no object storage, no
|
||||||
|
# clustering — right for a three-node hobby fleet, and everything lives on one
|
||||||
|
# volume you can back up or blow away.
|
||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
grpc_listen_port: 9096
|
||||||
|
log_level: warn
|
||||||
|
|
||||||
|
common:
|
||||||
|
instance_addr: 127.0.0.1
|
||||||
|
path_prefix: /loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2024-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
# A hobby fleet's volume is tiny; keep 30 days and cap ingestion generously.
|
||||||
|
retention_period: 720h
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
max_query_series: 5000
|
||||||
|
allow_structured_metadata: true
|
||||||
|
volume_enabled: true
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
|
||||||
|
# No analytics phone-home from a private box.
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
Loading…
Reference in a new issue