f2fd8f6835
git-subtree-dir: observability git-subtree-mainline:ae1d9bb534git-subtree-split:19e74af9ca
96 lines
3.3 KiB
Text
96 lines
3.3 KiB
Text
// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN
|
|
// dev). It gathers three sources and ships them to the central Loki on beta:
|
|
//
|
|
// 1. Every Docker container's stdout/stderr (app, db, and on beta forgejo)
|
|
// via the Docker socket.
|
|
// 2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs
|
|
// on the host, not in a container, so its logs aren't in Docker.
|
|
// 3. The app's structured JSON logs (errors/access/audit *.jsonl) from the
|
|
// `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels.
|
|
//
|
|
// Per-node settings come from the environment (see docker-compose.agent.yml):
|
|
// ALLOY_NODE — this node's name label (prod | beta | dev)
|
|
// LOKI_URL — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0)
|
|
|
|
livedebugging { enabled = false }
|
|
|
|
// --- 1. All Docker container logs ------------------------------------------------
|
|
discovery.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
}
|
|
|
|
// Turn Docker metadata into tidy labels: `container` (short name) and `service`
|
|
// (the compose service, e.g. app/db). Drop Alloy's own container to avoid a loop.
|
|
discovery.relabel "containers" {
|
|
targets = discovery.docker.containers.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_docker_container_name"]
|
|
regex = "/(.*)"
|
|
target_label = "container"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
|
target_label = "service"
|
|
}
|
|
rule {
|
|
source_labels = ["container"]
|
|
regex = ".*alloy.*"
|
|
action = "drop"
|
|
}
|
|
}
|
|
|
|
loki.source.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.relabel.containers.output
|
|
forward_to = [loki.write.central.receiver]
|
|
relabel_rules = discovery.relabel.containers.rules
|
|
labels = { job = "docker" }
|
|
}
|
|
|
|
// --- 2. Caddy host access logs ---------------------------------------------------
|
|
local.file_match "caddy" {
|
|
path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }]
|
|
}
|
|
|
|
loki.source.file "caddy" {
|
|
targets = local.file_match.caddy.targets
|
|
forward_to = [loki.write.central.receiver]
|
|
}
|
|
|
|
// --- 3. App structured JSON logs (errors / access / audit) -----------------------
|
|
// Mounted read-only from the app's `applogs` volume at /applogs (see the agent
|
|
// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable.
|
|
local.file_match "app_jsonl" {
|
|
path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }]
|
|
}
|
|
|
|
loki.source.file "app_jsonl" {
|
|
targets = local.file_match.app_jsonl.targets
|
|
forward_to = [loki.process.app_jsonl.receiver]
|
|
}
|
|
|
|
loki.process "app_jsonl" {
|
|
forward_to = [loki.write.central.receiver]
|
|
|
|
stage.json {
|
|
expressions = { level = "level", tag = "tag", phase = "phase", status = "status" }
|
|
}
|
|
// A record with no explicit level: an error-folder line is an error, else info.
|
|
stage.static_labels {
|
|
values = { source = "app" }
|
|
}
|
|
stage.labels {
|
|
values = { level = "", tag = "", phase = "" }
|
|
}
|
|
}
|
|
|
|
// --- Ship to the central Loki over the WireGuard mesh ----------------------------
|
|
loki.write "central" {
|
|
endpoint {
|
|
url = sys.env("LOKI_URL")
|
|
}
|
|
// Every line from this node is stamped with its node name, so one Grafana
|
|
// view can slice prod vs beta vs dev.
|
|
external_labels = { host = sys.env("ALLOY_NODE") }
|
|
}
|