140 lines
5.4 KiB
Text
140 lines
5.4 KiB
Text
// Grafana Alloy — the log collector that runs on EVERY node (prod, beta, LAN
|
|
// dev). It gathers three sources and ships them to the central Loki on beta:
|
|
//
|
|
// 1. Every Docker container's stdout/stderr (app, db, and on beta forgejo)
|
|
// via the Docker socket.
|
|
// 2. Caddy's host access logs (/var/log/caddy/*.log) — the reverse proxy runs
|
|
// on the host, not in a container, so its logs aren't in Docker.
|
|
// 3. The app's structured JSON logs (errors/access/audit *.jsonl) from the
|
|
// `applogs` Docker volume, parsed so `level`/`tag`/`phase` become labels.
|
|
//
|
|
// Per-node settings come from the environment (see docker-compose.agent.yml):
|
|
// ALLOY_NODE — this node's name label (prod | beta | dev)
|
|
// LOKI_URL — where to push (http://10.10.0.2:3100/loki/api/v1/push over wg0)
|
|
|
|
livedebugging { enabled = false }
|
|
|
|
// --- 1. All Docker container logs ------------------------------------------------
|
|
// refresh_interval defaults to 60s; Swarm task churn reshuffles the target set on
|
|
// roughly that cadence, which restarts tailers ~every 90s and was costing ~11% of
|
|
// Alloy's own CPU in tailer restarts alone. 5m is still fast enough to pick up a
|
|
// real deploy without paying that churn cost.
|
|
discovery.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
refresh_interval = "5m"
|
|
}
|
|
|
|
// Turn Docker metadata into tidy labels: `container` (short name) and `service`
|
|
// (the compose service, e.g. app/db). Drop noise/duplicate containers so Loki never
|
|
// ingests them: Alloy itself (loop), the autoscaler, throwaway `thermograph-test_*`
|
|
// stacks, the loopback LB bridge (`thermograph-lb`, pure plumbing, nothing to debug
|
|
// from its logs), and the app's own worker (`thermograph_worker`'s stdout is 100%
|
|
// `/healthz` poll noise — the app's `access/*.jsonl` under source #3 is a strict
|
|
// superset of anything useful it logs).
|
|
discovery.relabel "containers" {
|
|
targets = discovery.docker.containers.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_docker_container_name"]
|
|
regex = "/(.*)"
|
|
target_label = "container"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
|
|
target_label = "service"
|
|
}
|
|
rule {
|
|
source_labels = ["container"]
|
|
regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*"
|
|
action = "drop"
|
|
}
|
|
}
|
|
|
|
// Pass RAW targets here (not discovery.relabel.containers.output) alongside
|
|
// relabel_rules: loki.source.docker applies relabel_rules itself, once, using the
|
|
// __meta_docker_* metadata it still holds at collection time. Passing the
|
|
// already-relabelled output *and* relabel_rules ran the same rules twice per log
|
|
// entry, and made the drop rules above a no-op on the second pass since the
|
|
// __meta_docker_* labels are already gone from the pre-relabelled output.
|
|
loki.source.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.docker.containers.targets
|
|
forward_to = [loki.write.central.receiver]
|
|
relabel_rules = discovery.relabel.containers.rules
|
|
labels = { job = "docker" }
|
|
}
|
|
|
|
// --- 2. Caddy host access logs ---------------------------------------------------
|
|
// sync_period (glob rescan) defaults to 10s; 1m is plenty for a log file that only
|
|
// appears/rotates on the order of hours.
|
|
local.file_match "caddy" {
|
|
path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }]
|
|
sync_period = "1m"
|
|
}
|
|
|
|
loki.source.file "caddy" {
|
|
targets = local.file_match.caddy.targets
|
|
forward_to = [loki.process.caddy.receiver]
|
|
|
|
// PollingFileWatcher defaults (250ms/250ms) stat every tailed file 4x/second
|
|
// forever. Caddy's access log doesn't need sub-second latency into Loki.
|
|
file_watch {
|
|
min_poll_frequency = "2s"
|
|
max_poll_frequency = "10s"
|
|
}
|
|
}
|
|
|
|
// Drop well-known crawler/bot traffic before it hits Loki. Caddy itself can't do
|
|
// this cheaply (log_skip needs Caddy >= 2.7; both hosts run older Caddy, and an
|
|
// upgrade is out of scope here), so filter it at the shipper instead.
|
|
loki.process "caddy" {
|
|
forward_to = [loki.write.central.receiver]
|
|
|
|
stage.drop {
|
|
expression = "(?i)(semrushbot|claudebot|ahrefsbot|yandexbot|bytespider|mj12bot|petalbot)"
|
|
drop_counter_reason = "crawler"
|
|
}
|
|
}
|
|
|
|
// --- 3. App structured JSON logs (errors / access / audit) -----------------------
|
|
// Mounted read-only from the app's `applogs` volume at /applogs (see the agent
|
|
// compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable.
|
|
local.file_match "app_jsonl" {
|
|
path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }]
|
|
sync_period = "1m"
|
|
}
|
|
|
|
loki.source.file "app_jsonl" {
|
|
targets = local.file_match.app_jsonl.targets
|
|
forward_to = [loki.process.app_jsonl.receiver]
|
|
|
|
file_watch {
|
|
min_poll_frequency = "2s"
|
|
max_poll_frequency = "10s"
|
|
}
|
|
}
|
|
|
|
loki.process "app_jsonl" {
|
|
forward_to = [loki.write.central.receiver]
|
|
|
|
stage.json {
|
|
expressions = { level = "level", tag = "tag", phase = "phase", status = "status" }
|
|
}
|
|
// A record with no explicit level: an error-folder line is an error, else info.
|
|
stage.static_labels {
|
|
values = { source = "app" }
|
|
}
|
|
stage.labels {
|
|
values = { level = "", tag = "", phase = "" }
|
|
}
|
|
}
|
|
|
|
// --- Ship to the central Loki over the WireGuard mesh ----------------------------
|
|
loki.write "central" {
|
|
endpoint {
|
|
url = sys.env("LOKI_URL")
|
|
}
|
|
// Every line from this node is stamped with its node name, so one Grafana
|
|
// view can slice prod vs beta vs dev.
|
|
external_labels = { host = sys.env("ALLOY_NODE") }
|
|
}
|