65 lines
2.4 KiB
YAML
65 lines
2.4 KiB
YAML
|
# The routing tree. Provisioning `policies:` REPLACES Grafana's root policy, so
|
||
|
|
# applying this file is what actually severs the factory default
|
||
|
|
# (`receiver: grafana-default-email` -> "<example@email.com>") and points the
|
||
|
|
# whole estate at Discord. Everything lands in the same #ops-alerts channel; the
|
||
|
|
# severity branches differ only in how *insistent* they are.
|
||
|
|
#
|
||
|
|
# Timing rationale, for a solo operator on a phone:
|
||
|
|
# critical — 10s wait so a burst arrives as one message, re-nag hourly while
|
||
|
|
# it is still broken. You want to be pestered about a dead prod.
|
||
|
|
# warning — 2m wait (lets a flapping thing settle), re-nag twice a day.
|
||
|
|
# Not worth waking up for; worth not forgetting.
|
||
|
|
# watchdog — the dead-man's-switch. Fires forever by design, so it is throttled
|
||
|
|
# to one message per day. Its VALUE IS ITS ABSENCE: if #ops-alerts
|
||
|
|
# is silent for >24h, Grafana/Loki/the mesh is down and no other
|
||
|
|
# alert in this file can reach you either.
|
||
|
|
#
|
||
|
|
# There is deliberately no catch-all branch: an alert whose `severity` label is
|
||
|
|
# missing or unrecognised matches none of the routes and falls through to the
|
||
|
|
# ROOT receiver, which is also Discord. Nothing can be silently swallowed by a
|
||
|
|
# typo in a label — it just arrives on the default (4h) cadence instead.
|
||
|
|
apiVersion: 1
|
||
|
|
|
||
|
|
policies:
|
||
|
|
- orgId: 1
|
||
|
|
receiver: thermograph-ops-discord
|
||
|
|
# Collapse by rule + host so a whole-node outage (every prod rule firing at
|
||
|
|
# once) still arrives as a handful of messages, not thirty.
|
||
|
|
group_by:
|
||
|
|
- alertname
|
||
|
|
- host
|
||
|
|
group_wait: 30s
|
||
|
|
group_interval: 5m
|
||
|
|
repeat_interval: 4h
|
||
|
|
|
||
|
|
routes:
|
||
|
|
- receiver: thermograph-ops-discord
|
||
|
|
object_matchers:
|
||
|
|
- ['severity', '=', 'watchdog']
|
||
|
|
group_wait: 0s
|
||
|
|
group_interval: 24h
|
||
|
|
repeat_interval: 24h
|
||
|
|
continue: false
|
||
|
|
|
||
|
|
- receiver: thermograph-ops-discord
|
||
|
|
object_matchers:
|
||
|
|
- ['severity', '=', 'critical']
|
||
|
|
group_by:
|
||
|
|
- alertname
|
||
|
|
- host
|
||
|
|
group_wait: 10s
|
||
|
|
group_interval: 5m
|
||
|
|
repeat_interval: 1h
|
||
|
|
continue: false
|
||
|
|
|
||
|
|
- receiver: thermograph-ops-discord
|
||
|
|
object_matchers:
|
||
|
|
- ['severity', '=', 'warning']
|
||
|
|
group_by:
|
||
|
|
- alertname
|
||
|
|
- host
|
||
|
|
group_wait: 2m
|
||
|
|
group_interval: 10m
|
||
|
|
repeat_interval: 12h
|
||
|
|
continue: false
|