# The routing tree. Provisioning `policies:` REPLACES Grafana's root policy, so # applying this file is what actually severs the factory default # (`receiver: grafana-default-email` -> "") and points the # whole estate at Discord. Everything lands in the same #ops-alerts channel; the # severity branches differ only in how *insistent* they are. # # Timing rationale, for a solo operator on a phone: # critical — 10s wait so a burst arrives as one message, re-nag hourly while # it is still broken. You want to be pestered about a dead prod. # warning — 2m wait (lets a flapping thing settle), re-nag twice a day. # Not worth waking up for; worth not forgetting. # watchdog — the dead-man's-switch. Fires forever by design, so it is throttled # to one message per day. Its VALUE IS ITS ABSENCE: if #ops-alerts # is silent for >24h, Grafana/Loki/the mesh is down and no other # alert in this file can reach you either. # # There is deliberately no catch-all branch: an alert whose `severity` label is # missing or unrecognised matches none of the routes and falls through to the # ROOT receiver, which is also Discord. Nothing can be silently swallowed by a # typo in a label — it just arrives on the default (4h) cadence instead. apiVersion: 1 policies: - orgId: 1 receiver: thermograph-ops-discord # Collapse by rule + host so a whole-node outage (every prod rule firing at # once) still arrives as a handful of messages, not thirty. group_by: - alertname - host group_wait: 30s group_interval: 5m repeat_interval: 4h routes: - receiver: thermograph-ops-discord object_matchers: - ['severity', '=', 'watchdog'] group_wait: 0s group_interval: 24h repeat_interval: 24h continue: false - receiver: thermograph-ops-discord object_matchers: - ['severity', '=', 'critical'] group_by: - alertname - host group_wait: 10s group_interval: 5m repeat_interval: 1h continue: false - receiver: thermograph-ops-discord object_matchers: - ['severity', '=', 'warning'] group_by: - alertname - host group_wait: 2m group_interval: 10m repeat_interval: 12h continue: false