diff --git a/observability/grafana/provisioning/alerting/contact-points.yml b/observability/grafana/provisioning/alerting/contact-points.yml index 42f1f10..bdddbe2 100644 --- a/observability/grafana/provisioning/alerting/contact-points.yml +++ b/observability/grafana/provisioning/alerting/contact-points.yml @@ -37,12 +37,26 @@ contactPoints: # Deliberately plain. A template error here breaks EVERY notification # silently, so this uses only functions verified against Grafana # 11.6.1 via /api/alertmanager/grafana/config/api/v1/receivers/test. + # + # `.ValueString` is guarded because it is not always populated. It + # carries the evaluated refIds only when the notification came from an + # evaluation that produced them; an instance resolved by Grafana's + # STALENESS handling — no evaluation ever returns it again, so the + # state manager expires it — resolves with an empty ValueString. That + # happens whenever a rule's `title` changes, since the title becomes + # the alertname label and the old label set is orphaned. Renaming + # ProdWorkerContainerSilent -> ProdWorkerHeartbeatMissing on + # 2026-07-25 did exactly that and posted a resolved notice reading + # "value:" with nothing after it. An empty labelled field is worse + # than an absent one: it reads as a value that failed to compute. + # Unguarded `{{ .ValueString }}` on its own would be safe; printing + # the label unconditionally is the bug. message: |- {{ range .Alerts }}**severity:** {{ .Labels.severity }} · **host:** {{ .Labels.host }} {{ .Annotations.summary }} {{ .Annotations.description }} - `value: {{ .ValueString }}` - {{ end }} + {{ if .ValueString }}`value: {{ .ValueString }}` + {{ end }}{{ end }} # --- The factory-default contact point -------------------------------------------