observability/grafana/provisioning/alerting/contact-points.yml

@@ -37,12 +37,26 @@ contactPoints:
           # Deliberately plain. A template error here breaks EVERY notification
           # silently, so this uses only functions verified against Grafana
           # 11.6.1 via /api/alertmanager/grafana/config/api/v1/receivers/test.
+          #
+          # `.ValueString` is guarded because it is not always populated. It
+          # carries the evaluated refIds only when the notification came from an
+          # evaluation that produced them; an instance resolved by Grafana's
+          # STALENESS handling — no evaluation ever returns it again, so the
+          # state manager expires it — resolves with an empty ValueString. That
+          # happens whenever a rule's `title` changes, since the title becomes
+          # the alertname label and the old label set is orphaned. Renaming
+          # ProdWorkerContainerSilent -> ProdWorkerHeartbeatMissing on
+          # 2026-07-25 did exactly that and posted a resolved notice reading
+          # "value:" with nothing after it. An empty labelled field is worse
+          # than an absent one: it reads as a value that failed to compute.
+          # Unguarded `{{ .ValueString }}` on its own would be safe; printing
+          # the label unconditionally is the bug.
           message: |-
             {{ range .Alerts }}**severity:** {{ .Labels.severity }} · **host:** {{ .Labels.host }}
             {{ .Annotations.summary }}
             {{ .Annotations.description }}
-            `value: {{ .ValueString }}`
-            {{ end }}
+            {{ if .ValueString }}`value: {{ .ValueString }}`
+            {{ end }}{{ end }}
             <https://dashboard.thermograph.org/alerting/list>
 
 # --- The factory-default contact point -------------------------------------------