thermograph/observability/grafana/provisioning/alerting/contact-points.yml

# Where alerts go. ONE destination: a Discord webhook posting into the private
# #ops-alerts channel (Owners category) of the Thermograph.org server.
#
# WHY NOT EMAIL. The Grafana factory default routes to the literal string
# "<example@email.com>", i.e. nowhere. Fixing it by pointing at a real mailbox
# would still be wrong: vps1's Grafana (the monitoring host — not the beta
# environment, which runs on vps2) relays SMTP through vps2's Postfix at
# 10.10.0.1:25 (see the live docker-compose.override.yml on vps1), so email
# alerts travel *through the box most likely to be on fire* and are silently
# lost whenever Postfix is down — which is exactly when you need them. Discord
# is off-estate: it works when prod is dead, and it works from a phone.
#
# THE WEBHOOK URL IS A SECRET. Anyone holding it can post into the channel, so
# it is NOT in this repo. It is read from the Grafana process environment, which
# docker-compose.yml feeds from vps1's gitignored .env (see .env.example).
# Grafana expands $VAR / $__env{VAR} when it reads provisioning files; if the
# variable is unset the contact point ends up with a literal "$DISCORD_..."
# string and every notification fails — see README "Verify alerting".
#
# To rotate: make a new webhook on the #ops-alerts channel, replace the value in
# vps1's .env, `docker restart observability-grafana-1`, delete the old webhook
# in Discord.
apiVersion: 1

contactPoints:
  - orgId: 1
    name: thermograph-ops-discord
    receivers:
      - uid: tg_ops_discord
        type: discord
        # Send a green "resolved" message too — a page you never see close is a
        # page you stop trusting.
        disableResolveMessage: false
        settings:
          url: $DISCORD_ALERT_WEBHOOK_URL
          use_discord_username: false
          title: '{{ if eq .Status "firing" }}[FIRING]{{ else }}[RESOLVED]{{ end }} {{ .CommonLabels.alertname }}'
          # Deliberately plain. A template error here breaks EVERY notification
          # silently, so this uses only functions verified against Grafana
          # 11.6.1 via /api/alertmanager/grafana/config/api/v1/receivers/test.
          #
          # `.ValueString` is guarded because it is not always populated. It
          # carries the evaluated refIds only when the notification came from an
          # evaluation that produced them; an instance resolved by Grafana's
          # STALENESS handling — no evaluation ever returns it again, so the
          # state manager expires it — resolves with an empty ValueString. That
          # happens whenever a rule's `title` changes, since the title becomes
          # the alertname label and the old label set is orphaned. Renaming
          # ProdWorkerContainerSilent -> ProdWorkerHeartbeatMissing on
          # 2026-07-25 did exactly that and posted a resolved notice reading
          # "value:" with nothing after it. An empty labelled field is worse
          # than an absent one: it reads as a value that failed to compute.
          # Unguarded `{{ .ValueString }}` on its own would be safe; printing
          # the label unconditionally is the bug.
          message: |-
            {{ range .Alerts }}**severity:** {{ .Labels.severity }} · **host:** {{ .Labels.host }}
            {{ .Annotations.summary }}
            {{ .Annotations.description }}
            {{ if .ValueString }}`value: {{ .ValueString }}`
            {{ end }}{{ end }}
            <https://dashboard.thermograph.org/alerting/list>            

# --- The factory-default contact point -------------------------------------------
# Grafana ships a built-in receiver "grafana-default-email" whose only integration
# is `email receiver` -> "<example@email.com>". It is created by Grafana's default
# Alertmanager config, NOT by provisioning, and its uid is the empty string:
#
#   GET /api/v1/provisioning/contact-points
#   -> [{"name":"email receiver","type":"email","settings":{...},"uid":""}]
#
# Because `deleteContactPoints` matches on uid, provisioning CANNOT remove it —
# DELETE /api/v1/provisioning/contact-points/ with an empty uid 404s (verified
# against 11.6.1). What notification-policies.yml *does* do is stop anything ever
# routing to it, which makes it inert: it can no longer receive anything.
#
# Deleting the row itself is cosmetic and manual, and only works once no policy
# references it (i.e. after this config is live) — either the UI (Alerting ->
# Contact points -> grafana-default-email -> Delete) or the Alertmanager config
# API. The README's "The leftover default contact point" has the exact command.