thermograph/infra/deploy/provision-mail-supervision.sh

#!/usr/bin/env bash
# Postfix SUPERVISION layer — restart policy, start-time interface gate, and a
# self-heal watchdog. Called by provision-mail.sh; also safe to run standalone
# (it touches nothing in main.cf, so it will not fight a mail-config change):
#
#   sudo bash infra/deploy/provision-mail-supervision.sh
#
# WHY THIS EXISTS — the 2026-07-24 prod outage.
#
# Postfix treats an `inet_interfaces` address with no matching local interface
# as FATAL, and takes down EVERY listener with it — loopback and mesh included,
# not just the missing one. prod lists three addresses, and two of them are
# created by other daemons: 10.10.0.1 (wg-quick@wg0) and 172.18.0.1
# (docker_gwbridge, created by dockerd).
#
# The stock postfix@.service is ordered only `After=network-online.target`,
# which says nothing about dockerd or wg-quick, and it ships NO `Restart=`.
# At the 08:09 reboot that cost 13 hours of mail:
#
#   08:09:49  postfix@-.service starts
#   08:09:51  wg-quick@wg0 finishes  <- 10.10.0.1 appears, wins by one second
#   08:09:52  fatal: inet_interfaces: no local interface found for 172.18.0.1
#   08:09:53  docker.service only BEGINS starting
#   08:10:16  docker.service active  <- docker_gwbridge finally exists
#
# Note this was a BOOT-ORDERING RACE, not a Docker renumbering: 172.18.0.1 is
# the pinned docker_gwbridge gateway and has always had that address. Postfix
# simply asked for it 24 seconds before dockerd created it.
#
# Nobody noticed for 13h because `systemctl is-active postfix` said "active"
# the whole time — see the umbrella-unit note below.
#
# THREE LAYERS, because no single one of them is sufficient:
#
#   1. ORDERING (After=docker.service wg-quick@wg0.service) — makes the race
#      very unlikely. Not a guarantee: dockerd reporting "active" is not a
#      promise that a specific bridge is already numbered.
#   2. A START-TIME GATE (postfix-wait-interfaces, an ExecStartPre) — makes it
#      deterministic. It blocks until every address in inet_interfaces actually
#      exists, so a late interface costs seconds inside one start attempt
#      instead of burning a restart. This is what makes a BOUNDED start limit
#      safe; see the StartLimit discussion in the drop-in below.
#   3. A WATCHDOG TIMER (postfix-watchdog) — the backstop. It is not redundant
#      with Restart=: postfix@.service is Type=forking with GuessMainPID=no and
#      no PIDFile=, so systemd tracks NO main PID for it. `Restart=` therefore
#      only covers failures during START. If the master process dies at
#      runtime, systemd does not reliably notice, and no restart policy fires.
#      The watchdog is the only thing that catches that.
#
# THE UMBRELLA-UNIT TRAP. `postfix.service` is a wrapper whose ExecStart is
# /bin/true and which sets RemainAfterExit=yes. It reports `active (exited)`
# forever, whether or not any mail is being handled. The unit that actually
# binds sockets is the instance `postfix@-.service`. Never health-check the
# umbrella — that is precisely the check that reported green for 13 hours.
set -euo pipefail

if [[ $EUID -ne 0 ]]; then
  echo "run as root (sudo)" >&2
  exit 1
fi

# How long the start-time gate waits for a missing interface before giving up
# and letting the restart policy take over. Overridable for testing.
POSTFIX_WAIT_TIMEOUT="${POSTFIX_WAIT_TIMEOUT:-60}"

echo "==> installing /usr/local/sbin/postfix-wait-interfaces"
cat > /usr/local/sbin/postfix-wait-interfaces <<'WAITSCRIPT'
#!/usr/bin/env bash
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
#
# ExecStartPre gate for postfix@.service. Blocks until every address listed in
# this instance's inet_interfaces actually exists on some local interface.
#
# Postfix treats a missing inet_interfaces address as fatal and drops ALL its
# listeners, so at boot it must not be allowed to ask before dockerd/wg-quick
# have finished. Unit ordering makes that unlikely; this makes it deterministic.
#
# FAIL-OPEN BY DESIGN: if this script cannot introspect the config, it exits 0
# and lets Postfix try anyway. A supervision gate must only ever DELAY a start,
# never be the reason mail stays down.
set -uo pipefail

TIMEOUT="${POSTFIX_WAIT_TIMEOUT:-60}"
INSTANCE="${1:--}"
# Instance '-' is the default instance and always lives in /etc/postfix. A
# second config dir can be passed explicitly; this estate runs one instance.
CONFDIR="${2:-/etc/postfix}"
log() { echo "postfix-wait-interfaces: $*"; }

# MUST be plain `postconf -c`, NOT `postmulti -x postconf`.
#
# postmulti RESOLVES inet_interfaces, so it fatals with rc=1 and prints nothing
# whenever an address is missing -- which is precisely the situation this gate
# exists to detect. Using it here made the gate fail open and silently never
# wait at all. Plain postconf only reads main.cf and returns the value happily
# whether or not the address exists. (Same trap applies to postqueue/postfix;
# see postfix-health.)
addrs=$(postconf -c "$CONFDIR" -h inet_interfaces 2>/dev/null) || {
  log "cannot read inet_interfaces from $CONFDIR; not blocking startup"
  exit 0
}

# Only literal IP addresses can be waited for. Keywords (all, loopback-only)
# and hostnames are left to Postfix to resolve.
want=()
for a in ${addrs//,/ }; do
  case "$a" in
    all|loopback-only|localhost) continue ;;
  esac
  if [[ "$a" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ || "$a" == *:* ]]; then
    want+=("$a")
  fi
done

if (( ${#want[@]} == 0 )); then
  log "no literal addresses to wait for (inet_interfaces = $addrs)"
  exit 0
fi

deadline=$(( SECONDS + TIMEOUT ))
missing=()
while :; do
  present=$(ip -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1)
  missing=()
  for a in "${want[@]}"; do
    grep -qxF "$a" <<<"$present" || missing+=("$a")
  done
  (( ${#missing[@]} == 0 )) && break
  (( SECONDS >= deadline )) && break
  sleep 1
done

if (( ${#missing[@]} == 0 )); then
  log "all addresses present after ${SECONDS}s: ${want[*]}"
  exit 0
fi

# Deliberately non-zero: let the unit's Restart=/StartLimit policy decide what
# happens next, and make the reason visible in the journal.
log "TIMEOUT after ${TIMEOUT}s; still missing: ${missing[*]} (inet_interfaces = $addrs)"
log "Postfix will fatal on this. Either the interface provider is broken, or"
log "inet_interfaces names an address this host no longer has."
exit 1
WAITSCRIPT
chmod 0755 /usr/local/sbin/postfix-wait-interfaces

echo "==> installing /usr/local/sbin/postfix-health"
cat > /usr/local/sbin/postfix-health <<'HEALTHSCRIPT'
#!/usr/bin/env bash
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
#
# THE canonical "is this box's mail actually working" check. One implementation,
# so the watchdog, a human, a Grafana/Loki alert and a future `mail_health` tool
# all agree on what healthy means.
#
#   exit 0 = healthy      exit 1 = unhealthy
#
# Output is one logfmt line on stdout (parseable with LogQL `| logfmt`),
# followed by human-readable detail on stderr.
#
# WHAT IT DELIBERATELY DOES NOT DO: check `postfix.service`. That umbrella unit
# is ExecStart=/bin/true + RemainAfterExit=yes and reports active(exited)
# forever. It reported green through the entire 13h outage of 2026-07-24.
set -uo pipefail
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

INSTANCE="${1:--}"
CONFDIR="${2:-/etc/postfix}"
UNIT="postfix@${INSTANCE}.service"
problems=()

# --- 1. the INSTANCE unit, never the umbrella --------------------------------
unit_state=$(systemctl is-active "$UNIT" 2>/dev/null || true)
[[ "$unit_state" == "active" ]] || problems+=("unit-not-active:$unit_state")

# --- 2. every configured address is actually bound on :25 --------------------
#
# MUST be plain `postconf -c`, NOT `postmulti -x postconf` / `postqueue`.
#
# Every Postfix command that RESOLVES inet_interfaces (postmulti, postqueue,
# postfix) fatals when one of the addresses is missing from the host -- exactly
# the condition this check exists to find. An earlier version of this script
# used postmulti, got an empty string back, concluded there were 0 addresses to
# verify, and cheerfully reported "status=ok bound=0/0" while main.cf named a
# ghost address that would kill all mail at the next reboot. A health check that
# fails OPEN is worse than no health check.
#
# Plain postconf only reads main.cf and never touches an interface.
addrs=$(postconf -c "$CONFDIR" -h inet_interfaces 2>/dev/null || echo "")
if [[ -z "${addrs// /}" ]]; then
  problems+=("cannot-read-inet_interfaces")
fi
listening=$(ss -lnt 2>/dev/null | awk '{print $4}')
expected=(); bound=0; unbound=()
for a in ${addrs//,/ }; do
  case "$a" in
    all)           expected+=("0.0.0.0") ;;
    loopback-only) expected+=("127.0.0.1") ;;
    localhost)     expected+=("127.0.0.1") ;;
    *)             expected+=("$a") ;;
  esac
done
for a in "${expected[@]}"; do
  if grep -qxF "${a}:25" <<<"$listening"; then bound=$(( bound + 1 )); else unbound+=("$a"); fi
done
(( ${#unbound[@]} == 0 )) || problems+=("not-bound:$(IFS=,; echo "${unbound[*]}")")
# Never let "I could not work out what to check" read as healthy.
(( ${#expected[@]} > 0 )) || problems+=("no-configured-addresses")

# --- 3. LATENT FATAL: config names an address the host does not have ---------
# This is the state that survives silently until the next reboot and then kills
# all mail. Catching it here turns a future outage into a warning today.
present=$(ip -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1)
ghosts=()
for a in "${expected[@]}"; do
  [[ "$a" == "0.0.0.0" ]] && continue
  grep -qxF "$a" <<<"$present" || ghosts+=("$a")
done
(( ${#ghosts[@]} == 0 )) || problems+=("address-missing-from-host:$(IFS=,; echo "${ghosts[*]}")")

# --- 4. a real SMTP greeting, not just an open socket ------------------------
banner="none"
if command -v timeout >/dev/null 2>&1; then
  greeting=$(timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/25 || exit 1; head -1 <&3; printf "QUIT\r\n" >&3' 2>/dev/null || true)
  case "$greeting" in
    220*) banner="ok" ;;
    *)    banner="bad"; problems+=("no-220-greeting") ;;
  esac
fi

# --- 5. SEND-ONLY INVARIANT: must NOT be bound on a public address -----------
# A regression here is worse than an outage: it is an internet-facing MTA.
public_ips=$(ip -4 -o addr show scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 \
             | grep -vE '^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.)' || true)
exposed=()
for p in $public_ips; do
  grep -qxF "${p}:25" <<<"$listening" && exposed+=("$p")
done
grep -qxF "0.0.0.0:25" <<<"$listening" && exposed+=("0.0.0.0")
(( ${#exposed[@]} == 0 )) || problems+=("PUBLICLY-BOUND:$(IFS=,; echo "${exposed[*]}")")

# --- 6. queue depth and age, excluding the known-benign null-sender bounces ---
# Read arrival_time from `postqueue -j`, NOT the queue file's mtime: Postfix
# sets a deferred file's mtime to the NEXT RETRY time, which is in the future,
# so mtime-based ages come out negative and meaningless.
#
# Null-sender bounces (postqueue reports sender "MAILER-DAEMON") addressed to
# our own no-reply are undeliverable by design -- this host accepts no inbound
# mail -- so they sit in the queue until they expire. They must not hold an
# alert open forever, so they are excluded from queue_real/oldest. Both the raw
# and filtered counts are reported so a monitor can see either.
#
# NOTE: postqueue RESOLVES inet_interfaces and so fatals (rc=69) in the very
# latent-fatal state flagged above. When that happens it yields no JSON, so fall
# back to counting spool files directly -- a count that needs no Postfix command
# and therefore still works when Postfix itself will not run.
queue_json=$(postqueue -j 2>/dev/null || true)
read -r queue_total queue_real oldest_age <<<"$(
  printf '%s\n' "$queue_json" | awk -v now="$(date +%s)" '
    NF {
      total++
      sender=""; arr=0
      if (match($0, /"sender": "[^"]*"/)) {
        s = substr($0, RSTART, RLENGTH); gsub(/"sender": "|"/, "", s); sender = s
      }
      if (match($0, /"arrival_time": [0-9]+/)) {
        a = substr($0, RSTART, RLENGTH); gsub(/[^0-9]/, "", a); arr = a + 0
      }
      if (sender != "MAILER-DAEMON" && sender != "") {
        real++
        age = now - arr
        if (age > oldest) oldest = age
      }
    }
    END { printf "%d %d %d", total + 0, real + 0, oldest + 0 }
  '
)"
queue_total=${queue_total:-0}; queue_real=${queue_real:-0}; oldest_age=${oldest_age:-0}
if [[ -z "${queue_json// /}" ]]; then
  queue_total=$(find /var/spool/postfix/incoming /var/spool/postfix/active \
                     /var/spool/postfix/deferred -type f 2>/dev/null | wc -l)
  queue_real=-1   # unknown: cannot classify senders without postqueue
fi

status=ok; rc=0
(( ${#problems[@]} == 0 )) || { status=fail; rc=1; }

printf 'mail_health status=%s unit=%s unit_state=%s bound=%d/%d banner=%s queue=%d queue_real=%d oldest_deferred_s=%d reason=%s\n' \
  "$status" "$UNIT" "${unit_state:-unknown}" "$bound" "${#expected[@]}" "$banner" \
  "$queue_total" "$queue_real" "$oldest_age" \
  "$( (( ${#problems[@]} )) && (IFS=,; echo "${problems[*]}") || echo none )"

if (( rc != 0 )); then
  { echo "--- postfix-health detail ---"
    echo "inet_interfaces : $addrs"
    echo "host addresses  : $(echo "$present" | tr '\n' ' ')"
    echo "listening on 25 : $(grep ':25$' <<<"$listening" | tr '\n' ' ')"
    systemctl status "$UNIT" --no-pager -l 2>/dev/null | head -20
  } >&2
fi
exit $rc
HEALTHSCRIPT
chmod 0755 /usr/local/sbin/postfix-health

echo "==> installing /usr/local/sbin/postfix-watchdog"
cat > /usr/local/sbin/postfix-watchdog <<'WATCHSCRIPT'
#!/usr/bin/env bash
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
#
# Self-heal backstop, run every 5 minutes by postfix-watchdog.timer.
#
# It exists for two failure modes that `Restart=on-failure` provably cannot
# cover:
#
#   1. RUNTIME DEATH. postfix@.service is Type=forking, GuessMainPID=no, and
#      declares no PIDFile=, so systemd tracks no main PID. If the Postfix
#      master dies while running, no restart policy fires.
#   2. GIVING UP. The unit uses a BOUNDED start limit on purpose, so a broken
#      config lands in `failed` instead of thrashing forever. Without a
#      backstop, "failed" would mean "dead until a human looks" — which is the
#      13-hour outage again. This retries every 5 minutes, forever, and clears
#      the start-limit counter so the bound never becomes permanent death.
#
# Cadence is deliberately 20x slower than RestartSec: fast enough that nothing
# is ever down for more than ~5 minutes, slow enough that a genuinely broken
# config produces ~288 log lines a day instead of ~5,760.
set -uo pipefail
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

INSTANCE=-
UNIT="postfix@${INSTANCE}.service"

# Escape hatch: `sudo touch /etc/postfix/maintenance` to stop the watchdog
# fighting a deliberate `systemctl stop postfix@-`.
if [[ -e /etc/postfix/maintenance ]]; then
  echo "postfix_watchdog action=skipped reason=maintenance-flag"
  exit 0
fi

# Do not interfere with a start already in progress.
state=$(systemctl show "$UNIT" -p ActiveState --value 2>/dev/null || echo unknown)
if [[ "$state" == "activating" || "$state" == "deactivating" ]]; then
  echo "postfix_watchdog action=skipped reason=transitional:$state"
  exit 0
fi

if health=$(postfix-health "$INSTANCE" 2>/dev/null); then
  echo "postfix_watchdog action=ok ${health#mail_health }"
  exit 0
fi

echo "postfix_watchdog action=recovering ${health#mail_health }"
# reset-failed clears both the failed state and the start-limit counter, so the
# bounded StartLimitBurst can never become a permanent death sentence.
systemctl reset-failed "$UNIT" 2>/dev/null || true
systemctl start "$UNIT" 2>/dev/null || true
sleep 5

if health=$(postfix-health "$INSTANCE" 2>/dev/null); then
  echo "postfix_watchdog action=recovered ${health#mail_health }"
  exit 0
fi
# Distinctive, greppable line: this is the one that should page a human.
echo "postfix_watchdog action=RECOVERY_FAILED ${health#mail_health }"
exit 1
WATCHSCRIPT
chmod 0755 /usr/local/sbin/postfix-watchdog

echo "==> installing postfix@.service drop-in"
mkdir -p /etc/systemd/system/postfix@.service.d
# Quoted heredoc + a placeholder, NOT shell interpolation: this block contains
# backticks and $-signs in its prose, and an unquoted heredoc would run them.
cat > /etc/systemd/system/postfix@.service.d/10-wait-for-interfaces.conf <<'DROPIN'
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
#
# A DROP-IN, never an edit to /usr/lib/systemd/system/postfix@.service: the
# packaged unit is replaced wholesale by an apt upgrade.
#
# ORDERING. inet_interfaces may name a Docker bridge gateway and/or a WireGuard
# mesh address; neither exists until dockerd / wg-quick has run, and Postfix
# treats a missing address as fatal for ALL listeners. Stock ordering is only
# After=network-online.target, which says nothing about either daemon.
#
# Wants=, not Requires=, on purpose. Requires= would mean a dockerd failure
# takes mail down entirely — including the loopback and mesh listeners, which
# do not depend on Docker at all. That escalates someone else's outage into a
# total mail outage. Wants= keeps the ordering when both are present without
# making mail's availability a hostage to Docker's.
#
# THE GATE. ExecStartPre blocks until every configured address actually exists
# (bounded by POSTFIX_WAIT_TIMEOUT). Ordering alone is a strong heuristic —
# dockerd reporting "active" is not a promise that a specific bridge is
# numbered — and this turns the residual race into a deterministic wait.
# It is what makes the bounded start limit below safe.
#
# THE START LIMIT — the deliberate tradeoff.
#
# StartLimitIntervalSec=0 (retry forever) was considered and rejected. With the
# gate in place, the transient case is absorbed INSIDE a single start attempt,
# so restarts now overwhelmingly mean "genuinely broken", not "too early". For
# that case, retry-forever is the worse option: the unit never reaches `failed`
# (so the strongest local signal never exists), and a malformed main.cf
# re-fatals every RestartSec forever.
#
#   * 5 attempts / 600s. A broken config reaches `failed` in ~100s (fast
#     failure: ~5s start + 15s wait) or ~5min (slow failure: the full 60s gate
#     + 15s wait, 5 times) — either way, loudly, and then it stops.
#   * Normal operation never approaches this: an isolated crash is 1 restart.
#   * "Failed" does NOT mean "dead until a human looks" — postfix-watchdog.timer
#     retries every 5 minutes forever and clears the counter. Bounded fast
#     retry for transients, slow unbounded retry for everything else.
#
# TimeoutStartSec covers the worst case: the full @WAIT_TIMEOUT@s gate plus
# instance configure plus master start, with headroom. The default 90s would be
# uncomfortably tight against a @WAIT_TIMEOUT@s gate.
[Unit]
After=docker.service wg-quick@wg0.service
Wants=docker.service wg-quick@wg0.service
StartLimitIntervalSec=600
StartLimitBurst=5

[Service]
Environment=POSTFIX_WAIT_TIMEOUT=@WAIT_TIMEOUT@
ExecStartPre=/usr/local/sbin/postfix-wait-interfaces %i
TimeoutStartSec=180s
Restart=on-failure
RestartSec=15s
DROPIN
sed -i "s/@WAIT_TIMEOUT@/${POSTFIX_WAIT_TIMEOUT}/g" \
  /etc/systemd/system/postfix@.service.d/10-wait-for-interfaces.conf

echo "==> installing postfix-watchdog.service / .timer"
cat > /etc/systemd/system/postfix-watchdog.service <<'WDSERVICE'
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
[Unit]
Description=Postfix health watchdog (self-heal backstop)
Documentation=file:///usr/local/sbin/postfix-watchdog
# Never race the thing we are supervising.
After=postfix@-.service

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/postfix-watchdog
# The watchdog failing is itself a signal worth seeing, but it must never be
# able to wedge the timer.
SuccessExitStatus=0 1
WDSERVICE

cat > /etc/systemd/system/postfix-watchdog.timer <<'WDTIMER'
# Managed by deploy/provision-mail-supervision.sh -- do not hand-edit.
#
# Every 5 minutes: the recovery-latency ceiling for mail on this box. The
# 2026-07-24 outage ran 13h; this bounds the same failure at ~5min even with
# no alerting configured at all.
[Unit]
Description=Run the Postfix health watchdog every 5 minutes

[Timer]
OnBootSec=5min
OnUnitActiveSec=5min
# Spread the load a little so it never lands exactly on other cron-ish work.
RandomizedDelaySec=20s
AccuracySec=10s

[Install]
WantedBy=timers.target
WDTIMER

systemctl daemon-reload
systemctl enable --now postfix-watchdog.timer

echo "==> supervision layer installed"
echo "    drop-in : /etc/systemd/system/postfix@.service.d/10-wait-for-interfaces.conf"
echo "    gate    : /usr/local/sbin/postfix-wait-interfaces"
echo "    health  : /usr/local/sbin/postfix-health"
echo "    watchdog: /usr/local/sbin/postfix-watchdog  (postfix-watchdog.timer)"