From 97d0e53d778ab4671bd0e6559f44b2fb3d66043d Mon Sep 17 00:00:00 2001 From: emi Date: Wed, 22 Jul 2026 23:58:49 +0000 Subject: [PATCH 1/3] Port the Discord gateway bot from the archived app repo (its PR #23) (#3) --- notifications/discord_bot.py | 291 ++++++++++++++++++++++++ requirements.txt | 3 + tests/notifications/test_discord_bot.py | 98 ++++++++ web/app.py | 11 + 4 files changed, 403 insertions(+) create mode 100644 notifications/discord_bot.py create mode 100644 tests/notifications/test_discord_bot.py diff --git a/notifications/discord_bot.py b/notifications/discord_bot.py new file mode 100644 index 0000000..cc718d4 --- /dev/null +++ b/notifications/discord_bot.py @@ -0,0 +1,291 @@ +"""Discord **gateway** bot — reacts to @mentions (and DMs) in real time. + +The other two Discord paths need no persistent connection: slash commands arrive as +signed HTTP POSTs (discord_interactions.py) and the daily post goes out over an +incoming webhook (discord.py). This one is different — to answer an *ordinary* +message that mentions the bot, we have to hold a live websocket to Discord's +gateway and listen for MESSAGE_CREATE events. + +Single-connection invariant. Discord permits exactly one gateway connection per bot +token (per shard), so — like the subscription notifier and the daily post — this +runs in exactly one process: web/app.py starts it only when the process wins the +leader election (see core/singleton.claim_leader). No new container or daemon; it +rides the app's asyncio event loop as a background task. + +Least privilege / no privileged intent. We subscribe to GUILD_MESSAGES + +DIRECT_MESSAGES and act only on messages that @mention the bot or DM it. Discord +delivers message *content* for exactly those cases even without the privileged +MESSAGE_CONTENT intent, so the bot works with no portal review. To trigger on +arbitrary keywords in channels, enable the MESSAGE_CONTENT intent in the Developer +Portal and extend `_response_for_message` — the rest of the machinery is unchanged. + +Enable with THERMOGRAPH_DISCORD_BOT=1 (plus THERMOGRAPH_DISCORD_BOT_TOKEN, the same +bot token discord.py uses for DMs). Unset => no gateway connection; everything here +no-ops, so an unconfigured deploy pays nothing. + +Behaviour: mention the bot with a city name ("@Thermograph Phoenix") and it replies +with the same grade embed the /grade slash command returns, reusing +discord_interactions._grade_message so the two never drift. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import random + +from notifications import discord + +log = logging.getLogger("thermograph.discord_bot") + +# v10 JSON gateway. On a resume we reconnect to the session's resume_gateway_url +# (from the READY payload) instead, with the same query string appended. +_GATEWAY_URL = "wss://gateway.discord.gg/?v=10&encoding=json" +_GATEWAY_QS = "/?v=10&encoding=json" + +# Gateway intents (bitfield). GUILDS gives guild/channel context; GUILD_MESSAGES and +# DIRECT_MESSAGES deliver message-create events in servers and DMs. We deliberately +# do NOT request MESSAGE_CONTENT (privileged) — Discord still includes content for +# messages that mention us or DM us, which is all we act on. +_INTENT_GUILDS = 1 << 0 +_INTENT_GUILD_MESSAGES = 1 << 9 +_INTENT_DIRECT_MESSAGES = 1 << 12 +INTENTS = _INTENT_GUILDS | _INTENT_GUILD_MESSAGES | _INTENT_DIRECT_MESSAGES + +# Gateway opcodes (discord.com/developers/docs/topics/gateway-events#opcodes). +_OP_DISPATCH = 0 +_OP_HEARTBEAT = 1 +_OP_IDENTIFY = 2 +_OP_RESUME = 6 +_OP_RECONNECT = 7 +_OP_INVALID_SESSION = 9 +_OP_HELLO = 10 +_OP_HEARTBEAT_ACK = 11 + +# Close codes for unrecoverable conditions — reconnecting just loops, so we stop: +# 4004 auth failed (bad token), 4010 invalid shard, 4011 sharding required, 4012 +# invalid API version, 4013 invalid intents, 4014 disallowed (privileged) intents. +_FATAL_CLOSE = {4004, 4010, 4011, 4012, 4013, 4014} + +_TRUTHY = {"1", "true", "yes", "on"} + +_HELP = ("Mention me with a city name — e.g. `@Thermograph Phoenix` — and I'll grade " + "today's weather against ~45 years of local history.") + +# Reconnect backoff bounds (seconds). +_BACKOFF_START = 1.0 +_BACKOFF_MAX = 60.0 + + +def enabled() -> bool: + """True iff the gateway bot is switched on AND a bot token is configured. Off by + default: a gateway connection is a new persistent behaviour, so it takes an + explicit opt-in rather than piggybacking on the token being present for DMs.""" + flag = os.environ.get("THERMOGRAPH_DISCORD_BOT", "").strip().lower() in _TRUTHY + return flag and bool(discord.BOT_TOKEN) + + +# --- message handling (pure; unit-tested without a live gateway) -------------- +def _mentions_bot(msg: dict, bot_user_id: str) -> bool: + return any(str(u.get("id")) == str(bot_user_id) for u in (msg.get("mentions") or [])) + + +def _is_dm(msg: dict) -> bool: + # A DM has no guild_id; a guild message always carries one. + return not msg.get("guild_id") + + +def _strip_mentions(content: str, bot_user_id: str) -> str: + """Drop every form of the bot's own mention (<@id> and the legacy <@!id>) and + collapse the surrounding whitespace, leaving just the user's query text.""" + out = content.replace(f"<@{bot_user_id}>", " ").replace(f"<@!{bot_user_id}>", " ") + return " ".join(out.split()) + + +def _response_for_message(msg: dict, bot_user_id: str) -> dict | None: + """The reply payload for one MESSAGE_CREATE, or None to stay silent. + + Silent unless the message DMs the bot or @mentions it, and never for a bot + author (that includes ourselves — the guard against a mention-loop). The text + after the mention is treated as a city query and answered with the same embed + the /grade slash command builds; an empty query gets the help line.""" + author = msg.get("author") or {} + if author.get("bot") or str(author.get("id")) == str(bot_user_id): + return None + dm = _is_dm(msg) + if not dm and not _mentions_bot(msg, bot_user_id): + return None + content = msg.get("content") or "" + query = content.strip() if dm else _strip_mentions(content, bot_user_id) + if not query: + return {"content": _HELP} + # Import here (not at module top) to avoid importing the FastAPI web layer's + # transitive deps for a module that also loads in the notifier-only context. + from notifications import discord_interactions + data = discord_interactions._grade_message(query) + # Gateway messages can't be ephemeral — the flag is interactions-only. + data.pop("flags", None) + return data + + +# --- REST reply --------------------------------------------------------------- +async def _send_reply(channel_id, message_id, data: dict) -> None: + """Post the reply in-channel as a proper reply to the triggering message. + + Reuses discord._bot_post (sync httpx with a 429 retry) off the event loop via a + thread so the gateway read-loop never blocks. allowed_mentions is locked down so + a crafted query can't turn the reply into an @everyone/role ping; only the person + who asked is pinged, via the reply reference.""" + if not channel_id: + return + payload = dict(data) + if message_id: + payload["message_reference"] = {"message_id": str(message_id)} + payload["allowed_mentions"] = {"parse": [], "replied_user": True} + try: + await asyncio.to_thread( + discord._bot_post, f"/channels/{channel_id}/messages", payload) + except Exception: # noqa: BLE001 - a failed reply must never kill the read loop + log.warning("discord_bot: reply failed", exc_info=True) + + +# --- gateway session ---------------------------------------------------------- +class _Session: + """Mutable per-connection state that survives a resume: the last sequence + number, the resume token/URL, and our own user id (for mention detection).""" + + __slots__ = ("seq", "session_id", "resume_url", "user_id") + + def __init__(self) -> None: + self.seq: int | None = None + self.session_id: str | None = None + self.resume_url: str | None = None + self.user_id: str = "" + + +async def _heartbeat(ws, interval: float, sess: _Session, acked: dict) -> None: + """Send op-1 heartbeats every `interval` seconds (first beat jittered per + Discord's guidance). If the previous beat was never ACKed the link is a zombie — + close with a non-1000 code so the next connect can resume, and stop.""" + await asyncio.sleep(interval * random.random()) + while True: + if not acked["ok"]: + await ws.close(code=4000) + return + acked["ok"] = False + try: + await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq})) + except Exception: # noqa: BLE001 - closed socket; the read loop handles reconnect + return + await asyncio.sleep(interval) + + +async def _dispatch(msg: dict, sess: _Session) -> None: + """Handle an op-0 DISPATCH event we care about.""" + t = msg.get("t") + d = msg.get("d") or {} + if t == "READY": + sess.session_id = d.get("session_id") + resume = (d.get("resume_gateway_url") or "").rstrip("/") + sess.resume_url = f"{resume}{_GATEWAY_QS}" if resume else None + sess.user_id = str((d.get("user") or {}).get("id") or "") + log.info("discord_bot: gateway ready (user %s)", sess.user_id) + elif t == "MESSAGE_CREATE" and sess.user_id: + reply = _response_for_message(d, sess.user_id) + if reply: + await _send_reply(d.get("channel_id"), d.get("id"), reply) + + +async def _connection(ws, sess: _Session, resume: bool) -> str: + """Drive one live connection until it closes. Returns the mode for the next + connect: "resume" to reconnect and RESUME, "identify" to start a fresh session.""" + hello = json.loads(await ws.recv()) + if hello.get("op") != _OP_HELLO: + return "identify" # protocol out of step — start clean + interval = float(hello["d"]["heartbeat_interval"]) / 1000.0 + acked = {"ok": True} + hb = asyncio.create_task(_heartbeat(ws, interval, sess, acked)) + try: + if resume and sess.session_id and sess.seq is not None: + await ws.send(json.dumps({"op": _OP_RESUME, "d": { + "token": discord.BOT_TOKEN, + "session_id": sess.session_id, + "seq": sess.seq, + }})) + else: + await ws.send(json.dumps({"op": _OP_IDENTIFY, "d": { + "token": discord.BOT_TOKEN, + "intents": INTENTS, + "properties": {"os": "linux", "browser": "thermograph", "device": "thermograph"}, + }})) + async for raw in ws: + msg = json.loads(raw) + if msg.get("s") is not None: + sess.seq = msg["s"] + op = msg.get("op") + if op == _OP_DISPATCH: + await _dispatch(msg, sess) + elif op == _OP_HEARTBEAT: # server asked for an immediate beat + await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq})) + elif op == _OP_HEARTBEAT_ACK: + acked["ok"] = True + elif op == _OP_RECONNECT: # planned reconnect — resume + return "resume" + elif op == _OP_INVALID_SESSION: + # d == True => the session can still be resumed; else it's dead and + # the next connect must IDENTIFY fresh. + if not msg.get("d"): + sess.session_id = None + sess.resume_url = None + await asyncio.sleep(1 + random.random()) + return "resume" if sess.session_id else "identify" + finally: + hb.cancel() + return "resume" if sess.session_id else "identify" + + +def _close_code(exc: Exception): + """Best-effort websocket close code across websockets library versions (older + exposes .code; newer nests it under .rcvd).""" + code = getattr(exc, "code", None) + if code: + return code + return getattr(getattr(exc, "rcvd", None), "code", None) + + +async def run() -> None: + """Maintain the gateway connection for the app's lifetime: connect, IDENTIFY (or + RESUME), dispatch events, and reconnect with capped exponential backoff. Returns + only on a fatal close (bad token / disallowed intents) or task cancellation, and + never raises out — a misbehaving gateway must not take the app down.""" + if not enabled(): + return + try: + import websockets + except Exception: # noqa: BLE001 - shipped via uvicorn[standard]; guard anyway + log.warning("discord_bot: 'websockets' unavailable; gateway bot disabled") + return + sess = _Session() + mode = "identify" + backoff = _BACKOFF_START + while True: + use_resume = mode == "resume" and bool(sess.resume_url) + url = sess.resume_url if use_resume else _GATEWAY_URL + try: + async with websockets.connect(url, max_size=2 ** 20, open_timeout=30) as ws: + mode = await _connection(ws, sess, resume=use_resume) + backoff = _BACKOFF_START # server-requested reconnect: come back promptly + continue + except asyncio.CancelledError: + log.info("discord_bot: shutting down") + raise + except Exception as exc: # noqa: BLE001 - any drop -> back off and reconnect + if _close_code(exc) in _FATAL_CLOSE: + log.error("discord_bot: fatal gateway close %s; stopping", _close_code(exc)) + return + mode = "resume" if sess.session_id else "identify" + log.warning("discord_bot: gateway dropped (%s); reconnecting in %.0fs", + type(exc).__name__, backoff) + await asyncio.sleep(backoff + random.random()) + backoff = min(backoff * 2, _BACKOFF_MAX) diff --git a/requirements.txt b/requirements.txt index b8bb685..a1841c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,9 @@ alembic==1.14.0 pywebpush==2.0.0 # Ed25519 verification of Discord interaction webhooks (see notifications/discord_interactions.py). PyNaCl==1.5.0 +# Discord gateway bot's websocket client (notifications/discord_bot.py). Already +# pulled in transitively by uvicorn[standard]; pinned here since we import it directly. +websockets==16.0 # Recurring worker-tier jobs — city warming, IndexNow pings (see # notifications/scheduler.py). In-process; no broker/queue needed at this scale. apscheduler==3.11.3 diff --git a/tests/notifications/test_discord_bot.py b/tests/notifications/test_discord_bot.py new file mode 100644 index 0000000..89a1c00 --- /dev/null +++ b/tests/notifications/test_discord_bot.py @@ -0,0 +1,98 @@ +"""Discord gateway bot: the message-handling logic that decides when to reply and +with what. No live gateway — we feed MESSAGE_CREATE payloads straight to the pure +handler and stub the shared /grade lookup so these stay data-independent.""" +import pytest + +from notifications import discord +from notifications import discord_bot as bot +from notifications import discord_interactions as di + +BOT_ID = "999" + + +@pytest.fixture(autouse=True) +def _stub_grade(monkeypatch): + """Reply payload for /grade, mirroring the real shape (embeds + an ephemeral + flag the gateway path must drop).""" + monkeypatch.setattr( + di, "_grade_message", + lambda query: {"embeds": [{"title": f"grade:{query}"}], "flags": 64}) + + +def _msg(content="", *, author_id="1", bot_author=False, mentions=(), guild=True): + m = { + "content": content, + "id": "42", + "channel_id": "77", + "author": {"id": author_id, "bot": bot_author}, + "mentions": [{"id": mid} for mid in mentions], + } + if guild: + m["guild_id"] = "5" + return m + + +# --- when the bot stays silent ---------------------------------------------- + +def test_ignores_other_bots(): + assert bot._response_for_message( + _msg("<@999> Phoenix", author_id="8", bot_author=True, mentions=[BOT_ID]), BOT_ID) is None + + +def test_ignores_its_own_messages(): + assert bot._response_for_message( + _msg("hello", author_id=BOT_ID, mentions=[BOT_ID]), BOT_ID) is None + + +def test_ignores_guild_message_without_a_mention(): + assert bot._response_for_message(_msg("just chatting", mentions=[]), BOT_ID) is None + + +# --- when the bot replies ---------------------------------------------------- + +def test_mention_with_city_returns_grade_without_ephemeral_flag(): + out = bot._response_for_message(_msg("<@999> Phoenix", mentions=[BOT_ID]), BOT_ID) + assert out == {"embeds": [{"title": "grade:Phoenix"}]} # flags dropped + assert "flags" not in out + + +def test_legacy_nickname_mention_is_stripped(): + out = bot._response_for_message(_msg("<@!999> Tokyo", mentions=[BOT_ID]), BOT_ID) + assert out["embeds"][0]["title"] == "grade:Tokyo" + + +def test_dm_needs_no_mention(): + out = bot._response_for_message(_msg("Berlin", guild=False, mentions=[]), BOT_ID) + assert out["embeds"][0]["title"] == "grade:Berlin" + + +def test_mention_with_no_query_gives_help(): + out = bot._response_for_message(_msg("<@999>", mentions=[BOT_ID]), BOT_ID) + assert "content" in out and "mention me" in out["content"].lower() + + +# --- helpers ----------------------------------------------------------------- + +def test_strip_mentions_handles_both_forms(): + assert bot._strip_mentions("<@999> <@!999> Sao Paulo", BOT_ID) == "Sao Paulo" + + +def test_is_dm_is_true_without_guild_id(): + assert bot._is_dm(_msg("x", guild=False)) is True + assert bot._is_dm(_msg("x", guild=True)) is False + + +# --- enabled() gating -------------------------------------------------------- + +def test_enabled_is_off_by_default(monkeypatch): + monkeypatch.delenv("THERMOGRAPH_DISCORD_BOT", raising=False) + monkeypatch.setattr(discord, "BOT_TOKEN", "a-token") + assert bot.enabled() is False + + +def test_enabled_needs_both_flag_and_token(monkeypatch): + monkeypatch.setenv("THERMOGRAPH_DISCORD_BOT", "1") + monkeypatch.setattr(discord, "BOT_TOKEN", "") + assert bot.enabled() is False + monkeypatch.setattr(discord, "BOT_TOKEN", "a-token") + assert bot.enabled() is True diff --git a/web/app.py b/web/app.py index 0de9cf8..990a122 100644 --- a/web/app.py +++ b/web/app.py @@ -1,4 +1,5 @@ """Thermograph API — grade recent local weather against ~45 years of climatology.""" +import asyncio import contextlib import datetime import functools @@ -22,6 +23,7 @@ from api import content_routes from core import audit from data import climate from notifications import digest +from notifications import discord_bot from notifications import discord_interactions as discord_interactions_mod from notifications import discord_link from accounts import db @@ -162,16 +164,25 @@ async def _lifespan(app): # restricts this to processes that are allowed to own it at all — "web" replicas # never start it even if they'd otherwise win the leader election, so a stateless # N-replica web tier can scale without also scaling notifier instances. + bot_task = None if _should_run_notifier(): notify.start() # Same leader gate as the notifier above — the recurring worker jobs # (city warming, IndexNow) must likewise run in exactly one process. scheduler.start() + # Discord gateway bot (opt-in). Discord allows one gateway connection per + # bot token, so it belongs on the same single leader; it rides this event + # loop as a background task rather than a new process. No-op unless + # THERMOGRAPH_DISCORD_BOT is set with a bot token (discord_bot.enabled()). + if discord_bot.enabled(): + bot_task = asyncio.create_task(discord_bot.run(), name="discord-bot") # Deploy/restart marker: one line correlates a metric shift with a boot. audit.log_activity("app.start", {"version": app.version, "role": ROLE, "base": BASE, "notifier": _should_run_notifier(), "pid": os.getpid()}) yield audit.log_activity("app.stop", {"role": ROLE, "pid": os.getpid()}) + if bot_task is not None: + bot_task.cancel() notify.stop() scheduler.stop() await _frontend_client.aclose() From c2ce93fad6a6875b0c78f27733e19a689e9ca500 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 00:40:13 +0000 Subject: [PATCH 2/3] test: reproducible local runner + image boot-smoke (#2) --- .gitignore | 1 + Makefile | 14 +++++++++++++ docker-compose.test.yml | 40 ++++++++++++++++++++++++++++++++++++ scripts/smoke.sh | 45 +++++++++++++++++++++++++++++++++++++++++ scripts/test.sh | 31 ++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+) create mode 100644 Makefile create mode 100644 docker-compose.test.yml create mode 100755 scripts/smoke.sh create mode 100755 scripts/test.sh diff --git a/.gitignore b/.gitignore index 70bf14c..015606a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv/ +.venv-test/ __pycache__/ *.pyc data/cache/*.parquet diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ee12ad2 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +# thermograph-backend — local dev/test targets. +.PHONY: help test smoke clean-venv + +help: ## List targets + @grep -hE '^[a-z][a-zA-Z0-9_-]*:.*## ' $(MAKEFILE_LIST) | awk -F':.*## ' '{printf " %-14s %s\n", $$1, $$2}' + +test: ## Build a py3.12 dev venv and run the hermetic pytest suite (pass ARGS=...) + ./scripts/test.sh $(ARGS) + +smoke: ## Boot the backend image + a throwaway db and assert /healthz + /api/version + ./scripts/smoke.sh + +clean-venv: ## Remove the test venv + rm -rf .venv-test diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..d110466 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,40 @@ +# Boot-smoke harness: run the backend's OWN image against a throwaway TimescaleDB and +# assert it actually boots + serves its contract (build alone doesn't prove boot). Used +# by scripts/smoke.sh; not a deploy file. DB data is tmpfs (ephemeral) — no volume, no +# residue. See Makefile `smoke` and README. +services: + db: + image: timescale/timescaledb:${TIMESCALEDB_TAG:-latest-pg18} + environment: + POSTGRES_USER: thermograph + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-smoke} + POSTGRES_DB: thermograph + tmpfs: + # pg18 images place PGDATA in a major-version subdir under the mount, so mount at + # /var/lib/postgresql (matches the infra compose), not /var/lib/postgresql/data. + - /var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U thermograph -d thermograph"] + interval: 3s + timeout: 5s + retries: 20 + + backend: + # Defaults to a locally-built `:local` image (scripts/smoke.sh builds it). In CI, + # set BACKEND_IMAGE_TAG=sha-<12hex> to smoke the exact published image. + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG:-local} + depends_on: + db: + condition: service_healthy + environment: + THERMOGRAPH_DATABASE_URL: postgresql+asyncpg://thermograph:${POSTGRES_PASSWORD:-smoke}@db:5432/thermograph + # Required at import (web/app.py). The smoke never exercises a proxied page, so an + # unreachable placeholder is fine — it only hits /healthz and {BASE}/api/version. + THERMOGRAPH_FRONTEND_BASE_INTERNAL: http://127.0.0.1:1 + THERMOGRAPH_BASE: "/" + THERMOGRAPH_ENABLE_NOTIFIER: "0" + PORT: "8137" + WORKERS: "1" + ports: + # Host port defaults to 18137 to avoid colliding with a running dev server on 8137. + - "127.0.0.1:${SMOKE_HOST_PORT:-18137}:8137" diff --git a/scripts/smoke.sh b/scripts/smoke.sh new file mode 100755 index 0000000..4854c3b --- /dev/null +++ b/scripts/smoke.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Image boot-smoke: prove the backend IMAGE actually boots and serves its contract +# (a `docker build` succeeding does not prove the container starts). Builds the image +# locally (or uses a provided BACKEND_IMAGE_TAG), runs it + a throwaway TimescaleDB via +# docker-compose.test.yml, waits for /healthz, then asserts /healthz and the version +# endpoint. Always tears the stack down. +# +# ./scripts/smoke.sh # build :local and smoke it +# BACKEND_IMAGE_TAG=sha-abc123 ./scripts/smoke.sh # smoke a published image (CI) +set -euo pipefail +cd "$(dirname "$0")/.." + +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-smoke}" +# The image to smoke, pulled from the registry (no local build). CI passes the just- +# pushed sha (BACKEND_IMAGE_TAG=sha-<12hex>); locally it defaults to a published tag. +export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-v0.0.2-split-ci}" +IMG="${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph-backend/app}:${BACKEND_IMAGE_TAG}" +export SMOKE_HOST_PORT="${SMOKE_HOST_PORT:-18137}" +COMPOSE=(docker compose -f docker-compose.test.yml) +PORT_URL="http://127.0.0.1:${SMOKE_HOST_PORT}" + +cleanup() { echo "==> tearing down"; "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true; } +trap cleanup EXIT + +echo "==> pulling $IMG" +"${COMPOSE[@]}" pull backend + +echo "==> starting backend + db ($IMG)" +"${COMPOSE[@]}" up -d + +echo "==> waiting for /healthz (up to 90s)" +for i in $(seq 1 45); do + if curl -fsS -o /dev/null "$PORT_URL/healthz"; then ok=1; break; fi + sleep 2 +done +[ "${ok:-}" = 1 ] || { echo "!! backend never became healthy"; "${COMPOSE[@]}" logs --tail=40 backend; exit 1; } + +echo "==> asserting contract" +curl -fsS "$PORT_URL/healthz" >/dev/null && echo " /healthz 200 ok" +ver=$(curl -fsS "$PORT_URL/api/version") +echo " /api/version -> $ver" +echo "$ver" | grep -q '"backend_version"[[:space:]]*:[[:space:]]*"2"' \ + || { echo "!! /api/version did not report backend_version=2"; exit 1; } + +echo "==> SMOKE PASSED" diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..1cc6b1f --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Reproducible local test runner: build a Python 3.12 dev venv, install the pinned +# dev deps, and run the hermetic pytest suite. Matches what CI should run. +# +# Why 3.12 explicitly: the runtime deps pin versions that need 3.12, and a common box +# python (pyenv 3.10) ships without the `_sqlite3` extension, which the suite's +# conftest imports — so we don't just use whatever `python3` is on PATH. +# +# ./scripts/test.sh # full suite +# ./scripts/test.sh tests/web # pass pytest args through +set -euo pipefail +cd "$(dirname "$0")/.." + +VENV="${VENV:-.venv-test}" + +if [ ! -x "$VENV/bin/pytest" ]; then + echo "==> Building dev venv ($VENV) on Python 3.12" + if command -v uv >/dev/null 2>&1; then + uv venv --python 3.12 "$VENV" + VIRTUAL_ENV="$VENV" uv pip install -q -r requirements-dev.txt + else + PY="$(command -v python3.12 || echo "$HOME/.local/bin/python3.12")" + "$PY" -c 'import sqlite3' 2>/dev/null || { echo "!! $PY lacks the sqlite3 module — install a 3.12 with sqlite (e.g. via uv)"; exit 1; } + "$PY" -m venv "$VENV" + "$VENV/bin/pip" install -q --upgrade pip + "$VENV/bin/pip" install -q -r requirements-dev.txt + fi +fi + +echo "==> pytest ($("$VENV/bin/python" --version))" +exec "$VENV/bin/python" -m pytest "$@" From 90a7148165e9cbd5271d4b539993ab2493aa2534 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 00:59:03 +0000 Subject: [PATCH 3/3] CI: run the test suite inside the built image (#4) --- .forgejo/workflows/build.yml | 24 +++++++++++++++++++++--- .forgejo/workflows/deploy.yml | 5 ++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 6701686..64989cd 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -9,9 +9,9 @@ name: Build check # the runner's shared capacity=2 contention) without ever settling into a # reliably green check. The image DOES boot correctly standalone -- # independently verified by hand: `docker run` + real alembic migrations + -# `/healthz` returning 200 + a real `/api/v2/place` 200. Full pytest-suite -# migration is separate, deferred follow-up work too, same category as -# thermograph-copy's deferred vendoring. +# `/healthz` returning 200 + a real `/api/v2/place` 200. The full pytest +# suite DOES run here now -- inside the built image (see the test step below), +# which sidesteps those runner quirks entirely. on: # Reusable (uses: ./.forgejo/workflows/build.yml) by pr-build.yml and @@ -38,3 +38,21 @@ jobs: - name: Build run: docker build -t thermograph-backend:ci . + + # The suite is hermetic (sqlite/tmpdir; no network), and the image already + # contains tests/ + every runtime dep (COPY . /app/), so run pytest INSIDE + # the image we just built: the exact interpreter/deps that ship, none of + # the runner-environment quirks that sank the old host-side boot check. + # requirements-dev.txt only layers pytest on top, so the install is tiny. + # -u 0: the image's default user can't write to site-packages. + # --entrypoint sh: the image's entrypoint is alembic-migrate + uvicorn; + # without the override the test command is swallowed as entrypoint args + # and the container just boots the server forever. unset THERMOGRAPH_BASE: + # the image bakes the prod root-mount (/), which moves every route off the + # /thermograph prefix the tests are written against -- requests would fall + # through to the frontend-proxy catch-all and fail with ConnectError. + - name: Run tests in the built image + run: | + docker run --rm -u 0 --entrypoint sh thermograph-backend:ci \ + -c "unset THERMOGRAPH_BASE; pip install -q --no-cache-dir pytest==8.4.1 && python -m pytest tests -q" + diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index 217734f..8af668b 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -7,9 +7,8 @@ name: Deploy backend to beta VPS # versa. thermograph-infra/deploy/deploy.sh persists each service's live tag # host-side, so a single-service roll never disturbs the other. # -# The SSH_HOST/USER/KEY/PORT secrets point at beta. Prod is NOT deployed by a -# workflow -- it's deployed with `terraform apply` on the `release` branch, so -# there is deliberately no release-triggered workflow. appleboy/ssh-action is +# The SSH_HOST/USER/KEY/PORT secrets point at beta. Prod deploys the same way +# from deploy-prod.yml (release branch, PROD_SSH_* secrets). appleboy/ssh-action is # referenced by full GitHub URL because it isn't mirrored in Forgejo's default # action registry. #