From 370a4ffdc19bf4cfd73405740fc47718ed960e9e Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 21:20:35 +0000 Subject: [PATCH 01/15] ERA5 lake: bucket-hosted history primary + SQL indexer service (#15) --- backend/Dockerfile | 3 + backend/data/climate.py | 30 +++- backend/data/era5lake.py | 173 ++++++++++++++++++ backend/deploy/entrypoint.sh | 7 + backend/gen_era5_lake.py | 218 +++++++++++++++++++++++ backend/lake_app.py | 176 ++++++++++++++++++ backend/requirements-seed.txt | 7 + backend/requirements.txt | 3 + backend/seed_era5.py | 28 +-- backend/tests/data/test_climate.py | 38 ++++ backend/tests/data/test_era5lake.py | 90 ++++++++++ backend/tests/web/test_lake_app.py | 100 +++++++++++ infra/deploy/secrets/README.md | 2 +- infra/deploy/stack/autoscale.sh | 4 +- infra/deploy/stack/thermograph-stack.yml | 80 +++++++++ 15 files changed, 934 insertions(+), 25 deletions(-) create mode 100644 backend/data/era5lake.py create mode 100644 backend/gen_era5_lake.py create mode 100644 backend/lake_app.py create mode 100644 backend/tests/data/test_era5lake.py create mode 100644 backend/tests/web/test_lake_app.py diff --git a/backend/Dockerfile b/backend/Dockerfile index e5ed738..95d28ec 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -13,6 +13,9 @@ RUN apt-get update \ # Install deps first so this layer caches across code-only changes. COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt +# Bake DuckDB's httpfs extension so the lake role's S3 reads need no runtime +# extension download (tasks may roll while egress is degraded). +RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')" COPY . /app/ RUN chmod +x /app/deploy/entrypoint.sh diff --git a/backend/data/climate.py b/backend/data/climate.py index fc5e411..0a148ff 100644 --- a/backend/data/climate.py +++ b/backend/data/climate.py @@ -21,6 +21,7 @@ from core import audit from core import metrics import paths from data import climate_store +from data import era5lake from data import meteostat from data import store @@ -789,21 +790,32 @@ def _load_history(cell: dict) -> tuple[pl.DataFrame, dict]: def _serve_stale(): return _with_doy(stale), {"cached": True, "stale": True, "cache_age_days": None} - # Fetch. NASA POWER is the primary source (keyless, independent of Open-Meteo); - # the gusts it lacks are filled from Meteostat inside _fetch_history_nasa. - # Open-Meteo is the fallback when NASA is unavailable — still guarded by its - # rate-limit cooldown. Both sources must return a plausibly-full span before - # being accepted and cached as a complete record (a short/partial response is - # rejected rather than held indefinitely). + # Fetch. The ERA5 lake is first when configured (true ERA5 with measured + # gusts, served from our own bucket — no third-party API in the path); + # unconfigured or missing-point cells fall through at zero cost. NASA + # POWER is next (keyless, independent of Open-Meteo; its missing gusts + # are filled from Meteostat inside _fetch_history_nasa), then Open-Meteo + # — still guarded by its rate-limit cooldown. Every source must return a + # plausibly-full span before being accepted and cached as a complete + # record (a short/partial response is rejected rather than held + # indefinitely). df = None source = None try: - fetched = _fetch_history_nasa(cell) + fetched = era5lake.fetch_history_lake(cell, start=START_DATE) if fetched.height >= MIN_ARCHIVE_DAYS: df = fetched - source = "nasa-power" - except Exception: # noqa: BLE001 - primary unavailable; fall through to Open-Meteo + source = "era5-lake" + except Exception: # noqa: BLE001 - lake unconfigured/miss; fall through to NASA pass + if df is None: + try: + fetched = _fetch_history_nasa(cell) + if fetched.height >= MIN_ARCHIVE_DAYS: + df = fetched + source = "nasa-power" + except Exception: # noqa: BLE001 - primary unavailable; fall through to Open-Meteo + pass if df is None and time.time() >= _archive_cooldown_until: try: fetched = _fetch_history(cell) diff --git a/backend/data/era5lake.py b/backend/data/era5lake.py new file mode 100644 index 0000000..7914745 --- /dev/null +++ b/backend/data/era5lake.py @@ -0,0 +1,173 @@ +"""The ERA5 lake: layout, grid math, and read clients for the bucket-hosted archive. + +The lake is a set of parquet files in an S3-compatible bucket (Contabo Object +Storage, ``era5-thermograph``), extracted once from the public Earthmover ERA5 +Icechunk archive by ``gen_era5_lake.py`` and read three ways: + +- the **lake service** (``lake_app.py``, prod-only Swarm service) serves point + histories and SQL-style queries with a local disk cache in front; +- the **web/worker backend** asks the lake service first + (``THERMOGRAPH_LAKE_URL``), or reads the bucket directly when only S3 + credentials are configured (beta/LAN, no lake service); +- ad-hoc analysis hits the hive table straight from the bucket. + +Two layouts, one manifest (all under ``era5/``): + +- ``daily/tile=_/year=/month=/part.parquet`` — the analytical + table, hive-partitioned by location/year/month. The location grain is a + 12x12-point tile (~3°x3°), aligned to the Earthmover store's chunk grid; a + per-point x per-month grain would be ~170M objects, which no object store + enjoys. +- ``points/lat=/lon=.parquet`` — the serving projection: one file per + ERA5 grid point holding its full daily history, so the hot path (one cell's + history) is a single GET. +- ``manifest.parquet`` — Iceberg-manifest-style index of every point file: + grid indices, coordinates, row count, date span. Readers prune with it and + the extractor resumes from it. + +Columns everywhere: the finalized history-store schema (date, tmax, tmin, +precip, wind, gust, humid, fmax, fmin, feels, doy — °F/in/mph/%, as +``climate._finalize_approximated`` emits it), so a lake frame drops straight +into ``climate._write_history_backed`` exactly like a NASA fetch does. +""" +import io +import os + +import polars as pl + +# --- ERA5 native grid (0.25°, 721 x 1440, lat 90..-90, lon 0..359.75) --------- +LAT_N = 721 +LON_N = 1440 +STEP = 0.25 +TILE = 12 # matches the Earthmover chunk grid (8736h x 12 x 12) + +PREFIX = "era5" +MANIFEST_KEY = f"{PREFIX}/manifest.parquet" + +MIN_LAKE_DAYS = 10_000 # a plausibly-full 1980+ record, same spirit as + # climate.MIN_ARCHIVE_DAYS (rejects partial files) + + +def to_idx(lat: float, lon: float) -> tuple[int, int]: + """Nearest ERA5 grid indices for a lat/lon. lat_idx runs north->south.""" + i = round((90.0 - lat) / STEP) + j = round((lon % 360.0) / STEP) % LON_N + return min(max(i, 0), LAT_N - 1), j + + +def to_coords(i: int, j: int) -> tuple[float, float]: + """Center lat/lon ([-180, 180) longitude) of grid indices.""" + lon = j * STEP + return 90.0 - i * STEP, lon - 360.0 if lon >= 180.0 else lon + + +def tile_of(i: int, j: int) -> tuple[int, int]: + return i // TILE, j // TILE + + +def point_key(i: int, j: int) -> str: + return f"{PREFIX}/points/lat={i}/lon={j}.parquet" + + +def daily_part_key(ti: int, tj: int, year: int, month: int) -> str: + return f"{PREFIX}/daily/tile={ti}_{tj}/year={year}/month={month:02d}/part.parquet" + + +# --- Config (all optional: unset means "no lake configured") ------------------ + +def lake_url() -> str: + """Lake service base URL (prod: the Swarm VIP), '' when not deployed.""" + return os.environ.get("THERMOGRAPH_LAKE_URL", "").rstrip("/") + + +def s3_config() -> "dict | None": + """Bucket access for direct reads/writes, None unless fully configured. + Contabo is path-style; region is a signing formality there ('default').""" + key = os.environ.get("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "") + secret = os.environ.get("THERMOGRAPH_LAKE_S3_SECRET_KEY", "") + if not (key and secret): + return None + return { + "endpoint": os.environ.get("THERMOGRAPH_LAKE_S3_ENDPOINT", + "https://eu2.contabostorage.com"), + "bucket": os.environ.get("THERMOGRAPH_LAKE_S3_BUCKET", "era5-thermograph"), + "region": os.environ.get("THERMOGRAPH_LAKE_S3_REGION", "default"), + "access_key": key, + "secret_key": secret, + } + + +def local_dir() -> str: + """Filesystem mirror of the lake layout — tests and offline dev reads.""" + return os.environ.get("THERMOGRAPH_LAKE_LOCAL_DIR", "") + + +def storage_options(cfg: dict) -> dict: + """polars/object_store options for scan_parquet against the bucket.""" + return { + "aws_access_key_id": cfg["access_key"], + "aws_secret_access_key": cfg["secret_key"], + "aws_region": cfg["region"], + "endpoint_url": cfg["endpoint"], + # Contabo serves buckets on the path, not as a subdomain. + "aws_virtual_hosted_style_request": "false", + } + + +# --- Point-history reads (the app's hot path) ---------------------------------- + +class LakeUnavailable(Exception): + """No lake configured, point not in the lake, or the read failed.""" + + +def _read_local(i: int, j: int) -> pl.DataFrame: + path = os.path.join(local_dir(), point_key(i, j)) + if not os.path.exists(path): + raise LakeUnavailable(f"no local lake file for point ({i}, {j})") + return pl.read_parquet(path) + + +def _read_service(i: int, j: int) -> pl.DataFrame: + import httpx # noqa: PLC0415 - keep module import light for the extractor venv + + lat, lon = to_coords(i, j) + r = httpx.get(f"{lake_url()}/history", params={"lat": lat, "lon": lon}, + timeout=httpx.Timeout(20.0, connect=3.0)) + if r.status_code == 404: + raise LakeUnavailable(f"point ({i}, {j}) not in the lake") + r.raise_for_status() + return pl.read_parquet(io.BytesIO(r.content)) + + +def _read_bucket(i: int, j: int, cfg: dict) -> pl.DataFrame: + try: + return pl.read_parquet(f"s3://{cfg['bucket']}/{point_key(i, j)}", + storage_options=storage_options(cfg)) + except Exception as e: # noqa: BLE001 - object_store errors are not one type + raise LakeUnavailable(f"bucket read failed for point ({i}, {j}): {e}") from e + + +def fetch_history_lake(cell: dict, start: "str | None" = None) -> pl.DataFrame: + """Finalized daily history for the ERA5 point nearest a grid cell, from the + first configured source: local dir -> lake service -> bucket. The lake + stores each point's whole 1940+ record; pass ``start`` (the caller's + grading window, e.g. climate.START_DATE) to slice — grading semantics stay + the caller's, the deep record stays in the lake. Raises LakeUnavailable + when unconfigured, missing, or implausibly short.""" + import datetime # noqa: PLC0415 + + i, j = to_idx(cell["center_lat"], cell["center_lon"]) + if local_dir(): + df = _read_local(i, j) + elif lake_url(): + df = _read_service(i, j) + else: + cfg = s3_config() + if cfg is None: + raise LakeUnavailable("no lake configured") + df = _read_bucket(i, j, cfg) + if start is not None: + df = df.filter(pl.col("date") >= datetime.date.fromisoformat(start)) + if df.height < MIN_LAKE_DAYS: + raise LakeUnavailable(f"lake point ({i}, {j}) is short: {df.height} rows") + return df.drop([c for c in ("lat_idx", "lon_idx") if c in df.columns]) diff --git a/backend/deploy/entrypoint.sh b/backend/deploy/entrypoint.sh index 9c68987..9b6f399 100755 --- a/backend/deploy/entrypoint.sh +++ b/backend/deploy/entrypoint.sh @@ -62,6 +62,13 @@ run_migrations() { done } +# The lake role serves the ERA5 bucket only — no database, so no migrations: a +# lake replica must come up (and stay up) even when Postgres is down. +if [ "${THERMOGRAPH_ROLE:-}" = "lake" ]; then + echo "==> Starting lake service on 0.0.0.0:${PORT:-8141} with ${WORKERS:-1} worker(s)" + exec uvicorn lake_app:app --host 0.0.0.0 --port "${PORT:-8141}" --workers "${WORKERS:-1}" +fi + # One-shot migrate mode (`entrypoint.sh migrate`, or THERMOGRAPH_MIGRATE_ONLY=1): run # the migration and exit — a one-shot task, so it runs once against the DB rather # than racing per web replica. diff --git a/backend/gen_era5_lake.py b/backend/gen_era5_lake.py new file mode 100644 index 0000000..f058674 --- /dev/null +++ b/backend/gen_era5_lake.py @@ -0,0 +1,218 @@ +"""Extract the ERA5 lake: Earthmover Icechunk archive -> the Thermograph bucket. + +Builds the lake that ``data/era5lake.py`` reads (layout documented there): +per-point serving files, the tile/year/month hive table, and the manifest. +Pulls are tile-aligned to the source's chunk grid (8736h x 12 x 12) — one +46-year pull (~19s measured) decodes all 144 points of a tile, so per-point +pulls would waste 144x the wire. Roughly: ~600 tiles cover the curated +cities (hours), ~3000 cover land (a day or two); both resume from the +manifest, so ctrl-C and re-run is always safe. + + pip install -r requirements-seed.txt 'numcodecs[pcodec]' boto3 + python gen_era5_lake.py --cities --dry-run # one tile, no writes + python gen_era5_lake.py --cities # tiles covering cities.json + python gen_era5_lake.py --tiles 20:30,40:60 # explicit tile-index window + python gen_era5_lake.py --land # every land-containing tile + +Destination: the bucket from THERMOGRAPH_LAKE_S3_* (see era5lake.s3_config), +or a local mirror when THERMOGRAPH_LAKE_LOCAL_DIR is set (tests/rehearsal). +Seed-only deps (icechunk/xarray/zarr) stay out of the app image, same policy +as seed_era5.py. +""" +import argparse +import datetime +import io +import os +import sys +import time + +import polars as pl + +from data import cities +from data import climate +from data import era5lake +from seed_era5 import ERA5_BUCKET, ERA5_GROUP, ERA5_PREFIX, ERA5_REGION, ERA5_VARS, \ + hourly_to_daily + +# The lake keeps each location's WHOLE available record (the store reaches +# back to 1940); the app slices to its own grading window (climate.START_DATE) +# at read time, so the extra decades are analytical surface, not a grading +# change. +START_DATE = "1940-01-01" + + +def _open_store(): + import icechunk # noqa: PLC0415 - seed-only dep + import xarray as xr # noqa: PLC0415 + + storage = icechunk.s3_storage( + bucket=ERA5_BUCKET, prefix=ERA5_PREFIX, region=ERA5_REGION, anonymous=True) + repo = icechunk.Repository.open(storage) + return xr.open_zarr(repo.readonly_session("main").store, + group=ERA5_GROUP, consolidated=False) + + +class _Sink: + """Writes lake objects to the bucket (boto3) or a local mirror dir.""" + + def __init__(self): + self.local = era5lake.local_dir() + self.cfg = era5lake.s3_config() + if not self.local and self.cfg is None: + sys.exit("no destination: set THERMOGRAPH_LAKE_S3_* or " + "THERMOGRAPH_LAKE_LOCAL_DIR") + self._s3 = None + if not self.local: + import boto3 # noqa: PLC0415 - extractor-only dep + self._s3 = boto3.client( + "s3", endpoint_url=self.cfg["endpoint"], + region_name=self.cfg["region"], + aws_access_key_id=self.cfg["access_key"], + aws_secret_access_key=self.cfg["secret_key"]) + + def put(self, key: str, df: pl.DataFrame) -> None: + if self.local: + path = os.path.join(self.local, key) + os.makedirs(os.path.dirname(path), exist_ok=True) + df.write_parquet(path) + return + buf = io.BytesIO() + df.write_parquet(buf) + self._s3.put_object(Bucket=self.cfg["bucket"], Key=key, + Body=buf.getvalue()) + + def read_manifest(self) -> "pl.DataFrame | None": + try: + if self.local: + return pl.read_parquet( + os.path.join(self.local, era5lake.MANIFEST_KEY)) + obj = self._s3.get_object(Bucket=self.cfg["bucket"], + Key=era5lake.MANIFEST_KEY) + return pl.read_parquet(io.BytesIO(obj["Body"].read())) + except Exception: # noqa: BLE001 - a fresh lake has no manifest yet + return None + + +def _tile_frame(ds, ti: int, tj: int, end: str) -> "pl.DataFrame | None": + """Pull one 12x12 tile's hourly series and return the finalized daily frame + for all its points (lat_idx, lon_idx, date, ...). None for all-sea tiles.""" + t = era5lake.TILE + i0, j0 = ti * t, tj * t + lat_sl = slice(i0, min(i0 + t, era5lake.LAT_N)) + lon_sl = slice(j0, j0 + t) + if "lsm" in ds: # land-sea mask: skip all-sea tiles outright + lsm = ds["lsm"].isel(latitude=lat_sl, longitude=lon_sl) + if "valid_time" in lsm.dims: # static field, stored with a time axis + lsm = lsm.isel(valid_time=-1) + if float(lsm.max()) < 0.05: + return None + sub = (ds[list(ERA5_VARS)] + .isel(latitude=lat_sl, longitude=lon_sl) + .sel(valid_time=slice(START_DATE, end)) + .load()) + + frames = [] + for a, lat in enumerate(sub.latitude.values): + for b, lon in enumerate(sub.longitude.values): + cols = {"time": sub.valid_time.values} + for src, dst in ERA5_VARS.items(): + cols[dst] = sub[src].isel(latitude=a, longitude=b).values + daily = climate._finalize_approximated(hourly_to_daily(pl.DataFrame(cols))) + i, j = era5lake.to_idx(float(lat), float(lon)) + frames.append(daily.with_columns( + pl.lit(i, dtype=pl.Int32).alias("lat_idx"), + pl.lit(j, dtype=pl.Int32).alias("lon_idx"))) + return pl.concat(frames) + + +def _write_tile(sink: _Sink, ti: int, tj: int, tile_df: pl.DataFrame) -> pl.DataFrame: + """Write one tile's point files + hive partitions; return its manifest rows.""" + rows = [] + for (i, j), pdf in tile_df.group_by(["lat_idx", "lon_idx"]): + pdf = pdf.sort("date") + sink.put(era5lake.point_key(i, j), pdf.drop(["lat_idx", "lon_idx"])) + lat, lon = era5lake.to_coords(i, j) + rows.append({"lat_idx": i, "lon_idx": j, "lat": lat, "lon": lon, + "tile": f"{ti}_{tj}", "rows": pdf.height, + "date_min": str(pdf["date"].min()), + "date_max": str(pdf["date"].max())}) + parts = tile_df.with_columns( + pl.col("date").dt.year().alias("year"), + pl.col("date").dt.month().alias("month")) + for (year, month), mdf in parts.group_by(["year", "month"]): + sink.put(era5lake.daily_part_key(ti, tj, year, month), + mdf.drop(["year", "month"]).sort(["lat_idx", "lon_idx", "date"])) + return pl.DataFrame(rows) + + +def _city_tiles() -> list: + from data import grid # noqa: PLC0415 + + tiles = set() + for c in cities.all_cities(): + cell = grid.snap(c["lat"], c["lon"]) + tiles.add(era5lake.tile_of( + *era5lake.to_idx(cell["center_lat"], cell["center_lon"]))) + return sorted(tiles) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--cities", action="store_true") + ap.add_argument("--land", action="store_true") + ap.add_argument("--tiles", help="ti0:ti1,tj0:tj1 tile-index window") + ap.add_argument("--limit", type=int) + ap.add_argument("--overwrite", action="store_true") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + if args.cities: + todo = _city_tiles() + elif args.tiles: + (a, b), (c, d) = (p.split(":") for p in args.tiles.split(",")) + todo = [(ti, tj) for ti in range(int(a), int(b)) + for tj in range(int(c), int(d))] + elif args.land: + todo = [(ti, tj) for ti in range(era5lake.LAT_N // era5lake.TILE + 1) + for tj in range(era5lake.LON_N // era5lake.TILE)] + else: + ap.error("pick one of --cities / --tiles / --land") + if args.limit: + todo = todo[:args.limit] + + sink = _Sink() + ds = _open_store() + end = (datetime.date.today() + - datetime.timedelta(days=climate.ARCHIVE_LATENCY_DAYS)).isoformat() + + manifest = sink.read_manifest() + done = set() if manifest is None or args.overwrite else \ + set(manifest["tile"].unique().to_list()) + + written = skipped = empty = 0 + for n, (ti, tj) in enumerate(todo, 1): + if f"{ti}_{tj}" in done: + skipped += 1 + continue + t0 = time.time() + tile_df = _tile_frame(ds, ti, tj, end) + if tile_df is None: + empty += 1 + continue + if args.dry_run: + print(f"[dry-run] tile {ti}_{tj}: {tile_df.height} rows " + f"({time.time() - t0:.1f}s); nothing written") + print(tile_df.head()) + return + rows = _write_tile(sink, ti, tj, tile_df) + manifest = rows if manifest is None else pl.concat( + [manifest.filter(pl.col("tile") != f"{ti}_{tj}"), rows]) + sink.put(era5lake.MANIFEST_KEY, manifest) # after every tile: resumable + written += 1 + print(f"[{n}/{len(todo)}] tile {ti}_{tj}: {rows.height} points " + f"in {time.time() - t0:.1f}s") + print(f"done: tiles written={written} skipped(done)={skipped} sea={empty}") + + +if __name__ == "__main__": + main() diff --git a/backend/lake_app.py b/backend/lake_app.py new file mode 100644 index 0000000..6618b35 --- /dev/null +++ b/backend/lake_app.py @@ -0,0 +1,176 @@ +"""The lake service: low-latency SQL-ish reads over the ERA5 bucket (prod-only). + +A small FastAPI app run as the Swarm ``lake`` service (same backend image, +``THERMOGRAPH_ROLE=lake`` — the entrypoint launches this module instead of the +web app; no database, no migrations). Web replicas call it instead of touching +the bucket themselves, so a cell's history costs one LAN round-trip against a +warm disk cache instead of an S3 fetch, and analytical queries get +partition-pruned scans without shipping S3 credentials to every web task. + +Endpoints: +- ``GET /healthz`` liveness (also reports manifest row count) +- ``GET /history?lat=&lon=`` the hot path: nearest ERA5 point's full daily + history as parquet bytes, disk-cached per point + (a plain single-file read — no engine needed) +- ``POST /query {"sql": ..}`` SELECT-only SQL over two views: ``era5_daily`` + (the hive table, partition-pruned on + tile/year/month predicates) and ``manifest``. + Runs on DuckDB — a real lake engine (full SQL, + parallel scans, hive + row-group pushdown); the + httpfs extension is baked into the image so S3 + needs no runtime extension fetch. + +Scale-out safe: replicas share nothing — each keeps its own cache under +THERMOGRAPH_LAKE_CACHE (a per-task local volume), so 1..2 replicas behind the +Swarm VIP need no coordination. +""" +import os +import re +import threading +import time + +import polars as pl +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from data import era5lake + +CACHE_DIR = os.environ.get("THERMOGRAPH_LAKE_CACHE", "/state/lake-cache") +MANIFEST_TTL = 6 * 3600 # re-pull the manifest at most this often +QUERY_ROW_CAP = 10_000 # /query responses are JSON; keep them bounded + +app = FastAPI(title="thermograph-lake") + +_manifest_lock = threading.Lock() +_manifest: "pl.DataFrame | None" = None +_manifest_at = 0.0 + + +def _source(): + """(mode, cfg) for this process: a local lake dir (tests/dev) or the bucket.""" + if era5lake.local_dir(): + return "local", None + cfg = era5lake.s3_config() + return ("bucket", cfg) if cfg else ("none", None) + + +def _manifest_frame() -> "pl.DataFrame | None": + global _manifest, _manifest_at + with _manifest_lock: + if _manifest is not None and time.time() - _manifest_at < MANIFEST_TTL: + return _manifest + mode, cfg = _source() + try: + if mode == "local": + df = pl.read_parquet( + os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY)) + elif mode == "bucket": + df = pl.read_parquet( + f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}", + storage_options=era5lake.storage_options(cfg)) + else: + return None + except Exception: # noqa: BLE001 - empty/unreachable lake: serve health, 404 reads + return None + _manifest, _manifest_at = df, time.time() + return df + + +@app.get("/healthz") +def healthz(): + mode, _ = _source() + m = _manifest_frame() + return {"status": "ok", "source": mode, + "points": None if m is None else m.height} + + +@app.get("/history") +def history(lat: float, lon: float): + """One ERA5 point's full daily history, as parquet bytes (cached on disk — + the cache file IS the response body, so a warm hit is a sendfile).""" + i, j = era5lake.to_idx(lat, lon) + path = os.path.join(CACHE_DIR, f"p{i}_{j}.parquet") + if not os.path.exists(path): + mode, cfg = _source() + if mode == "none": + raise HTTPException(status_code=503, detail="lake not configured") + m = _manifest_frame() + if m is not None and m.filter( + (pl.col("lat_idx") == i) & (pl.col("lon_idx") == j)).is_empty(): + raise HTTPException(status_code=404, detail="point not in the lake") + try: + if mode == "local": + df = era5lake._read_local(i, j) + else: + df = era5lake._read_bucket(i, j, cfg) + except era5lake.LakeUnavailable as e: + raise HTTPException(status_code=404, detail=str(e)) from e + os.makedirs(CACHE_DIR, exist_ok=True) + tmp = f"{path}.{os.getpid()}.partial" + df.write_parquet(tmp) + os.replace(tmp, path) # atomic: concurrent requests see whole files only + return FileResponse(path, media_type="application/vnd.apache.parquet", + headers={"X-Lake-Point": f"{i},{j}"}) + + +class Query(BaseModel): + sql: str + + +_FORBIDDEN = re.compile( + r"\b(insert|update|delete|drop|create|alter|attach|copy|truncate|install|load" + r"|export|import|pragma|set|call)\b", re.I) + + +def _connect(): + """A fresh in-memory DuckDB with the lake views registered. Per-request: + connections are cheap, and a throwaway one means a query can't leave state + behind for the next. Returns None when no lake is configured.""" + import duckdb # noqa: PLC0415 - only the lake role pays the import + + mode, cfg = _source() + if mode == "local": + daily_glob = os.path.join(era5lake.local_dir(), era5lake.PREFIX, + "daily/*/*/*/*.parquet") + manifest_src = os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY) + elif mode == "bucket": + daily_glob = f"s3://{cfg['bucket']}/{era5lake.PREFIX}/daily/*/*/*/*.parquet" + manifest_src = f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}" + else: + return None + con = duckdb.connect() + if mode == "bucket": + con.execute("LOAD httpfs") # baked into the image at build time + host = cfg["endpoint"].split("://", 1)[-1] + con.execute(f"SET s3_endpoint='{host}'") + con.execute("SET s3_url_style='path'") + con.execute(f"SET s3_region='{cfg['region']}'") + con.execute("SET s3_access_key_id=?", [cfg["access_key"]]) + con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]]) + con.execute(f""" + CREATE VIEW era5_daily AS + SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""") + con.execute(f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')") + return con + + +@app.post("/query") +def query(q: Query): + sql = q.sql.strip().rstrip(";").strip() + if not sql.lower().startswith(("select", "with")) or ";" in sql \ + or _FORBIDDEN.search(sql): + raise HTTPException(status_code=400, + detail="only a single SELECT statement is allowed") + con = _connect() + if con is None: + raise HTTPException(status_code=503, detail="lake not configured") + try: + out = con.execute( + f"SELECT * FROM ({sql}) LIMIT {QUERY_ROW_CAP}").fetchall() + cols = [d[0] for d in con.description] + except Exception as e: # noqa: BLE001 - surface the engine's message to the caller + raise HTTPException(status_code=400, detail=f"query failed: {e}") from e + finally: + con.close() + return {"columns": cols, "rows": out, "row_cap": QUERY_ROW_CAP} diff --git a/backend/requirements-seed.txt b/backend/requirements-seed.txt index 5791fa4..5c33726 100644 --- a/backend/requirements-seed.txt +++ b/backend/requirements-seed.txt @@ -7,3 +7,10 @@ icechunk xarray zarr +# The live store compresses with PCodec; without this extra every read fails +# with "codec not available: 'pcodec'". +numcodecs[pcodec] +# gen_era5_lake.py's bucket writer. +boto3 +numcodecs[pcodec] +boto3 diff --git a/backend/requirements.txt b/backend/requirements.txt index 62fb0fc..127dc27 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -25,3 +25,6 @@ websockets==16.0 # Recurring worker-tier jobs — city warming, IndexNow pings (see # notifications/scheduler.py). In-process; no broker/queue needed at this scale. apscheduler==3.11.3 +# The lake role's SQL engine (lake_app.py /query): embedded, parallel +# partition-pruned scans over the ERA5 hive table. Only lake_app imports it. +duckdb==1.4.2 diff --git a/backend/seed_era5.py b/backend/seed_era5.py index 03be936..4d8e1b5 100644 --- a/backend/seed_era5.py +++ b/backend/seed_era5.py @@ -16,10 +16,10 @@ Idempotent by default: a cell that already has a cached history is skipped (pass --overwrite to reseed). Not run in CI or the app image — the icechunk/xarray/zarr stack lives only in requirements-seed.txt. -IMPORTANT — the S3/Icechunk *access* layer (_open_store / fetch_point) targets a -young, fast-moving API and the store's exact variable/coordinate names; it is NOT -exercised by the test suite. VERIFY it with `--dry-run` on the seed box before a -full run. The hourly→daily transform (hourly_to_daily) IS unit-tested and is the +The access layer was VERIFIED against the live store 2026-07-23 (prefix +`icechunkV2`, group `single/temporal`, coord `valid_time`, ECMWF short variable +names, and the pcodec codec — install `numcodecs[pcodec]`). Re-verify with +`--dry-run` if the store revs. The hourly→daily transform (hourly_to_daily) IS unit-tested and is the part that must be numerically correct. Source: https://registry.opendata.aws/earthmover-era5/ (group="temporal", anonymous) @@ -38,19 +38,19 @@ START_DATE = climate.START_DATE # "1980-01-01" # Anonymous AWS Open Data Icechunk store; overridable in case the registry moves. ERA5_BUCKET = os.environ.get("THERMOGRAPH_ERA5_BUCKET", "earthmover-icechunk-era5") -ERA5_PREFIX = os.environ.get("THERMOGRAPH_ERA5_PREFIX", "") # verify on the seed box +ERA5_PREFIX = os.environ.get("THERMOGRAPH_ERA5_PREFIX", "icechunkV2") ERA5_REGION = os.environ.get("THERMOGRAPH_ERA5_REGION", "us-east-1") -ERA5_GROUP = os.environ.get("THERMOGRAPH_ERA5_GROUP", "temporal") # time-contiguous layout +ERA5_GROUP = os.environ.get("THERMOGRAPH_ERA5_GROUP", "single/temporal") # time-contiguous layout # ERA5 store variable name -> our short name. ERA5 gusts (i10fg) are real here, so the # seed keeps genuine measured gusts rather than the Meteostat/estimate fallback. ERA5_VARS = { - "2m_temperature": "t2m", # K - "2m_dewpoint_temperature": "d2m", # K (→ RH with t2m) - "total_precipitation": "tp", # m, hourly accumulation - "10m_u_component_of_wind": "u10", # m/s - "10m_v_component_of_wind": "v10", # m/s - "instantaneous_10m_wind_gust": "i10fg", # m/s + "t2m": "t2m", # 2m temperature, K + "d2m": "d2m", # 2m dewpoint, K (→ RH with t2m) + "tp": "tp", # total precipitation, m, hourly accumulation + "u10": "u10", # 10m wind u, m/s + "v10": "v10", # 10m wind v, m/s + "fg10": "i10fg", # 10m wind gust, m/s (store short name; transform expects i10fg) } _M_TO_IN = 39.37007874 @@ -117,9 +117,9 @@ def fetch_point(ds, lat: float, lon: float, start: str, end: str) -> pl.DataFram may differ per store revision — verify with --dry-run.""" sub = (ds[list(ERA5_VARS)] .sel(latitude=lat, longitude=lon % 360.0, method="nearest") - .sel(time=slice(start, end)) + .sel(valid_time=slice(start, end)) .load()) - data = {"time": sub["time"].values} + data = {"time": sub["valid_time"].values} for src, dst in ERA5_VARS.items(): data[dst] = sub[src].values return pl.DataFrame(data) diff --git a/backend/tests/data/test_climate.py b/backend/tests/data/test_climate.py index 79c3d71..95e869f 100644 --- a/backend/tests/data/test_climate.py +++ b/backend/tests/data/test_climate.py @@ -311,6 +311,44 @@ def _full_history_frame(n=4000): })) +def test_lake_is_first_history_source(monkeypatch, tmp_path): + """A configured ERA5 lake short-circuits the whole third-party chain: neither + NASA POWER nor Open-Meteo is consulted.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "lake_primary", "center_lat": 47.6, "center_lon": -122.3} + + full = _full_history_frame() + monkeypatch.setattr(climate.era5lake, "fetch_history_lake", + lambda c, start=None: full) + def _nasa_should_not_run(c): raise AssertionError("NASA should not be called") + def _om_should_not_run(c): raise AssertionError("Open-Meteo should not be called") + monkeypatch.setattr(climate, "_fetch_history_nasa", _nasa_should_not_run) + monkeypatch.setattr(climate, "_fetch_history", _om_should_not_run) + + df, meta = climate._load_history(cell) + assert meta["source"] == "era5-lake" + assert df.height == full.height + + +def test_lake_miss_falls_through_to_nasa(monkeypatch, tmp_path): + """An unconfigured lake (or a point outside it) costs nothing: NASA serves + exactly as before the lake existed.""" + monkeypatch.setattr(climate, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(climate, "_archive_cooldown_until", 0.0) + cell = {"id": "lake_miss", "center_lat": 47.6, "center_lon": -122.3} + + def _no_lake(c, start=None): + raise climate.era5lake.LakeUnavailable("no lake configured") + monkeypatch.setattr(climate.era5lake, "fetch_history_lake", _no_lake) + full = _full_history_frame() + monkeypatch.setattr(climate, "_fetch_history_nasa", lambda c: full) + + df, meta = climate._load_history(cell) + assert meta["source"] == "nasa-power" + assert df.height == full.height + + def test_nasa_is_primary_history_source(monkeypatch, tmp_path): """NASA POWER is the primary history source: a full NASA frame is accepted and Open-Meteo is never consulted.""" diff --git a/backend/tests/data/test_era5lake.py b/backend/tests/data/test_era5lake.py new file mode 100644 index 0000000..198a5cf --- /dev/null +++ b/backend/tests/data/test_era5lake.py @@ -0,0 +1,90 @@ +"""The ERA5 lake: grid math, layout keys, and the read/slice client (no network +— the local-dir source stands in for the bucket).""" +import datetime + +import polars as pl +import pytest + +from data import era5lake + + +def _daily_frame(start="1940-01-01", days=32000): + d0 = datetime.date.fromisoformat(start) + dates = [d0 + datetime.timedelta(days=k) for k in range(days)] + n = len(dates) + return pl.DataFrame({ + "date": dates, + "tmax": [70.0] * n, "tmin": [50.0] * n, "precip": [0.1] * n, + "wind": [8.0] * n, "gust": [14.0] * n, "humid": [55.0] * n, + "fmax": [71.0] * n, "fmin": [48.0] * n, + }) + + +def test_grid_roundtrip_and_bounds(): + # Nearest-index snap, poles clamped, longitude wrapped to [0, 360). + assert era5lake.to_idx(90.0, 0.0) == (0, 0) + assert era5lake.to_idx(-90.0, 0.0) == (720, 0) + assert era5lake.to_idx(51.5074, -0.1278) == (154, 1439) + i, j = era5lake.to_idx(35.6762, 139.6503) + lat, lon = era5lake.to_coords(i, j) + assert abs(lat - 35.6762) <= era5lake.STEP / 2 + 1e-9 + assert abs(lon - 139.6503) <= era5lake.STEP / 2 + 1e-9 + + +def test_tile_and_keys_align_with_hive_layout(): + assert era5lake.tile_of(154, 1439) == (12, 119) + assert era5lake.point_key(154, 1439) == "era5/points/lat=154/lon=1439.parquet" + assert era5lake.daily_part_key(12, 119, 1994, 3) == \ + "era5/daily/tile=12_119/year=1994/month=03/part.parquet" + + +def test_s3_config_requires_both_keys(monkeypatch): + monkeypatch.delenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", raising=False) + assert era5lake.s3_config() is None + monkeypatch.setenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "k") + assert era5lake.s3_config() is None + monkeypatch.setenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", "s") + cfg = era5lake.s3_config() + assert cfg["bucket"] == "era5-thermograph" + opts = era5lake.storage_options(cfg) + assert opts["aws_virtual_hosted_style_request"] == "false" + + +def _write_point(tmp_path, cell, df): + i, j = era5lake.to_idx(cell["center_lat"], cell["center_lon"]) + path = tmp_path / era5lake.point_key(i, j) + path.parent.mkdir(parents=True, exist_ok=True) + df.write_parquet(path) + + +CELL = {"center_lat": 51.5, "center_lon": -0.125} + + +def test_fetch_slices_to_start_and_keeps_schema(monkeypatch, tmp_path): + monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path)) + _write_point(tmp_path, CELL, _daily_frame()) + df = era5lake.fetch_history_lake(CELL, start="1980-01-01") + assert df["date"].min() == datetime.date(1980, 1, 1) + assert set(df.columns) == {"date", "tmax", "tmin", "precip", "wind", + "gust", "humid", "fmax", "fmin"} + # Without a start, the whole 1940+ record comes back. + assert era5lake.fetch_history_lake(CELL).height == 32000 + + +def test_fetch_rejects_short_and_missing(monkeypatch, tmp_path): + monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path)) + with pytest.raises(era5lake.LakeUnavailable): + era5lake.fetch_history_lake(CELL) # nothing written + _write_point(tmp_path, CELL, _daily_frame(days=500)) + with pytest.raises(era5lake.LakeUnavailable): # implausibly short record + era5lake.fetch_history_lake(CELL) + + +def test_fetch_unconfigured_raises(monkeypatch): + monkeypatch.delenv("THERMOGRAPH_LAKE_LOCAL_DIR", raising=False) + monkeypatch.delenv("THERMOGRAPH_LAKE_URL", raising=False) + monkeypatch.delenv("THERMOGRAPH_LAKE_S3_ACCESS_KEY", raising=False) + monkeypatch.delenv("THERMOGRAPH_LAKE_S3_SECRET_KEY", raising=False) + with pytest.raises(era5lake.LakeUnavailable): + era5lake.fetch_history_lake(CELL) diff --git a/backend/tests/web/test_lake_app.py b/backend/tests/web/test_lake_app.py new file mode 100644 index 0000000..ea832c8 --- /dev/null +++ b/backend/tests/web/test_lake_app.py @@ -0,0 +1,100 @@ +"""The lake service over a local-dir lake fixture: health, the point hot path +(+ its disk cache), and the SELECT-only SQL surface.""" +import datetime +import io + +import polars as pl +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture +def lake(tmp_path, monkeypatch): + """A tiny two-point lake (points, hive partitions, manifest) on disk.""" + from data import era5lake + + points = [(154, 1439), (154, 0)] # around London + d0 = datetime.date(1940, 1, 1) + dates = [d0 + datetime.timedelta(days=k) for k in range(12000)] + n = len(dates) + rows = [] + for i, j in points: + df = pl.DataFrame({ + "date": dates, + "tmax": [70.0] * n, "tmin": [50.0] * n, "precip": [0.1] * n, + "wind": [8.0] * n, "gust": [14.0] * n, "humid": [55.0] * n, + "fmax": [71.0] * n, "fmin": [48.0] * n, + }) + p = tmp_path / era5lake.point_key(i, j) + p.parent.mkdir(parents=True, exist_ok=True) + df.write_parquet(p) + ti, tj = era5lake.tile_of(i, j) + part = df.with_columns(pl.lit(i, dtype=pl.Int32).alias("lat_idx"), + pl.lit(j, dtype=pl.Int32).alias("lon_idx")) + hp = tmp_path / era5lake.daily_part_key(ti, tj, 1940, 1) + hp.parent.mkdir(parents=True, exist_ok=True) + part.filter((pl.col("date").dt.year() == 1940) + & (pl.col("date").dt.month() == 1)).write_parquet(hp) + lat, lon = era5lake.to_coords(i, j) + rows.append({"lat_idx": i, "lon_idx": j, "lat": lat, "lon": lon, + "tile": f"{ti}_{tj}", "rows": n, + "date_min": "1940-01-01", "date_max": str(dates[-1])}) + (tmp_path / "era5").mkdir(exist_ok=True) + pl.DataFrame(rows).write_parquet(tmp_path / era5lake.MANIFEST_KEY) + + monkeypatch.setenv("THERMOGRAPH_LAKE_LOCAL_DIR", str(tmp_path)) + monkeypatch.setenv("THERMOGRAPH_LAKE_CACHE", str(tmp_path / "cache")) + import lake_app + monkeypatch.setattr(lake_app, "CACHE_DIR", str(tmp_path / "cache")) + monkeypatch.setattr(lake_app, "_manifest", None) + return TestClient(lake_app.app) + + +def test_healthz_reports_points(lake): + body = lake.get("/healthz").json() + assert body["status"] == "ok" + assert body["source"] == "local" + assert body["points"] == 2 + + +def test_history_serves_parquet_and_caches(lake, tmp_path): + r = lake.get("/history", params={"lat": 51.5074, "lon": -0.1278}) + assert r.status_code == 200 + assert r.headers["x-lake-point"] == "154,1439" + df = pl.read_parquet(io.BytesIO(r.content)) + assert df.height == 12000 + assert (tmp_path / "cache" / "p154_1439.parquet").exists() + # Second hit comes off the cache file and is byte-identical. + assert lake.get("/history", + params={"lat": 51.5074, "lon": -0.1278}).content == r.content + + +def test_history_404_for_point_outside_lake(lake): + r = lake.get("/history", params={"lat": -45.0, "lon": 170.0}) + assert r.status_code == 404 + + +def test_query_selects_with_partition_predicates(lake): + r = lake.post("/query", json={"sql": + "SELECT year, month, count(*) AS n FROM era5_daily " + "WHERE year = 1940 AND month = 1 GROUP BY year, month"}) + assert r.status_code == 200 + body = r.json() + assert body["columns"] == ["year", "month", "n"] + assert body["rows"][0][2] == 2 * 31 # two points x January 1940 + + +def test_query_manifest_view(lake): + r = lake.post("/query", json={"sql": "SELECT count(*) FROM manifest"}) + assert r.status_code == 200 + assert r.json()["rows"][0][0] == 2 + + +@pytest.mark.parametrize("sql", [ + "DROP TABLE era5_daily", + "SELECT 1; SELECT 2", + "INSERT INTO era5_daily VALUES (1)", + "COPY era5_daily TO 'x'", +]) +def test_query_rejects_non_select(lake, sql): + assert lake.post("/query", json={"sql": sql}).status_code == 400 diff --git a/infra/deploy/secrets/README.md b/infra/deploy/secrets/README.md index 3fc627d..afc09a6 100644 --- a/infra/deploy/secrets/README.md +++ b/infra/deploy/secrets/README.md @@ -15,7 +15,7 @@ no per-host duplication. | File | Holds | Encrypted? | |------|-------|------------| | `common.yaml` | Secrets shared by every host — Discord tokens/secret, VAPID keypair, `THERMOGRAPH_AUTH_SECRET`, metrics token, indexnow key, SMTP creds | ✅ | -| `prod.yaml` | Prod-only overrides — `POSTGRES_PASSWORD`, `REGISTRY_TOKEN` | ✅ | +| `prod.yaml` | Prod-only overrides — `POSTGRES_PASSWORD`, `REGISTRY_TOKEN`, ERA5 lake bucket creds (`THERMOGRAPH_LAKE_S3_ACCESS_KEY` / `THERMOGRAPH_LAKE_S3_SECRET_KEY`, consumed by the `lake` Swarm service and `gen_era5_lake.py`) | ✅ | | `beta.yaml` | Beta-only overrides | ✅ | | `example.yaml` | Format reference / CI fixture (fake values) | ✅ | | `../../.sops.yaml` | Which age recipient files are encrypted to (plaintext config) | — | diff --git a/infra/deploy/stack/autoscale.sh b/infra/deploy/stack/autoscale.sh index 2a36ade..0431ddc 100755 --- a/infra/deploy/stack/autoscale.sh +++ b/infra/deploy/stack/autoscale.sh @@ -21,7 +21,9 @@ set -eu STACK_NAME="${STACK_NAME:-thermograph}" -SERVICE="${STACK_NAME}_web" +# Which stack service to scale (the stack runs one autoscaler instance per +# scaled service — web and lake today; same loop, different bounds). +SERVICE="${STACK_NAME}_${TARGET_SERVICE:-web}" MIN="${MIN_REPLICAS:-1}" MAX="${MAX_REPLICAS:-3}" UP_AT="${SCALE_UP_CPU:-220}" diff --git a/infra/deploy/stack/thermograph-stack.yml b/infra/deploy/stack/thermograph-stack.yml index b34f77b..afb3c3f 100644 --- a/infra/deploy/stack/thermograph-stack.yml +++ b/infra/deploy/stack/thermograph-stack.yml @@ -82,6 +82,10 @@ services: # not the compose bridge's 172.19.0.1 (an overlay has no host gateway). # provision-mail.sh's DOCKER_MAIL_GATEWAY/SUBNET cover this listener. THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + # History reads ask the lake service (Swarm VIP over 1..2 replicas) + # before any third-party source; unset on beta/LAN, where compose has no + # lake service and climate.py falls straight through to NASA. + THERMOGRAPH_LAKE_URL: http://lake:8141 volumes: - appdata:/state - applogs:/app/logs @@ -121,6 +125,7 @@ services: THERMOGRAPH_SINGLETON_PG: "1" RUN_MIGRATIONS: "0" THERMOGRAPH_SMTP_HOST: ${STACK_SMTP_HOST:-172.18.0.1} + THERMOGRAPH_LAKE_URL: http://lake:8141 volumes: - appdata:/state - applogs:/app/logs @@ -138,6 +143,45 @@ services: restart_policy: condition: on-failure + # The ERA5 lake indexer: SQL-style, partition-pruned reads over the + # era5-thermograph bucket with a node-local parquet cache, so a cell's + # history is one LAN hop for web instead of an S3 fetch (or a NASA call). + # Same backend image; THERMOGRAPH_ROLE=lake makes the entrypoint start + # lake_app (no database, no migrations). Prod-only by construction: it only + # exists in this stack file, and its S3 credentials + # (THERMOGRAPH_LAKE_S3_ACCESS_KEY/_SECRET_KEY) arrive via the same rendered + # /host/thermograph.env as every other secret. Without them it stays + # healthy and answers 503/404, and web falls through to NASA — the lake is + # an accelerator, never a point of failure. + lake: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + entrypoint: ["/host/env-entrypoint.sh"] + environment: + THERMOGRAPH_ROLE: lake + PORT: 8141 + THERMOGRAPH_SERVICE_ROLE: backend + WORKERS: "1" + THERMOGRAPH_LAKE_CACHE: /state/lake-cache + volumes: + # Replicas share the cache; lake_app's tmp+rename writes make that safe. + - lakecache:/state + - /opt/thermograph/infra/deploy/stack/env-entrypoint.sh:/host/env-entrypoint.sh:ro + - /etc/thermograph/stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + replicas: ${LAKE_MIN_REPLICAS:-1} + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "${LAKE_CPUS:-2}" + restart_policy: + condition: on-failure + update_config: + order: start-first + failure_action: rollback + frontend: image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] @@ -200,6 +244,39 @@ services: restart_policy: condition: any + # Second autoscaler instance, watching `lake` (1..2 replicas). The lake is + # read-only and cache-heavy, so it saturates on decode CPU well below web's + # thresholds — scale up sooner, still down slowly. + autoscaler-lake: + image: docker:27-cli + entrypoint: ["/bin/sh", "/host/autoscale.sh"] + environment: + STACK_NAME: ${STACK_NAME:-thermograph} + TARGET_SERVICE: lake + MIN_REPLICAS: ${LAKE_MIN_REPLICAS:-1} + MAX_REPLICAS: ${LAKE_MAX_REPLICAS:-2} + SCALE_UP_CPU: ${LAKE_SCALE_UP_CPU:-120} + SCALE_DOWN_CPU: ${LAKE_SCALE_DOWN_CPU:-30} + POLL_SECONDS: ${POLL_SECONDS:-15} + UP_SAMPLES: ${UP_SAMPLES:-3} + DOWN_SAMPLES: ${DOWN_SAMPLES:-20} + COOLDOWN_SECONDS: ${COOLDOWN_SECONDS:-180} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /opt/thermograph/infra/deploy/stack/autoscale.sh:/host/autoscale.sh:ro + networks: + - internal + deploy: + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "0.2" + memory: 64m + restart_policy: + condition: any + networks: internal: driver: overlay @@ -221,3 +298,6 @@ volumes: applogs: external: true name: ${APPLOGS_VOLUME:-thermograph_applogs} + # Lake parquet cache: node-local and disposable (a cold cache just refills + # from the bucket), so NOT external — the stack owns its lifecycle. + lakecache: {} From d9537798c6461ca253741a3facc9e5ee7d5bc56d Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 21:27:32 +0000 Subject: [PATCH 02/15] Vault the ERA5 lake bucket credentials (prod) (#16) --- infra/deploy/secrets/prod.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/deploy/secrets/prod.yaml b/infra/deploy/secrets/prod.yaml index af4aa27..1557e40 100644 --- a/infra/deploy/secrets/prod.yaml +++ b/infra/deploy/secrets/prod.yaml @@ -24,6 +24,8 @@ THERMOGRAPH_DISCORD_BOT_TOKEN: ENC[AES256_GCM,data:zfVJLH8ddct2Jvstl/olZLQ9SM15c REGISTRY_TOKEN: ENC[AES256_GCM,data:3VETZR026YXvWA7DLly4/mz+hqEK7vkEMwKlDKU6CQZeZiFpbQXWig==,iv:ixZJi2VWAZak6dFPvVyErVRanp9bzqXyV3MVa4QsBuU=,tag:Pc2Mdw58S0tSdkgJ5yuVNw==,type:str] THERMOGRAPH_DISCORD_BOT: ENC[AES256_GCM,data:Kg==,iv:CPYN8aCu79PHaz3Zj6OgqyJhXdHLt0tLodgML2NuwQM=,tag:1B3IhlfjCwSlPq7FnKGeRg==,type:str] THERMOGRAPH_DISCORD_WEATHER_CHANNEL: ENC[AES256_GCM,data:WIxYcoOnBDdDcNj8uJ5eUqXjVQ==,iv:l6kU5PGVcW2Nv7EqBUGCRxBzaJOJ3AW1nEEH33aAbmI=,tag:/KgdCKmrlQioIWdz21PcTg==,type:str] +THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:goKTqruUdKUpSSrhu1Ivbh+yS7/DhGdCBeW/xb7AIkI=,iv:TmuHDgUi7dJZYv6uuUgqpnl5kG0tFNLgn11yizhHAV8=,tag:kkH+qBWA5rJwySIILk1cPg==,type:str] +THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:7q07fpPgjgJN1NivbhAHn5JRyhI/+f0nV7Oll9bxW3w=,iv:hKZNH4PLmyyhA/crJpy1RA6peL9HzosyiNi+Y4yMKwo=,tag:sb5ailD6K44d4m8TmzZD9g==,type:str] sops: age: - enc: | @@ -35,7 +37,7 @@ sops: PDXCB8EqKwdC3BnKlpD9ptYcuPHVUSgve1wIZYJ7iHIEhW9Kc6RXOw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-23T04:33:06Z" - mac: ENC[AES256_GCM,data:BjNMEudsS9ZViR55OjSXu5qrOJ9yu2kCGx61qu4vE1ZZJIZtnnyZ3CfpsPygvLLhiFRj76xcqFnuSqTdr17BCEHz3UzsHFv0vBi17lZ5ksN5Oz+9wr+3Pg3TvxC0VF2VHJVP84VH61CmB5+G4WOms8WqLtmIGSLYcTVtcIFoUhk=,iv:jTEB8Mxg8qIyNfhpSgJtUvULVx4+XFrPLyQKhNcbkTg=,tag:SIyvNltdGCMfDGvi4kD3MA==,type:str] + lastmodified: "2026-07-23T21:26:01Z" + mac: ENC[AES256_GCM,data:RHLp+0waxaKQXfAzipchk/vZnE3NtMOHAJwqddTlJYpxl9Sl47IqZcdV05j8kveySNoedZbpmu6+unso9HPaaJ9lmunZVmOPc9wqn+5JJNw8kkPIosrXL8UzKW9gO4OIbTNkEC4xSQP4qyQeUO/RaDP2aH0d+GCa7sKTePrd6Gg=,iv:9027dfABH2VuEJKWzY1+eTns+IdCUXW2KQTAdlj4iP4=,tag:JLLkj0g4gLMseLdUy896/g==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 From 21eea3a4c2aef83098963d758399373815053804 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 21:32:58 +0000 Subject: [PATCH 03/15] Parallelize lake tile uploads (#17) --- backend/gen_era5_lake.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/gen_era5_lake.py b/backend/gen_era5_lake.py index f058674..c7be634 100644 --- a/backend/gen_era5_lake.py +++ b/backend/gen_era5_lake.py @@ -20,6 +20,7 @@ Seed-only deps (icechunk/xarray/zarr) stay out of the app image, same policy as seed_era5.py. """ import argparse +import concurrent.futures import datetime import io import os @@ -126,11 +127,15 @@ def _tile_frame(ds, ti: int, tj: int, end: str) -> "pl.DataFrame | None": def _write_tile(sink: _Sink, ti: int, tj: int, tile_df: pl.DataFrame) -> pl.DataFrame: - """Write one tile's point files + hive partitions; return its manifest rows.""" + """Write one tile's point files + hive partitions; return its manifest rows. + Uploads run on a thread pool: a tile is ~1.2k small objects and sequential + PUTs are pure round-trip latency (measured 2.5 obj/s -> 8 min/tile; the + pool brings that to well under a minute). boto3 clients are thread-safe.""" + puts = [] rows = [] for (i, j), pdf in tile_df.group_by(["lat_idx", "lon_idx"]): pdf = pdf.sort("date") - sink.put(era5lake.point_key(i, j), pdf.drop(["lat_idx", "lon_idx"])) + puts.append((era5lake.point_key(i, j), pdf.drop(["lat_idx", "lon_idx"]))) lat, lon = era5lake.to_coords(i, j) rows.append({"lat_idx": i, "lon_idx": j, "lat": lat, "lon": lon, "tile": f"{ti}_{tj}", "rows": pdf.height, @@ -140,8 +145,11 @@ def _write_tile(sink: _Sink, ti: int, tj: int, tile_df: pl.DataFrame) -> pl.Data pl.col("date").dt.year().alias("year"), pl.col("date").dt.month().alias("month")) for (year, month), mdf in parts.group_by(["year", "month"]): - sink.put(era5lake.daily_part_key(ti, tj, year, month), - mdf.drop(["year", "month"]).sort(["lat_idx", "lon_idx", "date"])) + puts.append((era5lake.daily_part_key(ti, tj, year, month), + mdf.drop(["year", "month"]).sort(["lat_idx", "lon_idx", "date"]))) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + for f in [pool.submit(sink.put, k, df) for k, df in puts]: + f.result() # surface the first failure instead of hiding it return pl.DataFrame(rows) From de8e847f9f9424b531d476892c3b13c61ba759ab Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:26:05 +0000 Subject: [PATCH 04/15] shell: add shellcheck CI guard and drive the tree to zero findings (#19) Adds .forgejo/workflows/shell-lint.yml (pinned shellcheck v0.11.0 + sha256, -x, default severity, fail on any finding, not path-filtered) and drives all 26 scripts to zero findings. Two defects shellcheck cannot see: render-secrets.sh left DECRYPTED vault contents in /tmp whenever a sops decrypt failed -- the caller's set -e aborted the function before either cleanup ran. Now removed on every exit path, with `|| return 1` on both sops calls so a decrypt failure can never write a partial /etc/thermograph.env regardless of the caller's shell options. Explicitly not a `trap ... RETURN`: such a trap set in a sourced function persists into the caller's shell and re-fires when the caller's next `.`/source completes, where the function-local tmp is unset -- fatal and silent under deploy.sh's set -u. The file now records that reasoning. autoscale.sh ran `set -eu` without pipefail while piping docker stats into awk, so a failed left side was swallowed and the loop autoscaled on empty input. Promoted to pipefail with a missed sample treated as a skip; verified busybox ash in docker:27-cli supports it. Also: capture-fixtures.sh's `jq . || cat` ran cat after jq had consumed stdin, silently writing truncated fixtures; deploy.sh/deploy-stack.sh `# shellcheck source=` paths corrected for the monorepo layout. --- .forgejo/workflows/shell-lint.yml | 75 ++++++++++++++++++++++++++ backend/scripts/smoke.sh | 2 +- frontend/scripts/capture-fixtures.sh | 4 +- infra/deploy/deploy.sh | 14 +++-- infra/deploy/render-secrets.sh | 32 ++++++++--- infra/deploy/secrets/dry-run-render.sh | 4 ++ infra/deploy/stack/autoscale.sh | 11 +++- infra/deploy/stack/deploy-stack.sh | 10 +++- 8 files changed, 134 insertions(+), 18 deletions(-) create mode 100644 .forgejo/workflows/shell-lint.yml diff --git a/.forgejo/workflows/shell-lint.yml b/.forgejo/workflows/shell-lint.yml new file mode 100644 index 0000000..6aa69eb --- /dev/null +++ b/.forgejo/workflows/shell-lint.yml @@ -0,0 +1,75 @@ +name: shell-lint + +# Shellcheck over every *.sh in the tree. These scripts run as root over SSH +# against production hosts (deploy, secrets provisioning, host bootstrap) with +# no test suite in front of them -- a quoting bug or an unset-variable typo is +# found *on the host* or not at all. The tree was driven to zero findings in +# one pass; this guard keeps it there. +# +# Deliberately NOT path-filtered (same call as secrets-guard): a backstop that +# only runs when you expect it to isn't a backstop, and shellcheck over the +# ~26 scripts here is seconds. Scripts are discovered with `find`, not listed, +# so a new script is covered the moment it lands -- that is the entire point. +# +# Shellcheck is installed from the pinned official static-binary release, NOT +# `apt-get install shellcheck` (the observability-validate precedent): the +# distro version drifts with the job image, and a drifted shellcheck can grow +# NEW warnings that fail CI on an unrelated push. The pin (v0.11.0, matching +# the koalaman/shellcheck:stable image the zero-findings pass was run against) +# plus the sha256 makes the check reproducible; bump both together, and expect +# to fix any new findings in the same PR as the bump. + +on: + pull_request: + push: + branches: [dev, main, release] + +env: + SHELLCHECK_VERSION: v0.11.0 + SHELLCHECK_SHA256: 8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 + +jobs: + shellcheck: + runs-on: docker + steps: + - uses: actions/checkout@v4 + + - name: Install shellcheck ${{ env.SHELLCHECK_VERSION }} (pinned static binary) + run: | + set -euo pipefail + # node:20-bookworm (this runner's job image) ships curl, tar and xz, + # so the pinned tarball needs no apt-get at all -- faster than the + # distro install it replaces. + curl -fsSL -o /tmp/shellcheck.tar.xz \ + "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" + echo "${SHELLCHECK_SHA256} /tmp/shellcheck.tar.xz" | sha256sum -c - + tar -xJf /tmp/shellcheck.tar.xz -C /tmp + install -m 0755 "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck + shellcheck --version + + - name: Shellcheck every *.sh in the tree + run: | + set -euo pipefail + mapfile -t files < <(find . -name '*.sh' -not -path './.git/*' | sort) + if [ ${#files[@]} -eq 0 ]; then + echo "no *.sh files yet — nothing to lint" + exit 0 + fi + echo "shellcheck over ${#files[@]} scripts" + # -x follows sourced files: several scripts carry + # `# shellcheck source=` directives that only resolve with it. + # Default severity, no excludes: the tree is at zero findings, so + # anything shellcheck reports is a regression, not noise. + if shellcheck -x -f gcc "${files[@]}" > /tmp/findings.txt; then + echo "ok: all scripts clean" + exit 0 + fi + # gcc format is file:line:col: level: message — reshape each line + # into a ::error annotation so findings land on the diff in the PR + # view. `rest` soaks up any further colons inside the message. + while IFS=: read -r f l _ rest; do + [ -n "$l" ] || continue + echo "::error file=${f#./},line=${l}::${rest# }" + done < /tmp/findings.txt + echo "shellcheck found problems in the files above" + exit 1 diff --git a/backend/scripts/smoke.sh b/backend/scripts/smoke.sh index 4854c3b..7165756 100755 --- a/backend/scripts/smoke.sh +++ b/backend/scripts/smoke.sh @@ -29,7 +29,7 @@ echo "==> starting backend + db ($IMG)" "${COMPOSE[@]}" up -d echo "==> waiting for /healthz (up to 90s)" -for i in $(seq 1 45); do +for _ in $(seq 1 45); do if curl -fsS -o /dev/null "$PORT_URL/healthz"; then ok=1; break; fi sleep 2 done diff --git a/frontend/scripts/capture-fixtures.sh b/frontend/scripts/capture-fixtures.sh index de7578b..7641883 100755 --- a/frontend/scripts/capture-fixtures.sh +++ b/frontend/scripts/capture-fixtures.sh @@ -18,7 +18,7 @@ trap 'scripts/backend-for-tests.sh down >/dev/null 2>&1 || true' EXIT fetch() { # echo "==> $2.json <- $1" - curl -fsS "$BASE/$1" --max-time 40 | { command -v jq >/dev/null && jq . || cat; } > "$OUT/$2.json" + curl -fsS "$BASE/$1" --max-time 40 | { if command -v jq >/dev/null; then jq .; else cat; fi; } > "$OUT/$2.json" } fetch "api/v2/content/home" home @@ -28,4 +28,4 @@ fetch "api/v2/content/city/$SLUG" city fetch "api/v2/content/city/$SLUG/month/$MONTH" month fetch "api/v2/content/city/$SLUG/records" records -echo "==> captured $(ls "$OUT"/*.json | wc -l) fixtures for $SLUG into $OUT/" +echo "==> captured $(find "$OUT" -maxdepth 1 -name '*.json' | wc -l) fixtures for $SLUG into $OUT/" diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index b4a56ce..4b8f8f3 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -70,11 +70,16 @@ fi # existing /etc/thermograph.env. Then source it so a by-hand run interpolates the # same as the systemd unit does. See deploy/render-secrets.sh + deploy/secrets/. if [ -f "$INFRA_DIR/deploy/render-secrets.sh" ]; then - # shellcheck source=deploy/render-secrets.sh + # shellcheck source=infra/deploy/render-secrets.sh . "$INFRA_DIR/deploy/render-secrets.sh" render_thermograph_secrets "$INFRA_DIR" fi -set -a; . /etc/thermograph.env 2>/dev/null || true; set +a +# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it +# cannot exist at lint time, so don't ask shellcheck to follow it. +set -a +# shellcheck source=/dev/null +. /etc/thermograph.env 2>/dev/null || true +set +a # Pre-warm the ~750 city-page archives so /climate pages serve from cache and a # search-engine crawl never bursts the archive API quota. Detached inside the @@ -156,7 +161,10 @@ TAGS_FILE="$INFRA_DIR/deploy/.image-tags.env" _incoming_backend="${BACKEND_IMAGE_TAG:-}" _incoming_frontend="${FRONTEND_IMAGE_TAG:-}" if [ -f "$TAGS_FILE" ]; then - set -a; . "$TAGS_FILE"; set +a + set -a + # shellcheck source=/dev/null # untracked runtime artifact this script writes below + . "$TAGS_FILE" + set +a fi [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" diff --git a/infra/deploy/render-secrets.sh b/infra/deploy/render-secrets.sh index b44a965..d4a434e 100755 --- a/infra/deploy/render-secrets.sh +++ b/infra/deploy/render-secrets.sh @@ -48,15 +48,26 @@ render_thermograph_secrets() { key_env=("SOPS_AGE_KEY=$keymat") fi local tmp; tmp=$(mktemp) - : > "$tmp" - # set -e in the caller makes a decrypt failure fatal here (no partial env). common - # first, host second, so a host value overrides a shared one (last-wins). + # The tmp file holds DECRYPTED secrets, so every exit path below removes it. + # + # Deliberately NOT a `trap ... RETURN`, which looks like the tidier way to make + # that unconditional: a RETURN trap set inside a SOURCED function persists in + # the CALLER's shell after the function returns, and a RETURN trap also fires + # when a `.`/source completes. deploy.sh sources /etc/thermograph.env a few + # lines after calling us, which would re-fire the trap at top level where `tmp` + # (function-local) is unset -- fatal under deploy.sh's `set -u`, and silent, + # because that line already redirects stderr to /dev/null. Explicit removal on + # each path is duller and correct. + : > "$tmp" || { rm -f "$tmp"; return 1; } + # Decrypt failures return 1 explicitly, so a partial env is never written and + # correctness doesn't depend on the caller's shell options. common first, host + # second, so a host value overrides a shared one (last-wins). if [ -f "$repo/deploy/secrets/common.yaml" ]; then env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \ - "$repo/deploy/secrets/common.yaml" >> "$tmp" + "$repo/deploy/secrets/common.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; } fi env "${key_env[@]}" sops -d --input-type yaml --output-type dotenv \ - "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp" + "$repo/deploy/secrets/${env_name}.yaml" >> "$tmp" || { rm -f "$tmp"; return 1; } # Write /etc/thermograph.env. Prefer an in-place write when the existing file is # writable by us (e.g. a group-writable 0660 root: on a box whose CI @@ -67,14 +78,19 @@ render_thermograph_secrets() { # next line of deploy.sh (`. /etc/thermograph.env` as that non-root user) can't read # it, so POSTGRES_PASSWORD never enters the env and `docker compose` dies on # interpolation. Fail loudly rather than deploy against stale secrets. + # rc + a single cleanup point: the plaintext tmp must go whichever branch runs, + # but the old trailing `rm` was also the function's last command, so it masked a + # failed in-place `cat` write to status 0. Capture the status instead, so a + # half-written /etc/thermograph.env fails loudly rather than deploying stale. + local rc=0 if [ -f /etc/thermograph.env ] && [ -w /etc/thermograph.env ]; then - cat "$tmp" > /etc/thermograph.env + cat "$tmp" > /etc/thermograph.env || rc=1 elif install -m 0640 "$tmp" /etc/thermograph.env 2>/dev/null; then : elif sudo install -m 0640 -o "$(id -un)" -g "$(id -gn)" "$tmp" /etc/thermograph.env 2>/dev/null; then : else - rm -f "$tmp" echo "!! cannot write /etc/thermograph.env (need file write access or passwordless sudo)" >&2 - return 1 + rc=1 fi rm -f "$tmp" + return "$rc" } diff --git a/infra/deploy/secrets/dry-run-render.sh b/infra/deploy/secrets/dry-run-render.sh index 03533f1..44b41a1 100755 --- a/infra/deploy/secrets/dry-run-render.sh +++ b/infra/deploy/secrets/dry-run-render.sh @@ -19,6 +19,10 @@ tmp="$(mktemp)"; live="$(mktemp)"; trap 'rm -f "$tmp" "$live"' EXIT [ -f "$common" ] && SOPS_AGE_KEY="$keymat" sops -d --input-type yaml --output-type dotenv "$common" >> "$tmp" SOPS_AGE_KEY="$keymat" sops -d --input-type yaml --output-type dotenv "$host" >> "$tmp" +# sudo is only for the READ of the root-owned live env; the redirect target is our +# own mktemp file, so the redirect correctly runs as the invoking user (sudo tee +# would wrongly write it as root). +# shellcheck disable=SC2024 sudo cat /etc/thermograph.env > "$live" python3 - "$live" "$tmp" <<'PY' import sys diff --git a/infra/deploy/stack/autoscale.sh b/infra/deploy/stack/autoscale.sh index 2a36ade..b737bab 100755 --- a/infra/deploy/stack/autoscale.sh +++ b/infra/deploy/stack/autoscale.sh @@ -18,7 +18,11 @@ # - Clamped to [MIN_REPLICAS, MAX_REPLICAS]; scaling waits for convergence # (--detach=false), so a stuck rollout blocks further changes rather than # stacking them. -set -eu +# pipefail: without it a `docker stats` failure on the left of the avg_cpu +# pipe is silently swallowed by awk's exit 0 and the loop trusts the output. +# (Runs under busybox ash in docker:*-cli, which supports pipefail.) +# shellcheck disable=SC3040 # pipefail is not POSIX, but the busybox ash this runs under has it +set -euo pipefail STACK_NAME="${STACK_NAME:-thermograph}" SERVICE="${STACK_NAME}_web" @@ -64,7 +68,10 @@ while :; do cur=$(replicas) [ -n "$cur" ] || { log "service $SERVICE not found; waiting"; continue; } - cpu=$(avg_cpu) + # With pipefail, a transient `docker stats` failure makes avg_cpu return + # non-zero; treat that as a missed sample (don't trust partial output, and + # don't let set -e kill the autoscaler over a daemon hiccup). + cpu=$(avg_cpu) || cpu="" [ -n "$cpu" ] || continue # no running tasks visible this sample if [ "$cpu" -gt "$UP_AT" ]; then diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index 8833aec..8516a6b 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -49,11 +49,16 @@ export STACK_NAME # install the uid-10001-readable copy the tasks' env-entrypoint shim sources. # 10001 = the app images' `thermograph` user; the file is 0400 to that uid. if [ -f "$APP_DIR/infra/deploy/render-secrets.sh" ]; then - # shellcheck source=deploy/render-secrets.sh + # shellcheck source=infra/deploy/render-secrets.sh . "$APP_DIR/infra/deploy/render-secrets.sh" render_thermograph_secrets "$APP_DIR/infra" fi -set -a; . /etc/thermograph.env 2>/dev/null || true; set +a +# /etc/thermograph.env is rendered at deploy time from the SOPS vault — it +# cannot exist at lint time, so don't ask shellcheck to follow it. +set -a +# shellcheck source=/dev/null +. /etc/thermograph.env 2>/dev/null || true +set +a sudo install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env \ || install -o 10001 -g 0 -m 0400 /etc/thermograph.env /etc/thermograph/stack.env @@ -65,6 +70,7 @@ export REGISTRY_HOST TAGS_FILE="$APP_DIR/infra/deploy/.stack-image-tags.env" _incoming_backend="${BACKEND_IMAGE_TAG:-}" _incoming_frontend="${FRONTEND_IMAGE_TAG:-}" +# shellcheck source=/dev/null # untracked runtime artifact this script writes below if [ -f "$TAGS_FILE" ]; then set -a; . "$TAGS_FILE"; set +a; fi [ -n "$_incoming_backend" ] && BACKEND_IMAGE_TAG="$_incoming_backend" [ -n "$_incoming_frontend" ] && FRONTEND_IMAGE_TAG="$_incoming_frontend" From 8e09155aaf2fcccaf7faf17c7924327fde0c77c9 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:32:13 +0000 Subject: [PATCH 05/15] Run the ERA5 lake service in every environment (#20) --- .forgejo/workflows/backend-deploy-dev.yml | 6 ++++ infra/deploy/deploy.sh | 7 +++-- infra/deploy/secrets/beta.yaml | 6 ++-- infra/deploy/stack/deploy-stack.sh | 9 +++++- infra/docker-compose.dev.yml | 3 ++ infra/docker-compose.yml | 36 +++++++++++++++++++++++ 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/backend-deploy-dev.yml b/.forgejo/workflows/backend-deploy-dev.yml index 88227f1..94b247e 100644 --- a/.forgejo/workflows/backend-deploy-dev.yml +++ b/.forgejo/workflows/backend-deploy-dev.yml @@ -70,4 +70,10 @@ jobs: # thermograph-lan is host-native (the runner job runs directly on the # LAN box, not over SSH like beta/prod), so plain env vars on the # command reach deploy-dev.sh directly -- no ssh-action needed here. + # The lake creds ride the Actions S3 secrets: dev renders no vault + # (deploy-dev.sh's no-op secrets path), and compose interpolates + # ${THERMOGRAPH_LAKE_S3_*} straight from the deploy environment. + env: + THERMOGRAPH_LAKE_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} + THERMOGRAPH_LAKE_S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} run: SERVICE=backend BACKEND_IMAGE_TAG=${{ steps.tag.outputs.tag }} bash "$HOME/thermograph-dev/infra/deploy/deploy-dev.sh" diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index b4a56ce..b9eda1e 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -177,11 +177,12 @@ esac export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" -# Which compose services this run pulls/rolls. +# Which compose services this run pulls/rolls. lake runs the backend image +# with a different role, so it rolls with every backend deploy. case "$SERVICE" in - backend) TARGETS=(backend) ;; + backend) TARGETS=(backend lake) ;; frontend) TARGETS=(frontend) ;; - all) TARGETS=(backend frontend) ;; + all) TARGETS=(backend lake frontend) ;; esac # Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml, diff --git a/infra/deploy/secrets/beta.yaml b/infra/deploy/secrets/beta.yaml index d7a74c3..5162a7c 100644 --- a/infra/deploy/secrets/beta.yaml +++ b/infra/deploy/secrets/beta.yaml @@ -16,6 +16,8 @@ THERMOGRAPH_VAPID_PRIVATE_KEY: ENC[AES256_GCM,data:sTtkAbMVUtwvRssCZJa2PWUjlfnYS THERMOGRAPH_VAPID_PUBLIC_KEY: ENC[AES256_GCM,data:SnFw+pwNgZlxMEhMUH/1daMZlRB2QFNJHtL+4C3S1eFcJJdT2uNGkY7Ym9FPfx1V7cwt/3urHg1oVgdDzYaIal2wye4l5HhKBP5PjtQkDW+Pck24kxul,iv:Ku4MhsrAfyA2Y3CqhGIsYKrjhKQ4cqAYjCqjFeEvBWE=,tag:hw42iS+79D7gNf+BPTYM0g==,type:str] THERMOGRAPH_VAPID_CONTACT: ENC[AES256_GCM,data:ox+hHiFMTPZqYwICDSwPwcT13ZDSKiGcAVHfVVMNYQ==,iv:DssV/olPXERpdmXNMmzjVYvhn/0zmuSp2XlTs42pIi0=,tag:4YHXI1BFbGZ6JsdXqJuf3w==,type:str] REGISTRY_TOKEN: ENC[AES256_GCM,data:pC1Zzs56la8qjzWcbkuKyeVHwS5WfIe+mrLm5Sdi/LKYziAvRU5cOA==,iv:57PlJEBdgwNJsfFdNOGqAnKk8l6zpmMSWYFEKS6pn+4=,tag:pPftNHUVU6+CY2nWMaXO6w==,type:str] +THERMOGRAPH_LAKE_S3_ACCESS_KEY: ENC[AES256_GCM,data:ZEQ278P7azMKlYMjUraP6hIqnjymqL90zEbvTNC9iXU=,iv:5nXXZwx7RfqwCMqx5LWtxQy2nQcq+6Yr79ulBi6BOnE=,tag:jXcCCCcz4NsDoIZwa/ji4A==,type:str] +THERMOGRAPH_LAKE_S3_SECRET_KEY: ENC[AES256_GCM,data:p5qCrHWoH1i3smDZZBIO8zMSWWxDARYsTY12AA0E8ao=,iv:PVOEQ0kBFeYaE/FT2UswtSTY7S84U/YOPSbcsXR51f4=,tag:JFwaOwAYInPs199li66U/g==,type:str] sops: age: - enc: | @@ -27,7 +29,7 @@ sops: 0+PQdtDxom72BqrAoj5lhepxW8YWPqQIRXsVS/XYQEQfPJuuuiRMKw== -----END AGE ENCRYPTED FILE----- recipient: age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2 - lastmodified: "2026-07-22T03:17:28Z" - mac: ENC[AES256_GCM,data:npF+ngF4Khj3NJ2PNFcQjQYIrZVpdPM0drk43pAI6HfFaynX8JOqRFZRsH0sCfMoPkXB/ffCJxla1Tt3Mu3zgKTNSWcn+mjzl+dYvlruY1VylHLS+GnTO9zKeQcwbrYzTfZNWzQhm6/mw8KCL3791U/xNHwJc6g5VxRUWl6TBdo=,iv:JLzY8rm33NLG8CqyAlU6vBhlODUiRipKu9daoRvgyzg=,tag:yIsxVZAnw8FD7Bt/x55wMQ==,type:str] + lastmodified: "2026-07-23T22:30:10Z" + mac: ENC[AES256_GCM,data:JWfkQk4Z1ohPha9LID/hvZOycmDy0sUyjiqDdCidfCQEsPx+OwhPatcM31x1ZKFgh3TyYTxrauoSTlIeuwfy7+R9jATDdBBZe9zMqgs6Szu9WUdFiaZzIFpG7SEVw+8IqIyAuqZVSW3k5yHmchcmkNyDekV+1RIF8wxUrOeSCVQ=,iv:6PXUiCnYBiFyNPnCRAs6f7a+RuvWI4jSIMXwuMn7WjA=,tag:y+Ix2m65hQRkSt8FylfKSg==,type:str] unencrypted_suffix: _unencrypted version: 3.13.2 diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index 8833aec..e86a125 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -157,9 +157,16 @@ if [ "$SERVICE" = "all" ] || ! docker service inspect "${STACK_NAME}_web" >/dev/ else case "$SERVICE" in backend) - echo "==> Rolling web + worker to $BACKEND_IMAGE" + echo "==> Rolling web + worker + lake to $BACKEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" + # lake ships in the same image; a stack file predating it has no service + # yet — the next SERVICE=all stack deploy creates it, so don't fail here. + if docker service inspect "${STACK_NAME}_lake" >/dev/null 2>&1; then + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_lake" + else + echo " (no ${STACK_NAME}_lake service yet; created on the next full stack deploy)" + fi ;; frontend) echo "==> Rolling frontend to $FRONTEND_IMAGE" diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index c2107fd..b97e0e8 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -40,6 +40,9 @@ services: db: cpus: !reset null deploy: !reset null + lake: + cpus: !reset null + deploy: !reset null # Uncapped memory on dev too (prod ceilings it at 8g). The Postgres/DuckDB # memory *budget* is still the ~8 GB derived by deploy/db/init/20-tuning.sh from # the default DB_MEMORY (raise DB_MEMORY to give dev more); shm_size stays (it's diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 4de4659..7eb20a8 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -124,6 +124,11 @@ services: # (NOT /app/data) so it never shadows the Python `data/` package -- see the # volumes: note below and thermograph-backend paths.py. THERMOGRAPH_SINGLETON_LOCK: /state/notifier.lock + # History reads ask the lake service before any third-party source; if + # the lake has no bucket creds (LAN without them exported) it answers + # 503 and climate.py falls straight through to NASA — an accelerator, + # never a dependency. + THERMOGRAPH_LAKE_URL: http://lake:8141 # Prod secrets live in /etc/thermograph.env: POSTGRES_PASSWORD, # THERMOGRAPH_AUTH_SECRET, THERMOGRAPH_VAPID_PRIVATE_KEY/_PUBLIC_KEY, # THERMOGRAPH_COOKIE_SECURE=1, mail/Discord keys, ... (see @@ -157,6 +162,36 @@ services: - "127.0.0.1:8137:8137" restart: unless-stopped + # The ERA5 lake service: partition-pruned reads over the era5-thermograph + # bucket with a local parquet cache, same backend image with + # THERMOGRAPH_ROLE=lake (the entrypoint starts lake_app — no database, no + # migrations, so no depends_on: db). Bucket creds come from + # /etc/thermograph.env where the vault renders them (beta/prod) or from the + # deploy environment (LAN — deploy-dev.yml injects the Actions S3 secrets); + # without them the service stays healthy and every read falls through to + # NASA. Never published on a host port: only the backend talks to it. + lake: + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + environment: + THERMOGRAPH_ROLE: lake + PORT: 8141 + THERMOGRAPH_SERVICE_ROLE: backend + WORKERS: "1" + THERMOGRAPH_LAKE_CACHE: /state/lake-cache + THERMOGRAPH_LAKE_S3_ACCESS_KEY: ${THERMOGRAPH_LAKE_S3_ACCESS_KEY:-} + THERMOGRAPH_LAKE_S3_SECRET_KEY: ${THERMOGRAPH_LAKE_S3_SECRET_KEY:-} + env_file: + - path: /etc/thermograph.env + required: false + volumes: + - lakecache:/state + cpus: ${LAKE_CPUS:-2} + deploy: + resources: + limits: + cpus: "${LAKE_CPUS:-2}" + restart: unless-stopped + frontend: # Frontend's OWN image, published by thermograph-frontend's build-push.yml # from its own Dockerfile (which starts `uvicorn app:app` directly -- no @@ -187,6 +222,7 @@ volumes: pgdata: {} appdata: {} applogs: {} + lakecache: {} networks: # Pin the default network's subnet + gateway so the host Postfix can rely on a From a787f79f9299042faee173d051a902f9a355798d Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:47:26 +0000 Subject: [PATCH 06/15] Survive Contabo PUT throttling in the lake extractor (#22) --- backend/gen_era5_lake.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/gen_era5_lake.py b/backend/gen_era5_lake.py index c7be634..891249b 100644 --- a/backend/gen_era5_lake.py +++ b/backend/gen_era5_lake.py @@ -65,11 +65,16 @@ class _Sink: self._s3 = None if not self.local: import boto3 # noqa: PLC0415 - extractor-only dep + from botocore.config import Config # noqa: PLC0415 + # Contabo throttles bursts with 429s (seen live at 16 concurrent + # PUTs); adaptive mode rate-limits client-side and retries with + # backoff instead of dying after botocore's default 4 attempts. self._s3 = boto3.client( "s3", endpoint_url=self.cfg["endpoint"], region_name=self.cfg["region"], aws_access_key_id=self.cfg["access_key"], - aws_secret_access_key=self.cfg["secret_key"]) + aws_secret_access_key=self.cfg["secret_key"], + config=Config(retries={"max_attempts": 10, "mode": "adaptive"})) def put(self, key: str, df: pl.DataFrame) -> None: if self.local: @@ -147,7 +152,7 @@ def _write_tile(sink: _Sink, ti: int, tj: int, tile_df: pl.DataFrame) -> pl.Data for (year, month), mdf in parts.group_by(["year", "month"]): puts.append((era5lake.daily_part_key(ti, tj, year, month), mdf.drop(["year", "month"]).sort(["lat_idx", "lon_idx", "date"]))) - with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: for f in [pool.submit(sink.put, k, df) for k, df in puts]: f.result() # surface the first failure instead of hiding it return pl.DataFrame(rows) @@ -197,29 +202,35 @@ def main() -> None: done = set() if manifest is None or args.overwrite else \ set(manifest["tile"].unique().to_list()) - written = skipped = empty = 0 + written = skipped = empty = failed = 0 for n, (ti, tj) in enumerate(todo, 1): if f"{ti}_{tj}" in done: skipped += 1 continue t0 = time.time() - tile_df = _tile_frame(ds, ti, tj, end) - if tile_df is None: - empty += 1 + try: + tile_df = _tile_frame(ds, ti, tj, end) + if tile_df is None: + empty += 1 + continue + if args.dry_run: + print(f"[dry-run] tile {ti}_{tj}: {tile_df.height} rows " + f"({time.time() - t0:.1f}s); nothing written") + print(tile_df.head()) + return + rows = _write_tile(sink, ti, tj, tile_df) + except Exception as e: # noqa: BLE001 - keep going; the tile isn't in the + failed += 1 # manifest, so the next run retries it + print(f"[{n}/{len(todo)}] FAILED tile {ti}_{tj}: {e}") continue - if args.dry_run: - print(f"[dry-run] tile {ti}_{tj}: {tile_df.height} rows " - f"({time.time() - t0:.1f}s); nothing written") - print(tile_df.head()) - return - rows = _write_tile(sink, ti, tj, tile_df) manifest = rows if manifest is None else pl.concat( [manifest.filter(pl.col("tile") != f"{ti}_{tj}"), rows]) sink.put(era5lake.MANIFEST_KEY, manifest) # after every tile: resumable written += 1 print(f"[{n}/{len(todo)}] tile {ti}_{tj}: {rows.height} points " f"in {time.time() - t0:.1f}s") - print(f"done: tiles written={written} skipped(done)={skipped} sea={empty}") + print(f"done: tiles written={written} skipped(done)={skipped} sea={empty} " + f"failed={failed}") if __name__ == "__main__": From 2d3f37c474962d37c8c9d280ba383bb98dbcf4c1 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:49:54 +0000 Subject: [PATCH 07/15] daemon: move the Discord gateway and scheduler out of the web process into Go (#21) The gateway bot and APScheduler were long-lived stateful I/O loops running inside the async web app under a leader election. They move into a single Go binary that owns ONLY that I/O -- websocket, RESUME, heartbeat, backoff, timers. It owns no grading logic. Anything needing data calls back over a new internal-only surface (/internal/discord/grade, /internal/jobs/*). Grading depends on polars and the parquet cache; reimplementing it in Go would let the bot's grades drift from the API's. The grade route returns gateway-ready JSON and Go relays the bytes verbatim. The binary ships in the backend image and runs as a second compose service off the same tag, so the two ends of the /internal/* contract can never skew. deploy.sh rolls daemon alongside backend -- without that the service would never be created, since a single-service deploy uses --no-deps. It also probes the image first and skips the daemon when rolling a tag that predates the binary: infra tracks main while image tags are env-staged, so a host can legitimately be asked to roll an older backend image, and creating the service anyway would leave a container crash-looping on a missing binary. replicas: 1 with order: stop-first replaces the leader election -- Discord permits one gateway connection per bot token. THERMOGRAPH_INTERNAL_TOKEN is optional: both ends derive it from THERMOGRAPH_AUTH_SECRET via HMAC under a domain-separation label, so this needs no new vault entry. The derivation is pinned to a shared cross-language test vector asserted on both sides, so drift fails CI instead of 401ing every call. Fail closed when neither secret is set. Improvements over the Python: a close intended for RESUME uses 4000 rather than 1000 (Discord invalidates a session closed 1000, so the old default defeated its own resume); MESSAGE_CREATE runs on a bounded worker pool; and a malformed HELLO returns an error rather than a clean reconnect, which would otherwise reset backoff and hot-loop against the gateway. 365 Python tests pass; Go build/vet/test -race clean; shellcheck 0 findings. --- backend/.dockerignore | 10 + backend/Dockerfile | 26 + backend/api/internal_routes.py | 158 ++++++ backend/daemon/README.md | 50 ++ backend/daemon/go.mod | 5 + backend/daemon/go.sum | 2 + backend/daemon/internal/apiclient/client.go | 107 ++++ .../daemon/internal/apiclient/client_test.go | 124 ++++ backend/daemon/internal/config/config.go | 138 +++++ backend/daemon/internal/config/config_test.go | 96 ++++ backend/daemon/internal/config/derive_test.go | 66 +++ backend/daemon/internal/cron/cron.go | 92 +++ backend/daemon/internal/cron/cron_test.go | 204 +++++++ backend/daemon/internal/gateway/gateway.go | 529 ++++++++++++++++++ .../daemon/internal/gateway/gateway_test.go | 182 ++++++ backend/daemon/internal/gateway/message.go | 118 ++++ .../daemon/internal/gateway/message_test.go | 162 ++++++ backend/daemon/internal/gateway/reply.go | 138 +++++ backend/daemon/internal/gateway/reply_test.go | 278 +++++++++ backend/daemon/main.go | 130 +++++ backend/indexnow.py | 3 +- backend/notifications/discord_bot.py | 295 ---------- backend/notifications/scheduler.py | 68 --- backend/requirements.txt | 11 +- backend/tests/api/test_internal_routes.py | 134 +++++ .../api/test_internal_token_derivation.py | 70 +++ .../tests/notifications/test_discord_bot.py | 125 ----- backend/tests/notifications/test_scheduler.py | 79 --- backend/web/app.py | 35 +- infra/.env.example | 15 + infra/deploy/deploy.sh | 37 +- infra/deploy/stack/thermograph-stack.yml | 54 ++ infra/deploy/thermograph.env.example | 51 +- infra/docker-compose.dev.yml | 5 + infra/docker-compose.yml | 63 +++ 35 files changed, 3049 insertions(+), 611 deletions(-) create mode 100644 backend/.dockerignore create mode 100644 backend/api/internal_routes.py create mode 100644 backend/daemon/README.md create mode 100644 backend/daemon/go.mod create mode 100644 backend/daemon/go.sum create mode 100644 backend/daemon/internal/apiclient/client.go create mode 100644 backend/daemon/internal/apiclient/client_test.go create mode 100644 backend/daemon/internal/config/config.go create mode 100644 backend/daemon/internal/config/config_test.go create mode 100644 backend/daemon/internal/config/derive_test.go create mode 100644 backend/daemon/internal/cron/cron.go create mode 100644 backend/daemon/internal/cron/cron_test.go create mode 100644 backend/daemon/internal/gateway/gateway.go create mode 100644 backend/daemon/internal/gateway/gateway_test.go create mode 100644 backend/daemon/internal/gateway/message.go create mode 100644 backend/daemon/internal/gateway/message_test.go create mode 100644 backend/daemon/internal/gateway/reply.go create mode 100644 backend/daemon/internal/gateway/reply_test.go create mode 100644 backend/daemon/main.go delete mode 100644 backend/notifications/discord_bot.py delete mode 100644 backend/notifications/scheduler.py create mode 100644 backend/tests/api/test_internal_routes.py create mode 100644 backend/tests/api/test_internal_token_derivation.py delete mode 100644 backend/tests/notifications/test_discord_bot.py delete mode 100644 backend/tests/notifications/test_scheduler.py diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..7e8c7c1 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,10 @@ +# Local virtualenvs are disposable dev state (see CLAUDE.md), but without this +# file `COPY . /app/` on a dev checkout ships ~400MB of site-packages the image +# already installs via pip — and the chown -R layer then duplicates it AGAIN. +# CI contexts never contain a .venv, so this only protects `:local` dev builds. +# Do NOT add daemon/ here: the daemon-builder stage COPYs it from this same +# context. +.venv* +__pycache__/ +*.py[co] +.pytest_cache/ diff --git a/backend/Dockerfile b/backend/Dockerfile index e5ed738..5adaa68 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,25 @@ # Thermograph backend: FastAPI API + accounts/notifications + the SSR content # JSON API frontend consumes. Split from the monorepo (repo-split Stage 7). + +# thermograph-daemon (daemon/): the Go process that owns the Discord gateway +# websocket and the recurring-job timers, calling back into this app's +# /internal/* routes for anything that needs data. It is built INTO this image +# on purpose: daemon and backend share the internal API contract, so shipping +# one image (compose picks the process per-service) makes version skew between +# them impossible. CGO_ENABLED=0 gives a fully static binary that drops into +# the python:3.12-slim final stage with no runtime deps; go.mod/go.sum are +# copied and downloaded before the sources so the Go dep layer caches across +# daemon code-only changes, same reasoning as the pip layer below. +FROM golang:1.26 AS daemon-builder +WORKDIR /src +COPY daemon/go.mod daemon/go.sum ./ +RUN go mod download +COPY daemon/ ./ +# -trimpath keeps the build reproducible (identical sources -> identical +# binary), which is what lets the final stage's COPY layer cache-hit when only +# Python code changed. +RUN CGO_ENABLED=0 go build -trimpath -o /out/thermograph-daemon . + FROM python:3.12-slim # curl is only for the container HEALTHCHECK below. Everything Python needs @@ -14,6 +34,12 @@ RUN apt-get update \ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt +# The daemon binary lands before the app tree: it changes far less often than +# the Python code, so this layer usually cache-hits and only the COPY below +# rebuilds. NOT the entrypoint — deploy/entrypoint.sh stays that; compose +# selects this binary per-service for the daemon container. +COPY --from=daemon-builder /out/thermograph-daemon /usr/local/bin/thermograph-daemon + COPY . /app/ RUN chmod +x /app/deploy/entrypoint.sh diff --git a/backend/api/internal_routes.py b/backend/api/internal_routes.py new file mode 100644 index 0000000..9ecded4 --- /dev/null +++ b/backend/api/internal_routes.py @@ -0,0 +1,158 @@ +"""Internal control surface for the Go daemon (backend/daemon/). + +The gateway bot and the recurring worker jobs moved out of this process into +`thermograph-daemon` — Go owns only the long-lived stateful I/O (the Discord +websocket, RESUME/heartbeat/backoff, interval timers). Anything that needs +*data* calls back in here, because grading depends on polars + the parquet +cache: reimplementing it out-of-process would let the bot's grades drift from +the API's, and the slash-command path deliberately shares one grade builder so +the two never drift. These routes hand the daemon finished answers, keeping it +entirely out of the grading contract. + +Not publicly reachable: the deploy Caddyfile routes only /api/*, /digest and +/discord/interactions to this backend, so /internal/* never resolves through +the public domain. The shared-secret header is defence in depth on top of +that, not the only control — and it fails closed: with no token configured +the whole surface answers 404, never "open". + +Deliberately NOT under BASE (same posture as /healthz): the daemon targets +THERMOGRAPH_API_BASE_INTERNAL directly and shouldn't need to know +THERMOGRAPH_BASE to construct a path. +""" +import hashlib +import hmac +import os +import threading + +from fastapi import APIRouter, Depends, Header, HTTPException +from pydantic import BaseModel + +_TOKEN_ENV = "THERMOGRAPH_INTERNAL_TOKEN" +_HEADER = "X-Thermograph-Internal-Token" + +# Domain-separation label for the derived token. MUST match the Go daemon's +# internal/config constant byte-for-byte or every callback 401s. +_DERIVE_LABEL = b"thermograph/internal-api/v1" + + +def resolve_internal_token() -> str: + """The shared secret guarding /internal/*, or "" if it can't be established. + + An explicit THERMOGRAPH_INTERNAL_TOKEN always wins. Otherwise it is DERIVED + from THERMOGRAPH_AUTH_SECRET, which every environment already provisions, so + standing up the daemon needs no new vault entry and no operator step -- one + less secret to rotate, leak, or forget on a new host. + + HMAC with a distinct label rather than reusing AUTH_SECRET directly: that is + ordinary domain separation, so a leak of this token can't be replayed as the + session-signing key it was derived from, and the derivation is one-way. + + Deliberately reads THERMOGRAPH_AUTH_SECRET from the environment rather than + users.py's effective value: users.py falls back to a random per-process + secret when the env var is unset, and two processes would then derive two + different tokens and fail every call in a way that looks like a bug rather + than missing config. Unset env var => "" => the surface stays closed.""" + explicit = os.environ.get(_TOKEN_ENV, "").strip() + if explicit: + return explicit + auth_secret = os.environ.get("THERMOGRAPH_AUTH_SECRET", "").strip() + if not auth_secret: + return "" + return hmac.new(auth_secret.encode(), _DERIVE_LABEL, hashlib.sha256).hexdigest() + + +def _require_internal_token( + x_thermograph_internal_token: str = Header(default=""), +) -> None: + """Gate every internal route on the shared secret. + + Read from the environment per request (not at import) so the fail-closed + behaviour is a property of the running config, not of import order — and so + tests can exercise both states without re-importing the app. An unset/empty + token means the surface was never provisioned: 404 (the route doesn't + exist), matching the metrics endpoint's don't-reveal posture. A present but + wrong header is a real auth failure: 401. compare_digest, not ==, so the + comparison doesn't leak the token's prefix through response timing.""" + token = resolve_internal_token() + if not token: + raise HTTPException(status_code=404) + if not hmac.compare_digest(x_thermograph_internal_token, token): + raise HTTPException(status_code=401, detail="bad internal token") + + +# include_in_schema=False: an internal surface has no business in the public +# OpenAPI document, same as /api/v2/metrics. +router = APIRouter(prefix="/internal", include_in_schema=False, + dependencies=[Depends(_require_internal_token)]) + + +# Every handler below is a plain `def`, not `async def`, on purpose: they all do +# blocking work (sync parquet/DB reads, grading math, upstream HTTP fetches). +# FastAPI runs sync handlers on its threadpool, so a slow grade or a long warm +# run never stalls the event loop the rest of the API is serving on — the same +# reason web/app.py wraps discord_interactions.handle in run_in_threadpool. + + +class _GradeBody(BaseModel): + query: str + + +@router.post("/discord/grade") +def discord_grade(body: _GradeBody) -> dict: + """The gateway-ready reply for one bot mention/DM: the same embed the /grade + slash command builds, via the shared discord_interactions._grade_message — + reusing one builder is what keeps the two paths from ever drifting. Local + import for the same reason the in-process gateway bot used one: don't drag + the interactions module's transitive deps into contexts that only mount the + router.""" + from notifications import discord_interactions + data = discord_interactions._grade_message(body.query) + # Gateway messages can't be ephemeral — the flag is interactions-only, and + # Discord rejects it outside an interaction response. + data.pop("flags", None) + return data + + +# One in-flight run per job, enforced here rather than trusted to the daemon's +# timers: warm_cities spends the Open-Meteo quota, so a stacked second run +# double-spends it, and a slow run must never let invocations pile up holding +# requests open. Non-blocking acquire => the loser answers 409 immediately. +_JOB_LOCKS: dict[str, threading.Lock] = { + "warm-cities": threading.Lock(), + "indexnow": threading.Lock(), +} + + +def _run_exclusive(job: str, fn) -> dict: + lock = _JOB_LOCKS[job] + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail=f"{job} is already running") + try: + fn() + finally: + lock.release() + return {"ok": True} + + +@router.post("/jobs/warm-cities") +def jobs_warm_cities() -> dict: + """(Re-)warm the curated city set. The daemon owns the cadence (default + daily — every already-cached city is a cheap skip); this endpoint just runs + one pass. Local import: warm_cities is a script-style top-level module, so + only the process that actually runs the job pays its import cost.""" + def _run(): + import warm_cities + warm_cities.main() + return _run_exclusive("warm-cities", _run) + + +@router.post("/jobs/indexnow") +def jobs_indexnow() -> dict: + """Check whether the indexable URL set changed and, if so, ping IndexNow. + submit_if_changed() is a cheap no-op when nothing changed, which is why the + daemon can tick this more often than warm-cities without spending anything + on most runs.""" + def _run(): + import indexnow + indexnow.submit_if_changed() + return _run_exclusive("indexnow", _run) diff --git a/backend/daemon/README.md b/backend/daemon/README.md new file mode 100644 index 0000000..c653dbc --- /dev/null +++ b/backend/daemon/README.md @@ -0,0 +1,50 @@ +# thermograph-daemon + +Single Go binary that owns the backend's long-lived stateful I/O: the Discord +gateway connection (IDENTIFY/RESUME/heartbeat/reconnect backoff) and the +recurring-job timers (warm-cities, IndexNow). It replaces the two in-process +background jobs `web/app.py` used to run under leader election — one daemon +replica makes the single-connection invariant a deployment fact instead of a +runtime election. + +## The Go/Python split + +Go here owns **no** climate or grading logic. Grading depends on polars and the +parquet cache, and the slash-command path deliberately shares one grade builder +with the API so bot grades and API grades never drift. So anything data-shaped +is a POST back into the Python app's internal-only routes: + +| Route | Purpose | +| --- | --- | +| `POST /internal/discord/grade` | `{"query": "phoenix"}` → gateway-ready Discord message JSON, relayed to Discord verbatim | +| `POST /internal/jobs/warm-cities` | trigger the warm-cities job | +| `POST /internal/jobs/indexnow` | trigger the IndexNow check-and-submit | + +Every request carries `X-Thermograph-Internal-Token`. Caddy never routes +`/internal/*` to the backend, so the token is defence in depth, not the only +control. If `THERMOGRAPH_INTERNAL_TOKEN` is unset, the Python side disables the +routes and this daemon refuses to start (fail closed on both ends). + +## Configuration (env) + +| Var | Required | Meaning | +| --- | --- | --- | +| `THERMOGRAPH_INTERNAL_TOKEN` | yes | shared secret for the internal routes | +| `THERMOGRAPH_API_BASE_INTERNAL` | yes | backend base URL, e.g. `http://web:8137` | +| `THERMOGRAPH_DISCORD_BOT` | no | `1`/`true`/`yes`/`on` enables the gateway | +| `THERMOGRAPH_DISCORD_BOT_TOKEN` | no | bot token (gateway stays off without it) | +| `THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS` | no | default 24; malformed → default | +| `THERMOGRAPH_INDEXNOW_INTERVAL_HOURS` | no | default 6; malformed → default | + +## Run locally + +```sh +cd backend/daemon +go build ./... && go vet ./... && go test ./... + +THERMOGRAPH_INTERNAL_TOKEN=dev-token \ +THERMOGRAPH_API_BASE_INTERNAL=http://localhost:8137 \ +go run . +``` + +Deployed as `/usr/local/bin/thermograph-daemon` in the backend image. diff --git a/backend/daemon/go.mod b/backend/daemon/go.mod new file mode 100644 index 0000000..e8a6593 --- /dev/null +++ b/backend/daemon/go.mod @@ -0,0 +1,5 @@ +module thermograph/daemon + +go 1.26 + +require github.com/coder/websocket v1.8.15 diff --git a/backend/daemon/go.sum b/backend/daemon/go.sum new file mode 100644 index 0000000..4e0a48d --- /dev/null +++ b/backend/daemon/go.sum @@ -0,0 +1,2 @@ +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= diff --git a/backend/daemon/internal/apiclient/client.go b/backend/daemon/internal/apiclient/client.go new file mode 100644 index 0000000..ba998c4 --- /dev/null +++ b/backend/daemon/internal/apiclient/client.go @@ -0,0 +1,107 @@ +// Package apiclient is the daemon's ONLY channel back into the Python app. +// The daemon owns long-lived I/O (gateway socket, timers) and nothing else: +// grading depends on polars + the parquet cache, and the slash-command path +// deliberately shares one grade builder with the API so the two never drift. +// Reimplementing any of that here would reintroduce exactly that drift, so +// everything data-shaped is a POST to the backend's /internal/* routes. +package apiclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// defaultTimeout bounds every callback. Grading does synchronous parquet/DB +// reads on the Python side (a cold city can take a while), so be generous — +// but never infinite: a hung backend must not wedge the gateway's event loop. +const defaultTimeout = 60 * time.Second + +// errBodyLimit caps how much of an error response we echo into logs; enough +// to see a traceback's headline without dumping megabytes. +const errBodyLimit = 512 + +// Client calls the backend's internal-only routes. The token is defence in +// depth — Caddy never routes /internal/* to the backend in the first place — +// but it is still required on every request so a misrouted or LAN-local call +// cannot hit these endpoints unauthenticated. +type Client struct { + base string + token string + http *http.Client +} + +// New returns a Client for the backend at base (e.g. http://web:8137), +// authenticating with token. +func New(base, token string) *Client { + return &Client{ + base: strings.TrimRight(base, "/"), + token: token, + http: &http.Client{Timeout: defaultTimeout}, + } +} + +// post sends body to path and returns the raw response body on 2xx. +func (c *Client) post(ctx context.Context, path string, body any) (json.RawMessage, error) { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("apiclient: encode %s body: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("apiclient: build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Thermograph-Internal-Token", c.token) + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("apiclient: POST %s: %w", path, err) + } + defer resp.Body.Close() + + // Read one byte past the limit so the truncation marker only appears when + // something was actually cut. + raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("apiclient: POST %s: read body: %w", path, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + snippet := raw + truncated := "" + if len(snippet) > errBodyLimit { + snippet = snippet[:errBodyLimit] + truncated = "... (truncated)" + } + return nil, fmt.Errorf("apiclient: POST %s: status %d: %s%s", path, resp.StatusCode, snippet, truncated) + } + return json.RawMessage(raw), nil +} + +// Grade asks Python to grade a city query and returns the gateway-ready +// Discord message JSON VERBATIM. The raw bytes are relayed to Discord without +// parsing or reshaping — that verbatim relay is what keeps Go out of the +// grading/embed contract: any change to the message shape lives entirely on +// the Python side. +func (c *Client) Grade(ctx context.Context, query string) (json.RawMessage, error) { + return c.post(ctx, "/internal/discord/grade", map[string]string{"query": query}) +} + +// WarmCities triggers the warm-cities job (the Python side owns what "warm" +// means and which cities are curated). +func (c *Client) WarmCities(ctx context.Context) error { + _, err := c.post(ctx, "/internal/jobs/warm-cities", map[string]string{}) + return err +} + +// IndexNow triggers the IndexNow check-and-submit job; a no-op on the Python +// side when the indexable URL set has not changed. +func (c *Client) IndexNow(ctx context.Context) error { + _, err := c.post(ctx, "/internal/jobs/indexnow", map[string]string{}) + return err +} diff --git a/backend/daemon/internal/apiclient/client_test.go b/backend/daemon/internal/apiclient/client_test.go new file mode 100644 index 0000000..9fa456b --- /dev/null +++ b/backend/daemon/internal/apiclient/client_test.go @@ -0,0 +1,124 @@ +package apiclient + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestGradeHappyPathRelaysBodyVerbatim(t *testing.T) { + // Deliberately odd formatting/key order: Grade must hand back the exact + // bytes, never a re-marshalled reshaping of them. + const body = `{ "content":null, "embeds": [ {"title":"Phoenix"} ] }` + var gotPath, gotToken, gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotToken = r.Header.Get("X-Thermograph-Internal-Token") + var req map[string]string + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode request: %v", err) + } + gotQuery = req["query"] + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write([]byte(body)); err != nil { + t.Errorf("write response: %v", err) + } + })) + defer srv.Close() + + c := New(srv.URL, "tok123") + raw, err := c.Grade(context.Background(), "phoenix") + if err != nil { + t.Fatalf("Grade: %v", err) + } + if string(raw) != body { + t.Errorf("body not relayed verbatim:\n got %q\nwant %q", raw, body) + } + if gotPath != "/internal/discord/grade" { + t.Errorf("path = %q", gotPath) + } + if gotToken != "tok123" { + t.Errorf("token header = %q, want %q", gotToken, "tok123") + } + if gotQuery != "phoenix" { + t.Errorf("query = %q", gotQuery) + } +} + +func TestJobsSendTokenAndHitRightPaths(t *testing.T) { + paths := map[string]bool{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if r.Header.Get("X-Thermograph-Internal-Token") != "tok" { + t.Errorf("missing/wrong token header on %s", r.URL.Path) + } + paths[r.URL.Path] = true + if _, err := w.Write([]byte(`{"ok":true}`)); err != nil { + t.Errorf("write response: %v", err) + } + })) + defer srv.Close() + + c := New(srv.URL+"/", "tok") // trailing slash must not produce "//internal" + if err := c.WarmCities(context.Background()); err != nil { + t.Fatalf("WarmCities: %v", err) + } + if err := c.IndexNow(context.Background()); err != nil { + t.Fatalf("IndexNow: %v", err) + } + for _, p := range []string{"/internal/jobs/warm-cities", "/internal/jobs/indexnow"} { + if !paths[p] { + t.Errorf("path %s was never hit (saw %v)", p, paths) + } + } +} + +func TestNon2xxReturnsStatusAndTruncatedBody(t *testing.T) { + long := strings.Repeat("x", 2000) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, long, http.StatusBadGateway) + })) + defer srv.Close() + + c := New(srv.URL, "tok") + _, err := c.Grade(context.Background(), "phoenix") + if err == nil { + t.Fatal("expected error on 502, got nil") + } + msg := err.Error() + if !strings.Contains(msg, "502") { + t.Errorf("error should include status, got: %s", msg) + } + if !strings.Contains(msg, "truncated") { + t.Errorf("error should mark truncation, got %d chars: %.100s...", len(msg), msg) + } + if len(msg) > 1000 { + t.Errorf("error message not truncated: %d chars", len(msg)) + } +} + +func TestContextCancellation(t *testing.T) { + block := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-block // hold the request open until the test ends + })) + defer srv.Close() + defer close(block) + + c := New(srv.URL, "tok") + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err := c.Grade(ctx, "phoenix") + if err == nil { + t.Fatal("expected error from cancelled context, got nil") + } + if !strings.Contains(err.Error(), "deadline") && !strings.Contains(err.Error(), "cancel") { + t.Errorf("expected a context-shaped error, got: %v", err) + } +} diff --git a/backend/daemon/internal/config/config.go b/backend/daemon/internal/config/config.go new file mode 100644 index 0000000..c7d60e6 --- /dev/null +++ b/backend/daemon/internal/config/config.go @@ -0,0 +1,138 @@ +// Package config loads and validates the daemon's environment. All knobs are +// env vars because the daemon runs as a sibling container to the web service +// and shares its /etc/thermograph.env — a config file would be a second source +// of truth for the same values. +package config + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "strconv" + "strings" + "time" +) + +// Defaults mirror the Python scheduler they replace (notifications/scheduler.py): +// warm-cities daily — a warm cache mostly stays warm, and every already-cached +// city is a cheap skip; IndexNow every 6h — submit_if_changed() is a no-op when +// nothing changed, so it can tick more often without spending anything. +const ( + DefaultWarmCitiesIntervalHours = 24 + DefaultIndexNowIntervalHours = 6 +) + +// Config is the daemon's fully-resolved configuration. Construct it only via +// Load so the required/optional split is enforced in one place. +type Config struct { + // InternalToken is the shared secret sent as X-Thermograph-Internal-Token + // on every callback into Python. Required: if it is unset the web app + // disables its /internal/* routes (fail closed), so a daemon without it + // could only ever fail every call — better to refuse to start. + InternalToken string + + // APIBase is the in-network base URL of the FastAPI app (e.g. + // http://web:8137). Same env var the frontend already uses for its + // server-side calls, so there is one name for "where is the backend". + APIBase string + + // DiscordEnabled gates the gateway connection. Off by default: a gateway + // connection is a new persistent behaviour, so it takes an explicit opt-in + // (THERMOGRAPH_DISCORD_BOT truthy AND a token) rather than piggybacking on + // the token being present for DMs. + DiscordEnabled bool + + // DiscordToken is the bot token; only meaningful when DiscordEnabled. + DiscordToken string + + WarmCitiesInterval time.Duration + IndexNowInterval time.Duration +} + +// truthy matches the Python side's _TRUTHY set exactly, so the same env file +// enables/disables the bot regardless of which process reads it. +func truthy(s string) bool { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "true", "yes", "on": + return true + } + return false +} + +// intervalHours parses an interval env var. A malformed or non-positive value +// falls back to the default rather than erroring — mirrors the Python +// `int(os.environ.get(..., "24") or 24)` behaviour where a bad knob degrades +// to the documented default instead of taking the process down. +func intervalHours(raw string, def int) time.Duration { + hours := def + if v := strings.TrimSpace(raw); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + hours = n + } + } + return time.Duration(hours) * time.Hour +} + +// Load reads the environment and returns a validated Config, or an error that +// should abort startup. The daemon must never run half-configured: a missing +// token or API base means every callback would fail, silently, forever. +func Load() (*Config, error) { + return load(os.Getenv) +} + +// load takes a getenv func so tests can inject an environment without mutating +// the real one. +func load(getenv func(string) string) (*Config, error) { + cfg := &Config{ + InternalToken: strings.TrimSpace(getenv("THERMOGRAPH_INTERNAL_TOKEN")), + APIBase: strings.TrimSpace(getenv("THERMOGRAPH_API_BASE_INTERNAL")), + DiscordToken: strings.TrimSpace(getenv("THERMOGRAPH_DISCORD_BOT_TOKEN")), + } + if cfg.InternalToken == "" { + cfg.InternalToken = deriveInternalToken(strings.TrimSpace(getenv("THERMOGRAPH_AUTH_SECRET"))) + } + if cfg.InternalToken == "" { + return nil, errors.New("neither THERMOGRAPH_INTERNAL_TOKEN nor THERMOGRAPH_AUTH_SECRET is set; refusing to start (the web app disables /internal/* without a token, so every callback would fail)") + } + if cfg.APIBase == "" { + return nil, errors.New("THERMOGRAPH_API_BASE_INTERNAL is not set; refusing to start (no way to reach the backend)") + } + // The gateway needs both the opt-in flag and a token; missing either just + // disables it — the cron half is still worth running on its own. + cfg.DiscordEnabled = truthy(getenv("THERMOGRAPH_DISCORD_BOT")) && cfg.DiscordToken != "" + + cfg.WarmCitiesInterval = intervalHours(getenv("THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS"), DefaultWarmCitiesIntervalHours) + cfg.IndexNowInterval = intervalHours(getenv("THERMOGRAPH_INDEXNOW_INTERVAL_HOURS"), DefaultIndexNowIntervalHours) + return cfg, nil +} + +// deriveLabel is the domain-separation label for the derived internal token. +// It MUST match backend/api/internal_routes.py's _DERIVE_LABEL byte-for-byte, +// or the daemon and the web app compute different tokens and every callback +// 401s. TestDeriveInternalTokenMatchesPython pins the pair to a shared vector. +const deriveLabel = "thermograph/internal-api/v1" + +// deriveInternalToken computes the shared secret from THERMOGRAPH_AUTH_SECRET, +// which every environment already provisions. This is what lets the daemon +// stand up with no new vault entry and no operator step -- one less secret to +// rotate, leak, or forget on a new host. An explicit THERMOGRAPH_INTERNAL_TOKEN +// still wins when set. +// +// HMAC under a distinct label rather than reusing the auth secret directly: +// ordinary domain separation, so a leak of this token cannot be replayed as the +// session-signing key it came from, and the derivation is one-way. +// +// Returns "" for an empty secret, which the caller turns into a refusal to +// start. It must never silently invent a token: the web app would have derived +// a different one (or none), and every call would fail as an auth error rather +// than as the missing configuration it actually is. +func deriveInternalToken(authSecret string) string { + if authSecret == "" { + return "" + } + mac := hmac.New(sha256.New, []byte(authSecret)) + mac.Write([]byte(deriveLabel)) + return hex.EncodeToString(mac.Sum(nil)) +} diff --git a/backend/daemon/internal/config/config_test.go b/backend/daemon/internal/config/config_test.go new file mode 100644 index 0000000..c5257c3 --- /dev/null +++ b/backend/daemon/internal/config/config_test.go @@ -0,0 +1,96 @@ +package config + +import ( + "testing" + "time" +) + +// env builds a getenv func over a map, defaulting the two required vars so +// each test only states what it cares about. +func env(overrides map[string]string) func(string) string { + base := map[string]string{ + "THERMOGRAPH_INTERNAL_TOKEN": "sekrit", + "THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137", + } + for k, v := range overrides { + base[k] = v + } + return func(k string) string { return base[k] } +} + +func TestRequiredVarsRefuseToStart(t *testing.T) { + for _, missing := range []string{"THERMOGRAPH_INTERNAL_TOKEN", "THERMOGRAPH_API_BASE_INTERNAL"} { + if _, err := load(env(map[string]string{missing: ""})); err == nil { + t.Errorf("expected error when %s is unset, got nil", missing) + } + } + if cfg, err := load(env(nil)); err != nil || cfg == nil { + t.Fatalf("both required vars set: unexpected error %v", err) + } +} + +func TestTruthyParsing(t *testing.T) { + cases := []struct { + flag, token string + want bool + }{ + // Must match the Python _TRUTHY set exactly. + {"1", "tok", true}, + {"true", "tok", true}, + {"TRUE", "tok", true}, + {"yes", "tok", true}, + {"on", "tok", true}, + {" on ", "tok", true}, // whitespace tolerated, like .strip() in Python + {"0", "tok", false}, + {"false", "tok", false}, + {"", "tok", false}, + {"enabled", "tok", false}, // not in the truthy set + // Flag alone is not enough: no token means no gateway. + {"1", "", false}, + } + for _, c := range cases { + cfg, err := load(env(map[string]string{ + "THERMOGRAPH_DISCORD_BOT": c.flag, + "THERMOGRAPH_DISCORD_BOT_TOKEN": c.token, + })) + if err != nil { + t.Fatalf("flag=%q token=%q: unexpected error %v", c.flag, c.token, err) + } + if cfg.DiscordEnabled != c.want { + t.Errorf("flag=%q token=%q: DiscordEnabled=%v, want %v", c.flag, c.token, cfg.DiscordEnabled, c.want) + } + } +} + +func TestIntervalDefaultsAndFallbacks(t *testing.T) { + cases := []struct { + name string + warm, indexnow string + wantWarm time.Duration + wantIndexNow time.Duration + }{ + {"unset uses defaults", "", "", 24 * time.Hour, 6 * time.Hour}, + {"explicit values win", "12", "3", 12 * time.Hour, 3 * time.Hour}, + // Malformed values degrade to the default instead of erroring, like + // the Python int(... or 24) they replace. + {"garbage falls back", "soon", "1.5", 24 * time.Hour, 6 * time.Hour}, + {"non-positive falls back", "0", "-2", 24 * time.Hour, 6 * time.Hour}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + cfg, err := load(env(map[string]string{ + "THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS": c.warm, + "THERMOGRAPH_INDEXNOW_INTERVAL_HOURS": c.indexnow, + })) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.WarmCitiesInterval != c.wantWarm { + t.Errorf("WarmCitiesInterval=%v, want %v", cfg.WarmCitiesInterval, c.wantWarm) + } + if cfg.IndexNowInterval != c.wantIndexNow { + t.Errorf("IndexNowInterval=%v, want %v", cfg.IndexNowInterval, c.wantIndexNow) + } + }) + } +} diff --git a/backend/daemon/internal/config/derive_test.go b/backend/daemon/internal/config/derive_test.go new file mode 100644 index 0000000..7e185e6 --- /dev/null +++ b/backend/daemon/internal/config/derive_test.go @@ -0,0 +1,66 @@ +package config + +import "testing" + +// CROSS-LANGUAGE CONTRACT. The daemon and the web app must derive byte-identical +// tokens from the same THERMOGRAPH_AUTH_SECRET, or every callback 401s -- and it +// would surface as an auth error, which reads like a bug rather than like the +// configuration drift it actually is. The same vector is asserted on the Python +// side in backend/tests/api/test_internal_routes.py::test_derived_token_matches_go. +// If deriveLabel or the construction changes, BOTH must change together. +const ( + sharedVectorSecret = "test-auth-secret" + sharedVectorToken = "4c3830b2158941ff52720d198b65f7924372a3a482b8da52540a4d9be38247ea" +) + +func TestDeriveInternalTokenMatchesPython(t *testing.T) { + if got := deriveInternalToken(sharedVectorSecret); got != sharedVectorToken { + t.Fatalf("deriveInternalToken(%q) = %q; want %q (must match internal_routes.py)", + sharedVectorSecret, got, sharedVectorToken) + } +} + +// An empty secret must yield nothing rather than some fixed "empty HMAC" value: +// the caller turns "" into a refusal to start, and inventing a token here would +// let the daemon run against a web app that computed a different one. +func TestDeriveInternalTokenEmptySecretYieldsNothing(t *testing.T) { + if got := deriveInternalToken(""); got != "" { + t.Fatalf("deriveInternalToken(%q) = %q; want empty so Load refuses to start", "", got) + } +} + +func TestLoadPrefersExplicitTokenOverDerived(t *testing.T) { + env := map[string]string{ + "THERMOGRAPH_INTERNAL_TOKEN": "explicit-wins", + "THERMOGRAPH_AUTH_SECRET": sharedVectorSecret, + "THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137", + } + cfg, err := load(func(k string) string { return env[k] }) + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.InternalToken != "explicit-wins" { + t.Fatalf("InternalToken = %q; an explicit env var must win over derivation", cfg.InternalToken) + } +} + +func TestLoadDerivesTokenWhenOnlyAuthSecretIsSet(t *testing.T) { + env := map[string]string{ + "THERMOGRAPH_AUTH_SECRET": sharedVectorSecret, + "THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137", + } + cfg, err := load(func(k string) string { return env[k] }) + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.InternalToken != sharedVectorToken { + t.Fatalf("InternalToken = %q; want the derived %q", cfg.InternalToken, sharedVectorToken) + } +} + +func TestLoadRefusesWhenNeitherTokenNorAuthSecretIsSet(t *testing.T) { + env := map[string]string{"THERMOGRAPH_API_BASE_INTERNAL": "http://web:8137"} + if _, err := load(func(k string) string { return env[k] }); err == nil { + t.Fatal("load succeeded with no token and no auth secret; it must refuse to start") + } +} diff --git a/backend/daemon/internal/cron/cron.go b/backend/daemon/internal/cron/cron.go new file mode 100644 index 0000000..945323f --- /dev/null +++ b/backend/daemon/internal/cron/cron.go @@ -0,0 +1,92 @@ +// Package cron is the Go replacement for notifications/scheduler.py's +// APScheduler pair (warm-cities + IndexNow). The Python side chose APScheduler +// over a Postgres-native queue because exactly one process ever runs these +// jobs — no fan-out, backpressure, or dead-letter need — and that shape +// carries over unchanged: this daemon is the sole scheduler, so plain +// per-job tickers are all the machinery required. procrastinate/pgqueuer +// remain the documented upgrade path if that ever changes — see +// docs/architecture/repo-topology-and-infrastructure.md §7. +// +// The jobs themselves stay in Python (called via the internal HTTP routes); +// this package owns only the timing. +package cron + +import ( + "context" + "log/slog" + "sync" + "time" +) + +// Job is one recurring task: a name for logs, an interval, and the callback +// (in practice an apiclient method hitting a /internal/jobs/* route). +type Job struct { + Name string + Interval time.Duration + Run func(ctx context.Context) error +} + +// Run drives every job on its own ticker until ctx is cancelled, then returns +// once all job goroutines have stopped. It never returns early: a failing job +// logs and keeps ticking — one bad tick (backend briefly down, timeout) must +// never kill the schedule, because nothing restarts it short of a container +// restart. +func Run(ctx context.Context, logger *slog.Logger, jobs ...Job) { + var wg sync.WaitGroup + for _, job := range jobs { + wg.Add(1) + go func(job Job) { + defer wg.Done() + runJob(ctx, logger, job) + }(job) + } + wg.Wait() +} + +func runJob(ctx context.Context, logger *slog.Logger, job Job) { + // A plain ticker's first fire is one interval from now, NOT at startup — + // deliberately kept from the Python scheduler (its add_job had no + // next_run_time override for the same reason): warm-cities already runs at + // deploy time via the deploy hook, so an immediate run on every daemon + // boot/restart would just re-spend work (and, for warm-cities, archive-fetch + // quota) that the deploy already paid for. + ticker := time.NewTicker(job.Interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + start := time.Now() + // Run synchronously in this loop: a job can never overlap with + // itself by construction. That matters most for warm-cities, + // which spends the archive-fetch quota — two overlapping runs would + // double-spend it. + if err := job.Run(ctx); err != nil { + if ctx.Err() != nil { + // Shutdown raced the job; the cancellation is the story, + // not the (expected) aborted call. + return + } + logger.Error("cron job failed; will retry next tick", + "job", job.Name, "err", err, "elapsed", time.Since(start)) + } else { + logger.Info("cron job ok", + "job", job.Name, "elapsed", time.Since(start)) + } + // If the job overran its interval, a tick may have fired mid-run. + // Skip it rather than running again back-to-back — a late tick is + // the same double-spend as an overlapping one, just serialized. + // (Go 1.23+ tickers already drop fires with no ready receiver; + // this drain pins the skip semantics rather than relying on the + // runtime's channel buffering.) + select { + case <-ticker.C: + logger.Warn("cron job overran its interval; skipping the tick that fired mid-run", + "job", job.Name, "interval", job.Interval, "elapsed", time.Since(start)) + default: + } + } + } +} diff --git a/backend/daemon/internal/cron/cron_test.go b/backend/daemon/internal/cron/cron_test.go new file mode 100644 index 0000000..f2c5675 --- /dev/null +++ b/backend/daemon/internal/cron/cron_test.go @@ -0,0 +1,204 @@ +package cron + +import ( + "context" + "errors" + "io" + "log/slog" + "sync/atomic" + "testing" + "time" +) + +// Intervals here are tens of milliseconds: long enough that scheduler jitter +// cannot invert an assertion, short enough that the whole file runs in well +// under a second. + +func discard() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// The first run must be one interval after start, never at startup — the +// deploy hook already warmed the cache, so a boot-time run is pure waste. +func TestFirstRunIsDeferredOneInterval(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var runs atomic.Int32 + done := make(chan struct{}) + go func() { + defer close(done) + Run(ctx, discard(), Job{ + Name: "test", + Interval: 60 * time.Millisecond, + Run: func(context.Context) error { + runs.Add(1) + return nil + }, + }) + }() + + // Well inside the first interval: nothing may have run yet. + time.Sleep(20 * time.Millisecond) + if got := runs.Load(); got != 0 { + t.Fatalf("job ran %d time(s) before the first interval elapsed; first run must be deferred", got) + } + + // Well past the first interval: it must have run by now. + deadline := time.After(500 * time.Millisecond) + for runs.Load() == 0 { + select { + case <-deadline: + t.Fatal("job never ran after the first interval elapsed") + case <-time.After(5 * time.Millisecond): + } + } + + cancel() + <-done +} + +// A slow run must not overlap with itself, and ticks that fired mid-run must +// be skipped, not queued — an immediate back-to-back run would double-spend +// the archive-fetch quota just like a concurrent one. +func TestSlowJobNeverOverlapsAndSkipsMissedTicks(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const interval = 30 * time.Millisecond + + var inFlight, maxInFlight, runs atomic.Int32 + release := make(chan struct{}) + started := make(chan struct{}, 16) + + done := make(chan struct{}) + go func() { + defer close(done) + Run(ctx, discard(), Job{ + Name: "slow", + Interval: interval, + Run: func(context.Context) error { + n := inFlight.Add(1) + for { + m := maxInFlight.Load() + if n <= m || maxInFlight.CompareAndSwap(m, n) { + break + } + } + runs.Add(1) + started <- struct{}{} + <-release // block until the test lets each run finish + inFlight.Add(-1) + return nil + }, + }) + }() + + // First run starts; hold it across several intervals. + <-started + time.Sleep(4 * interval) + if got := maxInFlight.Load(); got != 1 { + t.Fatalf("job overlapped with itself: max in-flight = %d", got) + } + if got := runs.Load(); got != 1 { + t.Fatalf("expected exactly 1 run while the first is still blocked, got %d", got) + } + release <- struct{}{} + + // After release the schedule resumes, but the ticks missed during the + // block must have been dropped: the next run arrives roughly one interval + // later, and only one more within that window (not a burst of catch-ups). + select { + case <-started: + case <-time.After(500 * time.Millisecond): + t.Fatal("job never resumed after the blocked run finished") + } + if got := runs.Load(); got != 2 { + t.Fatalf("missed ticks were replayed as a burst: %d runs total, want 2", got) + } + release <- struct{}{} + + cancel() + <-done + if got := maxInFlight.Load(); got != 1 { + t.Fatalf("job overlapped with itself: max in-flight = %d", got) + } +} + +// One failed tick must never kill the ticker — nothing restarts the schedule +// short of a container restart. +func TestFailedTickDoesNotStopSchedule(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var runs atomic.Int32 + done := make(chan struct{}) + go func() { + defer close(done) + Run(ctx, discard(), Job{ + Name: "flaky", + Interval: 20 * time.Millisecond, + Run: func(context.Context) error { + if runs.Add(1) == 1 { + return errors.New("backend briefly down") + } + return nil + }, + }) + }() + + deadline := time.After(1 * time.Second) + for runs.Load() < 3 { + select { + case <-deadline: + t.Fatalf("schedule stalled after a failure: only %d run(s)", runs.Load()) + case <-time.After(5 * time.Millisecond): + } + } + + cancel() + <-done +} + +// Cancellation must stop Run promptly even while a job is mid-call: the job +// callbacks are ctx-aware HTTP calls, so cancelling the context aborts the +// in-flight request and the loop must then exit instead of ticking again. +func TestCancelStopsRunWhileJobInFlight(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + + started := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + Run(ctx, discard(), + Job{ + Name: "blocking", + Interval: 10 * time.Millisecond, + Run: func(jobCtx context.Context) error { + close(started) + <-jobCtx.Done() // behaves like an HTTP call aborted by cancellation + return jobCtx.Err() + }, + }, + // A second, idle job proves Run waits for ALL jobs to stop. + Job{ + Name: "idle", + Interval: time.Hour, + Run: func(context.Context) error { return nil }, + }, + ) + }() + + <-started + cancel() + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("Run did not return promptly after context cancellation") + } +} diff --git a/backend/daemon/internal/gateway/gateway.go b/backend/daemon/internal/gateway/gateway.go new file mode 100644 index 0000000..123652e --- /dev/null +++ b/backend/daemon/internal/gateway/gateway.go @@ -0,0 +1,529 @@ +// Package gateway holds Discord's **gateway** bot — it reacts to @mentions +// (and DMs) in real time. The other two Discord paths need no persistent +// connection: slash commands arrive as signed HTTP POSTs and the daily post +// goes out over an incoming webhook, both handled on the Python side. This one +// is different — to answer an *ordinary* message that mentions the bot, we +// have to hold a live websocket to Discord's gateway and listen for +// MESSAGE_CREATE events. +// +// Single-connection invariant. Discord permits exactly one gateway connection +// per bot token (per shard). The Python predecessor enforced that with a +// leader election inside web/app.py; here it holds by deployment shape — the +// daemon runs as exactly one replica, and this package owns its only +// connection. +// +// Least privilege / no privileged intent. We subscribe to GUILDS + +// GUILD_MESSAGES + DIRECT_MESSAGES and act only on messages that @mention the +// bot or DM it. Discord delivers message *content* for exactly those cases +// even without the privileged MESSAGE_CONTENT intent, so the bot needs no +// privileged-intent portal review — a real design constraint, not trivia. To +// trigger on arbitrary keywords in channels, enable MESSAGE_CONTENT in the +// Developer Portal and extend triage; the rest of the machinery is unchanged. +// +// The gateway owns NO climate/grading logic. A mention with a city query is +// answered by calling back into the Python app (Grader / apiclient), which +// shares one grade builder with the /grade slash command so the two never +// drift. +package gateway + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/coder/websocket" + + "thermograph/daemon/internal/config" +) + +// Run is the package entrypoint: build a Bot from the resolved config and +// drive it for ctx's lifetime. The caller has already checked +// cfg.DiscordEnabled — an unconfigured deploy never gets this far, so an +// unconfigured deploy pays nothing. +func Run(ctx context.Context, cfg *config.Config, api Grader) error { + return New(cfg.DiscordToken, api).Run(ctx) +} + +// v10 JSON gateway. On a resume we reconnect to the session's +// resume_gateway_url (from the READY payload) instead, with the same query +// string appended. +const ( + gatewayURL = "wss://gateway.discord.gg/?v=10&encoding=json" + gatewayQS = "/?v=10&encoding=json" +) + +// Gateway intents (bitfield). GUILDS gives guild/channel context; +// GUILD_MESSAGES and DIRECT_MESSAGES deliver message-create events in servers +// and DMs. We deliberately do NOT request MESSAGE_CONTENT (privileged) — +// Discord still includes content for messages that mention us or DM us, which +// is all we act on. +const ( + intentGuilds = 1 << 0 + intentGuildMessages = 1 << 9 + intentDirectMessages = 1 << 12 + intents = intentGuilds | intentGuildMessages | intentDirectMessages +) + +// Gateway opcodes (discord.com/developers/docs/topics/gateway-events#opcodes). +const ( + opDispatch = 0 + opHeartbeat = 1 + opIdentify = 2 + opResume = 6 + opReconnect = 7 + opInvalidSession = 9 + opHello = 10 + opHeartbeatACK = 11 +) + +// isFatalClose reports whether a close code is unrecoverable — reconnecting +// just loops, so we stop: 4004 auth failed (bad token), 4010 invalid shard, +// 4011 sharding required, 4012 invalid API version, 4013 invalid intents, +// 4014 disallowed (privileged) intents. +func isFatalClose(code websocket.StatusCode) bool { + switch code { + case 4004, 4010, 4011, 4012, 4013, 4014: + return true + } + return false +} + +// Reconnect backoff bounds. A server-requested reconnect (op 7) resets to the +// start so a planned reconnect comes back promptly. +const ( + backoffStart = 1 * time.Second + backoffMax = 60 * time.Second +) + +// nextBackoff doubles the reconnect delay, capped at backoffMax. +func nextBackoff(d time.Duration) time.Duration { + d *= 2 + if d > backoffMax { + d = backoffMax + } + return d +} + +// maxConcurrentReplies bounds in-flight MESSAGE_CREATE handling. The Python +// version pushed each message onto a thread (asyncio.to_thread) so a slow +// grading lookup couldn't stall heartbeats, but its queue was unbounded; the +// semaphore here keeps the same never-block-the-read-loop property while +// making sure a flood of mentions can't spawn unbounded goroutines against +// the backend. +const maxConcurrentReplies = 4 + +// dialTimeout mirrors the Python's open_timeout=30. +const dialTimeout = 30 * time.Second + +// maxFrameSize mirrors the Python's max_size=2**20 — a grade embed is tiny, +// so anything near a megabyte is not a frame we want to buffer. +const maxFrameSize = 1 << 20 + +const ( + modeIdentify = "identify" + modeResume = "resume" +) + +// Grader is the one slice of the internal API the gateway needs: a city query +// in, gateway-ready Discord message JSON out. Satisfied by *apiclient.Client. +type Grader interface { + Grade(ctx context.Context, query string) (json.RawMessage, error) +} + +// Bot owns the single gateway connection and everything hanging off it. +type Bot struct { + token string + api Grader + sem chan struct{} // bounds concurrent MESSAGE_CREATE handlers + rest string // Discord REST base; swapped for a test server in tests + http *http.Client // REST replies (not the gateway socket) +} + +// New returns a Bot that authenticates with token and answers city queries +// through api. +func New(token string, api Grader) *Bot { + return &Bot{ + token: token, + api: api, + sem: make(chan struct{}, maxConcurrentReplies), + rest: discordAPIBase, + http: &http.Client{Timeout: 30 * time.Second}, + } +} + +// payload is the gateway envelope: opcode, event data, sequence number, +// dispatch type. +type payload struct { + Op int `json:"op"` + D json.RawMessage `json:"d"` + S *int64 `json:"s"` + T string `json:"t"` +} + +// session is the per-connection state that must survive a resume: the last +// sequence number (what RESUME replays from), the resume token/URL, and our +// own user id (for mention detection). seq is mutex-guarded because the read +// loop advances it while the heartbeat goroutine reads it; the other fields +// are only ever touched from the read/run loop. +type session struct { + mu sync.Mutex + seq int64 + haveSeq bool + sessionID string + resumeURL string + userID string +} + +func (s *session) setSeq(v int64) { + s.mu.Lock() + s.seq, s.haveSeq = v, true + s.mu.Unlock() +} + +func (s *session) lastSeq() (int64, bool) { + s.mu.Lock() + defer s.mu.Unlock() + return s.seq, s.haveSeq +} + +// wsConn is a thin JSON send/receive wrapper. coder/websocket allows +// concurrent calls to everything except Read, and only the read loop reads, +// so no extra locking is needed for the heartbeat goroutine's writes. +type wsConn struct { + c *websocket.Conn +} + +func (w *wsConn) send(ctx context.Context, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + return w.c.Write(ctx, websocket.MessageText, data) +} + +func (w *wsConn) read(ctx context.Context) ([]byte, error) { + _, data, err := w.c.Read(ctx) + return data, err +} + +func (w *wsConn) close(code websocket.StatusCode, reason string) { + // Best-effort: the connection may already be gone, and a failed close on + // a dead socket changes nothing about what happens next. + _ = w.c.Close(code, reason) +} + +// jitter is the 0..1s randomness added to every reconnect sleep so a fleet of +// clients dropped by the same incident doesn't reconnect in lockstep. +func jitter() time.Duration { + return time.Duration(rand.Float64() * float64(time.Second)) +} + +// sleepCtx sleeps for d, returning false if ctx was cancelled first. +func sleepCtx(ctx context.Context, d time.Duration) bool { + select { + case <-ctx.Done(): + return false + case <-time.After(d): + return true + } +} + +// Run maintains the gateway connection for the daemon's lifetime: connect, +// IDENTIFY (or RESUME), dispatch events, and reconnect with capped +// exponential backoff. It returns nil on context cancellation and an error +// only on a fatal close (bad token / disallowed intents) — reconnecting after +// those loops forever, so the caller must not restart us. +func (b *Bot) Run(ctx context.Context) error { + sess := &session{} + mode := modeIdentify + backoff := backoffStart + for { + useResume := mode == modeResume && sess.resumeURL != "" + url := gatewayURL + if useResume { + url = sess.resumeURL + } + dialCtx, cancel := context.WithTimeout(ctx, dialTimeout) + conn, _, err := websocket.Dial(dialCtx, url, nil) + cancel() + if err != nil { + if ctx.Err() != nil { + log.Print("gateway: shutting down") + return nil + } + // A failed dial carries no gateway close code, so it is never + // fatal — back off and retry. + mode = modeIdentify + if sess.sessionID != "" { + mode = modeResume + } + log.Printf("gateway: connect failed (%v); reconnecting in %s", err, backoff) + if !sleepCtx(ctx, backoff+jitter()) { + return nil + } + backoff = nextBackoff(backoff) + continue + } + conn.SetReadLimit(maxFrameSize) + ws := &wsConn{c: conn} + + next, err := b.connection(ctx, ws, sess, useResume) + if ctx.Err() != nil { + // Shutdown: close clean (1000) — we are gone for good, so the + // session should not be held open for a RESUME that never comes. + ws.close(websocket.StatusNormalClosure, "shutting down") + log.Print("gateway: shutting down") + return nil + } + if err != nil { + if code := websocket.CloseStatus(err); isFatalClose(code) { + log.Printf("gateway: fatal gateway close %d; stopping", code) + return fmt.Errorf("gateway: fatal close code %d", code) + } + mode = modeIdentify + if sess.sessionID != "" { + mode = modeResume + } + log.Printf("gateway: connection dropped (%v); reconnecting in %s", err, backoff) + if !sleepCtx(ctx, backoff+jitter()) { + return nil + } + backoff = nextBackoff(backoff) + continue + } + + // Deliberate reconnect (op 7 / op 9): close it ourselves. Non-1000 + // when we intend to RESUME — Discord invalidates the session on a + // 1000/1001 close, which would force a fresh IDENTIFY. + if next == modeResume { + ws.close(websocket.StatusCode(4000), "resuming") + } else { + ws.close(websocket.StatusNormalClosure, "reidentifying") + } + mode = next + backoff = backoffStart // planned reconnect: come back promptly + } +} + +// parseHello validates the op-10 HELLO that opens every connection and returns +// its heartbeat interval. +// +// Every failure here is an ERROR rather than a clean "reidentify". That +// distinction is load-bearing: a clean return is Run's PLANNED-reconnect path, +// which resets the backoff to 1s and redials with no sleep at all. That is +// correct for a server-requested reconnect (op 7), but a gateway persistently +// answering with a malformed HELLO would turn it into a tight reconnect loop +// against Discord -- precisely what the capped backoff exists to prevent. The +// error path applies backoff and still picks the next mode from surviving +// session state. +func parseHello(raw []byte) (time.Duration, error) { + var hello payload + if err := json.Unmarshal(raw, &hello); err != nil { + return 0, fmt.Errorf("gateway: unparseable HELLO: %w", err) + } + if hello.Op != opHello { + return 0, fmt.Errorf("gateway: expected HELLO (op %d), got op %d", opHello, hello.Op) + } + var hd struct { + HeartbeatInterval float64 `json:"heartbeat_interval"` + } + if err := json.Unmarshal(hello.D, &hd); err != nil { + return 0, fmt.Errorf("gateway: unparseable HELLO payload: %w", err) + } + // A non-positive interval would busy-loop the heartbeat goroutine. + if hd.HeartbeatInterval <= 0 { + return 0, fmt.Errorf("gateway: invalid heartbeat_interval %v", hd.HeartbeatInterval) + } + return time.Duration(hd.HeartbeatInterval * float64(time.Millisecond)), nil +} + +// connection drives one live connection until it ends. A clean return carries +// the mode for the next connect ("resume" to reconnect and RESUME, "identify" +// to start a fresh session); an error return means the socket died and the +// caller picks the next mode from the surviving session state. +func (b *Bot) connection(ctx context.Context, ws *wsConn, sess *session, resume bool) (string, error) { + raw, err := ws.read(ctx) + if err != nil { + return "", err + } + interval, err := parseHello(raw) + if err != nil { + return modeIdentify, err + } + + // The heartbeat lives exactly as long as this connection. + hbCtx, stopHB := context.WithCancel(ctx) + defer stopHB() + acked := &atomic.Bool{} + acked.Store(true) + beatNow := make(chan struct{}, 1) + go b.heartbeat(hbCtx, ws, interval, sess, acked, beatNow) + + seq, haveSeq := sess.lastSeq() + if resume && sess.sessionID != "" && haveSeq { + err = ws.send(ctx, map[string]any{"op": opResume, "d": map[string]any{ + "token": b.token, + "session_id": sess.sessionID, + "seq": seq, + }}) + } else { + err = ws.send(ctx, map[string]any{"op": opIdentify, "d": map[string]any{ + "token": b.token, + "intents": intents, + "properties": map[string]string{"os": "linux", "browser": "thermograph", "device": "thermograph"}, + }}) + } + if err != nil { + return "", err + } + + for { + data, err := ws.read(ctx) + if err != nil { + return "", err + } + var p payload + if err := json.Unmarshal(data, &p); err != nil { + log.Printf("gateway: dropping unparseable frame: %v", err) + continue + } + if p.S != nil { + // Every payload that carries a sequence number advances it — it + // is what RESUME replays from. + sess.setSeq(*p.S) + } + switch p.Op { + case opDispatch: + b.dispatch(ctx, p, sess) + case opHeartbeat: + // Server asked for an immediate beat; route it through the + // heartbeat goroutine, which owns beat sending. + select { + case beatNow <- struct{}{}: + default: + } + case opHeartbeatACK: + acked.Store(true) + case opReconnect: + return modeResume, nil // planned reconnect — resume + case opInvalidSession: + // d == true => the session can still be resumed; d == false => + // it is dead and the next connect must IDENTIFY fresh. + var resumable bool + _ = json.Unmarshal(p.D, &resumable) + if !resumable { + sess.sessionID = "" + sess.resumeURL = "" + } + // Discord wants a short randomized wait before re-authing. + if !sleepCtx(ctx, time.Second+jitter()) { + return "", ctx.Err() + } + if sess.sessionID != "" { + return modeResume, nil + } + return modeIdentify, nil + } + } +} + +// heartbeat sends op-1 heartbeats every interval, the first one jittered by +// interval*rand per Discord's guidance. If the previous beat was never ACKed +// the link is a zombie — a half-open TCP connection where our writes +// "succeed" but nothing comes back — so close with a non-1000 code (1000 +// would invalidate the session) and stop; the next connect can then RESUME. +func (b *Bot) heartbeat(ctx context.Context, ws *wsConn, interval time.Duration, sess *session, acked *atomic.Bool, beatNow <-chan struct{}) { + timer := time.NewTimer(time.Duration(rand.Float64() * float64(interval))) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-beatNow: + // Server-requested beat (op 1): send immediately, outside the + // zombie accounting — it is not the reply to one of ours. + if err := b.sendHeartbeat(ctx, ws, sess); err != nil { + return // socket gone; the read loop owns reconnecting + } + case <-timer.C: + if !acked.Load() { + ws.close(websocket.StatusCode(4000), "heartbeat ack missed") + return + } + acked.Store(false) + if err := b.sendHeartbeat(ctx, ws, sess); err != nil { + return + } + timer.Reset(interval) + } + } +} + +func (b *Bot) sendHeartbeat(ctx context.Context, ws *wsConn, sess *session) error { + var d any // null until the first sequence number arrives + if seq, ok := sess.lastSeq(); ok { + d = seq + } + return ws.send(ctx, map[string]any{"op": opHeartbeat, "d": d}) +} + +// dispatch handles an op-0 DISPATCH event we care about. +func (b *Bot) dispatch(ctx context.Context, p payload, sess *session) { + switch p.T { + case "READY": + var d struct { + SessionID string `json:"session_id"` + ResumeGatewayURL string `json:"resume_gateway_url"` + User struct { + ID snowflake `json:"id"` + } `json:"user"` + } + if err := json.Unmarshal(p.D, &d); err != nil { + log.Printf("gateway: unparseable READY: %v", err) + return + } + sess.sessionID = d.SessionID + sess.resumeURL = "" + if r := strings.TrimRight(d.ResumeGatewayURL, "/"); r != "" { + sess.resumeURL = r + gatewayQS + } + sess.userID = string(d.User.ID) + log.Printf("gateway: ready (user %s)", sess.userID) + case "MESSAGE_CREATE": + // Until READY tells us our own user id we can't detect mentions (or + // guard against answering ourselves), so stay silent. + if sess.userID == "" { + return + } + var m gwMessage + if err := json.Unmarshal(p.D, &m); err != nil { + log.Printf("gateway: unparseable MESSAGE_CREATE: %v", err) + return + } + // Grading is a synchronous parquet/DB round trip on the Python side, + // so it must never run on the read loop — a slow lookup would stall + // heartbeats (the Python used asyncio.to_thread for the same + // reason). Each message gets its own goroutine, with the semaphore + // bounding in-flight replies so a flood of mentions can't spawn + // unbounded goroutines against the backend — an improvement over + // to_thread's unbounded queue. Over the bound we drop and log rather + // than block the read loop. + botID := sess.userID + select { + case b.sem <- struct{}{}: + go func() { + defer func() { <-b.sem }() + b.handleMessage(ctx, &m, botID) + }() + default: + log.Printf("gateway: dropping message %s: too many replies in flight", m.ID) + } + } +} diff --git a/backend/daemon/internal/gateway/gateway_test.go b/backend/daemon/internal/gateway/gateway_test.go new file mode 100644 index 0000000..3f8cfdf --- /dev/null +++ b/backend/daemon/internal/gateway/gateway_test.go @@ -0,0 +1,182 @@ +// Connection-machinery tests, restricted to the pure parts (backoff, fatal +// close codes, READY/session bookkeeping, sequence tracking). Deliberately no +// live-gateway test — that path is exercised in staging, not CI. + +package gateway + +import ( + "context" + "encoding/json" + "net/http" + "testing" + "time" + + "github.com/coder/websocket" +) + +func TestBackoffDoublesAndCaps(t *testing.T) { + want := []time.Duration{ + 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second, + 32 * time.Second, 60 * time.Second, 60 * time.Second, 60 * time.Second, + } + d := backoffStart + for i, w := range want { + d = nextBackoff(d) + if d != w { + t.Fatalf("step %d: want %s, got %s", i, w, d) + } + } +} + +func TestFatalCloseCodes(t *testing.T) { + for _, c := range []websocket.StatusCode{4004, 4010, 4011, 4012, 4013, 4014} { + if !isFatalClose(c) { + t.Errorf("close code %d must be fatal (reconnecting loops forever)", c) + } + } + // 4008 (rate limited) and 4009 (session timed out) are resumable; 4000 is + // our own zombie close; -1 is CloseStatus's not-a-close-error sentinel. + for _, c := range []websocket.StatusCode{-1, 1000, 1001, 4000, 4001, 4008, 4009} { + if isFatalClose(c) { + t.Errorf("close code %d must not be fatal", c) + } + } +} + +func TestReadyCapturesResumeState(t *testing.T) { + b := New("tok", &fakeGrader{}) + sess := &session{} + d := []byte(`{"session_id":"abc","resume_gateway_url":"wss://resume.example/","user":{"id":"999"}}`) + + b.dispatch(context.Background(), payload{Op: opDispatch, T: "READY", D: d}, sess) + + if sess.sessionID != "abc" { + t.Fatalf("want session id abc, got %q", sess.sessionID) + } + // Trailing slash trimmed, gateway query string appended — RESUME must go + // to the session's own URL with the same v10/json parameters. + if sess.resumeURL != "wss://resume.example/?v=10&encoding=json" { + t.Fatalf("bad resume url: %q", sess.resumeURL) + } + if sess.userID != botID { + t.Fatalf("want user id %s, got %q", botID, sess.userID) + } +} + +func TestReadyWithoutResumeURLLeavesItEmpty(t *testing.T) { + b := New("tok", &fakeGrader{}) + sess := &session{resumeURL: "wss://stale.example/?v=10&encoding=json"} + d := []byte(`{"session_id":"abc","user":{"id":"999"}}`) + + b.dispatch(context.Background(), payload{Op: opDispatch, T: "READY", D: d}, sess) + + if sess.resumeURL != "" { + t.Fatalf("a READY without resume_gateway_url must clear the stale url, got %q", sess.resumeURL) + } +} + +func TestMessageCreateBeforeReadyIsIgnored(t *testing.T) { + // Until READY supplies our user id we cannot detect mentions (or guard + // against answering ourselves), so nothing may be handled. + g := &fakeGrader{resp: json.RawMessage(`{}`)} + b := New("tok", g) + sess := &session{} // no userID yet + d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID))) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess) + + time.Sleep(20 * time.Millisecond) + if g.callCount() != 0 { + t.Fatal("pre-READY message must be ignored") + } +} + +func TestPayloadSequenceIsNullable(t *testing.T) { + var withSeq, withoutSeq payload + if err := json.Unmarshal([]byte(`{"op":0,"t":"X","s":7,"d":{}}`), &withSeq); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if withSeq.S == nil || *withSeq.S != 7 { + t.Fatalf("want seq 7, got %v", withSeq.S) + } + if err := json.Unmarshal([]byte(`{"op":11,"s":null,"d":null}`), &withoutSeq); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if withoutSeq.S != nil { + t.Fatalf("null seq must stay nil, got %v", *withoutSeq.S) + } +} + +func TestSessionSeqTracking(t *testing.T) { + sess := &session{} + if _, ok := sess.lastSeq(); ok { + t.Fatal("fresh session must have no sequence number") + } + sess.setSeq(41) + sess.setSeq(42) + if seq, ok := sess.lastSeq(); !ok || seq != 42 { + t.Fatalf("want seq 42, got %d (ok=%v)", seq, ok) + } +} + +// A malformed HELLO must be an ERROR, not a clean "reidentify". Run treats a +// clean return as a PLANNED reconnect: backoff resets to 1s and it redials with +// no sleep at all. So if these ever regress to returning nil, a gateway stuck +// sending a bad HELLO becomes a tight reconnect loop against Discord. +func TestParseHelloRejectsMalformedSoBackoffApplies(t *testing.T) { + for _, tc := range []struct { + name string + raw string + }{ + {"not json", `{`}, + {"wrong op", `{"op":0,"d":{"heartbeat_interval":41250}}`}, + {"missing interval", `{"op":10,"d":{}}`}, + {"zero interval", `{"op":10,"d":{"heartbeat_interval":0}}`}, + {"negative interval", `{"op":10,"d":{"heartbeat_interval":-1}}`}, + {"interval wrong type", `{"op":10,"d":{"heartbeat_interval":"soon"}}`}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := parseHello([]byte(tc.raw)) + if err == nil { + t.Fatalf("parseHello(%s) = %v, nil; want an error so Run applies backoff", tc.raw, got) + } + if got != 0 { + t.Fatalf("parseHello(%s) interval = %v; want 0 on error", tc.raw, got) + } + }) + } +} + +func TestParseHelloAcceptsValid(t *testing.T) { + got, err := parseHello([]byte(`{"op":10,"d":{"heartbeat_interval":41250}}`)) + if err != nil { + t.Fatalf("parseHello: unexpected error %v", err) + } + if want := 41250 * time.Millisecond; got != want { + t.Fatalf("interval = %v; want %v", got, want) + } +} + +// Discord's advertised retry_after is clamped to 5s, matching the Python REST +// helper's _MAX_BACKOFF_S. Replies run on a small fixed worker pool, so a bogus +// server value must not park a slot for minutes. +func TestRetryAfterClamp(t *testing.T) { + for _, tc := range []struct { + name string + body string + want time.Duration + }{ + {"honours a sane value", `{"retry_after":2}`, 2 * time.Second}, + {"clamps a hostile value", `{"retry_after":600}`, 5 * time.Second}, + {"defaults when absent", `{}`, 1 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := retryAfter(http.Header{}, []byte(tc.body)); got != tc.want { + t.Fatalf("retryAfter(%s) = %v; want %v", tc.body, got, tc.want) + } + }) + } +} diff --git a/backend/daemon/internal/gateway/message.go b/backend/daemon/internal/gateway/message.go new file mode 100644 index 0000000..762f846 --- /dev/null +++ b/backend/daemon/internal/gateway/message.go @@ -0,0 +1,118 @@ +// Message triage: the pure decision of whether — and how — to answer one +// MESSAGE_CREATE. Ported from the Python bot's _response_for_message / +// _mentions_bot / _is_dm / _strip_mentions; it is plain string/JSON logic +// with no data dependency, so it lives in Go. Anything needing climate data +// (the actual grade) goes back to Python via Grader. + +package gateway + +import ( + "encoding/json" + "fmt" + "strings" +) + +// helpText answers an empty query. Go owns this constant because it needs no +// data; the wording matches the Python original exactly. +const helpText = "Mention me with a city name — e.g. `@Thermograph Phoenix` — and I'll grade " + + "today's weather against ~45 years of local history." + +// snowflake decodes a Discord id that may arrive as a JSON string (what v10 +// sends) or a bare number. The Python compared ids through str() coercion; +// this keeps that tolerance instead of betting the payload shape never +// wobbles. +type snowflake string + +func (s *snowflake) UnmarshalJSON(b []byte) error { + if string(b) == "null" { + *s = "" + return nil + } + var str string + if err := json.Unmarshal(b, &str); err == nil { + *s = snowflake(str) + return nil + } + var n json.Number + if err := json.Unmarshal(b, &n); err == nil { + *s = snowflake(n.String()) + return nil + } + return fmt.Errorf("snowflake: cannot decode %s", b) +} + +// gwMessage is the slice of a MESSAGE_CREATE payload the triage needs. +type gwMessage struct { + ID snowflake `json:"id"` + ChannelID snowflake `json:"channel_id"` + GuildID snowflake `json:"guild_id"` + Content string `json:"content"` + Author gwAuthor `json:"author"` + Mentions []gwAuthor `json:"mentions"` +} + +type gwAuthor struct { + ID snowflake `json:"id"` + Bot bool `json:"bot"` +} + +// action is triage's verdict for one message. +type action int + +const ( + actSilent action = iota // no reply + actHelp // reply with helpText + actGrade // call back into Python with the query +) + +// mentionsBot reports whether the message @mentions our own user id. +func mentionsBot(m *gwMessage, botID string) bool { + for _, u := range m.Mentions { + if string(u.ID) == botID { + return true + } + } + return false +} + +// isDM: a DM has no guild_id; a guild message always carries one. +func isDM(m *gwMessage) bool { + return m.GuildID == "" +} + +// stripMentions drops every form of the bot's own mention (<@id> and the +// legacy <@!id>) and collapses the surrounding whitespace, leaving just the +// user's query text. +func stripMentions(content, botID string) string { + out := strings.ReplaceAll(content, "<@"+botID+">", " ") + out = strings.ReplaceAll(out, "<@!"+botID+">", " ") + return strings.Join(strings.Fields(out), " ") +} + +// triage decides the reply for one MESSAGE_CREATE: silence, the help line, or +// a grade lookup for the returned query. +// +// Silent unless the message DMs the bot or @mentions it, and never for a bot +// author — which includes ourselves, the guard against a mention-loop (our +// reply quoting the mention must not trigger another reply). The text after +// the mention is treated as a city query; DMs are the query as-is (trimmed +// but not collapsed, matching the Python's .strip()). +func triage(m *gwMessage, botID string) (action, string) { + if m.Author.Bot || string(m.Author.ID) == botID { + return actSilent, "" + } + dm := isDM(m) + if !dm && !mentionsBot(m, botID) { + return actSilent, "" + } + var query string + if dm { + query = strings.TrimSpace(m.Content) + } else { + query = stripMentions(m.Content, botID) + } + if query == "" { + return actHelp, "" + } + return actGrade, query +} diff --git a/backend/daemon/internal/gateway/message_test.go b/backend/daemon/internal/gateway/message_test.go new file mode 100644 index 0000000..294d363 --- /dev/null +++ b/backend/daemon/internal/gateway/message_test.go @@ -0,0 +1,162 @@ +// Triage tests: every behaviour asserted by the Python suite +// (backend/tests/notifications/test_discord_bot.py) has an equivalent here, +// minus the ephemeral-flag drop — the /internal/discord/grade route now +// returns gateway-ready JSON, so that concern moved to the Python side. + +package gateway + +import ( + "encoding/json" + "strings" + "testing" +) + +const botID = "999" + +// testMsg mirrors the Python suite's _msg fixture: author "1" (not a bot), +// message id 42 in channel 77, guild 5 unless overridden. +func testMsg(content string, opts ...func(*gwMessage)) *gwMessage { + m := &gwMessage{ + ID: "42", + ChannelID: "77", + GuildID: "5", + Content: content, + Author: gwAuthor{ID: "1"}, + } + for _, opt := range opts { + opt(m) + } + return m +} + +func fromBot() func(*gwMessage) { + return func(m *gwMessage) { m.Author.Bot = true } +} + +func fromID(id string) func(*gwMessage) { + return func(m *gwMessage) { m.Author.ID = snowflake(id) } +} + +func mentioning(ids ...string) func(*gwMessage) { + return func(m *gwMessage) { + for _, id := range ids { + m.Mentions = append(m.Mentions, gwAuthor{ID: snowflake(id)}) + } + } +} + +func asDM() func(*gwMessage) { + return func(m *gwMessage) { m.GuildID = "" } +} + +// --- when the bot stays silent ---------------------------------------------- + +func TestIgnoresOtherBots(t *testing.T) { + act, _ := triage(testMsg("<@999> Phoenix", fromID("8"), fromBot(), mentioning(botID)), botID) + if act != actSilent { + t.Fatalf("bot author should be ignored, got action %d", act) + } +} + +func TestIgnoresItsOwnMessages(t *testing.T) { + act, _ := triage(testMsg("hello", fromID(botID), mentioning(botID)), botID) + if act != actSilent { + t.Fatalf("own message should be ignored, got action %d", act) + } +} + +func TestIgnoresGuildMessageWithoutAMention(t *testing.T) { + act, _ := triage(testMsg("just chatting"), botID) + if act != actSilent { + t.Fatalf("unmentioned guild message should be ignored, got action %d", act) + } +} + +// --- when the bot replies ---------------------------------------------------- + +func TestMentionWithCityGrades(t *testing.T) { + act, query := triage(testMsg("<@999> Phoenix", mentioning(botID)), botID) + if act != actGrade || query != "Phoenix" { + t.Fatalf("want grade %q, got action %d query %q", "Phoenix", act, query) + } +} + +func TestLegacyNicknameMentionIsStripped(t *testing.T) { + act, query := triage(testMsg("<@!999> Tokyo", mentioning(botID)), botID) + if act != actGrade || query != "Tokyo" { + t.Fatalf("want grade %q, got action %d query %q", "Tokyo", act, query) + } +} + +func TestDMNeedsNoMention(t *testing.T) { + act, query := triage(testMsg("Berlin", asDM()), botID) + if act != actGrade || query != "Berlin" { + t.Fatalf("want grade %q, got action %d query %q", "Berlin", act, query) + } +} + +func TestDMTrimsButKeepsInnerWhitespace(t *testing.T) { + // Parity with the Python DM path (.strip(), not a collapse): only the + // edges are trimmed. + act, query := triage(testMsg(" New York ", asDM()), botID) + if act != actGrade || query != "New York" { + t.Fatalf("want grade %q, got action %d query %q", "New York", act, query) + } +} + +func TestMentionWithNoQueryGivesHelp(t *testing.T) { + act, query := triage(testMsg("<@999>", mentioning(botID)), botID) + if act != actHelp || query != "" { + t.Fatalf("want help, got action %d query %q", act, query) + } + if !strings.Contains(strings.ToLower(helpText), "mention me") { + t.Fatalf("help text lost its instruction: %q", helpText) + } +} + +// --- helpers ----------------------------------------------------------------- + +func TestStripMentionsHandlesBothForms(t *testing.T) { + if got := stripMentions("<@999> <@!999> Sao Paulo", botID); got != "Sao Paulo" { + t.Fatalf("want %q, got %q", "Sao Paulo", got) + } +} + +func TestIsDMIsTrueWithoutGuildID(t *testing.T) { + if !isDM(testMsg("x", asDM())) { + t.Fatal("message without guild_id should be a DM") + } + if isDM(testMsg("x")) { + t.Fatal("message with guild_id should not be a DM") + } +} + +func TestSnowflakeAcceptsStringOrNumber(t *testing.T) { + // The Python compared ids through str() coercion; a numeric id in the + // payload must still count as a mention. + var m gwMessage + raw := `{"id":42,"channel_id":"77","guild_id":"5","content":"<@999> Lima", + "author":{"id":1,"bot":false},"mentions":[{"id":999}]}` + if err := json.Unmarshal([]byte(raw), &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if m.ID != "42" || string(m.Author.ID) != "1" { + t.Fatalf("numeric ids mis-decoded: id=%q author=%q", m.ID, m.Author.ID) + } + act, query := triage(&m, botID) + if act != actGrade || query != "Lima" { + t.Fatalf("want grade %q, got action %d query %q", "Lima", act, query) + } +} + +func TestIntentsExcludeMessageContent(t *testing.T) { + // The no-privileged-intent constraint is load-bearing: adding + // MESSAGE_CONTENT (1<<15) silently would make Discord close the gateway + // with 4014 until the portal review happens. + if intents != intentGuilds|intentGuildMessages|intentDirectMessages { + t.Fatalf("unexpected intents bitfield: %d", intents) + } + if intents&(1<<15) != 0 { + t.Fatal("intents must not request MESSAGE_CONTENT") + } +} diff --git a/backend/daemon/internal/gateway/reply.go b/backend/daemon/internal/gateway/reply.go new file mode 100644 index 0000000..0c7b52d --- /dev/null +++ b/backend/daemon/internal/gateway/reply.go @@ -0,0 +1,138 @@ +// The REST half of answering a message: replies go out over plain HTTP with +// the bot token, not the gateway socket (the gateway is receive-only for us). + +package gateway + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "strconv" + "time" +) + +const discordAPIBase = "https://discord.com/api/v10" + +// replyBodyLimit caps how much of a failed reply's response body we echo into +// logs. +const replyBodyLimit = 4096 + +// injectReplyFields adds message_reference + allowed_mentions to an otherwise +// verbatim message body. Only the top level is decoded — every value that +// came from Python (embeds and all) passes through as raw bytes, unparsed, so +// the grading/embed contract stays entirely on the Python side. +// +// allowed_mentions is a SECURITY control, not decoration: the graded reply +// echoes the user's query text, so without {"parse":[]} a crafted query could +// turn our reply into an @everyone/role ping. Only the person who asked is +// pinged, via the reply reference. +func injectReplyFields(body []byte, messageID string) ([]byte, error) { + var top map[string]json.RawMessage + if err := json.Unmarshal(body, &top); err != nil { + return nil, err + } + if messageID != "" { + ref, err := json.Marshal(map[string]string{"message_id": messageID}) + if err != nil { + return nil, err + } + top["message_reference"] = ref + top["allowed_mentions"] = json.RawMessage(`{"parse":[],"replied_user":true}`) + } + return json.Marshal(top) +} + +// reply posts body to the triggering message's channel as a proper reply. A +// failed reply is logged and swallowed — it must never kill the read loop. +func (b *Bot) reply(ctx context.Context, channelID, messageID string, body json.RawMessage) { + if channelID == "" { + return + } + payload, err := injectReplyFields(body, messageID) + if err != nil { + log.Printf("gateway: reply body is not a JSON object: %v", err) + return + } + url := b.rest + "/channels/" + channelID + "/messages" + for attempt := 0; ; attempt++ { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + log.Printf("gateway: reply failed: %v", err) + return + } + req.Header.Set("Authorization", "Bot "+b.token) + req.Header.Set("Content-Type", "application/json") + resp, err := b.http.Do(req) + if err != nil { + log.Printf("gateway: reply failed: %v", err) + return + } + raw, _ := io.ReadAll(io.LimitReader(resp.Body, replyBodyLimit)) + resp.Body.Close() + if resp.StatusCode == http.StatusTooManyRequests && attempt == 0 { + // One retry honouring the advertised wait — parity with the + // Python REST helper's 429 handling. + if !sleepCtx(ctx, retryAfter(resp.Header, raw)) { + return + } + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Printf("gateway: reply failed: status %d: %s", resp.StatusCode, raw) + } + return + } +} + +// retryAfter extracts Discord's requested wait from a 429 (JSON retry_after +// in seconds, falling back to the Retry-After header), clamped so a bogus +// server value can't park the handler for minutes. +func retryAfter(h http.Header, body []byte) time.Duration { + seconds := 1.0 + var d struct { + RetryAfter float64 `json:"retry_after"` + } + if err := json.Unmarshal(body, &d); err == nil && d.RetryAfter > 0 { + seconds = d.RetryAfter + } else if v, err := strconv.ParseFloat(h.Get("Retry-After"), 64); err == nil && v > 0 { + seconds = v + } + // 5s matches the Python REST helper's _MAX_BACKOFF_S. It also bounds the cost + // of a bogus/hostile retry_after: replies run on a small fixed worker pool, so + // a parked handler holds one of very few slots and mentions start being + // dropped that much sooner. + if seconds > 5 { + seconds = 5 + } + return time.Duration(seconds * float64(time.Second)) +} + +// handleMessage triages one MESSAGE_CREATE and sends whatever reply it calls +// for. Runs in its own goroutine (see dispatch) so a slow grade lookup can +// never stall heartbeats or the read loop. +func (b *Bot) handleMessage(ctx context.Context, m *gwMessage, botID string) { + act, query := triage(m, botID) + switch act { + case actSilent: + case actHelp: + body, err := json.Marshal(map[string]string{"content": helpText}) + if err != nil { + return + } + b.reply(ctx, string(m.ChannelID), string(m.ID), body) + case actGrade: + // The callback owns all grading; its JSON is relayed VERBATIM (no + // parsing, no reshaping) so the bot's grades can never drift from + // the API's. A failed callback stays silent — better no reply than a + // made-up one. + raw, err := b.api.Grade(ctx, query) + if err != nil { + log.Printf("gateway: grade callback failed for %q: %v", query, err) + return + } + b.reply(ctx, string(m.ChannelID), string(m.ID), raw) + } +} diff --git a/backend/daemon/internal/gateway/reply_test.go b/backend/daemon/internal/gateway/reply_test.go new file mode 100644 index 0000000..e5d2f40 --- /dev/null +++ b/backend/daemon/internal/gateway/reply_test.go @@ -0,0 +1,278 @@ +// Reply-path tests: a fake Grader plus an httptest server standing in for +// Discord's REST API. No live gateway is involved anywhere. + +package gateway + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +type fakeGrader struct { + mu sync.Mutex + resp json.RawMessage + err error + calls []string +} + +func (f *fakeGrader) Grade(_ context.Context, query string) (json.RawMessage, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, query) + return f.resp, f.err +} + +func (f *fakeGrader) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +type captured struct { + method, path, auth string + body []byte +} + +// newTestBot wires a Bot to a Grader and an httptest stand-in for Discord. +func newTestBot(t *testing.T, g Grader, handler http.HandlerFunc) *Bot { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + b := New("tok-123", g) + b.rest = srv.URL + return b +} + +// capture records each request and replies 200 {}. +func capture(got chan<- captured) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + got <- captured{r.Method, r.URL.Path, r.Header.Get("Authorization"), body} + _, _ = w.Write([]byte(`{}`)) + } +} + +func TestGradeReplyIsRelayedVerbatimAsAReply(t *testing.T) { + grade := json.RawMessage(`{"embeds":[{"title":"grade:Phoenix"}]}`) + got := make(chan captured, 1) + b := newTestBot(t, &fakeGrader{resp: grade}, capture(got)) + + b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID) + + c := <-got + if c.method != http.MethodPost || c.path != "/channels/77/messages" { + t.Fatalf("want POST /channels/77/messages, got %s %s", c.method, c.path) + } + if c.auth != "Bot tok-123" { + t.Fatalf("want bot-token auth, got %q", c.auth) + } + var top map[string]json.RawMessage + if err := json.Unmarshal(c.body, &top); err != nil { + t.Fatalf("reply body not JSON: %v", err) + } + // The embed must be byte-for-byte what Python returned — the verbatim + // relay is the no-drift guarantee. + if string(top["embeds"]) != `[{"title":"grade:Phoenix"}]` { + t.Fatalf("embeds not relayed verbatim: %s", top["embeds"]) + } + if string(top["message_reference"]) != `{"message_id":"42"}` { + t.Fatalf("bad message_reference: %s", top["message_reference"]) + } + // The locked-down allowed_mentions is the anti-@everyone control; it must + // be present exactly. + if string(top["allowed_mentions"]) != `{"parse":[],"replied_user":true}` { + t.Fatalf("bad allowed_mentions: %s", top["allowed_mentions"]) + } +} + +func TestEmptyQueryRepliesWithHelpWithoutGrading(t *testing.T) { + got := make(chan captured, 1) + g := &fakeGrader{} + b := newTestBot(t, g, capture(got)) + + b.handleMessage(context.Background(), testMsg("<@999>", mentioning(botID)), botID) + + c := <-got + var body struct { + Content string `json:"content"` + } + if err := json.Unmarshal(c.body, &body); err != nil { + t.Fatalf("help body not JSON: %v", err) + } + if body.Content != helpText { + t.Fatalf("want help text, got %q", body.Content) + } + if g.callCount() != 0 { + t.Fatal("help reply must not hit the grade callback") + } +} + +func TestSilentMessagesMakeNoRequests(t *testing.T) { + var hits atomic.Int32 + g := &fakeGrader{resp: json.RawMessage(`{}`)} + b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + }) + + b.handleMessage(context.Background(), testMsg("just chatting"), botID) + + if g.callCount() != 0 || hits.Load() != 0 { + t.Fatalf("silent message leaked: grades=%d posts=%d", g.callCount(), hits.Load()) + } +} + +func TestGradeFailureStaysSilent(t *testing.T) { + var hits atomic.Int32 + g := &fakeGrader{err: context.DeadlineExceeded} + b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + }) + + // Must log-and-return, not panic or post a broken reply. + b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID) + + if hits.Load() != 0 { + t.Fatal("failed grade must not produce a reply") + } +} + +func TestFailedReplyIsSwallowed(t *testing.T) { + g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)} + b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + }) + + // A failed reply must never kill the caller (in production: the read + // loop) — returning normally is the assertion. + b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID) +} + +func TestReplyRetriesOnceOn429(t *testing.T) { + var hits atomic.Int32 + bodies := make(chan []byte, 2) + g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)} + b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + bodies <- body + if hits.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"retry_after":0.01}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + }) + + b.handleMessage(context.Background(), testMsg("<@999> Phoenix", mentioning(botID)), botID) + + if hits.Load() != 2 { + t.Fatalf("want 1 retry after 429, got %d requests", hits.Load()) + } + first, second := <-bodies, <-bodies + if string(first) != string(second) { + t.Fatal("retry must resend the identical payload") + } +} + +func TestNoChannelMeansNoPost(t *testing.T) { + var hits atomic.Int32 + g := &fakeGrader{resp: json.RawMessage(`{"content":"x"}`)} + b := newTestBot(t, g, func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + }) + + b.handleMessage(context.Background(), + testMsg("<@999> Phoenix", mentioning(botID), func(m *gwMessage) { m.ChannelID = "" }), botID) + + if hits.Load() != 0 { + t.Fatal("a message without a channel id must not be answered") + } +} + +func TestInjectReplyFieldsWithoutMessageIDLeavesBodyAlone(t *testing.T) { + out, err := injectReplyFields([]byte(`{"content":"x"}`), "") + if err != nil { + t.Fatalf("inject: %v", err) + } + var top map[string]json.RawMessage + if err := json.Unmarshal(out, &top); err != nil { + t.Fatalf("out not JSON: %v", err) + } + if _, ok := top["message_reference"]; ok { + t.Fatal("no triggering id => no message_reference") + } + if _, ok := top["allowed_mentions"]; ok { + t.Fatal("no triggering id => no allowed_mentions") + } +} + +// --- dispatch: same reply, off the read loop --------------------------------- + +func TestDispatchDeliversTheSameReplyViaAGoroutine(t *testing.T) { + // Mirrors the Python test_dispatch_delivers_the_same_reply_via_a_thread: + // the concurrency offload (goroutine + semaphore here, asyncio.to_thread + // there) must not change what gets sent. + grade := json.RawMessage(`{"embeds":[{"title":"grade:Phoenix"}]}`) + got := make(chan captured, 1) + b := newTestBot(t, &fakeGrader{resp: grade}, capture(got)) + sess := &session{userID: botID} + d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID))) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess) + + select { + case c := <-got: + if c.path != "/channels/77/messages" { + t.Fatalf("want /channels/77/messages, got %s", c.path) + } + var top map[string]json.RawMessage + if err := json.Unmarshal(c.body, &top); err != nil { + t.Fatalf("reply body not JSON: %v", err) + } + if string(top["embeds"]) != `[{"title":"grade:Phoenix"}]` { + t.Fatalf("embeds not relayed verbatim: %s", top["embeds"]) + } + if string(top["message_reference"]) != `{"message_id":"42"}` { + t.Fatalf("bad message_reference: %s", top["message_reference"]) + } + case <-time.After(2 * time.Second): + t.Fatal("dispatched message never produced a reply") + } +} + +func TestMessageFloodIsBoundedWithoutBlockingTheReadLoop(t *testing.T) { + g := &fakeGrader{resp: json.RawMessage(`{}`)} + b := New("tok", g) + b.sem = make(chan struct{}, 1) + b.sem <- struct{}{} // occupy the only slot: simulate a reply in flight + sess := &session{userID: botID} + d, err := json.Marshal(testMsg("<@999> Phoenix", mentioning(botID))) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + done := make(chan struct{}) + go func() { + b.dispatch(context.Background(), payload{Op: opDispatch, T: "MESSAGE_CREATE", D: d}, sess) + close(done) + }() + select { + case <-done: + // dispatch returned promptly — the read loop would not have stalled + case <-time.After(time.Second): + t.Fatal("dispatch blocked on a full semaphore") + } + time.Sleep(20 * time.Millisecond) + if g.callCount() != 0 { + t.Fatal("over-the-bound message must be dropped, not queued") + } +} diff --git a/backend/daemon/main.go b/backend/daemon/main.go new file mode 100644 index 0000000..c264b14 --- /dev/null +++ b/backend/daemon/main.go @@ -0,0 +1,130 @@ +// thermograph-daemon owns the backend's long-lived stateful I/O — the Discord +// gateway websocket (IDENTIFY/RESUME/heartbeat/backoff) and the recurring-job +// timers — and nothing else. Anything that needs data calls back into the +// Python app over its internal-only HTTP routes (see internal/apiclient). +// +// This replaces the two in-process background jobs web/app.py used to start +// under leader election. One daemon process replaces the election entirely: +// there is exactly one replica of this container, so the single-connection +// invariant (Discord tolerates only one gateway session per token; duplicated +// APScheduler replicas would repeat every job) holds by deployment shape +// rather than by runtime coordination. +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "thermograph/daemon/internal/apiclient" + "thermograph/daemon/internal/config" + "thermograph/daemon/internal/cron" + "thermograph/daemon/internal/gateway" +) + +// shutdownGrace bounds how long we wait for the gateway and cron halves to +// stop after a signal. Long enough for the gateway to close its session +// cleanly (a dirty exit leaves Discord holding a stale session), short enough +// to stay inside a container runtime's stop timeout so we exit on our own +// terms rather than being SIGKILLed mid-close. +const shutdownGrace = 10 * time.Second + +func main() { + os.Exit(run()) +} + +// run is main minus os.Exit, so deferred cleanup actually runs. +func run() int { + // Everything to stdout: the container collects stdout, nothing else. + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + // Refuse to start on bad config rather than running half-configured: a + // daemon that cannot authenticate to the backend can only fail silently. + cfg, err := config.Load() + if err != nil { + logger.Error("invalid configuration; refusing to start", "err", err) + return 1 + } + + // One context for everything long-lived; SIGINT/SIGTERM cancels it so the + // gateway can close its session cleanly (Discord treats an abrupt drop + // worse than a clean close for RESUME purposes). + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + api := apiclient.New(cfg.APIBase, cfg.InternalToken) + + logger.Info("thermograph-daemon starting", + "api", cfg.APIBase, + "discord", cfg.DiscordEnabled, + "warm_cities_interval", cfg.WarmCitiesInterval, + "indexnow_interval", cfg.IndexNowInterval) + + var wg sync.WaitGroup + + // The cron half runs unconditionally — it is the floor of what this + // daemon does, gateway or not. + wg.Add(1) + go func() { + defer wg.Done() + cron.Run(ctx, logger, + cron.Job{ + Name: "warm-cities", + Interval: cfg.WarmCitiesInterval, + Run: func(ctx context.Context) error { return api.WarmCities(ctx) }, + }, + cron.Job{ + Name: "indexnow", + Interval: cfg.IndexNowInterval, + Run: func(ctx context.Context) error { return api.IndexNow(ctx) }, + }, + ) + }() + + if cfg.DiscordEnabled { + wg.Add(1) + go func() { + defer wg.Done() + // A fatal gateway error (bad token, disallowed intents) must not + // take the cron half down: the schedule is independently useful, + // and exiting here would turn a Discord misconfiguration into a + // crash-loop that also stops warm-cities/IndexNow. Log loudly and + // keep running instead. + if err := gateway.Run(ctx, cfg, api); err != nil && ctx.Err() == nil { + logger.Error("discord gateway exited with a fatal error; cron jobs keep running without it", + "err", err) + } + }() + } else { + // An unconfigured deploy must pay nothing, same as the Python side: + // no connection, no retries — just say so once and run cron-only. + logger.Info("discord gateway disabled (THERMOGRAPH_DISCORD_BOT not truthy or no bot token); running cron-only") + } + + <-ctx.Done() + // Restore default signal handling immediately: a second SIGINT/SIGTERM + // during the grace period kills us the ordinary way instead of being + // swallowed by the (already-cancelled) NotifyContext. + stop() + logger.Info("shutting down", "grace", shutdownGrace) + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + logger.Info("shutdown complete") + case <-time.After(shutdownGrace): + // This process is the sole holder of the gateway connection, so a + // clean close matters — but a wedged goroutine must not hold the + // container's stop hostage forever. + logger.Warn("shutdown grace period elapsed before all loops stopped; exiting anyway") + } + return 0 +} diff --git a/backend/indexnow.py b/backend/indexnow.py index 1bf3422..a33d987 100644 --- a/backend/indexnow.py +++ b/backend/indexnow.py @@ -179,7 +179,8 @@ def _write_state(sig: str) -> None: def submit_if_changed(base_url: str | None = None) -> dict | None: """Submit every indexable page, but only when the URL set changed since the last successful submission — the shared logic behind the CLI's --if-changed - flag and the worker scheduler's periodic ping (notifications/scheduler.py), so + flag and the daemon's periodic ping (thermograph-daemon's indexnow timer, + via api/internal_routes.py), so a code-only deploy or an unremarkable scheduled tick never resubmits. Returns the submit_all() result, or None when skipped (unchanged, or a partial/failed submission that leaves the prior state in place so the next attempt retries).""" diff --git a/backend/notifications/discord_bot.py b/backend/notifications/discord_bot.py deleted file mode 100644 index 6e32780..0000000 --- a/backend/notifications/discord_bot.py +++ /dev/null @@ -1,295 +0,0 @@ -"""Discord **gateway** bot — reacts to @mentions (and DMs) in real time. - -The other two Discord paths need no persistent connection: slash commands arrive as -signed HTTP POSTs (discord_interactions.py) and the daily post goes out over an -incoming webhook (discord.py). This one is different — to answer an *ordinary* -message that mentions the bot, we have to hold a live websocket to Discord's -gateway and listen for MESSAGE_CREATE events. - -Single-connection invariant. Discord permits exactly one gateway connection per bot -token (per shard), so — like the subscription notifier and the daily post — this -runs in exactly one process: web/app.py starts it only when the process wins the -leader election (see core/singleton.claim_leader). No new container or daemon; it -rides the app's asyncio event loop as a background task. - -Least privilege / no privileged intent. We subscribe to GUILD_MESSAGES + -DIRECT_MESSAGES and act only on messages that @mention the bot or DM it. Discord -delivers message *content* for exactly those cases even without the privileged -MESSAGE_CONTENT intent, so the bot works with no portal review. To trigger on -arbitrary keywords in channels, enable the MESSAGE_CONTENT intent in the Developer -Portal and extend `_response_for_message` — the rest of the machinery is unchanged. - -Enable with THERMOGRAPH_DISCORD_BOT=1 (plus THERMOGRAPH_DISCORD_BOT_TOKEN, the same -bot token discord.py uses for DMs). Unset => no gateway connection; everything here -no-ops, so an unconfigured deploy pays nothing. - -Behaviour: mention the bot with a city name ("@Thermograph Phoenix") and it replies -with the same grade embed the /grade slash command returns, reusing -discord_interactions._grade_message so the two never drift. -""" -from __future__ import annotations - -import asyncio -import json -import logging -import os -import random - -from notifications import discord - -log = logging.getLogger("thermograph.discord_bot") - -# v10 JSON gateway. On a resume we reconnect to the session's resume_gateway_url -# (from the READY payload) instead, with the same query string appended. -_GATEWAY_URL = "wss://gateway.discord.gg/?v=10&encoding=json" -_GATEWAY_QS = "/?v=10&encoding=json" - -# Gateway intents (bitfield). GUILDS gives guild/channel context; GUILD_MESSAGES and -# DIRECT_MESSAGES deliver message-create events in servers and DMs. We deliberately -# do NOT request MESSAGE_CONTENT (privileged) — Discord still includes content for -# messages that mention us or DM us, which is all we act on. -_INTENT_GUILDS = 1 << 0 -_INTENT_GUILD_MESSAGES = 1 << 9 -_INTENT_DIRECT_MESSAGES = 1 << 12 -INTENTS = _INTENT_GUILDS | _INTENT_GUILD_MESSAGES | _INTENT_DIRECT_MESSAGES - -# Gateway opcodes (discord.com/developers/docs/topics/gateway-events#opcodes). -_OP_DISPATCH = 0 -_OP_HEARTBEAT = 1 -_OP_IDENTIFY = 2 -_OP_RESUME = 6 -_OP_RECONNECT = 7 -_OP_INVALID_SESSION = 9 -_OP_HELLO = 10 -_OP_HEARTBEAT_ACK = 11 - -# Close codes for unrecoverable conditions — reconnecting just loops, so we stop: -# 4004 auth failed (bad token), 4010 invalid shard, 4011 sharding required, 4012 -# invalid API version, 4013 invalid intents, 4014 disallowed (privileged) intents. -_FATAL_CLOSE = {4004, 4010, 4011, 4012, 4013, 4014} - -_TRUTHY = {"1", "true", "yes", "on"} - -_HELP = ("Mention me with a city name — e.g. `@Thermograph Phoenix` — and I'll grade " - "today's weather against ~45 years of local history.") - -# Reconnect backoff bounds (seconds). -_BACKOFF_START = 1.0 -_BACKOFF_MAX = 60.0 - - -def enabled() -> bool: - """True iff the gateway bot is switched on AND a bot token is configured. Off by - default: a gateway connection is a new persistent behaviour, so it takes an - explicit opt-in rather than piggybacking on the token being present for DMs.""" - flag = os.environ.get("THERMOGRAPH_DISCORD_BOT", "").strip().lower() in _TRUTHY - return flag and bool(discord.BOT_TOKEN) - - -# --- message handling (pure; unit-tested without a live gateway) -------------- -def _mentions_bot(msg: dict, bot_user_id: str) -> bool: - return any(str(u.get("id")) == str(bot_user_id) for u in (msg.get("mentions") or [])) - - -def _is_dm(msg: dict) -> bool: - # A DM has no guild_id; a guild message always carries one. - return not msg.get("guild_id") - - -def _strip_mentions(content: str, bot_user_id: str) -> str: - """Drop every form of the bot's own mention (<@id> and the legacy <@!id>) and - collapse the surrounding whitespace, leaving just the user's query text.""" - out = content.replace(f"<@{bot_user_id}>", " ").replace(f"<@!{bot_user_id}>", " ") - return " ".join(out.split()) - - -def _response_for_message(msg: dict, bot_user_id: str) -> dict | None: - """The reply payload for one MESSAGE_CREATE, or None to stay silent. - - Silent unless the message DMs the bot or @mentions it, and never for a bot - author (that includes ourselves — the guard against a mention-loop). The text - after the mention is treated as a city query and answered with the same embed - the /grade slash command builds; an empty query gets the help line.""" - author = msg.get("author") or {} - if author.get("bot") or str(author.get("id")) == str(bot_user_id): - return None - dm = _is_dm(msg) - if not dm and not _mentions_bot(msg, bot_user_id): - return None - content = msg.get("content") or "" - query = content.strip() if dm else _strip_mentions(content, bot_user_id) - if not query: - return {"content": _HELP} - # Import here (not at module top) to avoid importing the FastAPI web layer's - # transitive deps for a module that also loads in the notifier-only context. - from notifications import discord_interactions - data = discord_interactions._grade_message(query) - # Gateway messages can't be ephemeral — the flag is interactions-only. - data.pop("flags", None) - return data - - -# --- REST reply --------------------------------------------------------------- -async def _send_reply(channel_id, message_id, data: dict) -> None: - """Post the reply in-channel as a proper reply to the triggering message. - - Reuses discord._bot_post (sync httpx with a 429 retry) off the event loop via a - thread so the gateway read-loop never blocks. allowed_mentions is locked down so - a crafted query can't turn the reply into an @everyone/role ping; only the person - who asked is pinged, via the reply reference.""" - if not channel_id: - return - payload = dict(data) - if message_id: - payload["message_reference"] = {"message_id": str(message_id)} - payload["allowed_mentions"] = {"parse": [], "replied_user": True} - try: - await asyncio.to_thread( - discord._bot_post, f"/channels/{channel_id}/messages", payload) - except Exception: # noqa: BLE001 - a failed reply must never kill the read loop - log.warning("discord_bot: reply failed", exc_info=True) - - -# --- gateway session ---------------------------------------------------------- -class _Session: - """Mutable per-connection state that survives a resume: the last sequence - number, the resume token/URL, and our own user id (for mention detection).""" - - __slots__ = ("seq", "session_id", "resume_url", "user_id") - - def __init__(self) -> None: - self.seq: int | None = None - self.session_id: str | None = None - self.resume_url: str | None = None - self.user_id: str = "" - - -async def _heartbeat(ws, interval: float, sess: _Session, acked: dict) -> None: - """Send op-1 heartbeats every `interval` seconds (first beat jittered per - Discord's guidance). If the previous beat was never ACKed the link is a zombie — - close with a non-1000 code so the next connect can resume, and stop.""" - await asyncio.sleep(interval * random.random()) - while True: - if not acked["ok"]: - await ws.close(code=4000) - return - acked["ok"] = False - try: - await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq})) - except Exception: # noqa: BLE001 - closed socket; the read loop handles reconnect - return - await asyncio.sleep(interval) - - -async def _dispatch(msg: dict, sess: _Session) -> None: - """Handle an op-0 DISPATCH event we care about.""" - t = msg.get("t") - d = msg.get("d") or {} - if t == "READY": - sess.session_id = d.get("session_id") - resume = (d.get("resume_gateway_url") or "").rstrip("/") - sess.resume_url = f"{resume}{_GATEWAY_QS}" if resume else None - sess.user_id = str((d.get("user") or {}).get("id") or "") - log.info("discord_bot: gateway ready (user %s)", sess.user_id) - elif t == "MESSAGE_CREATE" and sess.user_id: - # _response_for_message reaches into discord_interactions._grade_message for - # anything but the help line -- sync parquet/DB reads + grading math, same as - # web/app.py's discord_interactions route -- keep it off the (single, shared) - # gateway event loop so one grading lookup can't stall heartbeats/reads. - reply = await asyncio.to_thread(_response_for_message, d, sess.user_id) - if reply: - await _send_reply(d.get("channel_id"), d.get("id"), reply) - - -async def _connection(ws, sess: _Session, resume: bool) -> str: - """Drive one live connection until it closes. Returns the mode for the next - connect: "resume" to reconnect and RESUME, "identify" to start a fresh session.""" - hello = json.loads(await ws.recv()) - if hello.get("op") != _OP_HELLO: - return "identify" # protocol out of step — start clean - interval = float(hello["d"]["heartbeat_interval"]) / 1000.0 - acked = {"ok": True} - hb = asyncio.create_task(_heartbeat(ws, interval, sess, acked)) - try: - if resume and sess.session_id and sess.seq is not None: - await ws.send(json.dumps({"op": _OP_RESUME, "d": { - "token": discord.BOT_TOKEN, - "session_id": sess.session_id, - "seq": sess.seq, - }})) - else: - await ws.send(json.dumps({"op": _OP_IDENTIFY, "d": { - "token": discord.BOT_TOKEN, - "intents": INTENTS, - "properties": {"os": "linux", "browser": "thermograph", "device": "thermograph"}, - }})) - async for raw in ws: - msg = json.loads(raw) - if msg.get("s") is not None: - sess.seq = msg["s"] - op = msg.get("op") - if op == _OP_DISPATCH: - await _dispatch(msg, sess) - elif op == _OP_HEARTBEAT: # server asked for an immediate beat - await ws.send(json.dumps({"op": _OP_HEARTBEAT, "d": sess.seq})) - elif op == _OP_HEARTBEAT_ACK: - acked["ok"] = True - elif op == _OP_RECONNECT: # planned reconnect — resume - return "resume" - elif op == _OP_INVALID_SESSION: - # d == True => the session can still be resumed; else it's dead and - # the next connect must IDENTIFY fresh. - if not msg.get("d"): - sess.session_id = None - sess.resume_url = None - await asyncio.sleep(1 + random.random()) - return "resume" if sess.session_id else "identify" - finally: - hb.cancel() - return "resume" if sess.session_id else "identify" - - -def _close_code(exc: Exception): - """Best-effort websocket close code across websockets library versions (older - exposes .code; newer nests it under .rcvd).""" - code = getattr(exc, "code", None) - if code: - return code - return getattr(getattr(exc, "rcvd", None), "code", None) - - -async def run() -> None: - """Maintain the gateway connection for the app's lifetime: connect, IDENTIFY (or - RESUME), dispatch events, and reconnect with capped exponential backoff. Returns - only on a fatal close (bad token / disallowed intents) or task cancellation, and - never raises out — a misbehaving gateway must not take the app down.""" - if not enabled(): - return - try: - import websockets - except Exception: # noqa: BLE001 - shipped via uvicorn[standard]; guard anyway - log.warning("discord_bot: 'websockets' unavailable; gateway bot disabled") - return - sess = _Session() - mode = "identify" - backoff = _BACKOFF_START - while True: - use_resume = mode == "resume" and bool(sess.resume_url) - url = sess.resume_url if use_resume else _GATEWAY_URL - try: - async with websockets.connect(url, max_size=2 ** 20, open_timeout=30) as ws: - mode = await _connection(ws, sess, resume=use_resume) - backoff = _BACKOFF_START # server-requested reconnect: come back promptly - continue - except asyncio.CancelledError: - log.info("discord_bot: shutting down") - raise - except Exception as exc: # noqa: BLE001 - any drop -> back off and reconnect - if _close_code(exc) in _FATAL_CLOSE: - log.error("discord_bot: fatal gateway close %s; stopping", _close_code(exc)) - return - mode = "resume" if sess.session_id else "identify" - log.warning("discord_bot: gateway dropped (%s); reconnecting in %.0fs", - type(exc).__name__, backoff) - await asyncio.sleep(backoff + random.random()) - backoff = min(backoff * 2, _BACKOFF_MAX) diff --git a/backend/notifications/scheduler.py b/backend/notifications/scheduler.py deleted file mode 100644 index 2677f33..0000000 --- a/backend/notifications/scheduler.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Recurring worker-tier jobs: keep the curated city set warm and ping IndexNow -on a schedule, so neither needs a human to run it after every deploy anymore. - -Only the process that already won leader election for the notifier runs this -too (gated by the same check in web/app.py's _lifespan) — APScheduler has no -cross-host coordination of its own, so running it unguarded on every worker -replica would repeat every job once per replica, same reason the notifier -itself needs a leader. - -APScheduler, not a Postgres-native queue: there is exactly one process that -ever runs this, no fan-out/backpressure/dead-letter need, and it needs no new -infrastructure. procrastinate/pgqueuer are the documented upgrade path if that -changes — see docs/architecture/repo-topology-and-infrastructure.md §7. -""" -import os - -from apscheduler.schedulers.background import BackgroundScheduler - -# How often the curated city set is (re-)warmed. warm_cities.py's own docstring -# frames it as a deploy-time script ("Run at/after deploy") because a warm cache -# mostly just stays warm; scheduling it periodically instead catches a newly -# added city, or one that fell out of cache, without waiting for the next -# deploy. Daily is plenty — every already-cached city is a cheap skip. -WARM_CITIES_INTERVAL_HOURS = int(os.environ.get("THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS", "24") or 24) - -# How often to check whether the indexable URL set changed and, if so, ping -# IndexNow. submit_if_changed() is a cheap no-op when nothing changed, so this -# can run more often than warm_cities without spending anything on most ticks. -INDEXNOW_INTERVAL_HOURS = int(os.environ.get("THERMOGRAPH_INDEXNOW_INTERVAL_HOURS", "6") or 6) - -_scheduler: "BackgroundScheduler | None" = None - - -def _warm_cities_job() -> None: - import warm_cities # local import: a script-style top-level module — only - # pay its import cost on the process that runs the job - warm_cities.main() - - -def _indexnow_job() -> None: - import indexnow - indexnow.submit_if_changed() - - -def start() -> None: - """Start the scheduler (idempotent, mirroring notify.start()). The caller is - responsible for the leader-election gate — see web/app.py's _lifespan.""" - global _scheduler - if _scheduler is not None: - return - sched = BackgroundScheduler(daemon=True) - # No next_run_time override: the default first run is one interval from now, - # not immediately — warm_cities already runs at deploy time (warm_cities.py / - # the deploy hook), so an immediate duplicate run on every worker boot/restart - # would just be wasted work. - sched.add_job(_warm_cities_job, "interval", hours=WARM_CITIES_INTERVAL_HOURS, - id="warm_cities") - sched.add_job(_indexnow_job, "interval", hours=INDEXNOW_INTERVAL_HOURS, - id="indexnow") - sched.start() - _scheduler = sched - - -def stop() -> None: - global _scheduler - if _scheduler is not None: - _scheduler.shutdown(wait=False) - _scheduler = None diff --git a/backend/requirements.txt b/backend/requirements.txt index 62fb0fc..6e8b175 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -19,9 +19,8 @@ alembic==1.14.0 pywebpush==2.0.0 # Ed25519 verification of Discord interaction webhooks (see notifications/discord_interactions.py). PyNaCl==1.5.0 -# Discord gateway bot's websocket client (notifications/discord_bot.py). Already -# pulled in transitively by uvicorn[standard]; pinned here since we import it directly. -websockets==16.0 -# Recurring worker-tier jobs — city warming, IndexNow pings (see -# notifications/scheduler.py). In-process; no broker/queue needed at this scale. -apscheduler==3.11.3 +# The Discord gateway bot and the recurring worker jobs moved to the +# thermograph-daemon Go binary (backend/daemon/), which calls back over +# api/internal_routes.py — so the direct `websockets` pin (only discord_bot.py +# imported it; uvicorn[standard] still pulls it transitively for its own ws +# protocol support) and `apscheduler` are gone from this list. diff --git a/backend/tests/api/test_internal_routes.py b/backend/tests/api/test_internal_routes.py new file mode 100644 index 0000000..3a940ef --- /dev/null +++ b/backend/tests/api/test_internal_routes.py @@ -0,0 +1,134 @@ +"""The internal control surface the thermograph-daemon Go binary calls back +into (api/internal_routes.py): shared-secret gating (fail closed when the token +was never provisioned), the gateway-ready /grade reply with the interactions-only +ephemeral flag dropped, the two job triggers, and the one-in-flight-run guard. +The underlying job/grade functions are stubbed — hermetic, no real network.""" +import pytest +from fastapi.testclient import TestClient + +from api import internal_routes +from notifications import discord_interactions as di +from web import app as appmod + +TOKEN = "test-internal-token" +HDR = {"X-Thermograph-Internal-Token": TOKEN} + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", TOKEN) + return TestClient(appmod.app) + + +@pytest.fixture +def grade_stub(monkeypatch): + """Reply payload for the shared grade builder, mirroring the real shape + (embeds + an ephemeral flag the gateway path must drop).""" + calls = [] + monkeypatch.setattr( + di, "_grade_message", + lambda query: calls.append(query) or {"embeds": [{"title": f"grade:{query}"}], + "flags": 64}) + return calls + + +# --- auth: fail closed, then reject before accept ---------------------------- + +def test_router_is_disabled_when_no_token_is_provisioned(monkeypatch): + """No THERMOGRAPH_INTERNAL_TOKEN => the surface doesn't exist (404), even + for a caller presenting a header — never fall open to no-auth.""" + monkeypatch.delenv("THERMOGRAPH_INTERNAL_TOKEN", raising=False) + client = TestClient(appmod.app) + assert client.post("/internal/discord/grade", json={"query": "x"}, + headers=HDR).status_code == 404 + assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 404 + assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 404 + + +def test_missing_header_is_rejected(client): + assert client.post("/internal/jobs/indexnow", json={}).status_code == 401 + + +def test_wrong_token_is_rejected(client): + r = client.post("/internal/jobs/indexnow", json={}, + headers={"X-Thermograph-Internal-Token": "not-the-token"}) + assert r.status_code == 401 + + +def test_correct_token_is_accepted(client, monkeypatch): + import indexnow + monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None) + assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200 + + +# --- /internal/discord/grade ------------------------------------------------- + +def test_grade_returns_the_shared_builder_message_without_ephemeral_flag(client, grade_stub): + r = client.post("/internal/discord/grade", json={"query": "Phoenix"}, headers=HDR) + assert r.status_code == 200 + # Gateway messages can't be ephemeral — the flag must be gone, everything + # else exactly as _grade_message built it. + assert r.json() == {"embeds": [{"title": "grade:Phoenix"}]} + assert grade_stub == ["Phoenix"] + + +def test_grade_requires_a_query(client, grade_stub): + assert client.post("/internal/discord/grade", json={}, headers=HDR).status_code == 422 + assert grade_stub == [] + + +# --- job triggers ------------------------------------------------------------ + +def test_warm_cities_invokes_warm_cities_main(client, monkeypatch): + import warm_cities + calls = [] + monkeypatch.setattr(warm_cities, "main", lambda: calls.append(1)) + r = client.post("/internal/jobs/warm-cities", json={}, headers=HDR) + assert r.status_code == 200 + assert r.json() == {"ok": True} + assert calls == [1] + + +def test_indexnow_invokes_submit_if_changed(client, monkeypatch): + import indexnow + calls = [] + monkeypatch.setattr(indexnow, "submit_if_changed", lambda: calls.append(1)) + r = client.post("/internal/jobs/indexnow", json={}, headers=HDR) + assert r.status_code == 200 + assert r.json() == {"ok": True} + assert calls == [1] + + +# --- one-in-flight-run guard ------------------------------------------------- + +def test_concurrent_job_invocation_answers_409(client, monkeypatch): + """A second trigger while a run is in flight must not stack — warm_cities + spends the Open-Meteo quota, so a stacked run double-spends it. Holding the + job's lock stands in for an in-flight run.""" + import indexnow + monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None) + lock = internal_routes._JOB_LOCKS["warm-cities"] + assert lock.acquire(blocking=False) + try: + assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 409 + # The guard is per job: an in-flight warm run doesn't block indexnow. + assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200 + finally: + lock.release() + # Released => the next trigger runs again. + import warm_cities + monkeypatch.setattr(warm_cities, "main", lambda: None) + assert client.post("/internal/jobs/warm-cities", json={}, headers=HDR).status_code == 200 + + +def test_job_lock_is_released_after_a_failing_run(monkeypatch): + """A crashed run must not wedge the job forever: the 500 surfaces, the lock + is released, and the next trigger runs.""" + monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", TOKEN) + client = TestClient(appmod.app, raise_server_exceptions=False) + import indexnow + monkeypatch.setattr(indexnow, "submit_if_changed", + lambda: (_ for _ in ()).throw(RuntimeError("boom"))) + assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 500 + monkeypatch.setattr(indexnow, "submit_if_changed", lambda: None) + assert client.post("/internal/jobs/indexnow", json={}, headers=HDR).status_code == 200 diff --git a/backend/tests/api/test_internal_token_derivation.py b/backend/tests/api/test_internal_token_derivation.py new file mode 100644 index 0000000..77734cc --- /dev/null +++ b/backend/tests/api/test_internal_token_derivation.py @@ -0,0 +1,70 @@ +"""The internal token's derivation, and its cross-language contract with the Go daemon. + +The daemon (backend/daemon/) and this app must compute byte-identical tokens from +the same THERMOGRAPH_AUTH_SECRET. If they drift, every callback fails with 401 -- +which reads like an auth bug rather than like the configuration drift it is, so +the shared vector below is pinned on both sides. +""" +import hashlib +import hmac + +import pytest + +from api import internal_routes + +# Must equal backend/daemon/internal/config/derive_test.go's sharedVector*. +SHARED_VECTOR_SECRET = "test-auth-secret" +SHARED_VECTOR_TOKEN = "4c3830b2158941ff52720d198b65f7924372a3a482b8da52540a4d9be38247ea" + + +@pytest.fixture(autouse=True) +def _clear_token_env(monkeypatch): + """Both knobs start unset so each test states exactly the config it exercises.""" + monkeypatch.delenv("THERMOGRAPH_INTERNAL_TOKEN", raising=False) + monkeypatch.delenv("THERMOGRAPH_AUTH_SECRET", raising=False) + + +def test_derived_token_matches_go(monkeypatch): + """The pinned cross-language vector. Changing the label or construction here + requires the identical change in the Go daemon's config package.""" + monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET) + assert internal_routes.resolve_internal_token() == SHARED_VECTOR_TOKEN + + +def test_vector_is_actually_hmac_of_the_label(): + """Guards against the vector and the implementation drifting together if + someone regenerates the constant from the code it is supposed to check.""" + expected = hmac.new( + SHARED_VECTOR_SECRET.encode(), + internal_routes._DERIVE_LABEL, + hashlib.sha256, + ).hexdigest() + assert expected == SHARED_VECTOR_TOKEN + + +def test_explicit_token_wins_over_derivation(monkeypatch): + monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET) + monkeypatch.setenv("THERMOGRAPH_INTERNAL_TOKEN", "explicit-wins") + assert internal_routes.resolve_internal_token() == "explicit-wins" + + +def test_no_secret_at_all_leaves_the_surface_closed(): + """Neither knob set => no token => the router 404s. Fail closed, never open.""" + assert internal_routes.resolve_internal_token() == "" + + +def test_derivation_is_not_the_auth_secret_itself(monkeypatch): + """Domain separation: a leak of the internal token must not hand over the + session-signing key it was derived from.""" + monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", SHARED_VECTOR_SECRET) + token = internal_routes.resolve_internal_token() + assert token != SHARED_VECTOR_SECRET + assert SHARED_VECTOR_SECRET not in token + + +def test_different_secrets_derive_different_tokens(monkeypatch): + monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", "secret-a") + a = internal_routes.resolve_internal_token() + monkeypatch.setenv("THERMOGRAPH_AUTH_SECRET", "secret-b") + b = internal_routes.resolve_internal_token() + assert a != b diff --git a/backend/tests/notifications/test_discord_bot.py b/backend/tests/notifications/test_discord_bot.py deleted file mode 100644 index 9692542..0000000 --- a/backend/tests/notifications/test_discord_bot.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Discord gateway bot: the message-handling logic that decides when to reply and -with what. No live gateway — we feed MESSAGE_CREATE payloads straight to the pure -handler and stub the shared /grade lookup so these stay data-independent.""" -import asyncio - -import pytest - -from notifications import discord -from notifications import discord_bot as bot -from notifications import discord_interactions as di - -BOT_ID = "999" - - -@pytest.fixture(autouse=True) -def _stub_grade(monkeypatch): - """Reply payload for /grade, mirroring the real shape (embeds + an ephemeral - flag the gateway path must drop).""" - monkeypatch.setattr( - di, "_grade_message", - lambda query: {"embeds": [{"title": f"grade:{query}"}], "flags": 64}) - - -def _msg(content="", *, author_id="1", bot_author=False, mentions=(), guild=True): - m = { - "content": content, - "id": "42", - "channel_id": "77", - "author": {"id": author_id, "bot": bot_author}, - "mentions": [{"id": mid} for mid in mentions], - } - if guild: - m["guild_id"] = "5" - return m - - -# --- when the bot stays silent ---------------------------------------------- - -def test_ignores_other_bots(): - assert bot._response_for_message( - _msg("<@999> Phoenix", author_id="8", bot_author=True, mentions=[BOT_ID]), BOT_ID) is None - - -def test_ignores_its_own_messages(): - assert bot._response_for_message( - _msg("hello", author_id=BOT_ID, mentions=[BOT_ID]), BOT_ID) is None - - -def test_ignores_guild_message_without_a_mention(): - assert bot._response_for_message(_msg("just chatting", mentions=[]), BOT_ID) is None - - -# --- when the bot replies ---------------------------------------------------- - -def test_mention_with_city_returns_grade_without_ephemeral_flag(): - out = bot._response_for_message(_msg("<@999> Phoenix", mentions=[BOT_ID]), BOT_ID) - assert out == {"embeds": [{"title": "grade:Phoenix"}]} # flags dropped - assert "flags" not in out - - -def test_legacy_nickname_mention_is_stripped(): - out = bot._response_for_message(_msg("<@!999> Tokyo", mentions=[BOT_ID]), BOT_ID) - assert out["embeds"][0]["title"] == "grade:Tokyo" - - -def test_dm_needs_no_mention(): - out = bot._response_for_message(_msg("Berlin", guild=False, mentions=[]), BOT_ID) - assert out["embeds"][0]["title"] == "grade:Berlin" - - -def test_mention_with_no_query_gives_help(): - out = bot._response_for_message(_msg("<@999>", mentions=[BOT_ID]), BOT_ID) - assert "content" in out and "mention me" in out["content"].lower() - - -# --- helpers ----------------------------------------------------------------- - -def test_strip_mentions_handles_both_forms(): - assert bot._strip_mentions("<@999> <@!999> Sao Paulo", BOT_ID) == "Sao Paulo" - - -def test_is_dm_is_true_without_guild_id(): - assert bot._is_dm(_msg("x", guild=False)) is True - assert bot._is_dm(_msg("x", guild=True)) is False - - -# --- _dispatch: same reply, off the event loop ------------------------------- - -def test_dispatch_delivers_the_same_reply_via_a_thread(monkeypatch): - """_dispatch now runs _response_for_message through asyncio.to_thread (the - event-loop fix) -- confirm that still produces the exact same reply and gets - it to _send_reply, i.e. the offload didn't change behaviour.""" - sent = {} - - async def _fake_send_reply(channel_id, message_id, data): - sent["channel_id"] = channel_id - sent["message_id"] = message_id - sent["data"] = data - - monkeypatch.setattr(bot, "_send_reply", _fake_send_reply) - sess = bot._Session() - sess.user_id = BOT_ID - msg = {"t": "MESSAGE_CREATE", "d": _msg("<@999> Phoenix", mentions=[BOT_ID])} - - asyncio.run(bot._dispatch(msg, sess)) - - assert sent["channel_id"] == "77" - assert sent["message_id"] == "42" - assert sent["data"] == {"embeds": [{"title": "grade:Phoenix"}]} - - -# --- enabled() gating -------------------------------------------------------- - -def test_enabled_is_off_by_default(monkeypatch): - monkeypatch.delenv("THERMOGRAPH_DISCORD_BOT", raising=False) - monkeypatch.setattr(discord, "BOT_TOKEN", "a-token") - assert bot.enabled() is False - - -def test_enabled_needs_both_flag_and_token(monkeypatch): - monkeypatch.setenv("THERMOGRAPH_DISCORD_BOT", "1") - monkeypatch.setattr(discord, "BOT_TOKEN", "") - assert bot.enabled() is False - monkeypatch.setattr(discord, "BOT_TOKEN", "a-token") - assert bot.enabled() is True diff --git a/backend/tests/notifications/test_scheduler.py b/backend/tests/notifications/test_scheduler.py deleted file mode 100644 index 3aeb929..0000000 --- a/backend/tests/notifications/test_scheduler.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Tests for the recurring worker-tier jobs (scheduler.py): job registration, -cadence from env, idempotent start/stop, and that each job calls the right -underlying function. No real hours-long wait — inspects APScheduler's own job -registry rather than letting anything actually fire.""" -import datetime - -import pytest - -from notifications import scheduler - - -@pytest.fixture(autouse=True) -def _clean_scheduler(): - """Every test starts from "not running" and leaves nothing behind, even on - failure — scheduler._scheduler is module-global state shared across tests.""" - scheduler.stop() - yield - scheduler.stop() - - -def test_start_registers_both_jobs_with_configured_intervals(monkeypatch): - monkeypatch.setattr(scheduler, "WARM_CITIES_INTERVAL_HOURS", 24) - monkeypatch.setattr(scheduler, "INDEXNOW_INTERVAL_HOURS", 6) - scheduler.start() - jobs = {j.id: j for j in scheduler._scheduler.get_jobs()} - assert set(jobs) == {"warm_cities", "indexnow"} - assert jobs["warm_cities"].trigger.interval == datetime.timedelta(hours=24) - assert jobs["indexnow"].trigger.interval == datetime.timedelta(hours=6) - assert jobs["warm_cities"].func is scheduler._warm_cities_job - assert jobs["indexnow"].func is scheduler._indexnow_job - - -def test_start_does_not_run_a_job_immediately(): - """warm_cities already runs at deploy time; an immediate duplicate run on - every worker boot/restart would just be wasted work — the first scheduled - run must be one interval away, not "now".""" - scheduler.start() - now = datetime.datetime.now(datetime.timezone.utc) - for job in scheduler._scheduler.get_jobs(): - assert job.next_run_time > now - - -def test_start_is_idempotent(monkeypatch): - scheduler.start() - first = scheduler._scheduler - calls = [] - monkeypatch.setattr(scheduler.BackgroundScheduler, "__init__", - lambda self, **kw: calls.append(1) or None) - scheduler.start() - assert scheduler._scheduler is first - assert calls == [] # a second BackgroundScheduler was never constructed - - -def test_stop_shuts_down_and_clears_state(): - scheduler.start() - assert scheduler._scheduler is not None - scheduler.stop() - assert scheduler._scheduler is None - - -def test_stop_without_start_is_a_safe_no_op(): - scheduler.stop() # must not raise - assert scheduler._scheduler is None - - -def test_warm_cities_job_calls_warm_cities_main(monkeypatch): - import warm_cities - calls = [] - monkeypatch.setattr(warm_cities, "main", lambda: calls.append(1)) - scheduler._warm_cities_job() - assert calls == [1] - - -def test_indexnow_job_calls_submit_if_changed(monkeypatch): - import indexnow - calls = [] - monkeypatch.setattr(indexnow, "submit_if_changed", lambda: calls.append(1)) - scheduler._indexnow_job() - assert calls == [1] diff --git a/backend/web/app.py b/backend/web/app.py index 6953b73..0253319 100644 --- a/backend/web/app.py +++ b/backend/web/app.py @@ -1,5 +1,4 @@ """Thermograph API — grade recent local weather against ~45 years of climatology.""" -import asyncio import contextlib import datetime import hashlib @@ -19,17 +18,16 @@ from fastapi.responses import JSONResponse, RedirectResponse from accounts import api_accounts from api import content_routes +from api import internal_routes from core import audit from data import climate from notifications import digest -from notifications import discord_bot from notifications import discord_interactions as discord_interactions_mod from notifications import discord_link from accounts import db from data import grid from core import metrics from notifications import notify -from notifications import scheduler import paths from data import places from core import singleton @@ -163,33 +161,19 @@ async def _lifespan(app): # restricts this to processes that are allowed to own it at all — "web" replicas # never start it even if they'd otherwise win the leader election, so a stateless # N-replica web tier can scale without also scaling notifier instances. - bot_task = None + # The Discord gateway bot and the recurring worker jobs (city warming, + # IndexNow) used to ride this same leader gate in-process; both now live in + # the thermograph-daemon Go binary, which owns the connection/timers and + # calls back over api/internal_routes.py for anything needing data. The + # notifier is the one singleton still running here. if _should_run_notifier(): notify.start() - # Same leader gate as the notifier above — the recurring worker jobs - # (city warming, IndexNow) must likewise run in exactly one process. - scheduler.start() - # Discord gateway bot (opt-in). Discord allows one gateway connection per - # bot token, so it belongs on the same single leader; it rides this event - # loop as a background task rather than a new process. No-op unless - # THERMOGRAPH_DISCORD_BOT is set with a bot token (discord_bot.enabled()). - if discord_bot.enabled(): - bot_task = asyncio.create_task(discord_bot.run(), name="discord-bot") # Deploy/restart marker: one line correlates a metric shift with a boot. audit.log_activity("app.start", {"version": app.version, "role": ROLE, "base": BASE, "notifier": _should_run_notifier(), "pid": os.getpid()}) yield audit.log_activity("app.stop", {"role": ROLE, "pid": os.getpid()}) - if bot_task is not None: - bot_task.cancel() - # Wait for the cancellation to actually land before tearing anything else - # down -- otherwise shutdown races the bot coroutine's own teardown (closing - # the websocket, cancelling its heartbeat task) and the CancelledError it - # raises on its way out surfaces as unhandled-task noise in the logs. - with contextlib.suppress(asyncio.CancelledError): - await bot_task notify.stop() - scheduler.stop() await _frontend_client.aclose() @@ -838,6 +822,13 @@ app.include_router(v2, prefix=f"{BASE}/api/v2") # service (Stage 3) instead of anything in this process. See # backend/api/content_routes.py. app.include_router(content_routes.router, prefix=f"{BASE}/api/v2") +# Internal control surface for the thermograph-daemon Go binary (the gateway +# bot + recurring jobs that used to run in _lifespan). Deliberately NOT under +# BASE — the daemon hits THERMOGRAPH_API_BASE_INTERNAL directly, same posture +# as /healthz — and never publicly routed (the Caddyfile only forwards +# /api/*, /digest and /discord/interactions here). Registered before the +# catch-all frontend proxy below so /internal/* can't fall through to it. +app.include_router(internal_routes.router) # --- proxy to the frontend SSR service (repo-split Stage 4) ------------------ diff --git a/infra/.env.example b/infra/.env.example index e95b86a..3253125 100644 --- a/infra/.env.example +++ b/infra/.env.example @@ -9,6 +9,21 @@ # build the app's THERMOGRAPH_DATABASE_URL. Change it before first `up`. POSTGRES_PASSWORD=change-me +# OPTIONAL. Shared secret between the daemon service (Discord gateway + job +# timers) and the backend's /internal/* routes. +# +# Leave it empty and both ends DERIVE the same token from THERMOGRAPH_AUTH_SECRET +# (HMAC-SHA256 under a fixed label), which every environment already provisions — +# so the daemon needs no new vault entry and no operator step. Set it only to +# override that, e.g. to rotate this surface independently of the auth secret: +# openssl rand -hex 32 +# +# With neither this nor THERMOGRAPH_AUTH_SECRET set, both ends fail closed: the +# backend disables the internal routes and the daemon refuses to start. Never +# routed publicly — Caddy only forwards /api/*, /digest and +# /discord/interactions to the backend. +THERMOGRAPH_INTERNAL_TOKEN= + # --- Everything below is optional -- docker-compose.yml already defaults each # --- of these, so a plain `make up` works with none of it set. Uncomment to # --- override. diff --git a/infra/deploy/deploy.sh b/infra/deploy/deploy.sh index 4b8f8f3..5e7c451 100755 --- a/infra/deploy/deploy.sh +++ b/infra/deploy/deploy.sh @@ -186,10 +186,19 @@ export BACKEND_IMAGE_TAG="${BACKEND_IMAGE_TAG:-local}" export FRONTEND_IMAGE_TAG="${FRONTEND_IMAGE_TAG:-local}" # Which compose services this run pulls/rolls. +# +# `daemon` rides with `backend` and is never a target on its own. It runs the +# SAME image at the SAME tag (it is a second binary inside the backend image), +# and it talks to the web process over the /internal/* contract -- so rolling one +# without the other is exactly the version skew that contract has no negotiation +# for. Pairing them here is also what makes the service exist at all: a +# single-service deploy runs `up -d --no-deps `, so a `daemon` listed +# only in compose but absent from TARGETS would never be created on a +# backend-only deploy, and the gateway bot would silently never start. case "$SERVICE" in - backend) TARGETS=(backend) ;; + backend) TARGETS=(backend daemon) ;; frontend) TARGETS=(frontend) ;; - all) TARGETS=(backend frontend) ;; + all) TARGETS=(backend frontend daemon) ;; esac # Login only when a token is supplied. The SSH/CI deploy paths (deploy.yml, @@ -226,6 +235,30 @@ if [ "$pull_ok" != 1 ]; then exit 1 fi +# The daemon binary ships INSIDE the backend image, but infra and app images +# advance on different axes by design: this checkout tracks `main`, while image +# tags are env-staged (prod's come from `release`). So a host can legitimately +# be asked to roll a backend image OLDER than this compose file -- one built +# before the daemon existed and with no /usr/local/bin/thermograph-daemon in it. +# Creating the service anyway would leave a container crash-looping on a missing +# binary, on a deploy that otherwise succeeded. Probe the image we are actually +# about to roll and drop the daemon from this run if it can't support it; the +# next deploy of a tag that has it picks it up with no further action. +case " ${TARGETS[*]} " in + *" daemon "*) + daemon_img="$(docker compose config --images daemon 2>/dev/null | head -1 || true)" + if [ -n "$daemon_img" ] \ + && ! docker run --rm --entrypoint sh "$daemon_img" -c 'test -x /usr/local/bin/thermograph-daemon' 2>/dev/null; then + echo "==> $daemon_img predates the daemon binary; rolling without the daemon service this run" + kept=() + for t in "${TARGETS[@]}"; do + [ "$t" = daemon ] || kept+=("$t") + done + TARGETS=("${kept[@]}") + fi + ;; +esac + # Roll only the target service(s). Backend schema migrations run inside its own # entrypoint (alembic upgrade head) before uvicorn, so there's no separate # migrate step; frontend is stateless. diff --git a/infra/deploy/stack/thermograph-stack.yml b/infra/deploy/stack/thermograph-stack.yml index b34f77b..bccb6b9 100644 --- a/infra/deploy/stack/thermograph-stack.yml +++ b/infra/deploy/stack/thermograph-stack.yml @@ -138,6 +138,60 @@ services: restart_policy: condition: on-failure + daemon: + # SAME image and tag as web/worker on purpose: the daemon binary + # (/usr/local/bin/thermograph-daemon, built into the backend image) and the + # app share the /internal/* API contract, so one BACKEND_IMAGE_TAG rolls + # them together and they can never skew. + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:?required} + # Not env-entrypoint.sh: that shim's handoff execs the image's + # deploy/entrypoint.sh (Alembic + uvicorn) whenever it exists, which is + # exactly what this service must never run — migrations belong to the + # one-shot task, and a second migrator racing it is a real hazard. Instead + # source the same host-rendered env directly (it is shell-sourceable by + # contract — deploy.sh sources it too) and exec the binary. stack.env holds + # vault secrets, never service-topology URLs, so it cannot collide with the + # `environment:` block below. + entrypoint: + - /bin/bash + - -c + - 'set -a; [ -f /host/thermograph.env ] && . /host/thermograph.env; set +a; exec /usr/local/bin/thermograph-daemon' + environment: + # The service VIP: the daemon's /internal/* calls load-balance across web + # replicas like any other client of the app. + THERMOGRAPH_API_BASE_INTERNAL: http://web:8137 + # Only the env mount — no appdata/applogs: warm-cities/IndexNow work + # executes inside the web process (the daemon just fires the internal + # endpoints on a timer), so the daemon holds no state and logs to stdout. + volumes: + - /etc/thermograph/stack.env:/host/thermograph.env:ro + networks: + - internal + deploy: + # EXACTLY 1, load-bearing — do not scale this service, ever. Discord + # permits ONE gateway connection per bot token; the Python bot enforced + # that with a leader election (core/singleton.claim_leader), and this + # pinned replica count is what REPLACES that election. A second replica + # means two gateway sessions fighting over the token (Discord kicks + # both into a reconnect storm) and every timer job firing twice. The + # autoscaler cannot touch this — it scales ${STACK_NAME}_web only. + replicas: 1 + placement: + constraints: ["node.role == manager"] + resources: + limits: + cpus: "0.5" + memory: 128m + restart_policy: + condition: on-failure + update_config: + # stop-first (the default), NOT start-first: a start-first roll would + # briefly run two daemons — two gateway sessions on one token, the + # exact thing replicas:1 exists to prevent. A few seconds of gateway + # downtime per deploy is the correct trade. + order: stop-first + failure_action: rollback + frontend: image: ${REGISTRY_HOST:-git.thermograph.org}/${FRONTEND_IMAGE_PATH:-emi/thermograph/frontend}:${FRONTEND_IMAGE_TAG:?required} entrypoint: ["/host/env-entrypoint.sh"] diff --git a/infra/deploy/thermograph.env.example b/infra/deploy/thermograph.env.example index 29a1f07..7020e2d 100644 --- a/infra/deploy/thermograph.env.example +++ b/infra/deploy/thermograph.env.example @@ -83,12 +83,35 @@ WORKERS=4 # Read once at process start; changing it needs a restart, not a live toggle. #THERMOGRAPH_ROLE=all -# The worker's own recurring jobs (city warming, IndexNow), gated by the same -# leader election as the notifier above. Hours between runs; both are cheap -# no-op skips when there's nothing to do, so the defaults rarely need changing. +# The recurring jobs (city warming, IndexNow). Hours between runs; both are +# cheap no-op skips when there's nothing to do, so the defaults rarely need +# changing. The daemon service (below) owns these timers and fires the work +# through the backend's internal routes. #THERMOGRAPH_WARM_CITIES_INTERVAL_HOURS=24 #THERMOGRAPH_INDEXNOW_INTERVAL_HOURS=6 +# --- Internal daemon<->backend API ------------------------------------------------ +# Shared secret authenticating the daemon service's calls to the backend's +# /internal/* routes (X-Thermograph-Internal-Token header). A credential: +# OPTIONAL — leave unset unless you specifically want to override the derivation. +# +# When unset, the backend and the daemon each DERIVE this token from +# THERMOGRAPH_AUTH_SECRET (HMAC-SHA256 under a fixed domain-separation label). +# That secret is already required and already in every vault, so the daemon +# stands up with NO new key to add, rotate, or forget on a new host. HMAC under a +# distinct label rather than reusing the auth secret directly, so a leak of this +# token can't be replayed as the session-signing key. +# +# Set it explicitly to rotate this surface independently of the auth secret +# (e.g. openssl rand -hex 32). On a SOPS host that means `sops edit` on +# deploy/secrets/*.yaml, never hand-editing /etc/thermograph.env — it is a +# rendered artifact. Both ends must agree, so set it on both or neither. +# +# With neither this nor THERMOGRAPH_AUTH_SECRET set, both ends fail closed: the +# backend disables the internal routes and the daemon refuses to start. Defence +# in depth: Caddy never routes /internal/* to the backend anyway. +#THERMOGRAPH_INTERNAL_TOKEN= + # Base path the app is served under. # / -> app at the domain root (Thermograph owns the whole domain — # this is what the Caddyfile expects: thermograph.org proxies "/") @@ -188,12 +211,13 @@ THERMOGRAPH_BASE_URL=https://thermograph.org # alone). Leave unset to disable. In the Thermograph.org server: # weather-events = 1529274516746932307 #THERMOGRAPH_DISCORD_WEATHER_CHANNEL= -# Discord gateway bot (@mention/DM grading): runs INSIDE the backend leader -# process (same singleton election as the notifier — see web/app.py's lifespan), -# riding its event loop; no separate bot process. Opt-in: any truthy value here -# starts it, provided BOT_TOKEN above is also set (flag alone is a silent no-op). -# Discord allows ONE gateway connection per bot token, so enable this in exactly -# one environment — prod, the only vault with the token. +# Discord gateway bot (@mention/DM grading): runs in the daemon service (the +# Go thermograph-daemon container — no longer inside the backend leader +# process). Opt-in: any truthy value here starts it, provided BOT_TOKEN above +# is also set (flag alone is a silent no-op). Discord allows ONE gateway +# connection per bot token, so enable this in exactly one environment — prod, +# the only vault with the token (the stack pins the daemon to 1 replica for +# the same reason). #THERMOGRAPH_DISCORD_BOT= # Account linking (OAuth2 "identify"): lets a signed-in user connect their Discord # account, storing their Discord user id for DM alerts. CLIENT_ID is the same App @@ -202,8 +226,9 @@ THERMOGRAPH_BASE_URL=https://thermograph.org #THERMOGRAPH_DISCORD_CLIENT_SECRET= # Gateway bot (opt-in): holds a live websocket so the bot replies to messages that # @mention it (or DM it) with a city grade — e.g. "@Thermograph Phoenix". Reuses -# THERMOGRAPH_DISCORD_BOT_TOKEN above. Runs on the single notifier leader only, so -# it needs THERMOGRAPH_ENABLE_NOTIFIER on (the default). No privileged intent -# required — Discord delivers content for mentions/DMs. Unset/0 => no gateway -# connection. (Code: thermograph-backend notifications/discord_bot.py.) +# THERMOGRAPH_DISCORD_BOT_TOKEN above. Runs in the single-replica daemon service +# (which also needs THERMOGRAPH_INTERNAL_TOKEN below — it grades via the +# backend's internal API). No privileged intent required — Discord delivers +# content for mentions/DMs. Unset/0 => no gateway connection. +# (Code: backend/daemon/, calling backend's /internal/discord/grade.) #THERMOGRAPH_DISCORD_BOT=1 diff --git a/infra/docker-compose.dev.yml b/infra/docker-compose.dev.yml index c2107fd..f136a09 100644 --- a/infra/docker-compose.dev.yml +++ b/infra/docker-compose.dev.yml @@ -37,6 +37,11 @@ services: ports: !reset null cpus: !reset null deploy: !reset null + # Same uncapped-on-dev treatment for the daemon; it has no ports to touch + # (outbound-only in every environment). + daemon: + cpus: !reset null + deploy: !reset null db: cpus: !reset null deploy: !reset null diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 4de4659..5f57cc2 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -124,6 +124,13 @@ services: # (NOT /app/data) so it never shadows the Python `data/` package -- see the # volumes: note below and thermograph-backend paths.py. THERMOGRAPH_SINGLETON_LOCK: /state/notifier.lock + # Shared secret for the /internal/* routes the daemon service calls. + # Interpolated (like POSTGRES_PASSWORD) so local runs get it from the + # repo-root .env; prod/beta have it in /etc/thermograph.env, which + # deploy.sh sources before compose so this resolves there too. Empty => + # the backend disables the routes and the daemon refuses to start -- fail + # closed on both ends. + THERMOGRAPH_INTERNAL_TOKEN: ${THERMOGRAPH_INTERNAL_TOKEN:-} # Prod secrets live in /etc/thermograph.env: POSTGRES_PASSWORD, # THERMOGRAPH_AUTH_SECRET, THERMOGRAPH_VAPID_PRIVATE_KEY/_PUBLIC_KEY, # THERMOGRAPH_COOKIE_SECURE=1, mail/Discord keys, ... (see @@ -157,6 +164,62 @@ services: - "127.0.0.1:8137:8137" restart: unless-stopped + daemon: + # The SAME image and tag as backend, on purpose: the daemon (Go, built into + # the backend image as /usr/local/bin/thermograph-daemon) and the backend + # share the /internal/* API contract, so they must roll together — a + # separate tag could skew them. It owns the Discord gateway websocket and + # the recurring-job timers; anything needing data calls back into backend. + image: ${REGISTRY_HOST:-git.thermograph.org}/${BACKEND_IMAGE_PATH:-emi/thermograph/backend}:${BACKEND_IMAGE_TAG:-local} + # Bypass deploy/entrypoint.sh entirely: that script runs Alembic, and the + # backend service's boot already owns migrations — two containers racing + # `alembic upgrade head` against one database is a real hazard, not a + # nicety. The daemon binary never touches the DB. + entrypoint: ["/usr/local/bin/thermograph-daemon"] + # Its first acts are calls to backend's /internal/* routes, so wait for a + # genuinely healthy backend, not just a started container. + depends_on: + backend: + condition: service_healthy + environment: + # Same var the frontend uses for the same purpose: the backend's URL on + # the compose-internal network. + THERMOGRAPH_API_BASE_INTERNAL: http://backend:8137 + # See the backend service's note on this var — same interpolation, same + # fail-closed-when-empty behavior on both ends. Normally EMPTY: with no + # explicit value both ends derive the same token from + # THERMOGRAPH_AUTH_SECRET, which arrives via the env_file below. That is + # why this service must keep reading that file even though it serves + # nothing — without the auth secret it has nothing to derive from and + # refuses to start. + THERMOGRAPH_INTERNAL_TOKEN: ${THERMOGRAPH_INTERNAL_TOKEN:-} + # THERMOGRAPH_AUTH_SECRET (derivation input, see above), the bot token/flag + # and the job intervals (THERMOGRAPH_DISCORD_BOT[_TOKEN], + # THERMOGRAPH_*_INTERVAL_HOURS) all arrive via the host env file, same as + # the backend's secrets do — so both processes read one source of truth and + # cannot disagree about the derived token. + env_file: + - path: /etc/thermograph.env + required: false + # The image's HEALTHCHECK curls /healthz on ${PORT} — right for the backend + # process, meaningless for the daemon, which serves nothing. Without this + # the container would sit permanently "unhealthy". + healthcheck: + disable: true + # No volumes: warm-cities/IndexNow work executes inside the BACKEND process + # (the daemon only fires the internal endpoints on a timer), the parquet + # cache and locks stay on backend's appdata volume, and the daemon logs to + # stdout. It holds no state a volume would protect. + # + # No ports: outbound-only (Discord gateway + calls to backend). Publishing + # anything here would only widen the surface for no benefit. + cpus: 0.5 + deploy: + resources: + limits: + cpus: "0.5" + restart: unless-stopped + frontend: # Frontend's OWN image, published by thermograph-frontend's build-push.yml # from its own Dockerfile (which starts `uvicorn app:app` directly -- no From 6396d553d293eec849eb2d644a10df81b9229dca Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:55:41 +0000 Subject: [PATCH 08/15] Own /state in the image; lake cache degrades instead of failing (#25) --- backend/Dockerfile | 4 ++-- backend/lake_app.py | 21 ++++++++++++++++----- backend/tests/web/test_lake_app.py | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 95d28ec..fecc193 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -26,8 +26,8 @@ RUN chmod +x /app/deploy/entrypoint.sh # Docker seeds them from this image content -- including this ownership -- # so they stay writable. RUN useradd --system --create-home --uid 10001 thermograph \ - && mkdir -p /app/data /app/logs \ - && chown -R thermograph:thermograph /app + && mkdir -p /app/data /app/logs /state \ + && chown -R thermograph:thermograph /app /state USER thermograph WORKDIR /app diff --git a/backend/lake_app.py b/backend/lake_app.py index 6618b35..37eb721 100644 --- a/backend/lake_app.py +++ b/backend/lake_app.py @@ -24,6 +24,7 @@ Scale-out safe: replicas share nothing — each keeps its own cache under THERMOGRAPH_LAKE_CACHE (a per-task local volume), so 1..2 replicas behind the Swarm VIP need no coordination. """ +import io import os import re import threading @@ -31,7 +32,7 @@ import time import polars as pl from fastapi import FastAPI, HTTPException -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, Response from pydantic import BaseModel from data import era5lake @@ -106,10 +107,20 @@ def history(lat: float, lon: float): df = era5lake._read_bucket(i, j, cfg) except era5lake.LakeUnavailable as e: raise HTTPException(status_code=404, detail=str(e)) from e - os.makedirs(CACHE_DIR, exist_ok=True) - tmp = f"{path}.{os.getpid()}.partial" - df.write_parquet(tmp) - os.replace(tmp, path) # atomic: concurrent requests see whole files only + try: + os.makedirs(CACHE_DIR, exist_ok=True) + tmp = f"{path}.{os.getpid()}.partial" + df.write_parquet(tmp) + os.replace(tmp, path) # atomic: concurrent readers see whole files only + except OSError: + # An unwritable cache (bad volume ownership, full disk) must never + # fail the hot path — serve from memory; the cache is an + # accelerator, not a dependency. + buf = io.BytesIO() + df.write_parquet(buf) + return Response(buf.getvalue(), + media_type="application/vnd.apache.parquet", + headers={"X-Lake-Point": f"{i},{j}"}) return FileResponse(path, media_type="application/vnd.apache.parquet", headers={"X-Lake-Point": f"{i},{j}"}) diff --git a/backend/tests/web/test_lake_app.py b/backend/tests/web/test_lake_app.py index ea832c8..2dd1f1e 100644 --- a/backend/tests/web/test_lake_app.py +++ b/backend/tests/web/test_lake_app.py @@ -69,6 +69,20 @@ def test_history_serves_parquet_and_caches(lake, tmp_path): params={"lat": 51.5074, "lon": -0.1278}).content == r.content +def test_history_serves_from_memory_when_cache_unwritable(lake, tmp_path, monkeypatch): + """An unwritable cache dir (bad volume ownership — seen live) degrades to + serving from memory, never a 500.""" + import lake_app + ro = tmp_path / "ro-cache" / "nested" # parent made read-only below + (tmp_path / "ro-cache").mkdir() + (tmp_path / "ro-cache").chmod(0o555) + monkeypatch.setattr(lake_app, "CACHE_DIR", str(ro)) + r = lake.get("/history", params={"lat": 51.5074, "lon": -0.1278}) + (tmp_path / "ro-cache").chmod(0o755) + assert r.status_code == 200 + assert r.headers["x-lake-point"] == "154,1439" + + def test_history_404_for_point_outside_lake(lake): r = lake.get("/history", params={"lat": -45.0, "lon": 170.0}) assert r.status_code == 404 From 16bddb662588e142c04348b500723b97edfd5ce9 Mon Sep 17 00:00:00 2001 From: emi Date: Thu, 23 Jul 2026 22:56:27 +0000 Subject: [PATCH 09/15] Iceberg conversion container for the ERA5 lake (infra/lake-iceberg) (#24) --- infra/.gitignore | 4 + infra/lake-iceberg/Dockerfile | 20 ++ infra/lake-iceberg/README.md | 123 ++++++++ infra/lake-iceberg/requirements.txt | 5 + infra/lake-iceberg/sync_iceberg.py | 451 ++++++++++++++++++++++++++++ infra/lake-iceberg/test_sync.py | 247 +++++++++++++++ 6 files changed, 850 insertions(+) create mode 100644 infra/lake-iceberg/Dockerfile create mode 100644 infra/lake-iceberg/README.md create mode 100644 infra/lake-iceberg/requirements.txt create mode 100644 infra/lake-iceberg/sync_iceberg.py create mode 100644 infra/lake-iceberg/test_sync.py diff --git a/infra/.gitignore b/infra/.gitignore index 5795e87..c3ac61f 100644 --- a/infra/.gitignore +++ b/infra/.gitignore @@ -15,6 +15,10 @@ age.key .DS_Store +# lake-iceberg's pytest suite runs in place. +__pycache__/ +*.pyc + # Host-side deploy state (deploy.sh): the live per-service image tags and the # cross-repo deploy lock. Untracked on purpose -- they must survive the # `git reset --hard` at the top of every deploy (deploy.sh's comments already diff --git a/infra/lake-iceberg/Dockerfile b/infra/lake-iceberg/Dockerfile new file mode 100644 index 0000000..da79b53 --- /dev/null +++ b/infra/lake-iceberg/Dockerfile @@ -0,0 +1,20 @@ +# One-shot Iceberg sync for the ERA5 lake: registers new hive tiles from the +# era5-thermograph bucket into the iceberg/era5_daily table (see +# sync_iceberg.py). Everything ships as manylinux wheels — no toolchain. +FROM python:3.12-slim + +# The pyiceberg extras pull sqlalchemy (sqlite catalog), pyarrow (FileIO + +# parquet footers) and s3fs (bucket listing/manifest reads). +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +WORKDIR /app +COPY sync_iceberg.py . + +# The sqlite catalog lives on a volume so runs share it; losing it is fine +# (the next run re-registers the table from its latest metadata JSON). +ENV THERMOGRAPH_ICEBERG_CATALOG_DB=/state/iceberg-catalog.db \ + PYTHONUNBUFFERED=1 +VOLUME /state + +ENTRYPOINT ["python", "sync_iceberg.py"] diff --git a/infra/lake-iceberg/README.md b/infra/lake-iceberg/README.md new file mode 100644 index 0000000..d3dac78 --- /dev/null +++ b/infra/lake-iceberg/README.md @@ -0,0 +1,123 @@ +# lake-iceberg — Iceberg conversion for the ERA5 lake + +Maintains an Apache Iceberg v2 table `era5_daily` at +`s3://era5-thermograph/iceberg/era5_daily` over the lake's existing hive +parquet (`era5/daily/tile=_/year=/month=/part.parquet`). + +Key properties: + +- **No data rewrite.** `sync_iceberg.py` uses pyiceberg `add_files`: the + Iceberg metadata references the existing part files in place. Storage is + not doubled and nothing under `era5/` is ever written, moved, or deleted — + the tool only writes under the `iceberg/` prefix. +- **Incremental + idempotent.** Each run diffs the tiles in + `era5/manifest.parquet` (the source of truth for *complete* tiles — the + extractor uploads all of a tile's objects before its manifest rows appear) + against the `thermograph.synced-tiles` table property, and registers only + the new ones. Files and the property update land in the same commit, so an + interrupted run never half-syncs a tile; re-running is always safe, even + while an extraction job is writing new tiles. +- **Self-contained catalog.** A pyiceberg SqlCatalog on local sqlite + (`THERMOGRAPH_ICEBERG_CATALOG_DB`, default `/state/iceberg-catalog.db`). + The catalog is only a pointer: if the db is lost, the next run finds the + newest `*.metadata.json` under the table location and re-registers it — + the synced-tiles set lives in the table metadata itself, so nothing is + re-added. +- **Partitioning `(tile, year, month)`** via order-preserving transforms of + real columns: `truncate[12](lat_idx)` / `truncate[12](lon_idx)` (partition + values are `ti*12` / `tj*12` — the part files carry no tile column, the 12 + matches the lake's 12x12-point tile grid) plus `year(date)` and + `month(date)`. Predicates on `lat_idx`/`lon_idx`/`date` prune partitions + directly — no synthetic tile/year/month columns needed in queries. + +## Build + +```sh +docker build -t thermograph-lake-iceberg infra/lake-iceberg/ +``` + +## Run (one-shot) + +Config is the backend's lake variables (same names and defaults as +`backend/data/era5lake.py`): `THERMOGRAPH_LAKE_S3_ACCESS_KEY`, +`THERMOGRAPH_LAKE_S3_SECRET_KEY`, and optionally +`THERMOGRAPH_LAKE_S3_ENDPOINT` / `_BUCKET` / `_REGION`. + +```sh +docker run --rm --env-file /path/to/lake.env \ + -v lake-iceberg-state:/state \ + thermograph-lake-iceberg +``` + +Flags: `--dry-run` (list pending tiles, write nothing), `--limit N` (cap +tiles per run), `--batch N` (tiles per Iceberg commit, default 5), +`--catalog-db PATH`. `PYICEBERG_MAX_WORKERS` controls the parquet-footer +read parallelism inside `add_files` (32 is a good value for the ~1000 small +footers per tile). + +Each run prints the row-count reconciliation (records added vs the +manifest's per-tile row counts) and ends with the table's current metadata +JSON path. It also refreshes `metadata/version-hint.text` (Hadoop-catalog +convention) so readers can scan the bare table root without knowing that +path. Transient S3 errors (Contabo intermittently returns SLOW_DOWN / +connection timeouts under load) are retried per batch with backoff +(`--retries`, default 3); a run that still dies resumes at the next batch +boundary on re-run. + +## Scheduling + +Meant to run as a one-shot on prod after extraction batches finish — e.g. a +cron or an ops-cron workflow step invoking the `docker run` above, with the +secrets rendered from the SOPS vault like every other consumer of the +`THERMOGRAPH_LAKE_S3_*` pair. Deliberately **not** wired into the Swarm +stack or CI: the sync is an offline maintenance job, not a service, and each +run costs one footer read per new part file (~1000 per tile), so it belongs +after extraction batches, not on a tight loop. + +## Querying from DuckDB + +No catalog server needed — the version-hint pointer lets DuckDB scan the +table root directly: + +```sql +INSTALL iceberg; LOAD iceberg; +INSTALL httpfs; LOAD httpfs; +CREATE SECRET lake ( + TYPE s3, + KEY_ID '', + SECRET '', + ENDPOINT 'eu2.contabostorage.com', + REGION 'default', + URL_STYLE 'path' +); +SELECT count(*) +FROM iceberg_scan('s3://era5-thermograph/iceberg/era5_daily'); +``` + +To pin an exact table version, scan the metadata JSON path the sync run +printed (`iceberg_scan('s3://…/metadata/.metadata.json')`). + +Partition pruning works on the raw columns, e.g. one tile-month: + +```sql +SELECT lat_idx, lon_idx, avg(tmax) +FROM iceberg_scan('s3://era5-thermograph/iceberg/era5_daily') +WHERE lat_idx BETWEEN 120 AND 131 AND lon_idx BETWEEN 120 AND 131 + AND date >= DATE '2020-07-01' AND date < DATE '2020-08-01' +GROUP BY 1, 2; +``` + +The part files carry no Iceberg field ids; the table sets +`schema.name-mapping.default` so engines resolve columns by name (validated +with pyiceberg 0.11.1 and DuckDB 1.5.5, values byte-identical to the hive +side). + +## Tests + +```sh +pip install -r infra/lake-iceberg/requirements.txt pytest +pytest infra/lake-iceberg/test_sync.py +``` + +Local-filesystem fixtures only (tiny parquet files, `file://` warehouse, +sqlite catalog in tmp) — no network, no credentials. diff --git a/infra/lake-iceberg/requirements.txt b/infra/lake-iceberg/requirements.txt new file mode 100644 index 0000000..22943aa --- /dev/null +++ b/infra/lake-iceberg/requirements.txt @@ -0,0 +1,5 @@ +# Pinned to the versions the sync was validated with (add_files partition +# inference from footer stats, sqlite catalog, path-style S3). +pyiceberg[sql-sqlite,pyarrow,s3fs]==0.11.1 +pyarrow==25.0.0 +s3fs==2026.6.0 diff --git a/infra/lake-iceberg/sync_iceberg.py b/infra/lake-iceberg/sync_iceberg.py new file mode 100644 index 0000000..9bf3921 --- /dev/null +++ b/infra/lake-iceberg/sync_iceberg.py @@ -0,0 +1,451 @@ +"""Maintain an Apache Iceberg table over the ERA5 lake's hive parquet. + +The lake extractor (backend/gen_era5_lake.py) writes plain hive-partitioned +parquet to the era5-thermograph bucket: + + era5/daily/tile=_/year=/month=/part.parquet + +This tool registers those files, in place, into an Iceberg v2 table +``era5_daily`` located at ``s3:///iceberg/era5_daily`` via pyiceberg +``add_files`` — no data is rewritten or copied, the Iceberg metadata simply +points at the existing objects. Everything this tool writes lives under the +``iceberg/`` prefix; the ``era5/`` prefixes are read-only to it by design. + +Partitioning: the part files carry no tile/year/month columns (those live in +the object path), so the table partitions by order-preserving transforms of +real columns instead — ``truncate[12](lat_idx)`` / ``truncate[12](lon_idx)`` +(a 12x12-point tile, so the partition value is ``ti*12``/``tj*12``) and +``year(date)`` / ``month(date)``. Each part file holds exactly one tile and +one calendar month, so every partition value is single-valued per file and +``add_files`` can derive it from the parquet footer statistics alone. + +Incremental and safe under concurrent extraction: ``era5/manifest.parquet`` +is the source of truth for complete tiles (the extractor uploads all of a +tile's objects before adding it to the manifest), so each run diffs the +manifest's tiles against the ``thermograph.synced-tiles`` table property and +adds only the difference. The property is updated in the same commit as the +files, so a run can die at any point and the next one resumes cleanly. + +Catalog: a local pyiceberg SqlCatalog (sqlite, path from +THERMOGRAPH_ICEBERG_CATALOG_DB, default /state/iceberg-catalog.db). The +catalog is a convenience pointer only — if it is lost, the next run finds the +latest ``*.metadata.json`` under the table location and re-registers it. Each +run ends by printing the current metadata JSON path, which DuckDB can read +directly: ``iceberg_scan('s3://.../metadata/NNNNN-....metadata.json')``. + +S3 config comes from the same THERMOGRAPH_LAKE_S3_* variables (and defaults) +as backend/data/era5lake.py; THERMOGRAPH_LAKE_LOCAL_DIR switches to a local +mirror of the lake layout (tests, rehearsal). +""" +import argparse +import os +import re +import sys +import time + +import pyarrow.parquet as pq +from pyiceberg.catalog.sql import SqlCatalog +from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.table.name_mapping import create_mapping_from_schema +from pyiceberg.transforms import MonthTransform, TruncateTransform, YearTransform +from pyiceberg.types import ( + DateType, + DoubleType, + FloatType, + IntegerType, + NestedField, +) + +PREFIX = "era5" +MANIFEST_KEY = f"{PREFIX}/manifest.parquet" +ICEBERG_PREFIX = "iceberg" + +NAMESPACE = "lake" +TABLE_NAME = "era5_daily" +IDENTIFIER = f"{NAMESPACE}.{TABLE_NAME}" +SYNCED_PROP = "thermograph.synced-tiles" + +TILE = 12 # grid points per tile side, as in era5lake.TILE +DEFAULT_BATCH = 5 # tiles per Iceberg commit +DEFAULT_RETRIES = 3 # per-batch retries on transient S3 errors +BACKOFF_SECONDS = 15 # first retry delay, doubled per attempt +DEFAULT_CATALOG_DB = "/state/iceberg-catalog.db" + + +class LakeInvariantError(RuntimeError): + """The lake violated an extractor invariant; never retried.""" + +# The part-file schema as the extractor writes it (verified against the +# bucket): float32 for the heat-index-derived and raw met columns, float64 +# where polars promoted (fmin/sun/feels), int16 doy, int32 grid indices. +# Iceberg has no 16-bit type, so doy maps to int; all fields optional. +SCHEMA = Schema( + NestedField(1, "date", DateType(), required=False), + NestedField(2, "tmax", FloatType(), required=False), + NestedField(3, "tmin", FloatType(), required=False), + NestedField(4, "precip", FloatType(), required=False), + NestedField(5, "wind", FloatType(), required=False), + NestedField(6, "gust", FloatType(), required=False), + NestedField(7, "humid", FloatType(), required=False), + NestedField(8, "fmax", FloatType(), required=False), + NestedField(9, "fmin", DoubleType(), required=False), + NestedField(10, "sun", DoubleType(), required=False), + NestedField(11, "feels", DoubleType(), required=False), + NestedField(12, "doy", IntegerType(), required=False), + NestedField(13, "lat_idx", IntegerType(), required=False), + NestedField(14, "lon_idx", IntegerType(), required=False), +) + +# (tile, year, month): truncate[12] of the grid indices is the tile (values +# are ti*12 / tj*12), year/month come from the date column. All four +# transforms preserve order, which is what lets add_files infer the partition +# value from footer min/max stats (single value per file by construction). +PARTITION_SPEC = PartitionSpec( + PartitionField(source_id=13, field_id=1000, + transform=TruncateTransform(TILE), name="tile_lat"), + PartitionField(source_id=14, field_id=1001, + transform=TruncateTransform(TILE), name="tile_lon"), + PartitionField(source_id=1, field_id=1002, + transform=YearTransform(), name="year"), + PartitionField(source_id=1, field_id=1003, + transform=MonthTransform(), name="month"), +) + + +def s3_config() -> "dict | None": + """Bucket access, or None when unconfigured. Same variable names and + defaults as backend/data/era5lake.py's s3_config (Contabo: path-style + addressing, region is a signing formality).""" + key = os.environ.get("THERMOGRAPH_LAKE_S3_ACCESS_KEY", "") + secret = os.environ.get("THERMOGRAPH_LAKE_S3_SECRET_KEY", "") + if not (key and secret): + return None + return { + "endpoint": os.environ.get("THERMOGRAPH_LAKE_S3_ENDPOINT", + "https://eu2.contabostorage.com"), + "bucket": os.environ.get("THERMOGRAPH_LAKE_S3_BUCKET", "era5-thermograph"), + "region": os.environ.get("THERMOGRAPH_LAKE_S3_REGION", "default"), + "access_key": key, + "secret_key": secret, + } + + +class HiveLake: + """Read-only view of the extracted lake (bucket or local mirror) plus the + iceberg/ prefix the table lives under. ``root`` is an fsspec path prefix + ("" or "/abs/dir"), ``uri`` the equivalent URI prefix pyiceberg + sees ("s3://" or "file:///abs/dir").""" + + def __init__(self, fs, root: str, uri: str, cfg: "dict | None"): + self.fs = fs + self.root = root.rstrip("/") + self.uri = uri.rstrip("/") + self.cfg = cfg + + @property + def warehouse(self) -> str: + return f"{self.uri}/{ICEBERG_PREFIX}" + + def to_uri(self, path: str) -> str: + rel = path.removeprefix(self.root).lstrip("/") + return f"{self.uri}/{rel}" + + def tile_rows(self) -> "dict[str, int]": + """tile -> row count, from the manifest (the source of truth for + complete tiles: the extractor uploads every object of a tile before + its manifest rows appear, so listing incomplete tiles is impossible + here even while extraction is running).""" + with self.fs.open(f"{self.root}/{MANIFEST_KEY}", "rb") as f: + m = pq.read_table(f, columns=["tile", "rows"]) + out: dict[str, int] = {} + for tile, rows in zip(m.column("tile").to_pylist(), + m.column("rows").to_pylist()): + out[tile] = out.get(tile, 0) + rows + return out + + def tile_files(self, tile: str) -> "list[str]": + """URIs of one complete tile's hive part files. A flat find() on the + tile prefix, not a glob: fsspec expands year=*/month=* by walking + every directory level (~1000 LIST calls per tile on S3), while find() + is one paginated listing.""" + prefix = f"{self.root}/{PREFIX}/daily/tile={tile}" + part = re.compile(r"/year=\d+/month=\d+/part\.parquet$") + try: + paths = self.fs.find(prefix) + except FileNotFoundError: + return [] + return sorted(self.to_uri(p) for p in paths if part.search(p)) + + def latest_metadata_uri(self) -> "str | None": + """Newest *.metadata.json under the table location, for re-registering + the table into a fresh catalog db (pyiceberg names them with a + zero-padded, monotonically increasing version prefix).""" + prefix = f"{self.root}/{ICEBERG_PREFIX}/{TABLE_NAME}/metadata" + try: + paths = [p for p in self.fs.find(prefix) + if p.endswith(".metadata.json")] + except FileNotFoundError: + return None + if not paths: + return None + + def version(p: str) -> int: + m = re.match(r"(\d+)", os.path.basename(p)) + return int(m.group(1)) if m else -1 + + return self.to_uri(max(paths, key=version)) + + +def lake_from_env() -> HiveLake: + local = os.environ.get("THERMOGRAPH_LAKE_LOCAL_DIR", "") + if local: + from fsspec.implementations.local import LocalFileSystem + root = os.path.abspath(local) + return HiveLake(LocalFileSystem(), root, f"file://{root}", None) + cfg = s3_config() + if cfg is None: + sys.exit("no lake configured: set THERMOGRAPH_LAKE_S3_ACCESS_KEY/" + "_SECRET_KEY (or THERMOGRAPH_LAKE_LOCAL_DIR for a local mirror)") + import s3fs + fs = s3fs.S3FileSystem( + key=cfg["access_key"], secret=cfg["secret_key"], + endpoint_url=cfg["endpoint"], + client_kwargs={"region_name": cfg["region"]}, + config_kwargs={"s3": {"addressing_style": "path"}}) + return HiveLake(fs, cfg["bucket"], f"s3://{cfg['bucket']}", cfg) + + +def open_catalog(catalog_db: str, lake: HiveLake) -> SqlCatalog: + parent = os.path.dirname(os.path.abspath(catalog_db)) + os.makedirs(parent, exist_ok=True) + props = {"uri": f"sqlite:///{os.path.abspath(catalog_db)}", + "warehouse": lake.warehouse} + if lake.cfg: + props |= { + "py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO", + "s3.endpoint": lake.cfg["endpoint"], + "s3.region": lake.cfg["region"], + "s3.access-key-id": lake.cfg["access_key"], + "s3.secret-access-key": lake.cfg["secret_key"], + # Contabo serves buckets on the path, not as a subdomain. + "s3.force-virtual-addressing": "false", + } + return SqlCatalog("thermograph", **props) + + +def ensure_table(catalog: SqlCatalog, lake: HiveLake, create: bool = True): + """Load the table; adopt existing S3 metadata into a fresh catalog db; + otherwise create it. ``create=False`` (dry runs) returns None instead of + creating anything.""" + try: + return catalog.load_table(IDENTIFIER) + except NoSuchTableError: + pass + existing = lake.latest_metadata_uri() + if existing: + catalog.create_namespace_if_not_exists(NAMESPACE) + print(f"registering existing table metadata: {existing}") + return catalog.register_table(IDENTIFIER, existing) + if not create: + return None + catalog.create_namespace_if_not_exists(NAMESPACE) + return catalog.create_table( + IDENTIFIER, + schema=SCHEMA, + location=f"{lake.warehouse}/{TABLE_NAME}", + partition_spec=PARTITION_SPEC, + properties={ + "format-version": "2", + # The part files carry no Iceberg field ids; every reader needs + # this name mapping to resolve columns (DuckDB included). + "schema.name-mapping.default": + create_mapping_from_schema(SCHEMA).model_dump_json(), + }) + + +def synced_tiles(table) -> "set[str]": + if table is None: + return set() + raw = table.properties.get(SYNCED_PROP, "") + return {t for t in raw.split(",") if t} + + +def tile_key(tile: str) -> "tuple[int, int]": + ti, tj = tile.split("_") + return int(ti), int(tj) + + +def pending_tiles(manifest_tiles, synced: "set[str]") -> "list[str]": + return sorted(set(manifest_tiles) - synced, key=tile_key) + + +def sync_batch(table, lake: HiveLake, chunk: "list[str]", + retries: int = DEFAULT_RETRIES) -> "list[str]": + """Commit one batch of tiles (files + synced-tiles property, atomically), + retrying transient S3 failures with backoff. Each attempt re-reads the + synced set from the (refreshed) table, so an attempt that committed but + errored afterwards is not re-added. Returns the tiles actually added.""" + for attempt in range(retries + 1): + try: + table.refresh() + left = [t for t in chunk if t not in synced_tiles(table)] + if not left: + return [] + files = [] + for t in left: + tf = lake.tile_files(t) + if not tf: + raise LakeInvariantError( + f"tile {t} is in the manifest but has no daily part " + f"files — refusing to mark it synced") + files.extend(tf) + new_prop = ",".join( + sorted(synced_tiles(table) | set(left), key=tile_key)) + with table.transaction() as tx: + tx.add_files( + file_paths=files, + snapshot_properties={"thermograph.tiles-added": + ",".join(left)}) + tx.set_properties({SYNCED_PROP: new_prop}) + return left + except LakeInvariantError: + raise + except Exception as e: # noqa: BLE001 - S3 throttling has many faces + if attempt >= retries: + raise + wait = BACKOFF_SECONDS * 2 ** attempt + print(f"batch {chunk[0]}..{chunk[-1]} attempt {attempt + 1} " + f"failed ({e}); retrying in {wait}s", file=sys.stderr) + time.sleep(wait) + return [] # unreachable + + +def write_version_hint(lake: HiveLake, table) -> "str | None": + """Hadoop-catalog-style pointer next to the metadata files so engines can + scan the bare table root — DuckDB: iceberg_scan('s3://…/era5_daily') — + without knowing the exact metadata path. Content is the current metadata + file's basename minus '.metadata.json' (DuckDB's default + version_name_format tries '%s.metadata.json', which resolves it). A + convenience pointer only: refreshed every run, so a failed write just + lags one version.""" + hint = os.path.basename(table.metadata_location) \ + .removesuffix(".metadata.json") + path = f"{lake.root}/{ICEBERG_PREFIX}/{TABLE_NAME}/metadata/version-hint.text" + for attempt in range(3): + try: + with lake.fs.open(path, "wb") as f: + f.write(hint.encode()) + return hint + except Exception as e: # noqa: BLE001 + if attempt == 2: + print(f"WARNING: version-hint.text write failed ({e}); " + f"table-root scans will lag one version", file=sys.stderr) + return None + time.sleep(BACKOFF_SECONDS) + return None + + +def run(lake: HiveLake, catalog_db: str, *, limit: "int | None" = None, + batch: int = DEFAULT_BATCH, retries: int = DEFAULT_RETRIES, + dry_run: bool = False) -> dict: + tile_rows = lake.tile_rows() + catalog = open_catalog(catalog_db, lake) + table = ensure_table(catalog, lake, create=not dry_run) + done = synced_tiles(table) + todo = pending_tiles(tile_rows, done) + if limit: + todo = todo[:limit] + print(f"manifest tiles={len(tile_rows)} synced={len(done)} " + f"pending this run={len(todo)}") + + added_files = added_records = 0 + mismatches = [] + if dry_run: + for t in todo: + files = lake.tile_files(t) + print(f"[dry-run] tile {t}: {len(files)} part files, " + f"{tile_rows[t]} manifest rows") + else: + for k in range(0, len(todo), batch): + chunk = todo[k:k + batch] + # One commit per batch: the files and the synced-tiles marker land + # atomically, so an interrupted run never half-syncs a tile. + added = sync_batch(table, lake, chunk, retries) + done |= set(chunk) + if not added: + print(f"[{min(k + batch, len(todo))}/{len(todo)}] tiles " + f"{chunk[0]}..{chunk[-1]}: already synced") + continue + summary = table.current_snapshot().summary + got = int(summary.get("added-records", "0")) + expected = sum(tile_rows[t] for t in added) + n_files = int(summary.get("added-data-files", "0")) + added_files += n_files + added_records += got + status = "ok" if got == expected else "MISMATCH" + if got != expected: + mismatches.append((added, expected, got)) + print(f"[{min(k + batch, len(todo))}/{len(todo)}] tiles " + f"{added[0]}..{added[-1]}: +{n_files} files, " + f"+{got} records (manifest says {expected}: {status})") + + result = {"pending": todo, "added_files": added_files, + "added_records": added_records, "mismatches": mismatches, + "metadata_location": None, "total_records": 0, + "total_files": 0} + if table is not None: + snap = table.current_snapshot() + if snap is not None: + result["total_records"] = int(snap.summary.get("total-records", "0")) + result["total_files"] = int(snap.summary.get("total-data-files", "0")) + result["metadata_location"] = table.metadata_location + if not dry_run: + result["version_hint"] = write_version_hint(lake, table) + expected_total = sum(tile_rows[t] for t in done if t in tile_rows) + print(f"table: {result['total_files']} data files, " + f"{result['total_records']} records " + f"(manifest rows for synced tiles: {expected_total})") + print(f"metadata: {table.metadata_location}") + if lake.cfg: + print("read it from DuckDB (no catalog needed):\n" + " LOAD iceberg; LOAD httpfs; -- plus an S3 secret for the " + "endpoint\n" + f" SELECT count(*) FROM iceberg_scan(" + f"'{lake.warehouse}/{TABLE_NAME}');\n" + f" -- or pin the exact version: iceberg_scan(" + f"'{table.metadata_location}')") + if mismatches: + print(f"WARNING: {len(mismatches)} batch(es) added a record count " + f"that differs from the manifest — investigate before trusting " + f"the table", file=sys.stderr) + return result + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Register new ERA5 lake tiles into the Iceberg table") + ap.add_argument("--catalog-db", + default=os.environ.get("THERMOGRAPH_ICEBERG_CATALOG_DB", + DEFAULT_CATALOG_DB), + help="sqlite catalog path (default: %(default)s)") + ap.add_argument("--batch", type=int, default=DEFAULT_BATCH, + help="tiles per Iceberg commit (default: %(default)s)") + ap.add_argument("--limit", type=int, + help="sync at most this many new tiles this run") + ap.add_argument("--retries", type=int, default=DEFAULT_RETRIES, + help="per-batch retries on transient S3 errors " + "(default: %(default)s)") + ap.add_argument("--dry-run", action="store_true", + help="list pending tiles; write nothing") + args = ap.parse_args() + + lake = lake_from_env() + result = run(lake, args.catalog_db, limit=args.limit, batch=args.batch, + retries=args.retries, dry_run=args.dry_run) + return 1 if result["mismatches"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/lake-iceberg/test_sync.py b/infra/lake-iceberg/test_sync.py new file mode 100644 index 0000000..fc29c84 --- /dev/null +++ b/infra/lake-iceberg/test_sync.py @@ -0,0 +1,247 @@ +"""Unit tests for the Iceberg lake sync: incremental tile diffing and +add_files planning against a local filesystem fixture (a miniature lake in a +tmp dir, a file:// warehouse, a sqlite catalog). No network.""" +import datetime +import os +import sys + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from fsspec.implementations.local import LocalFileSystem + +sys.path.insert(0, os.path.dirname(__file__)) +import sync_iceberg as si # noqa: E402 + +# Mirrors the real part-file schema (float32/float64 split, int16 doy, +# all-null float64 sun) so the compatibility check is exercised for real. +PART_SCHEMA = pa.schema([ + ("date", pa.date32()), + ("tmax", pa.float32()), + ("tmin", pa.float32()), + ("precip", pa.float32()), + ("wind", pa.float32()), + ("gust", pa.float32()), + ("humid", pa.float32()), + ("fmax", pa.float32()), + ("fmin", pa.float64()), + ("sun", pa.float64()), + ("feels", pa.float64()), + ("doy", pa.int16()), + ("lat_idx", pa.int32()), + ("lon_idx", pa.int32()), +]) + +POINTS = 2 # points per fixture tile +DAYS = 3 # rows per point per month + + +def write_tile(root, ti, tj, months): + """Write one tile's hive part files; return its total row count.""" + total = 0 + for year, month in months: + rows = {name: [] for name in PART_SCHEMA.names} + for p in range(POINTS): + i, j = ti * si.TILE + p, tj * si.TILE + p + for d in range(DAYS): + day = datetime.date(year, month, d + 1) + rows["date"].append(day) + for col in ("tmax", "tmin", "precip", "wind", "gust", + "humid", "fmax", "fmin", "feels"): + rows[col].append(float(10 * p + d)) + rows["sun"].append(None) + rows["doy"].append(day.timetuple().tm_yday) + rows["lat_idx"].append(i) + rows["lon_idx"].append(j) + part = pa.table(rows, schema=PART_SCHEMA) + path = os.path.join(root, si.PREFIX, "daily", f"tile={ti}_{tj}", + f"year={year}", f"month={month:02d}", + "part.parquet") + os.makedirs(os.path.dirname(path), exist_ok=True) + pq.write_table(part, path) + total += part.num_rows + return total + + +def write_manifest(root, tile_rows): + """Manifest with one row per (tile, point) like the extractor writes; + the sync only reads tile+rows.""" + tiles, rows = [], [] + for tile, total in tile_rows.items(): + per_point = total // POINTS + for _ in range(POINTS): + tiles.append(tile) + rows.append(per_point) + m = pa.table({ + "lat_idx": pa.array([0] * len(tiles), pa.int64()), + "lon_idx": pa.array([0] * len(tiles), pa.int64()), + "lat": pa.array([0.0] * len(tiles)), + "lon": pa.array([0.0] * len(tiles)), + "tile": pa.array(tiles, pa.large_string()), + "rows": pa.array(rows, pa.int64()), + "date_min": pa.array(["1940-01-01"] * len(tiles), pa.large_string()), + "date_max": pa.array(["2026-01-01"] * len(tiles), pa.large_string()), + }) + path = os.path.join(root, si.MANIFEST_KEY) + os.makedirs(os.path.dirname(path), exist_ok=True) + pq.write_table(m, path) + + +@pytest.fixture +def lake(tmp_path): + root = str(tmp_path / "lake") + counts = { + "10_10": write_tile(root, 10, 10, [(2000, 1), (2000, 2)]), + "10_11": write_tile(root, 10, 11, [(2000, 1)]), + } + write_manifest(root, counts) + hl = si.HiveLake(LocalFileSystem(), root, f"file://{root}", None) + hl.fixture_counts = counts + hl.fixture_root = root + return hl + + +@pytest.fixture +def catalog_db(tmp_path): + return str(tmp_path / "catalog.db") + + +def test_pending_tiles_diff(): + manifest = {"10_10": 1, "10_11": 1, "2_3": 1} + assert si.pending_tiles(manifest, set()) == ["2_3", "10_10", "10_11"] + assert si.pending_tiles(manifest, {"10_10"}) == ["2_3", "10_11"] + assert si.pending_tiles(manifest, set(manifest)) == [] + # numeric, not lexicographic, tile ordering + assert si.pending_tiles({"10_2": 1, "9_11": 1}, set()) == ["9_11", "10_2"] + + +def test_initial_sync_registers_all_files_in_place(lake, catalog_db): + result = si.run(lake, catalog_db, batch=1) + expected = sum(lake.fixture_counts.values()) + assert result["added_records"] == expected + assert result["total_records"] == expected + assert result["added_files"] == 3 # 2 months + 1 month + assert not result["mismatches"] + + table = si.open_catalog(catalog_db, lake).load_table(si.IDENTIFIER) + assert si.synced_tiles(table) == set(lake.fixture_counts) + # add_files, not a rewrite: every data file is an original hive part file + files = table.inspect.files().column("file_path").to_pylist() + assert len(files) == 3 + assert all(f"/{si.PREFIX}/daily/tile=" in f for f in files) + # nothing was written outside iceberg/ (the hive tree is untouched, the + # table tree holds only metadata) + fs = LocalFileSystem() + written = fs.find(os.path.join(lake.fixture_root, si.ICEBERG_PREFIX)) + assert written and all("/metadata/" in p for p in written) + + +def test_rerun_is_noop(lake, catalog_db): + first = si.run(lake, catalog_db) + again = si.run(lake, catalog_db) + assert again["pending"] == [] + assert again["added_records"] == 0 + assert again["total_records"] == first["total_records"] + assert again["metadata_location"] == first["metadata_location"] + + +def test_incremental_new_tile_only(lake, catalog_db): + si.run(lake, catalog_db) + counts = dict(lake.fixture_counts) + counts["11_10"] = write_tile(lake.fixture_root, 11, 10, [(2001, 6)]) + write_manifest(lake.fixture_root, counts) + + result = si.run(lake, catalog_db) + assert result["pending"] == ["11_10"] + assert result["added_records"] == counts["11_10"] + assert result["total_records"] == sum(counts.values()) + + +def test_version_hint_tracks_current_metadata(lake, catalog_db): + first = si.run(lake, catalog_db) + hint_path = os.path.join(lake.fixture_root, si.ICEBERG_PREFIX, + si.TABLE_NAME, "metadata", "version-hint.text") + with open(hint_path) as f: + assert f.read() == os.path.basename( + first["metadata_location"]).removesuffix(".metadata.json") + + counts = dict(lake.fixture_counts) + counts["11_10"] = write_tile(lake.fixture_root, 11, 10, [(2001, 6)]) + write_manifest(lake.fixture_root, counts) + second = si.run(lake, catalog_db) + assert second["metadata_location"] != first["metadata_location"] + with open(hint_path) as f: + assert f.read() == os.path.basename( + second["metadata_location"]).removesuffix(".metadata.json") + + +def test_partition_spec_and_pruned_scan(lake, catalog_db): + si.run(lake, catalog_db) + table = si.open_catalog(catalog_db, lake).load_table(si.IDENTIFIER) + assert [f.name for f in table.spec().fields] == \ + ["tile_lat", "tile_lon", "year", "month"] + + parts = table.inspect.partitions() + keys = {(r["partition"]["tile_lat"], r["partition"]["tile_lon"]) + for r in parts.to_pylist()} + assert keys == {(120, 120), (120, 132)} # ti*12, tj*12 + + # a tile-shaped predicate resolves to that tile's rows only + scan = table.scan(row_filter="lat_idx >= 120 and lat_idx < 132 " + "and lon_idx >= 132 and lon_idx < 144") + assert scan.to_arrow().num_rows == lake.fixture_counts["10_11"] + # and file planning pruned to that tile's single part file + assert len(list(scan.plan_files())) == 1 + + +def test_lost_catalog_db_reregisters_without_readding(lake, catalog_db): + first = si.run(lake, catalog_db) + os.remove(catalog_db) # simulate losing local state + again = si.run(lake, catalog_db) + # synced-tiles lives in table metadata, so nothing is re-added + assert again["pending"] == [] + assert again["added_records"] == 0 + assert again["total_records"] == first["total_records"] + + +def test_dry_run_writes_nothing(lake, catalog_db): + result = si.run(lake, catalog_db, dry_run=True) + assert sorted(result["pending"]) == sorted(lake.fixture_counts) + assert result["metadata_location"] is None + assert not os.path.exists( + os.path.join(lake.fixture_root, si.ICEBERG_PREFIX)) + + +def test_manifest_tile_without_files_refuses(lake, catalog_db): + counts = dict(lake.fixture_counts) + counts["50_50"] = 6 # in the manifest, no files + write_manifest(lake.fixture_root, counts) + # an invariant break aborts immediately: no retries, nothing marked synced + with pytest.raises(si.LakeInvariantError, match="50_50"): + si.run(lake, catalog_db) + + +def test_transient_failure_retries_and_succeeds(lake, catalog_db, monkeypatch): + monkeypatch.setattr(si, "BACKOFF_SECONDS", 0) + fails = {"n": 2} + original = si.HiveLake.tile_files + + def flaky(self, tile): + if fails["n"]: + fails["n"] -= 1 + raise OSError("AWS Error SLOW_DOWN during CompleteMultipartUpload") + return original(self, tile) + + monkeypatch.setattr(si.HiveLake, "tile_files", flaky) + result = si.run(lake, catalog_db, retries=2) + assert result["added_records"] == sum(lake.fixture_counts.values()) + assert not result["mismatches"] + + +def test_transient_failure_exhausts_retries(lake, catalog_db, monkeypatch): + monkeypatch.setattr(si, "BACKOFF_SECONDS", 0) + monkeypatch.setattr( + si.HiveLake, "tile_files", + lambda self, tile: (_ for _ in ()).throw(OSError("SLOW_DOWN"))) + with pytest.raises(OSError): + si.run(lake, catalog_db, retries=1) From eaaa4f1ad930636ba4c28bd7ec602703747be55d Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 03:47:39 +0000 Subject: [PATCH 10/15] Fix lake extension bake user; disable the daemon healthcheck in the stack (#30) --- backend/Dockerfile | 7 ++++--- infra/deploy/stack/thermograph-stack.yml | 7 +++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f422d16..5641a0c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -33,9 +33,6 @@ RUN apt-get update \ # Install deps first so this layer caches across code-only changes. COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt -# Bake DuckDB's httpfs extension so the lake role's S3 reads need no runtime -# extension download (tasks may roll while egress is degraded). -RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')" # The daemon binary lands before the app tree: it changes far less often than # the Python code, so this layer usually cache-hits and only the COPY below @@ -56,6 +53,10 @@ RUN useradd --system --create-home --uid 10001 thermograph \ && chown -R thermograph:thermograph /app /state USER thermograph +# Bake DuckDB's httpfs extension AS THE RUNTIME USER — extensions install to +# the invoking user's ~/.duckdb, and a root-time bake leaves uid 10001 unable +# to LOAD it (seen live: prod lake /query 500'd on exactly this). +RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')" WORKDIR /app ENV PORT=8137 \ diff --git a/infra/deploy/stack/thermograph-stack.yml b/infra/deploy/stack/thermograph-stack.yml index 8bbc3c2..12aef0b 100644 --- a/infra/deploy/stack/thermograph-stack.yml +++ b/infra/deploy/stack/thermograph-stack.yml @@ -208,6 +208,13 @@ services: # endpoints on a timer), so the daemon holds no state and logs to stdout. volumes: - /etc/thermograph/stack.env:/host/thermograph.env:ro + # The image HEALTHCHECK curls /healthz on ${PORT} — right for the app, + # meaningless for the daemon (it serves nothing). Without this override + # Swarm marks the task unhealthy and restarts it forever (seen live on the + # first prod stack deploy: gateway up, killed ~100s later, 0/1 loop). The + # compose file already disables it; the stack file must too. + healthcheck: + disable: true networks: - internal deploy: From 0862395dda683199838acc4639bff4bbdd50afc1 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 04:37:41 +0000 Subject: [PATCH 11/15] Log hygiene: Alloy CPU, Loki chunks/limits, Caddy field-stripping (#36) --- .forgejo/workflows/observability-validate.yml | 29 +++++++++-- infra/deploy/Caddyfile | 29 +++++++++-- .../templates/Caddyfile.tftpl | 30 +++++++++-- observability/alloy/config.alloy | 52 +++++++++++++++++-- observability/loki/config.yml | 20 +++++++ 5 files changed, 146 insertions(+), 14 deletions(-) diff --git a/.forgejo/workflows/observability-validate.yml b/.forgejo/workflows/observability-validate.yml index 7b6fcc0..157a3d9 100644 --- a/.forgejo/workflows/observability-validate.yml +++ b/.forgejo/workflows/observability-validate.yml @@ -12,10 +12,13 @@ name: Validate observability stack # path-filtered to the domain, and workflow_call lets pr-build.yml reuse this # as the domain's PR check under its single `gate` required status. # -# Not validated here: the Alloy config (observability/alloy/config.alloy, an -# HCL-like format, needs the `alloy` binary) and docker-compose *schema* -# (unknown-key checks, which would need the docker CLI). Add either if the -# churn warrants the extra tooling. +# The Alloy config (observability/alloy/config.alloy, an HCL-like format) IS +# validated below -- the static `alloy` binary is downloaded straight from its +# GitHub release (pinned to the same v1.9.1 the fleet runs; see +# observability/alloy/docker-compose.agent.yml), no docker CLI needed. +# +# Not validated here: docker-compose *schema* (unknown-key checks, which would +# need the docker CLI). Add it if the churn warrants the extra tooling. on: workflow_call: {} @@ -70,3 +73,21 @@ jobs: print("INVALID YAML", f, "-", e); bad = True sys.exit(1 if bad else 0) PY + + - name: Alloy config parses and validates (components, relabel/process wiring) + run: | + set -euo pipefail + apt-get update -qq && apt-get install -y -qq unzip >/dev/null + curl -sSL -o /tmp/alloy.zip \ + https://github.com/grafana/alloy/releases/download/v1.9.1/alloy-linux-amd64.zip + unzip -q /tmp/alloy.zip -d /tmp/alloybin + chmod +x /tmp/alloybin/alloy-linux-amd64 + + # `alloy validate` builds the real component graph (catches bad field + # names, dangling forward_to/receiver refs, malformed relabel/process + # stages) but doesn't recognize the top-level `livedebugging {}` singleton + # block as a component -- it only knows named components, even though + # `alloy run` loads that block fine. Known gap in the `validate` + # subcommand, not a config error, so strip that one line before checking. + grep -v '^livedebugging ' observability/alloy/config.alloy > /tmp/config-for-validate.alloy + /tmp/alloybin/alloy-linux-amd64 validate /tmp/config-for-validate.alloy diff --git a/infra/deploy/Caddyfile b/infra/deploy/Caddyfile index 0d6a31a..6baf615 100644 --- a/infra/deploy/Caddyfile +++ b/infra/deploy/Caddyfile @@ -31,10 +31,12 @@ thermograph.org { # Active health check on the same cheap /healthz route each container's own # HEALTHCHECK uses (Dockerfile) — so a deploy that's still restarting/booting # never gets proxied into (a reload alone has no gate, hop-1 runbook hazard #10). + # 15s (was 5s): plenty responsive for a process that only restarts on a deploy, + # and a quarter of the polling load. handle @backend_paths { reverse_proxy 127.0.0.1:8137 { health_uri /healthz - health_interval 5s + health_interval 15s health_timeout 3s health_status 2xx } @@ -43,14 +45,35 @@ thermograph.org { handle { reverse_proxy 127.0.0.1:8080 { health_uri /healthz - health_interval 5s + health_interval 15s health_timeout 3s health_status 2xx } } + # Access-log hygiene: the default JSON encoder serializes full request headers, + # the TLS block, and response headers on every line (measured ~1,133B/line) -- + # strip those with the `filter` format encoder. Also strip the query string from + # the logged URI: Caddy's default logger records request.uri *including* the + # query string, so every `?q=` a visitor typed sat in Loki next to + # their client IP for the full 30-day retention -- a real privacy leak, not just + # noise. Bot/crawler skipping stays out of here: `log_skip` needs Caddy >= 2.7 + # and an upgrade is out of scope, so that's handled downstream in Alloy's + # loki.process "caddy" stage instead (see observability/alloy/config.alloy). log { - output file /var/log/caddy/thermograph.log + output file /var/log/caddy/thermograph.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } } } diff --git a/infra/terraform/modules/thermograph-host/templates/Caddyfile.tftpl b/infra/terraform/modules/thermograph-host/templates/Caddyfile.tftpl index 340bdb7..95cd448 100644 --- a/infra/terraform/modules/thermograph-host/templates/Caddyfile.tftpl +++ b/infra/terraform/modules/thermograph-host/templates/Caddyfile.tftpl @@ -12,6 +12,16 @@ # # NOTE: the repo's deploy/Caddyfile additionally serves the emigriffith.dev portfolio # and legacy redirects; those are host-specific and intentionally not templated here. +# +# Access-log hygiene: the default JSON encoder serializes full request headers, the +# TLS block, and response headers on every line (measured ~1,133B/line on prod, +# ~922B on beta) -- strip those with the `filter` format encoder. Also strip the +# query string from the logged URI: Caddy's default logger records request.uri +# *including* the query string, so every `?q=` a visitor typed sat in +# Loki next to their client IP for the full 30-day retention -- a real privacy +# leak, not just noise. Bot/crawler skipping stays out of here: `log_skip` needs +# Caddy >= 2.7 and an upgrade is out of scope, so that's handled downstream in +# Alloy's loki.process "caddy" stage instead (see observability/alloy/config.alloy). ${domain} { encode zstd gzip @@ -21,10 +31,12 @@ ${domain} { # HEALTHCHECK uses (Dockerfile) — so a `docker compose up -d --build` deploy # that's still restarting/booting never gets proxied into (a reload alone has # no gate, hop-1 runbook hazard #10). health_uri is relative to the upstream. + # 15s (was 5s): plenty responsive for an active health check against a process + # that only restarts on a deploy, and a quarter of the polling load. handle @backend_paths { reverse_proxy 127.0.0.1:${port} { health_uri /healthz - health_interval 5s + health_interval 15s health_timeout 3s health_status 2xx } @@ -33,13 +45,25 @@ ${domain} { handle { reverse_proxy 127.0.0.1:${frontend_port} { health_uri /healthz - health_interval 5s + health_interval 15s health_timeout 3s health_status 2xx } } log { - output file /var/log/caddy/thermograph.log + output file /var/log/caddy/thermograph.log { + roll_size 20MiB + roll_keep 5 + } + format filter { + wrap json + fields { + request>headers delete + request>tls delete + resp_headers delete + request>uri regexp \?.* "" + } + } } } diff --git a/observability/alloy/config.alloy b/observability/alloy/config.alloy index 7ddbc29..3279da2 100644 --- a/observability/alloy/config.alloy +++ b/observability/alloy/config.alloy @@ -15,12 +15,22 @@ livedebugging { enabled = false } // --- 1. All Docker container logs ------------------------------------------------ +// refresh_interval defaults to 60s; Swarm task churn reshuffles the target set on +// roughly that cadence, which restarts tailers ~every 90s and was costing ~11% of +// Alloy's own CPU in tailer restarts alone. 5m is still fast enough to pick up a +// real deploy without paying that churn cost. discovery.docker "containers" { - host = "unix:///var/run/docker.sock" + host = "unix:///var/run/docker.sock" + refresh_interval = "5m" } // Turn Docker metadata into tidy labels: `container` (short name) and `service` -// (the compose service, e.g. app/db). Drop Alloy's own container to avoid a loop. +// (the compose service, e.g. app/db). Drop noise/duplicate containers so Loki never +// ingests them: Alloy itself (loop), the autoscaler, throwaway `thermograph-test_*` +// stacks, the loopback LB bridge (`thermograph-lb`, pure plumbing, nothing to debug +// from its logs), and the app's own worker (`thermograph_worker`'s stdout is 100% +// `/healthz` poll noise — the app's `access/*.jsonl` under source #3 is a strict +// superset of anything useful it logs). discovery.relabel "containers" { targets = discovery.docker.containers.targets @@ -35,27 +45,55 @@ discovery.relabel "containers" { } rule { source_labels = ["container"] - regex = ".*alloy.*" + regex = "(alloy|autoscaler|thermograph-test_.*|thermograph-lb|thermograph_worker).*" action = "drop" } } +// Pass RAW targets here (not discovery.relabel.containers.output) alongside +// relabel_rules: loki.source.docker applies relabel_rules itself, once, using the +// __meta_docker_* metadata it still holds at collection time. Passing the +// already-relabelled output *and* relabel_rules ran the same rules twice per log +// entry, and made the drop rules above a no-op on the second pass since the +// __meta_docker_* labels are already gone from the pre-relabelled output. loki.source.docker "containers" { host = "unix:///var/run/docker.sock" - targets = discovery.relabel.containers.output + targets = discovery.docker.containers.targets forward_to = [loki.write.central.receiver] relabel_rules = discovery.relabel.containers.rules labels = { job = "docker" } } // --- 2. Caddy host access logs --------------------------------------------------- +// sync_period (glob rescan) defaults to 10s; 1m is plenty for a log file that only +// appears/rotates on the order of hours. local.file_match "caddy" { path_targets = [{ __path__ = "/var/log/caddy/*.log", job = "caddy" }] + sync_period = "1m" } loki.source.file "caddy" { targets = local.file_match.caddy.targets + forward_to = [loki.process.caddy.receiver] + + // PollingFileWatcher defaults (250ms/250ms) stat every tailed file 4x/second + // forever. Caddy's access log doesn't need sub-second latency into Loki. + file_watch { + min_poll_frequency = "2s" + max_poll_frequency = "10s" + } +} + +// Drop well-known crawler/bot traffic before it hits Loki. Caddy itself can't do +// this cheaply (log_skip needs Caddy >= 2.7; both hosts run older Caddy, and an +// upgrade is out of scope here), so filter it at the shipper instead. +loki.process "caddy" { forward_to = [loki.write.central.receiver] + + stage.drop { + expression = "(?i)(semrushbot|claudebot|ahrefsbot|yandexbot|bytespider|mj12bot|petalbot)" + drop_counter_reason = "crawler" + } } // --- 3. App structured JSON logs (errors / access / audit) ----------------------- @@ -63,11 +101,17 @@ loki.source.file "caddy" { // compose). Lift `level`/`tag`/`phase` out of the JSON so they're queryable. local.file_match "app_jsonl" { path_targets = [{ __path__ = "/applogs/**/*.jsonl", job = "app-json" }] + sync_period = "1m" } loki.source.file "app_jsonl" { targets = local.file_match.app_jsonl.targets forward_to = [loki.process.app_jsonl.receiver] + + file_watch { + min_poll_frequency = "2s" + max_poll_frequency = "10s" + } } loki.process "app_jsonl" { diff --git a/observability/loki/config.yml b/observability/loki/config.yml index d17a37f..b2dfb46 100644 --- a/observability/loki/config.yml +++ b/observability/loki/config.yml @@ -32,6 +32,17 @@ schema_config: prefix: index_ period: 24h +# 30 low-rate streams almost always hit chunk_idle_period (30m default) long before +# they'd ever fill chunk_target_size, so chunks were flushed 1.98% full on average +# (972 near-empty chunks for 30MB of actual log data). Give idle streams far longer +# to accumulate before an idle flush, and cap age/size so a chunk still can't grow +# unbounded. +ingester: + chunk_idle_period: 2h + max_chunk_age: 12h + chunk_target_size: 1572864 + chunk_encoding: snappy + limits_config: # A hobby fleet's volume is tiny; keep 30 days and cap ingestion generously. retention_period: 720h @@ -40,6 +51,15 @@ limits_config: max_query_series: 5000 allow_structured_metadata: true volume_enabled: true + # No explicit ingestion/stream limits meant Loki fell back to its (much stricter) + # built-in defaults, which is why 25 pushes came back HTTP 429 with nothing in + # this file explaining why. These are sized for a three-node hobby fleet, not the + # multi-tenant defaults. + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + per_stream_rate_limit: 3MB + per_stream_rate_limit_burst: 10MB + max_global_streams_per_user: 1000 compactor: working_directory: /loki/compactor From 578fb563a4ab44de892cf9d47c47f5c33f325dac Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 18:59:46 +0000 Subject: [PATCH 12/15] Roll the daemon with backend deploys in the Swarm stack (#40) --- infra/deploy/stack/deploy-stack.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/infra/deploy/stack/deploy-stack.sh b/infra/deploy/stack/deploy-stack.sh index 15d8a99..c3badc2 100755 --- a/infra/deploy/stack/deploy-stack.sh +++ b/infra/deploy/stack/deploy-stack.sh @@ -166,13 +166,19 @@ else echo "==> Rolling web + worker + lake to $BACKEND_IMAGE" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_web" docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_worker" - # lake ships in the same image; a stack file predating it has no service - # yet — the next SERVICE=all stack deploy creates it, so don't fail here. - if docker service inspect "${STACK_NAME}_lake" >/dev/null 2>&1; then - docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_lake" - else - echo " (no ${STACK_NAME}_lake service yet; created on the next full stack deploy)" - fi + # lake and daemon ship in the same image; a stack file predating either + # has no service yet — the next SERVICE=all stack deploy creates it, so + # don't fail here. The daemon especially must roll with web: they share + # the /internal/* contract, and a version skew between them is exactly + # what pinning one BACKEND_IMAGE_TAG exists to prevent (seen live: the + # first post-creation backend roll left the daemon a release behind). + for extra in lake daemon; do + if docker service inspect "${STACK_NAME}_${extra}" >/dev/null 2>&1; then + docker service update --with-registry-auth --detach=false --image "$BACKEND_IMAGE" "${STACK_NAME}_${extra}" + else + echo " (no ${STACK_NAME}_${extra} service yet; created on the next full stack deploy)" + fi + done ;; frontend) echo "==> Rolling frontend to $FRONTEND_IMAGE" From 0f1b3a168c6500680e1153becc82717628a6b2a5 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:09:22 +0000 Subject: [PATCH 13/15] Lake /query reads era5_daily through Iceberg metadata (#44) --- backend/Dockerfile | 10 +++++--- backend/lake_app.py | 59 +++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 5641a0c..ab97519 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -53,10 +53,12 @@ RUN useradd --system --create-home --uid 10001 thermograph \ && chown -R thermograph:thermograph /app /state USER thermograph -# Bake DuckDB's httpfs extension AS THE RUNTIME USER — extensions install to -# the invoking user's ~/.duckdb, and a root-time bake leaves uid 10001 unable -# to LOAD it (seen live: prod lake /query 500'd on exactly this). -RUN python -c "import duckdb; duckdb.connect().execute('INSTALL httpfs')" +# Bake DuckDB's httpfs + iceberg extensions AS THE RUNTIME USER — extensions +# install to the invoking user's ~/.duckdb, and a root-time bake leaves uid +# 10001 unable to LOAD them (seen live: prod lake /query 500'd on exactly +# this). iceberg powers the era5_daily view via table metadata instead of a +# bucket-wide glob LIST. +RUN python -c "import duckdb; c = duckdb.connect(); c.execute('INSTALL httpfs'); c.execute('INSTALL iceberg')" WORKDIR /app ENV PORT=8137 \ diff --git a/backend/lake_app.py b/backend/lake_app.py index 37eb721..c6a6100 100644 --- a/backend/lake_app.py +++ b/backend/lake_app.py @@ -135,34 +135,51 @@ _FORBIDDEN = re.compile( def _connect(): - """A fresh in-memory DuckDB with the lake views registered. Per-request: - connections are cheap, and a throwaway one means a query can't leave state - behind for the next. Returns None when no lake is configured.""" + """A fresh in-memory DuckDB with the lake views registered. Returns None + when no lake is configured. + + Bucket mode reads era5_daily through the ICEBERG table, not a hive glob: + a glob over era5/daily/ must LIST every object under it (~1k per tile — + minutes at a few hundred tiles, seen live timing out /query at 390), + while Iceberg gets the file list from table metadata in a handful of + reads. Local mode (tests, small trees) keeps the plain hive glob — no + Iceberg table exists in the fixtures and listing a local dir is free.""" import duckdb # noqa: PLC0415 - only the lake role pays the import mode, cfg = _source() + if mode == "none": + return None + con = duckdb.connect() if mode == "local": daily_glob = os.path.join(era5lake.local_dir(), era5lake.PREFIX, "daily/*/*/*/*.parquet") + con.execute(f""" + CREATE VIEW era5_daily AS + SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""") manifest_src = os.path.join(era5lake.local_dir(), era5lake.MANIFEST_KEY) - elif mode == "bucket": - daily_glob = f"s3://{cfg['bucket']}/{era5lake.PREFIX}/daily/*/*/*/*.parquet" - manifest_src = f"s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}" - else: - return None - con = duckdb.connect() - if mode == "bucket": - con.execute("LOAD httpfs") # baked into the image at build time - host = cfg["endpoint"].split("://", 1)[-1] - con.execute(f"SET s3_endpoint='{host}'") - con.execute("SET s3_url_style='path'") - con.execute(f"SET s3_region='{cfg['region']}'") - con.execute("SET s3_access_key_id=?", [cfg["access_key"]]) - con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]]) - con.execute(f""" - CREATE VIEW era5_daily AS - SELECT * FROM read_parquet('{daily_glob}', hive_partitioning=1)""") - con.execute(f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')") + con.execute( + f"CREATE VIEW manifest AS SELECT * FROM read_parquet('{manifest_src}')") + return con + + con.execute("LOAD httpfs") # both baked into the image at build time, + con.execute("LOAD iceberg") # as the runtime user (see Dockerfile) + host = cfg["endpoint"].split("://", 1)[-1] + con.execute(f"SET s3_endpoint='{host}'") + con.execute("SET s3_url_style='path'") + con.execute(f"SET s3_region='{cfg['region']}'") + con.execute("SET s3_access_key_id=?", [cfg["access_key"]]) + con.execute("SET s3_secret_access_key=?", [cfg["secret_key"]]) + # Newest metadata JSON wins (one single-page LIST of metadata/ — cheap); + # pyiceberg names them NNNNN-uuid.metadata.json, so lexicographic max is + # the numeric max. + meta = con.execute( + f"SELECT max(file) FROM glob('s3://{cfg['bucket']}/iceberg/era5_daily/" + f"metadata/*.metadata.json')").fetchone()[0] + if meta is None: + return None # lake bucket exists but the Iceberg table doesn't yet + con.execute(f"CREATE VIEW era5_daily AS SELECT * FROM iceberg_scan('{meta}')") + con.execute(f"""CREATE VIEW manifest AS SELECT * FROM + read_parquet('s3://{cfg['bucket']}/{era5lake.MANIFEST_KEY}')""") return con From 4ff113f2cf3dd47f0087d7667ba54df80f2ae9d9 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:28:47 +0000 Subject: [PATCH 14/15] Stop logging /healthz + internal SSR hop, truncate logged IPs (#35) --- backend/core/metrics.py | 13 ++++ backend/deploy/entrypoint.sh | 5 +- backend/tests/core/test_metrics.py | 10 +++ backend/tests/web/test_request_logging.py | 87 +++++++++++++++++++++++ backend/web/app.py | 36 ++++++++-- 5 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 backend/tests/web/test_request_logging.py diff --git a/backend/core/metrics.py b/backend/core/metrics.py index 36b085b..7652bb5 100644 --- a/backend/core/metrics.py +++ b/backend/core/metrics.py @@ -96,6 +96,12 @@ def classify_inbound(path: str, base: str = "") -> str: the app's mount prefix (e.g. ``/thermograph`` or ``""``). """ p = path or "/" + # The liveness probe (Dockerfile HEALTHCHECK, Caddy health_uri) is by far the + # highest-volume single path in prod (measured ~47% of the access log) and is + # never real traffic — same posture as the metrics check below. Never under + # BASE (see /healthz's own docstring in web/app.py), so check before stripping. + if p.rstrip("/") == "/healthz": + return "health" # The dashboard polls the metrics endpoint; never count it as traffic, whatever base # prefix it arrives under — e.g. a `/thermograph/api/v2/metrics` probe against a # root-served prod app would otherwise land in "other" and show as an inbound error. @@ -118,6 +124,13 @@ def classify_inbound(path: str, base: str = "") -> str: # category keeps record_inbound from double-counting every interaction. if seg == "event": return "event" + # The SSR content API (backend/api/content_routes.py) is called only by + # the frontend_ssr service's own api_client.py, never by a browser — a + # server-to-server hop, not a page view. Its own category is what lets + # that (measured ~32% of the access log on prod) be excluded from the + # access log below without also hiding real external traffic. + if seg == "content": + return "internal" return f"api:{seg}" if seg else "api:other" if p.endswith((".js", ".css", ".html", ".webmanifest", ".png", ".svg", ".ico", ".json", ".woff", ".woff2", ".map", ".txt", ".xml")): diff --git a/backend/deploy/entrypoint.sh b/backend/deploy/entrypoint.sh index 9b6f399..9baf4b3 100755 --- a/backend/deploy/entrypoint.sh +++ b/backend/deploy/entrypoint.sh @@ -85,4 +85,7 @@ if [ "${RUN_MIGRATIONS:-1}" != "0" ]; then fi echo "==> Starting uvicorn on 0.0.0.0:${PORT:-8137} with ${WORKERS:-4} worker(s)" -exec uvicorn app:app --host 0.0.0.0 --port "${PORT:-8137}" --workers "${WORKERS:-4}" +# --no-access-log: the app's own request-logging middleware (audit.log_access, +# web/app.py) already writes a structured line per request; uvicorn's own access +# log just duplicated every one of them for no benefit. +exec uvicorn app:app --host 0.0.0.0 --port "${PORT:-8137}" --workers "${WORKERS:-4}" --no-access-log diff --git a/backend/tests/core/test_metrics.py b/backend/tests/core/test_metrics.py index 010f246..d58ad47 100644 --- a/backend/tests/core/test_metrics.py +++ b/backend/tests/core/test_metrics.py @@ -40,6 +40,16 @@ def test_classify_inbound_categories(): # dashboard probing /thermograph/... against a root-served (base="") prod app. assert c("/thermograph/api/v2/metrics", "") == "metrics" assert c("/api/v2/metrics", "") == "metrics" + # The SSR content API (backend/api/content_routes.py) is a server-to-server + # hop from frontend_ssr's own api_client.py, never a browser -- its own + # category is what lets it be excluded from the access log without also + # hiding real external traffic under the generic "api:*" bucket. + assert c("/thermograph/api/v2/content/hub", "/thermograph") == "internal" + assert c("/api/v2/content/city/tokyo", "") == "internal" + # The liveness probe: never under BASE, whatever base the app happens to be + # mounted at (it's registered at a fixed path -- see web/app.py's healthz). + assert c("/healthz") == "health" + assert c("/healthz", "/thermograph") == "health" # pages, static, seo assert c("/thermograph/", "/thermograph") == "page" assert c("/thermograph/calendar", "/thermograph") == "page" diff --git a/backend/tests/web/test_request_logging.py b/backend/tests/web/test_request_logging.py new file mode 100644 index 0000000..b14882b --- /dev/null +++ b/backend/tests/web/test_request_logging.py @@ -0,0 +1,87 @@ +"""The request-logging middleware (web/app.py's revalidate_static): what gets +counted, what gets written to the access log, and what gets truncated before +it's persisted. Companion to core/test_metrics.py's classify_inbound tests -- +these exercise the actual ASGI request path, not just the classifier function. +""" +import pytest +from fastapi.testclient import TestClient + +from web import app as appmod + +metrics = appmod.metrics # the exact module object app.py's own calls use -- +audit = appmod.audit # see the note on module-reload isolation below. + + +@pytest.fixture +def client(): + return TestClient(appmod.app) + + +def _patch_log_access(monkeypatch): + calls = [] + monkeypatch.setattr(audit, "log_access", lambda record: calls.append(record)) + return calls + + +def test_healthz_is_never_audited(monkeypatch, client): + """/healthz is ~47% of prod's daily access log; it must never even reach + audit.log_access (not just be cheap once there -- see classify_inbound's + "health" category and the exclusion in revalidate_static).""" + calls = _patch_log_access(monkeypatch) + r = client.get("/healthz") + assert r.status_code == 200 + assert calls == [] + + +def test_internal_category_is_never_audited(monkeypatch, client): + """The SSR content API's own category ("internal") is excluded from the + access log the same way -- it's a server-to-server hop (frontend_ssr's + api_client.py calling backend), not a page view.""" + calls = _patch_log_access(monkeypatch) + monkeypatch.setattr(metrics, "classify_inbound", lambda path, base: "internal") + r = client.get("/thermograph/api/version") + assert r.status_code == 200 + assert calls == [] + + +def test_access_log_ip_is_truncated_not_raw(monkeypatch, client): + calls = _patch_log_access(monkeypatch) + r = client.get("/thermograph/api/version", + headers={"X-Forwarded-For": "203.0.113.77, 10.0.0.1"}) + assert r.status_code == 200 + assert len(calls) == 1 + assert calls[0]["ip"] == "203.0.113.0" # /24, not the exact address + assert calls[0]["ip"] != "203.0.113.77" + + +def test_loggable_ip_truncation(): + assert appmod._loggable_ip("203.0.113.77") == "203.0.113.0" + assert appmod._loggable_ip("2001:db8:1234:5678::1") == "2001:db8:1234::" + assert appmod._loggable_ip(None) is None + assert appmod._loggable_ip("") == "" + assert appmod._loggable_ip("not-an-ip") == "not-an-ip" # best-effort, never raises + + +def test_rate_limiter_still_sees_the_full_precision_ip(monkeypatch, client): + """_client_ip (which feeds both the access log and the event rate limiter) + is never itself truncated -- only what audit.log_access persists is. A + /24-truncated rate-limit key would let one abuser exhaust a whole NAT'd + office's quota, which is exactly what must NOT happen here.""" + seen = {} + monkeypatch.setattr(metrics, "record_event", lambda name, **kw: seen.update(kw)) + r = client.post("/thermograph/api/v2/event", json={"event": "home.locate"}, + headers={"X-Forwarded-For": "203.0.113.77"}) + assert r.status_code == 204 + assert seen["ip"] == "203.0.113.77" + + +def test_event_route_records_end_to_end(client): + """Full HTTP round trip through the real ASGI route -- confirms the write + path backend/api/event -> metrics.record_event -> the counters store works + end to end (the existing suite only ever called record_event directly).""" + before = metrics.snapshot()["events_total"] + r = client.post("/thermograph/api/v2/event", json={"event": "home.locate"}) + assert r.status_code == 204 + after = metrics.snapshot() + assert after["events_total"] == before + 1 + assert after["events"]["home.locate"]["direct"] # no Referer sent -> "direct" diff --git a/backend/web/app.py b/backend/web/app.py index 0253319..b8a5d35 100644 --- a/backend/web/app.py +++ b/backend/web/app.py @@ -3,6 +3,7 @@ import contextlib import datetime import hashlib import hmac +import ipaddress import json import os import queue @@ -263,13 +264,34 @@ def api_version(): def _client_ip(request) -> "str | None": """The real client IP: the left-most X-Forwarded-For hop when a proxy fronts us - (Caddy on prod sets it), otherwise the direct peer address (LAN dev).""" + (Caddy on prod sets it), otherwise the direct peer address (LAN dev). Full + precision — the rate limiter (metrics._rate_ok) needs the exact address, since + a truncated key would let one abuser exhaust a whole NAT'd office's quota. + Truncate at the point of persistence instead (see _loggable_ip).""" xff = request.headers.get("x-forwarded-for") if xff: return xff.split(",")[0].strip() return request.client.host if request.client else None +def _loggable_ip(ip: "str | None") -> "str | None": + """Coarsen a client IP before it's written to the access log: /24 for IPv4, + /48 for IPv6. Keeps rough geo/abuse signal without keeping a full, joinable + address sitting in structured logs for the log's 30-day retention -- the + public privacy page promises IPs aren't logged beyond normal request handling, + and a raw address shipped to Loki was a live gap against that. Never the + input to the rate limiter (_client_ip's callers pass the untruncated value + there) -- this is only what gets persisted.""" + if not ip: + return ip + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return ip # not a parseable address (e.g. a test/placeholder value) -- pass through + prefix = 24 if addr.version == 4 else 48 + return str(ipaddress.ip_network(f"{addr}/{prefix}", strict=False).network_address) + + @app.middleware("http") async def revalidate_static(request, call_next): """Serve the frontend with no-cache (NOT no-store): browsers may keep a copy @@ -288,11 +310,15 @@ async def revalidate_static(request, call_next): # loop so one request's instrumentation can't stall every other request # this worker is serving concurrently. await run_in_threadpool(metrics.record_inbound, cat, response.status_code) - # Retain per-request client IPs for later analysis; skip static assets and the - # dashboard's own metrics polling to keep the log to real, meaningful traffic. - if cat not in ("static", "metrics", "event"): + # Retain per-request client IPs for later analysis; skip static assets, the + # dashboard's own metrics polling, the liveness probe (health -- by far the + # highest-volume single path, and never real traffic), and the internal + # SSR->API hop (internal -- frontend_ssr's own server-to-server calls, not + # a page view) to keep the log to real, meaningful traffic. The IP itself is + # truncated (see _loggable_ip) -- coarse geo/abuse signal, not a full address. + if cat not in ("static", "metrics", "event", "health", "internal"): await run_in_threadpool(audit.log_access, { - "ip": _client_ip(request), "method": request.method, + "ip": _loggable_ip(_client_ip(request)), "method": request.method, "path": path, "status": response.status_code, "cat": cat}) # Threshold-gated slow-request line: RunAudit only times the 7 graded # endpoints, so this is the only latency signal for auth/account/content From 864d38d772eaf96f6d930b2f482d29d32244f491 Mon Sep 17 00:00:00 2001 From: emi Date: Fri, 24 Jul 2026 19:28:58 +0000 Subject: [PATCH 15/15] ops/dbq.sh: read-only Postgres queries across all environments (#18) --- infra/ops/ICEBERG-HANDOFF.md | 133 +++++++++++++++++++++++++++++++++++ infra/ops/README.md | 65 +++++++++++++++++ infra/ops/dbq.sh | 68 ++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 infra/ops/ICEBERG-HANDOFF.md create mode 100644 infra/ops/README.md create mode 100755 infra/ops/dbq.sh diff --git a/infra/ops/ICEBERG-HANDOFF.md b/infra/ops/ICEBERG-HANDOFF.md new file mode 100644 index 0000000..d27810b --- /dev/null +++ b/infra/ops/ICEBERG-HANDOFF.md @@ -0,0 +1,133 @@ +# Iceberg: implementing the agent-queryable service + +Handoff spec for whoever builds the Iceberg layer. The Postgres equivalent is +already shipped (`infra/ops/dbq.sh`) — **mirror it**. This document is the +contract, the environment facts you'll trip over, and how the result gets +verified. + +## Definition of done + +`infra/ops/iceberg.sh` exists and behaves like `dbq.sh`: + +```sh +infra/ops/iceberg.sh prod -c "select count(*) from ." +infra/ops/iceberg.sh dev -c "show tables" +echo "select 1" | infra/ops/iceberg.sh beta -f - +``` + +An agent (or operator) can query Iceberg in any environment, read-only, with no +new network exposure and no interactive steps. + +## The contract (non-negotiable) + +1. **Same interface shape as `dbq.sh`** — `iceberg.sh [flags] + ""`. Extra flags pass through to the engine; **stdin is forwarded** so + `-f -` works. Non-interactive, deterministic, safe to call from CI. +2. **Read-only enforced by the system, not by convention.** A dedicated + read-only identity — not an admin/superuser credential. You must be able to + *demonstrate* a refused write (see Acceptance). +3. **No new network endpoints.** Run the engine where the data is already + reachable (containerised), rather than publishing ports or opening firewall + rules. See the prod constraint below — this is not negotiable, it's physics. +4. **Env-keyed dispatch, names resolved at call time.** Prod is Docker Swarm; + task container names change on **every redeploy**. Resolve via + `docker ps --filter name=…`; never hardcode a container name. +5. **No secrets in the repo** or in workflow files. + +## Environment facts you need + +- Monorepo `emi/thermograph` (domain dirs: `backend/ frontend/ infra/ + observability/`). Ops tooling lives in `infra/ops/`. +- Environments: **dev** = LAN compose stack on the dev machine; **beta** = + `75.119.132.91`; **prod** = `169.58.46.181`. SSH as `agent` with + `~/.ssh/thermograph_agent_ed25519` (passwordless sudo on both boxes). +- **Prod runs Docker Swarm.** Its app database sits on the overlay network + `thermograph_internal` (`10.0.2.0/24`), which **the prod host itself cannot + route to**. Consequence, learned the hard way: `ssh -L` tunnelling works for + beta but is *impossible* for prod. Any design that depends on a tunnel or a + published port will fail on prod — that's why `dbq.sh` execs into the + container instead. Assume the same for anything you deploy there. +- The overlay is `attachable=true`, and the **dev machine is a Swarm worker** in + the prod cluster (NodeAddr `10.10.0.3`) — so `docker run --network + thermograph_internal …` from the dev box is a plausible route to prod-side + services. Untested; verify before relying on it. +- WireGuard mesh: prod `10.10.0.1`, beta `10.10.0.2`, dev `10.10.0.3`. +- `rclone` and `age` are already installed on prod and beta. +- Reference implementation to copy: **`infra/ops/dbq.sh`** + `infra/ops/README.md`. + +## Storage — almost certainly your warehouse + +Contabo Object Storage (S3-compatible), already provisioned and in use: + +- Endpoint `https://eu2.contabostorage.com` · bucket `era5-thermograph` · + region `default` · ~2 TB · **private** (keep it that way). +- ⚠️ **Contabo requires PATH-STYLE addressing.** Set + `force_path_style=true` / `s3.path-style-access=true` (whatever your engine's + FileIO calls it) and `region=default`. Virtual-host-style addressing **fails**. + This is validated, not theoretical. +- ⚠️ **The `backups/` prefix is in use** by the nightly backup jobs (prod DB + + Forgejo). Put Iceberg data under its own prefix (e.g. `iceberg/` or + `warehouse/`). Do not write outside your prefix. +- Credentials already exist — **reuse them, don't mint new ones silently**: + - SOPS vault: `infra/deploy/secrets/{prod,beta}.yaml` as `THERMOGRAPH_S3_*` + (rendered to `/etc/thermograph.env` at deploy by + `infra/deploy/render-secrets.sh`). + - Forgejo Actions secrets: `S3_ENDPOINT`, `S3_BUCKET`, `S3_ACCESS_KEY`, + `S3_SECRET_KEY`. + - If read-only S3 keys are needed for the query identity, ask the operator to + mint a scoped keypair rather than reusing the read-write one. + +## Secrets handling + +- Host-side: SOPS + age. Recipient `age1xx4dzs0dxlwvkv9sjuqzsphl7lfrxannkfken374yu2qvvcte9sqzktqt2`; + the private key is on each host at `/etc/thermograph/age.key` and on the + operator's machine at `~/.config/sops/age/keys.txt`. +- CI-side: Forgejo Actions secrets. +- ⚠️ **beta gotcha:** `/etc/thermograph.env` on beta is `agent:agent 0640` — the + `deploy` user **cannot read it**. If your service or job runs as `deploy` on + beta, pull credentials from Actions secrets instead, or change the ownership + deliberately and say so. + +## Decisions you must make — and report back + +1. **Catalog**: REST (Lakekeeper / Nessie / Polaris), JDBC-on-Postgres, Hive, or + filesystem/hadoop — and where it runs per environment. +2. **Engine**: DuckDB + iceberg extension, Trino, Spark, or pyiceberg. Prefer the + lightest that satisfies the contract; it must run containerised and headless. +3. **Warehouse location**: bucket + prefix, and whether environments are isolated + by separate prefixes, namespaces, or buckets. +4. **Read-only mechanism**: scoped S3 keys? catalog RBAC? engine restricted mode? + State which, and how a write is refused. +5. **Where the engine runs** for each env (local container on dev, on the box for + beta/prod, or attached to the overlay) — consistent with constraint #3. + +## Acceptance criteria + +The work is accepted when all of these are demonstrated with pasted output: + +- [ ] `infra/ops/iceberg.sh -c "