db: raise max_connections to 200, cap work_mem at 64MB, trim async pool overflow

infra/deploy/db/init/20-tuning.sh

@@ -8,10 +8,10 @@
 # 10-timescaledb.sql enables timescaledb. Settings are written via ALTER SYSTEM
 # (persisted to postgresql.auto.conf, which the timescaledb image's own
 # timescaledb-tune postgresql.conf defers to); the container's post-init restart brings
-# restart-only settings (shared_buffers, …) into effect. NB: never ALTER SYSTEM SET
+# restart-only settings (shared_buffers, max_connections, …) into effect. NB: never
-# shared_preload_libraries here — that would land in auto.conf and override the image's
+# ALTER SYSTEM SET shared_preload_libraries here — that would land in auto.conf and
-# `timescaledb` preload. Init scripts do NOT re-run on an existing volume — to
+# override the image's `timescaledb` preload. Init scripts do NOT re-run on an existing
-# re-tune later, set DB_MEMORY and run this by hand, then restart:
+# volume — to re-tune later, set DB_MEMORY and run this by hand, then restart:
 #   docker compose exec -e DB_MEMORY=16g db bash /docker-entrypoint-initdb.d/20-tuning.sh
 #   docker compose restart db
 set -euo pipefail
@@ -33,13 +33,30 @@ if [ "$mb" -lt 1024 ]; then mb=8192; fi
 # maintenance_work_mem 512 MB) and scale linearly on a bigger box.
 shared_buffers=$((mb / 4))      # 25% — the shared page cache
 effective_cache=$((mb * 3 / 4)) # 75% — planner's view of total cache (PG + OS)
-work_mem=$((mb / 128))          # ~64 MB at 8 GB (per-operation; kept modest)
+maint_mem=$((mb / 16))          # 512 MB at 8 GB — index builds / VACUUM
+
+# work_mem is per *sort operation*, not per connection, so its real cost is
+# work_mem x concurrent sorts x max_connections (200, below) — and the db container
+# has a hard memory limit. Left uncapped the ratio gives 128 MB at prod's 16g
+# budget, which is more than a 200-connection instance should carry: a burst of
+# heavy sorts can walk into the container limit, and an OOM-killed Postgres is a
+# far worse day than a refused connection. Cap at 64 MB (dev's 8g is unaffected).
+work_mem=$((mb / 128))
+if [ "$work_mem" -gt 64 ]; then work_mem=64; fi
 if [ "$work_mem" -lt 16 ]; then work_mem=16; fi
-maint_mem=$((mb / 16)) # 512 MB at 8 GB — index builds / VACUUM
+
+# Connection ceiling. NOT derived from DB_MEMORY: it is bounded by how many pools
+# the app opens, not by RAM. One instance serves prod and beta, and every backend
+# process opens three pools (two async engines + the notifier's sync engine, see
+# backend/accounts/db.py), so the image default of 100 was below the configured
+# worst case — prod saturated it on 2026-07-26 and started refusing clients with
+# "sorry, too many clients already". 200 covers today's replica counts including
+# web's autoscale maximum. ~+1 GB of per-connection overhead at 200.
+max_conn=200
 
 echo "[tuning] DB_MEMORY=${budget} -> ${mb}MB: shared_buffers=${shared_buffers}MB" \
   "effective_cache_size=${effective_cache}MB work_mem=${work_mem}MB" \
-  "maintenance_work_mem=${maint_mem}MB"
+  "maintenance_work_mem=${maint_mem}MB max_connections=${max_conn}"
 
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<SQL
 -- Caching: the shared page cache, and the planner's view of total cache (PG + OS).
@@ -50,6 +67,10 @@ ALTER SYSTEM SET effective_cache_size = '${effective_cache}MB';
 ALTER SYSTEM SET work_mem = '${work_mem}MB';
 ALTER SYSTEM SET maintenance_work_mem = '${maint_mem}MB';
 
+-- Connections. Restart-only, like shared_buffers: changing this on a live instance
+-- takes an ALTER SYSTEM plus a db restart, it does not reload.
+ALTER SYSTEM SET max_connections = ${max_conn};
+
 -- Write throughput: fewer, larger checkpoints.
 ALTER SYSTEM SET wal_buffers = '16MB';
 ALTER SYSTEM SET min_wal_size = '1GB';