diff --git a/infra/deploy/db/init/20-tuning.sh b/infra/deploy/db/init/20-tuning.sh index 4791d10..78f8161 100755 --- a/infra/deploy/db/init/20-tuning.sh +++ b/infra/deploy/db/init/20-tuning.sh @@ -8,10 +8,10 @@ # 10-timescaledb.sql enables timescaledb. Settings are written via ALTER SYSTEM # (persisted to postgresql.auto.conf, which the timescaledb image's own # timescaledb-tune postgresql.conf defers to); the container's post-init restart brings -# restart-only settings (shared_buffers, …) into effect. NB: never ALTER SYSTEM SET -# shared_preload_libraries here — that would land in auto.conf and override the image's -# `timescaledb` preload. Init scripts do NOT re-run on an existing volume — to -# re-tune later, set DB_MEMORY and run this by hand, then restart: +# restart-only settings (shared_buffers, max_connections, …) into effect. NB: never +# ALTER SYSTEM SET shared_preload_libraries here — that would land in auto.conf and +# override the image's `timescaledb` preload. Init scripts do NOT re-run on an existing +# volume — to re-tune later, set DB_MEMORY and run this by hand, then restart: # docker compose exec -e DB_MEMORY=16g db bash /docker-entrypoint-initdb.d/20-tuning.sh # docker compose restart db set -euo pipefail @@ -33,13 +33,30 @@ if [ "$mb" -lt 1024 ]; then mb=8192; fi # maintenance_work_mem 512 MB) and scale linearly on a bigger box. shared_buffers=$((mb / 4)) # 25% — the shared page cache effective_cache=$((mb * 3 / 4)) # 75% — planner's view of total cache (PG + OS) -work_mem=$((mb / 128)) # ~64 MB at 8 GB (per-operation; kept modest) +maint_mem=$((mb / 16)) # 512 MB at 8 GB — index builds / VACUUM + +# work_mem is per *sort operation*, not per connection, so its real cost is +# work_mem x concurrent sorts x max_connections (200, below) — and the db container +# has a hard memory limit. Left uncapped the ratio gives 128 MB at prod's 16g +# budget, which is more than a 200-connection instance should carry: a burst of +# heavy sorts can walk into the container limit, and an OOM-killed Postgres is a +# far worse day than a refused connection. Cap at 64 MB (dev's 8g is unaffected). +work_mem=$((mb / 128)) +if [ "$work_mem" -gt 64 ]; then work_mem=64; fi if [ "$work_mem" -lt 16 ]; then work_mem=16; fi -maint_mem=$((mb / 16)) # 512 MB at 8 GB — index builds / VACUUM + +# Connection ceiling. NOT derived from DB_MEMORY: it is bounded by how many pools +# the app opens, not by RAM. One instance serves prod and beta, and every backend +# process opens three pools (two async engines + the notifier's sync engine, see +# backend/accounts/db.py), so the image default of 100 was below the configured +# worst case — prod saturated it on 2026-07-26 and started refusing clients with +# "sorry, too many clients already". 200 covers today's replica counts including +# web's autoscale maximum. ~+1 GB of per-connection overhead at 200. +max_conn=200 echo "[tuning] DB_MEMORY=${budget} -> ${mb}MB: shared_buffers=${shared_buffers}MB" \ "effective_cache_size=${effective_cache}MB work_mem=${work_mem}MB" \ - "maintenance_work_mem=${maint_mem}MB" + "maintenance_work_mem=${maint_mem}MB max_connections=${max_conn}" psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <