diff --git a/backend/tests/accounts/test_db_engines.py b/backend/tests/accounts/test_db_engines.py index 29454d5..cbe6e24 100644 --- a/backend/tests/accounts/test_db_engines.py +++ b/backend/tests/accounts/test_db_engines.py @@ -9,6 +9,28 @@ from sqlalchemy import select from accounts import db +# The server-side ceiling these pools spend from, and the deployment shape that +# spends it. Kept here so a pool change that stops fitting is a failing test +# rather than a 2am "sorry, too many clients already". Mirrors max_connections in +# infra/deploy/db/init/20-tuning.sh. ONE Postgres instance serves prod and beta. +MAX_CONNECTIONS = 200 +PROD_WEB_WORKERS = 4 # uvicorn workers per prod web replica +BETA_WEB_WORKERS = 2 +ASYNC_ENGINES_PER_PROCESS = 2 # RW + RO, both built from _ASYNC_POOL_KWARGS + + +def _async_ceiling_per_process() -> int: + """Connections one backend process can hold from its async pools at once.""" + k = db._ASYNC_POOL_KWARGS + return ASYNC_ENGINES_PER_PROCESS * (k["pool_size"] + k["max_overflow"]) + + +def _sync_ceiling() -> int: + """The notifier's sync pool. One process holds it cluster-wide (leader flock), + so it is NOT multiplied by workers or replicas.""" + k = db._SYNC_POOL_KWARGS + return k["pool_size"] + k["max_overflow"] + def test_defaults_to_sqlite_when_no_database_url(): assert db.IS_POSTGRES is False @@ -32,13 +54,18 @@ def test_postgres_pool_sizing_and_sync_timeouts(): """The Postgres engine-construction kwargs, defined at module scope so they're testable without a real Postgres connection (the engines built from them are only actually constructed when IS_POSTGRES, exercised by the docker-compose - stack -- see the module docstring). pool_size=1/max_overflow=1 (the old + stack -- see the module docstring). pool_size=1/max_overflow=1 (the original setting) let a 3rd concurrent request per worker wait out pool_timeout for a - slot; both async engines need real headroom, and the sync engine (the + slot, so the steady-state pool needs real headroom; the sync engine (the notifier's, driven off-event-loop) must bound connect + statement time so a wedged Postgres can't hang it forever under its leader flock.""" assert db._ASYNC_POOL_KWARGS["pool_size"] >= 5 - assert db._ASYNC_POOL_KWARGS["max_overflow"] >= 5 + + # The overflow is the part nobody budgets for -- it is multiplied by two async + # engines x uvicorn workers x replicas, against a ceiling shared with beta. + # Prod saturated max_connections on 2026-07-26 with this at 5. Keep it well + # under pool_size; the budget test below is the real constraint. + assert db._ASYNC_POOL_KWARGS["max_overflow"] <= 2 assert db._SYNC_POOL_KWARGS["pool_size"] < db._ASYNC_POOL_KWARGS["pool_size"] connect_args = db._SYNC_POOL_KWARGS["connect_args"] @@ -53,6 +80,27 @@ def test_postgres_pool_sizing_and_sync_timeouts(): engine.dispose() +def test_pool_budget_fits_under_max_connections(): + """At today's replica counts the whole estate's configured worst case must fit + inside the server's max_connections, with margin -- prod web + prod worker + + beta web + beta worker all draw on ONE Postgres instance. This is the test + that would have failed before 2026-07-26, when the configured worst case was + 170 against a ceiling of 100. + + Note this counts *configured maxima*: every pool at full overflow at the same + instant. Prod web autoscaling to its max of 3 replicas is deliberately NOT in + this sum -- that peak still does not fit, and closing it needs a connection + pooler or a lower autoscale ceiling rather than a bigger number here.""" + per_process = _async_ceiling_per_process() + prod = per_process * PROD_WEB_WORKERS + (per_process + _sync_ceiling()) + beta = per_process * BETA_WEB_WORKERS + (per_process + _sync_ceiling()) + + assert prod + beta <= MAX_CONNECTIONS * 0.8, ( + f"configured worst case {prod + beta} leaves too little room under " + f"max_connections={MAX_CONNECTIONS}" + ) + + def test_read_session_yields_a_working_session(): # On SQLite the read session is fully usable (no read-only enforcement); this # just confirms the dependency wiring resolves to a live AsyncSession.