From a183be7e6e33451ecf491690a0d850e9602933c1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 00:08:19 -0400 Subject: [PATCH] fix(infra): bump Postgres shm_size (vacuum DiskFull) + raise DB-backup time limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more maintenance-queue failures from the operator's 24h list: - vacuum_analyze died with "could not resize shared memory segment to 67MB: No space left on device" — Docker's default /dev/shm is 64MB, too small for VACUUM (ANALYZE)'s parallel-worker shared memory. Set the postgres service shm_size: 512m. - backup_db_task timed out at its 12-min limit once the DB grew; a pg_dump can't be chunked, so raise it to 30/35 min. (A long backup still briefly holds the concurrency-1 lane — the structural fix is a dedicated lane for long one-shots.) Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/tasks/backup.py | 6 +++++- docker-compose.yml | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/app/tasks/backup.py b/backend/app/tasks/backup.py index 57d8157..77ad1f0 100644 --- a/backend/app/tasks/backup.py +++ b/backend/app/tasks/backup.py @@ -41,7 +41,11 @@ def _mark_failed(session, row: BackupRun, exc: BaseException) -> None: bind=True, autoretry_for=(OperationalError, DBAPIError), retry_backoff=10, retry_backoff_max=120, max_retries=2, - soft_time_limit=600, time_limit=720, + # A pg_dump can't be chunked; the 12-min limit timed out once the DB grew + # (operator-flagged 2026-06-07). 30/35 min gives real headroom. (A long + # backup still briefly holds the concurrency-1 maintenance lane — the + # structural fix is a dedicated lane for the long one-shots.) + soft_time_limit=1800, time_limit=2100, ) def backup_db_task(self, *, tag: str | None = None, triggered_by: str = "manual") -> dict: diff --git a/docker-compose.yml b/docker-compose.yml index b8da8cd..f292640 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,11 @@ services: postgres: image: pgvector/pgvector:pg16 + # Docker's default /dev/shm is 64MB; VACUUM (ANALYZE) and parallel queries + # allocate larger shared-memory segments and fail with + # "could not resize shared memory segment ... No space left on device" + # (operator-flagged 2026-06-07, vacuum_analyze on import_task needed 67MB). + shm_size: 512m environment: POSTGRES_USER: ${DB_USER:-fabledcurator} POSTGRES_PASSWORD: ${DB_PASSWORD:-fabledcurator_dev}