From 6cf70e22db83a9da8da93be2c0ac1ab97591a9d7 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Wed, 20 May 2026 20:18:19 -0400
Subject: [PATCH] compose(db): lenient healthcheck + stop_grace_period to
 survive host stalls
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mitigation for the nightly fabledscribe Postgres outage on the
vdnt-docker02 Swarm node (incidents 2026-05-15/16/17 around 03:50 UTC).
Confirmed kill chain (not the trigger): a brief host-level setns/exec
stall makes the Docker healthcheck exec fail with exit 1 → unhealthy →
SIGKILL → fast-shutdown can't finish on NFS in 10s → exit 137 → swarm
restart_policy.max_attempts: 5 burns out → DB stays dead.

Hardens the `db` service so a transient host blip can't escalate to
killing the database:
- stop_grace_period: 120s (gives PG room to fsync on shutdown)
- healthcheck: interval 30s / timeout 10s / retries 10 / start_period 180s
  (only gates app startup order — not authoritative liveness)
- prod: restart_policy condition=on-failure, max_attempts=0, window=120s
- quickstart/dev: restart: unless-stopped

Host-side trigger (what stalls runc/exec at ~03:50 UTC) is still under
investigation — see project_pg_nightly_outage.md.

Note: the Portainer prod stack differs from docker-compose.prod.yml
here (NFS bind, traefik labels, no ollama). The same `db:` block needs
to be pasted into Portainer for the prod mitigation to apply.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 docker-compose.prod.yml       | 15 +++++++++++----
 docker-compose.quickstart.yml | 10 +++++++---
 docker-compose.yml            | 11 ++++++++---
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 890d0df..7d0fac1 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -24,6 +24,7 @@ services:
 
   db:
     image: postgres:16-alpine
+    stop_grace_period: 120s
     volumes:
       - pgdata:/var/lib/postgresql/data
     environment:
@@ -32,15 +33,21 @@ services:
       POSTGRES_DB: fabledassistant
     networks:
       - fabledassistant_backend
+    # Lenient by design: a transient host exec/healthcheck stall (incident:
+    # runc setns failures -> "unhealthy" -> SIGKILL -> crash loop) must never
+    # escalate to killing the DB. Health here only gates app startup order.
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U fabled"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 180s
     deploy:
       restart_policy:
         condition: on-failure
-        max_attempts: 5
+        delay: 10s
+        max_attempts: 0
+        window: 120s
 
   ollama:
     image: ollama/ollama
diff --git a/docker-compose.quickstart.yml b/docker-compose.quickstart.yml
index e2a9ee5..882e80b 100644
--- a/docker-compose.quickstart.yml
+++ b/docker-compose.quickstart.yml
@@ -39,17 +39,21 @@ services:
 
   db:
     image: postgres:16-alpine
+    stop_grace_period: 120s
     volumes:
       - pgdata:/var/lib/postgresql/data
     environment:
       POSTGRES_USER: fabled
       POSTGRES_PASSWORD: fabled
       POSTGRES_DB: fabledassistant
+    # Lenient by design: a transient host exec/healthcheck stall must never
+    # escalate to killing the DB. Health here only gates app startup order.
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U fabled"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 180s
     restart: unless-stopped
 
   ollama:
diff --git a/docker-compose.yml b/docker-compose.yml
index 7be1606..2a06564 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -34,17 +34,22 @@ services:
 
   db:
     image: postgres:16-alpine
+    stop_grace_period: 120s
+    restart: unless-stopped
     volumes:
       - pgdata:/var/lib/postgresql/data
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-fabled}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-fabled}
       POSTGRES_DB: ${POSTGRES_DB:-fabledassistant}
+    # Lenient by design: a transient host exec/healthcheck stall must never
+    # escalate to killing the DB. Health here only gates app startup order.
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-fabled}"]
-      interval: 5s
-      timeout: 5s
-      retries: 5
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 180s
 
   ollama:
     image: ollama/ollama