backend/app/scripts/wait_for_deps.py

@@ -0,0 +1,146 @@
+"""Block until Postgres and Redis accept connections. Exit 0 ready, 1 timed out.
+
+## Why the container has to do this itself
+
+Compose has `depends_on: {condition: service_healthy}`, and **Swarm ignores
+it**. `docker stack deploy` has no ordering primitive at all: every service in
+the stack starts at once, so FabledCurator races Postgres on every cold
+deploy and always has.
+
+The multi-service stack hid how sharp that is. `web` ran `alembic upgrade
+head`, failed against a Postgres that was still doing `initdb`, and the task
+died — but Swarm restarts a failed task forever, so the service came up a few
+seconds later and nobody saw a problem worth naming.
+
+Consolidation removes that safety net. supervisord gives each program
+`startretries=3`, so a web program that fails three times in the first
+seconds goes FATAL and **stays** FATAL: supervisord keeps running, the
+container keeps running, and the application never starts. The healthcheck
+catches it — but as a container that is permanently unhealthy for a reason
+that has nothing to do with the image, on a stack whose database simply took
+twenty seconds to initialise.
+
+Operator, 2026-09-23: *"it's a single container that need to connect
+successfully to redis and postgres before starting work shouldn't that simply
+be a check (with retries) at the start of the container."* Yes.
+
+## A TCP connect, not a query
+
+The same probe `ci.yml`'s integration lane and the build smoke already use.
+It answers the question that is actually being asked — is something listening
+— and it cannot fail for a reason that retrying will never fix.
+
+A real query would be a stronger readiness signal and a worse gate: a wrong
+password or a missing database is not a transient condition, and a loop that
+waits for one to heal turns a five-second misconfiguration into a two-minute
+timeout with a misleading message. Those belong to alembic, which runs
+seconds later and says exactly what is wrong.
+
+The Postgres image is well behaved here: during `initdb` it serves on a unix
+socket only and opens TCP when it is ready for clients, so the connect is a
+good proxy for "ready" rather than merely "process exists".
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import socket
+import sys
+import time
+from urllib.parse import urlparse
+
+# Long enough for a first-ever `initdb` on a slow disk, which is the worst
+# case this exists for and is measured in tens of seconds, not minutes. A
+# deploy that is genuinely misconfigured should fail while someone is still
+# watching it rather than hold the container open for a quarter of an hour.
+DEFAULT_TIMEOUT = 120.0
+CONNECT_TIMEOUT = 2.0
+RETRY_DELAY = 1.0
+# Progress every N attempts. `docker logs` on a container that is waiting must
+# say what it is waiting for — silence is indistinguishable from a hang.
+REPORT_EVERY = 5
+
+
+def _target(url: str | None, default_port: int) -> tuple[str, int] | None:
+    """(host, port) from a connection URL, or None if there is nothing to wait for."""
+    if not url:
+        return None
+    parsed = urlparse(url)
+    if not parsed.hostname:
+        return None
+    return parsed.hostname, parsed.port or default_port
+
+
+def targets() -> list[tuple[str, tuple[str, int]]]:
+    """What this container must reach, read from the same env the app reads.
+
+    Derived rather than passed in, so the wait cannot drift from what the
+    application will actually connect to — a gate that checks a different
+    host than the app uses is worse than no gate.
+    """
+    out: list[tuple[str, tuple[str, int]]] = []
+
+    host = os.environ.get("DB_HOST")
+    if host:
+        out.append(("postgres", (host, int(os.environ.get("DB_PORT") or 5432))))
+
+    broker = _target(os.environ.get("CELERY_BROKER_URL"), 6379)
+    if broker:
+        out.append(("redis", broker))
+
+    return out
+
+
+def _accepts(host: str, port: int) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=CONNECT_TIMEOUT):
+            return True
+    except OSError:
+        return False
+
+
+def wait(
+    name: str, host: str, port: int, deadline: float, now=time.monotonic,
+) -> bool:
+    attempt = 0
+    while True:
+        if _accepts(host, port):
+            print(f"[wait] {name} at {host}:{port} is accepting connections")
+            return True
+        attempt += 1
+        if now() >= deadline:
+            print(
+                f"[wait] TIMEOUT: {name} at {host}:{port} never accepted a "
+                f"connection ({attempt} attempts)",
+                file=sys.stderr,
+            )
+            return False
+        if attempt % REPORT_EVERY == 0:
+            left = int(deadline - now())
+            print(f"[wait] {name} at {host}:{port} not ready yet, {left}s left")
+        time.sleep(RETRY_DELAY)
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="Wait for Postgres and Redis.")
+    ap.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT)
+    args = ap.parse_args(argv)
+
+    wanted = targets()
+    if not wanted:
+        # Nothing configured to wait for. Not an error: `shell` and one-off
+        # runs are legitimate, and refusing to start would make this gate the
+        # reason a debugging container will not boot.
+        print("[wait] no database or broker configured; nothing to wait for")
+        return 0
+
+    deadline = time.monotonic() + args.timeout
+    for name, (host, port) in wanted:
+        if not wait(name, host, port, deadline):
+            return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

entrypoint.sh

@@ -51,6 +51,28 @@ if [ -z "${FC_ROLE:-}" ]; then
   export FC_ROLE="$ROLE"
   printf '%s\n' "$ROLE" > "${FC_ROLE_FILE:-/tmp/fc-role}" 2>/dev/null || true
 fi
+
+# WAIT FOR POSTGRES AND REDIS before doing anything that needs them.
+#
+# Swarm has no ordering primitive — it ignores `depends_on` entirely — so
+# every service in a stack starts at once and this container races its own
+# database on every cold deploy.
+#
+# The multi-service stack hid how sharp that is: a `web` task that failed
+# `alembic upgrade head` against a still-initialising Postgres simply died,
+# and Swarm restarted it until it worked. Consolidation removes that. Each
+# supervisord program gets `startretries=3`, so three quick failures put the
+# program in FATAL and leave it there — supervisord keeps running, the
+# container keeps running, and the application never starts. It would present
+# as a permanently unhealthy container whose image was fine and whose
+# database merely took twenty seconds to come up.
+#
+# Skipped for `shell`, which exists precisely for the case where something
+# else is broken and you want a prompt rather than a gate.
+case "$ROLE" in
+  shell|bash) ;;
+  *) python -m backend.app.scripts.wait_for_deps ;;
+esac
 shift || true
 
 case "$ROLE" in

tests/test_wait_for_deps.py

@@ -0,0 +1,91 @@
+"""The startup gate that stands in for Swarm's missing `depends_on`.
+
+Swarm ignores `depends_on`, so the container races its own database on every
+cold deploy. Under supervisord a program that fails three times goes FATAL and
+stays there, so losing that race does not self-heal the way it did when Swarm
+restarted a failed task — it leaves a running container with a dead
+application inside it.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from backend.app.scripts import wait_for_deps as w
+
+
+def test_it_waits_for_both_postgres_and_redis(monkeypatch):
+    monkeypatch.setenv("DB_HOST", "postgres")
+    monkeypatch.setenv("DB_PORT", "5432")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://redis:6379/0")
+
+    assert w.targets() == [
+        ("postgres", ("postgres", 5432)),
+        ("redis", ("redis", 6379)),
+    ]
+
+
+def test_the_targets_come_from_the_same_env_the_app_reads(monkeypatch):
+    """A gate that checks a different host than the application connects to is
+    worse than no gate — it would pass while the app still cannot reach its
+    database."""
+    monkeypatch.setenv("DB_HOST", "10.0.0.5")
+    monkeypatch.setenv("DB_PORT", "6543")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://broker.internal:6380/2")
+
+    assert w.targets() == [
+        ("postgres", ("10.0.0.5", 6543)),
+        ("redis", ("broker.internal", 6380)),
+    ]
+
+
+def test_a_broker_url_without_a_port_falls_back_to_the_default(monkeypatch):
+    monkeypatch.delenv("DB_HOST", raising=False)
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://redis/0")
+
+    assert w.targets() == [("redis", ("redis", 6379))]
+
+
+def test_nothing_configured_is_not_an_error(monkeypatch):
+    """`shell` and one-off runs are legitimate. Refusing to start would make
+    this gate the reason a debugging container will not boot."""
+    monkeypatch.delenv("DB_HOST", raising=False)
+    monkeypatch.delenv("CELERY_BROKER_URL", raising=False)
+
+    assert w.targets() == []
+    assert w.main([]) == 0
+
+
+def test_it_returns_as_soon_as_the_port_accepts(monkeypatch):
+    attempts = {"n": 0}
+
+    def accepts(host, port):
+        attempts["n"] += 1
+        return attempts["n"] >= 3
+
+    monkeypatch.setattr(w, "_accepts", accepts)
+    monkeypatch.setattr(w.time, "sleep", lambda _: None)
+
+    assert w.wait("postgres", "h", 5432, deadline=float("inf")) is True
+    assert attempts["n"] == 3
+
+
+def test_it_gives_up_at_the_deadline_rather_than_hanging(monkeypatch):
+    """A wait with no deadline is a bug (rule 156). A container that never
+    starts and never says why is the shape this must not have."""
+    monkeypatch.setattr(w, "_accepts", lambda h, p: False)
+    monkeypatch.setattr(w.time, "sleep", lambda _: None)
+
+    clock = iter([0.0, 0.0, 99.0])
+    assert w.wait(
+        "postgres", "h", 5432, deadline=10.0, now=lambda: next(clock),
+    ) is False
+
+
+def test_a_dependency_that_never_comes_up_fails_the_boot(monkeypatch):
+    monkeypatch.setenv("DB_HOST", "postgres")
+    monkeypatch.delenv("CELERY_BROKER_URL", raising=False)
+    monkeypatch.setattr(w, "_accepts", lambda h, p: False)
+    monkeypatch.setattr(w.time, "sleep", lambda _: None)
+
+    assert w.main(["--timeout", "0"]) == 1