backend/app/scripts/gen_supervisord.py

@@ -0,0 +1,184 @@
+"""Emit a supervisord config for the single-container layout.
+
+Milestone 422 step 5. Writes to stdout; `entrypoint.sh all` redirects it to a
+file and execs supervisord against it.
+
+## Why this is generated and not a checked-in .conf
+
+A static config would spell out each lane's `-Q` list, and that would be a
+FIFTH hand-kept copy of the queue names — after `celery_app.task_routes`, and
+the three collapsed in steps 1, 2 and 4 (`service_roster.ROLE_NAMES`,
+`system_activity._QUEUE_NAMES`, and the Activity filter). Every one of those
+had already drifted by the time it was found.
+
+Generating from `worker_lanes.LANES` makes a stronger guarantee than "they
+match today": the processes this container runs and the lanes the application
+believes in are the same list, so a lane added to `LANES` gets a process
+without anyone remembering to add one, and a queue can never end up with no
+consumer because a config file was missed.
+
+## Why supervisord
+
+It is one pip dependency on an image that is already Python, and it does the
+four things this needs without being clever: restart a program that exits,
+give each one its OWN stop timeout, signal the process GROUP rather than the
+leader, and put every program's output on one stdout.
+
+The process-group part is not a detail. Celery's prefork pool forks children,
+and a TERM delivered only to the parent leaves them running — which is how a
+"graceful" shutdown turns into orphaned workers holding tasks. `stopasgroup`
+and `killasgroup` are both set for every program.
+
+s6-overlay is the other standard answer and would work; it needs a build-time
+download and a second mental model, and its advantage (correct PID-1 signal
+and zombie handling) is available here from `init: true` in compose, which
+puts tini in front of supervisord. Neither choice reaches the application —
+nothing in FC talks to the supervisor — so this is reversible without touching
+a line of product code.
+
+## What this does NOT start
+
+The `ml` lane, unless `--with-ml` is passed. Until step 6 merges the images,
+torch and the ML requirements live only in `Dockerfile.ml`, so an `ml` program
+in the web image would fail to import on every restart forever. Step 6 is
+where that flag turns on.
+"""
+
+from __future__ import annotations
+
+import argparse
+import shlex
+import sys
+
+from ..services.worker_lanes import LANES, Lane
+
+# One number for the whole container, and it must cover the SLOWEST lane —
+# docker gives the container a single stop timeout, where compose today gives
+# each service its own (90/60/180/120s). `maintenance_long` is the 180s one:
+# DB backups, library audits and translation backfill. Anything less turns a
+# routine restart into a SIGKILL mid-backup.
+#
+# Per-program values below are the old per-service ones, preserved: supervisord
+# waits `stopwaitsecs` for each, and they stop in parallel, so the container's
+# own timeout needs to cover the max rather than the sum.
+STOP_WAIT_SECONDS: dict[str, int] = {
+    "worker": 90,
+    "scheduler": 60,
+    "maintenance_long": 180,
+    "ml": 120,
+}
+DEFAULT_STOP_WAIT = 60
+
+# Lanes whose code is not in this image yet. See the module docstring.
+_NEEDS_ML_DEPS = frozenset({"ml"})
+
+
+def _program(lane: Lane, *, slots: int) -> str:
+    """One [program:x] block.
+
+    `stdout_logfile=/dev/fd/1` with maxbytes 0 puts the lane's output straight
+    on the container's stdout unbuffered, so `docker logs` shows every lane
+    interleaved rather than supervisord swallowing them into rotated files.
+
+    The output is prefixed through `sed` so a line can be attributed to a lane
+    — four celery workers and hypercorn on one stream are otherwise
+    indistinguishable. The shell that the pipe requires is exactly why
+    `stopasgroup` matters: the signal has to reach the celery process, not the
+    `sh` holding the pipeline.
+    """
+    inner = f"./entrypoint.sh {lane.entrypoint_role}"
+    prefixed = f"{inner} 2>&1 | sed -u 's/^/[{lane.name}] /'"
+    stop_wait = STOP_WAIT_SECONDS.get(lane.name, DEFAULT_STOP_WAIT)
+    return "\n".join([
+        f"[program:{lane.name}]",
+        f"command=sh -c {shlex.quote(prefixed)}",
+        # QUOTED, and that is load-bearing. supervisord parses `environment`
+        # as a COMMA-separated KEY=VALUE list, so an unquoted queue list reads
+        # as CELERY_QUEUES=default followed by three malformed entries — and
+        # the lane would consume only its first queue. Silent: the worker
+        # starts, reports healthy, and simply never picks up `import`.
+        f'environment=CELERY_QUEUES="{",".join(lane.queues)}",'
+        f"CELERY_CONCURRENCY={slots}",
+        "autostart=true",
+        "autorestart=true",
+        # A lane that dies instantly and repeatedly is a broken image, not a
+        # transient fault. Backing off stops it burning a core in a restart
+        # loop while still recovering from a one-off crash.
+        "startretries=3",
+        "startsecs=5",
+        f"stopwaitsecs={stop_wait}",
+        "stopasgroup=true",
+        "killasgroup=true",
+        "stdout_logfile=/dev/fd/1",
+        "stdout_logfile_maxbytes=0",
+        "redirect_stderr=true",
+        "",
+    ])
+
+
+def _web_program() -> str:
+    """hypercorn. Started FIRST (priority) because its role runs
+    `alembic upgrade head`, and a worker that boots against an un-migrated
+    schema fails in a way that looks like application breakage."""
+    prefixed = "./entrypoint.sh web 2>&1 | sed -u 's/^/[web] /'"
+    return "\n".join([
+        "[program:web]",
+        f"command=sh -c {shlex.quote(prefixed)}",
+        "priority=1",
+        "autostart=true",
+        "autorestart=true",
+        "startretries=3",
+        "startsecs=5",
+        # Short: HTTP requests and the occasional file download. Matches the
+        # 30s the operator's production stack gives the web service.
+        "stopwaitsecs=30",
+        "stopasgroup=true",
+        "killasgroup=true",
+        "stdout_logfile=/dev/fd/1",
+        "stdout_logfile_maxbytes=0",
+        "redirect_stderr=true",
+        "",
+    ])
+
+
+def render(*, with_ml: bool = False) -> str:
+    parts = [
+        "\n".join([
+            "[supervisord]",
+            # PID 1 in the container, so it must not daemonise.
+            "nodaemon=true",
+            # supervisord's OWN log. /dev/fd/1 keeps it on the container's
+            # stdout beside the programs rather than in a file nobody reads.
+            "logfile=/dev/fd/1",
+            "logfile_maxbytes=0",
+            "loglevel=info",
+            "",
+        ]),
+        _web_program(),
+    ]
+    # Lanes after web, in LANES order, so the log reads in a stable sequence.
+    for lane in LANES:
+        if lane.name in _NEEDS_ML_DEPS and not with_ml:
+            continue
+        # A lane configured at zero slots still gets a PROCESS, at one slot
+        # with its consumers cancelled by the reconcile. Without a running
+        # worker there is nothing for `add_consumer` to reach, so enabling the
+        # lane from the UI could not work at all — the process has to exist for
+        # the switch to have something to switch.
+        parts.append(_program(lane, slots=max(1, lane.default_slots)))
+    return "\n".join(parts)
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument(
+        "--with-ml", action="store_true",
+        help="include the ml lane (only valid once the ML deps are in this image)",
+    )
+    args = ap.parse_args(argv)
+    sys.stdout.write(render(with_ml=args.with_ml))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/app/scripts/healthcheck_all.py

@@ -0,0 +1,83 @@
+"""Container healthcheck for the single-container layout.
+
+Milestone 422 step 5. Exit 0 healthy, non-zero unhealthy.
+
+## Why this is not just "does :8080 answer"
+
+In the multi-service stack every service has its OWN healthcheck, so a dead
+worker turns that service unhealthy while web stays green — docker knows which
+part failed. Collapsing them into one container collapses that too: a web-only
+check would report a perfectly healthy container while every lane inside it
+had crashed and been abandoned by supervisord after its retries.
+
+So this asserts both halves: hypercorn answers, AND every lane this container
+was configured to run is answering the broker.
+
+## What it deliberately does NOT do
+
+It does not read the database, and it does not consult the `enabled` flag. A
+DISABLED lane still has a running process with its consumers cancelled (see
+the config generator), so it answers `inspect` and is healthy. Health is
+"is the process alive", and whether it should be consuming is a settings
+question the reconcile owns — conflating them would make turning a lane off
+in the UI mark the container unhealthy.
+
+It also cannot distinguish "the broker is down" from "every lane is down",
+and reports unhealthy either way. That is correct: a container that cannot
+reach its broker is not serving, whichever half is at fault.
+"""
+
+from __future__ import annotations
+
+import sys
+import urllib.error
+import urllib.request
+
+WEB_URL = "http://localhost:8080/api/health"
+WEB_TIMEOUT = 5.0
+
+
+def _web_ok() -> tuple[bool, str]:
+    try:
+        with urllib.request.urlopen(WEB_URL, timeout=WEB_TIMEOUT) as resp:
+            if resp.status == 200:
+                return True, ""
+            return False, f"web returned {resp.status}"
+    except (urllib.error.URLError, OSError) as exc:
+        return False, f"web unreachable: {exc}"
+
+
+def _lanes_ok(*, with_ml: bool) -> tuple[bool, str]:
+    from ..services.worker_control import inspect_lanes_sync
+    from ..services.worker_lanes import LANES
+
+    expected = {
+        lane.name for lane in LANES
+        if with_ml or lane.name != "ml"
+    }
+    live = inspect_lanes_sync()
+    missing = sorted(n for n in expected if not live[n].present)
+    if missing:
+        return False, "lanes not answering: " + ", ".join(missing)
+    return True, ""
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    with_ml = "--with-ml" in argv
+
+    ok, detail = _web_ok()
+    if not ok:
+        print(detail, file=sys.stderr)
+        return 1
+
+    ok, detail = _lanes_ok(with_ml=with_ml)
+    if not ok:
+        print(detail, file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

backend/app/services/worker_lanes.py

@@ -71,6 +71,12 @@ class Lane:
     name: str
     display_name: str
     queues: tuple[str, ...]
+    # Which `entrypoint.sh` role starts this lane. NOT always the lane name:
+    # `maintenance_long` is the plain `worker` role pointed at a different
+    # queue, exactly as docker-compose starts it today (`command: ["worker"]`
+    # with CELERY_QUEUES=maintenance_long). Recorded here so the generated
+    # supervisord config and the compose file cannot disagree about it.
+    entrypoint_role: str
     default_slots: int
     # The cap a lane STARTS with, which is not the ceiling. Set low enough
     # that raising slots within it is an ordinary adjustment, and raising the
@@ -104,6 +110,7 @@ LANES: tuple[Lane, ...] = (
         name="worker",
         display_name="Worker",
         queues=("default", "import", "thumbnail", "download"),
+        entrypoint_role="worker",
         default_slots=1,
         default_slots_cap=4,
         default_enabled=True,
@@ -112,6 +119,7 @@ LANES: tuple[Lane, ...] = (
         name="scheduler",
         display_name="Scheduler",
         queues=("maintenance", "scan"),
+        entrypoint_role="scheduler",
         default_slots=1,
         default_slots_cap=2,
         default_enabled=True,
@@ -120,6 +128,7 @@ LANES: tuple[Lane, ...] = (
         name="maintenance_long",
         display_name="Long maintenance",
         queues=("maintenance_long",),
+        entrypoint_role="worker",
         default_slots=1,
         default_slots_cap=2,
         default_enabled=True,
@@ -128,6 +137,7 @@ LANES: tuple[Lane, ...] = (
         name="ml",
         display_name="ML tagging",
         queues=("ml",),
+        entrypoint_role="ml-worker",
         default_slots=0,
         default_slots_cap=1,
         default_enabled=False,

docker-compose.single.yml

@@ -0,0 +1,113 @@
+# FabledCurator in three containers — the install path.
+#
+#     docker compose -f docker-compose.single.yml up -d
+#
+# Milestone 422 step 5. FabledCurator runs web and every worker lane inside
+# ONE container, with Postgres and Redis beside it. How much work each lane
+# does is then a dial in the web UI (Settings -> Activity -> Worker lanes),
+# live, with no compose edit and no restart.
+#
+# THE MULTI-SERVICE STACK IS NOT REPLACED. `docker-compose.yml` still runs the
+# five app services separately and is the right shape for a Swarm deployment
+# spread across hosts, where per-service rolling rollback and placement
+# constraints matter. This file is the adopter path: one box, one command.
+#
+# What consolidating costs, stated here rather than discovered later:
+#   - Everything shares one host, so there is no spreading work across nodes.
+#   - Rollback is all-or-nothing; there is no rolling back `web` alone.
+#   - The ML lane cannot write to the library read-only any more — in the
+#     multi-service stack ml-worker mounts /images:ro, and one container
+#     cannot mount one path two ways.
+#   - One stop timeout for the whole container, sized to the slowest lane.
+#
+# FabledCurator has no authentication. Whatever can reach ${PORT} is an
+# administrator, including over the stored platform session cookies. Do not
+# publish this port beyond a network you trust — see "Before you expose it"
+# in README.md.
+
+services:
+  redis:
+    image: redis:7-alpine
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  postgres:
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_USER: ${DB_USER:-curator}
+      POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres}
+      POSTGRES_DB: ${DB_NAME:-curator}
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    # pgvector index builds and the gallery's TABLESAMPLE reads both want more
+    # shared memory than docker's 64MB default.
+    shm_size: 512m
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-curator} -d ${DB_NAME:-curator}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
+  fabledcurator:
+    image: git.fabledsword.com/bvandeusen/fabledcurator:latest
+    # Everything: hypercorn plus one celery process per lane, under
+    # supervisord, whose config is generated from the application's own lane
+    # table so the two cannot disagree.
+    command: ["all"]
+    # tini as PID 1, in front of supervisord. supervisord reaps its own
+    # children, but a container's PID 1 also inherits orphans from anywhere
+    # below — celery's prefork pool and gallery-dl's subprocesses both make
+    # them. Without this they accumulate as zombies for the life of the
+    # container.
+    init: true
+    # Sized to the SLOWEST lane, not the average. maintenance_long runs DB
+    # backups, library audits and translation backfill, and gets 180s to
+    # finish a chunk; the lanes stop in parallel, so this covers the max
+    # rather than their sum. Below this, a routine restart becomes a SIGKILL
+    # mid-backup — which is recoverable (the work is chunked and idempotent)
+    # but wastes however long it had run.
+    stop_grace_period: 200s
+    # BOTH halves: hypercorn answers AND every configured lane is answering
+    # the broker. A web-only check would report a healthy container while
+    # every lane inside it had crashed — the failure mode consolidation
+    # creates, since docker can no longer see the lanes as separate services.
+    healthcheck:
+      test: ["CMD", "python", "-m", "backend.app.scripts.healthcheck_all"]
+      interval: 30s
+      timeout: 15s
+      retries: 3
+      # Covers alembic + hypercorn boot + four celery workers registering.
+      start_period: 90s
+    environment:
+      DB_USER: ${DB_USER:-curator}
+      DB_PASSWORD: ${DB_PASSWORD:-postgres}
+      DB_HOST: postgres
+      DB_PORT: "5432"
+      DB_NAME: ${DB_NAME:-curator}
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+      SECRET_KEY: ${SECRET_KEY:-change-me-before-you-expose-this}
+      EXTENSION_API_KEY: ${EXTENSION_API_KEY:-}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+    ports:
+      - "${PORT:-8080}:8080"
+    volumes:
+      - ${IMAGES_DIR:-./images}:/images
+      # Read-only. The filesystem scan copies out of here and never writes to
+      # it, so a mistake cannot reach the source library.
+      - ${IMPORT_DIR:-./import}:/import:ro
+    depends_on:
+      postgres: { condition: service_healthy }
+      redis: { condition: service_healthy }
+    restart: unless-stopped
+
+volumes:
+  redis_data:
+  postgres_data:

entrypoint.sh

@@ -35,12 +35,18 @@ case "$ROLE" in
 
   scheduler)
     QUEUES="${CELERY_QUEUES:-maintenance,scan}"
-    echo "[entrypoint] Starting Celery beat+worker queues=$QUEUES"
+    # Honours CELERY_CONCURRENCY like the `worker` role does. It was hardcoded
+    # to 1, which was harmless while only compose started this lane and set no
+    # concurrency for it — but the generated supervisord config (milestone 422
+    # step 5) passes one, and a value silently ignored at boot would leave the
+    # lane at 1 until the reconcile sweep noticed, with nothing saying why.
+    CONCURRENCY="${CELERY_CONCURRENCY:-1}"
+    echo "[entrypoint] Starting Celery beat+worker queues=$QUEUES concurrency=$CONCURRENCY"
     exec celery -A backend.app.celery_app:celery worker \
       --beat \
       --loglevel=info \
       -Q "$QUEUES" \
-      --concurrency=1
+      --concurrency="$CONCURRENCY"
     ;;
 
   ml-worker)
@@ -53,6 +59,27 @@ case "$ROLE" in
       --concurrency=1
     ;;
 
+  all)
+    # The single-container layout (milestone 422 step 5): hypercorn plus one
+    # celery process per lane, under supervisord, in one container beside
+    # Postgres and Redis.
+    #
+    # The config is GENERATED from services/worker_lanes.LANES rather than
+    # checked in, so the processes this container runs and the lanes the
+    # application believes in cannot disagree — see the generator's docstring
+    # for why a static .conf would have been a fifth copy of the queue names.
+    #
+    # supervisord is PID 1 here and never reads the database. Every lane boots
+    # at its LANES default; the reconcile sweep raises it to whatever the
+    # operator stored, within one tick. That ordering is deliberate: settings
+    # adjust a baseline that already works, and can never prevent a boot.
+    CONF="${SUPERVISOR_CONF:-/tmp/supervisord.conf}"
+    echo "[entrypoint] Generating $CONF from the lane table"
+    python -m backend.app.scripts.gen_supervisord ${FC_WITH_ML:+--with-ml} > "$CONF"
+    echo "[entrypoint] Starting supervisord (web + worker lanes)"
+    exec supervisord -c "$CONF"
+    ;;
+
   shell|bash)
     exec /bin/bash "$@"
     ;;
@@ -63,7 +90,7 @@ case "$ROLE" in
 
   *)
     echo "[entrypoint] Unknown role: $ROLE" >&2
-    echo "[entrypoint] Valid roles: web | worker | scheduler | ml-worker | shell | alembic" >&2
+    echo "[entrypoint] Valid roles: all | web | worker | scheduler | maintenance_long | ml | ml-worker | shell | alembic" >&2
     exit 1
     ;;
 esac

requirements.txt

@@ -43,3 +43,14 @@ py7zr>=1,<2
 # #830). Handles Drive's confirm-token + virus-scan interstitial. mega.nz uses
 # the `megatools` binary instead (Debian apt pkg in the runtime image, not pip).
 gdown>=6,<7
+
+# Process supervisor for the single-container layout (milestone 422 step 5).
+# `entrypoint.sh all` generates its config from the lane table and execs it as
+# PID 1, running hypercorn plus one celery process per lane in one container.
+# Unused by the multi-service compose path, where docker supervises instead.
+#
+# Chosen over s6-overlay because it is a pip install on an image that is
+# already Python, and gives per-program stop timeouts plus stopasgroup —
+# celery's prefork pool forks children, and a TERM that reaches only the
+# parent leaves them orphaned holding tasks.
+supervisor>=4.2,<5

tests/test_gen_supervisord.py

@@ -0,0 +1,185 @@
+"""The generated supervisord config (milestone 422 step 5).
+
+Asserts the config against the LANE TABLE rather than against a fixture of
+expected text. A fixture would have to be updated whenever a lane changes,
+which is the same hand-kept coupling generating the config exists to remove —
+and it would pass while describing a container that does not match the
+application's own idea of what it runs.
+"""
+
+from __future__ import annotations
+
+import configparser
+
+from backend.app.scripts import gen_supervisord as gen
+from backend.app.services.worker_lanes import LANES, LANES_BY_NAME
+
+# --- structure ---------------------------------------------------------------
+
+
+def _parse(**kwargs) -> configparser.ConfigParser:
+    """supervisord's config is ini, so parse it rather than grepping strings.
+
+    A substring assertion passes on a line that is present but malformed —
+    inside a comment, in the wrong section, or with a typo'd key that
+    supervisord silently ignores.
+    """
+    cp = configparser.ConfigParser()
+    cp.read_string(gen.render(**kwargs))
+    return cp
+
+
+def test_it_is_valid_ini_with_a_supervisord_section():
+    cp = _parse()
+    assert cp.has_section("supervisord")
+    # PID 1 in a container: daemonising would exit immediately and take the
+    # container with it.
+    assert cp.get("supervisord", "nodaemon") == "true"
+
+
+def test_web_and_every_non_ml_lane_get_a_program():
+    cp = _parse()
+    expected = {"program:web"} | {
+        f"program:{lane.name}" for lane in LANES if lane.name != "ml"
+    }
+    assert set(cp.sections()) - {"supervisord"} == expected
+
+
+def test_the_ml_lane_is_absent_until_its_deps_are_in_the_image():
+    """The web image has no torch. An `ml` program here would fail to import
+    on every restart, forever — startretries would give up and the lane would
+    be permanently dead while the container reported healthy."""
+    assert not _parse().has_section("program:ml")
+    assert _parse(with_ml=True).has_section("program:ml")
+
+
+# --- the coupling this generator exists to guarantee -------------------------
+
+
+def test_each_program_serves_exactly_its_lane_s_queues():
+    """The whole point: the container's processes and the application's lane
+    table are one list. A queue in LANES with no program means work that
+    queues forever with nothing consuming it."""
+    cp = _parse(with_ml=True)
+    for lane in LANES:
+        env = cp.get(f"program:{lane.name}", "environment")
+        # The QUOTED form. supervisord splits `environment` on commas, so an
+        # unquoted multi-queue value silently degrades to its first queue —
+        # and an assertion on the bare string passes either way, which is how
+        # that would have shipped.
+        assert f'CELERY_QUEUES="{",".join(lane.queues)}"' in env
+
+
+def test_each_program_invokes_the_lane_s_entrypoint_role_not_its_name():
+    """`maintenance_long` is the plain `worker` role pointed at a different
+    queue — exactly as docker-compose starts it today. Invoking
+    `entrypoint.sh maintenance_long` would hit the unknown-role branch and
+    exit 1 on every restart."""
+    cp = _parse(with_ml=True)
+    for lane in LANES:
+        command = cp.get(f"program:{lane.name}", "command")
+        assert f"entrypoint.sh {lane.entrypoint_role}" in command
+
+
+def test_every_entrypoint_role_a_lane_names_actually_exists():
+    """Reads entrypoint.sh itself. The generator can only emit a role name;
+    whether the script handles it is a separate fact, and getting it wrong
+    fails at container start rather than here."""
+    from pathlib import Path
+
+    script = Path(__file__).resolve().parents[1] / "entrypoint.sh"
+    text = script.read_text()
+    for lane in LANES:
+        # Roles are `case` arms: `  worker)` possibly in an alternation.
+        assert f"  {lane.entrypoint_role})" in text or \
+            f"|{lane.entrypoint_role})" in text, \
+            f"{lane.name} names entrypoint role {lane.entrypoint_role!r}, which does not exist"
+
+
+# --- shutdown ----------------------------------------------------------------
+
+
+def test_every_program_signals_its_whole_process_group():
+    """Celery's prefork pool forks children. A TERM delivered only to the
+    parent leaves them running and holding tasks — a 'graceful' shutdown that
+    orphans workers. The `sh -c … | sed` wrapper makes this doubly necessary:
+    without it the signal reaches the shell holding the pipeline, not celery."""
+    cp = _parse(with_ml=True)
+    for section in cp.sections():
+        if not section.startswith("program:"):
+            continue
+        assert cp.get(section, "stopasgroup") == "true", section
+        assert cp.get(section, "killasgroup") == "true", section
+
+
+def test_the_long_maintenance_lane_keeps_its_180s_drain():
+    """The per-service stop_grace_period values from the multi-service stack
+    are preserved per program. maintenance_long runs DB backups and library
+    audits; cutting its drain turns a restart into a SIGKILL mid-backup."""
+    cp = _parse()
+    assert cp.getint("program:maintenance_long", "stopwaitsecs") == 180
+
+
+def test_no_program_waits_longer_than_the_compose_stop_grace_period():
+    """The container gets ONE timeout and the programs stop in parallel, so it
+    must cover the slowest. If a lane's stopwaitsecs ever exceeds what
+    docker-compose.single.yml allows, docker kills the container while that
+    lane still believes it has time to drain."""
+    import re
+    from pathlib import Path
+
+    compose = (Path(__file__).resolve().parents[1] / "docker-compose.single.yml").read_text()
+    m = re.search(r"stop_grace_period:\s*(\d+)s", compose)
+    assert m, "docker-compose.single.yml has no stop_grace_period"
+    grace = int(m.group(1))
+
+    cp = _parse(with_ml=True)
+    for section in cp.sections():
+        if section.startswith("program:"):
+            assert cp.getint(section, "stopwaitsecs") <= grace, section
+
+
+# --- what runs, and how much ------------------------------------------------
+
+
+def test_a_zero_slot_lane_still_gets_a_running_process():
+    """ML ships at 0 slots and disabled — but `add_consumer` needs something to
+    reach. With no process there would be nothing for the UI switch to switch,
+    and enabling tagging could not work at all."""
+    cp = _parse(with_ml=True)
+    assert LANES_BY_NAME["ml"].default_slots == 0
+    env = cp.get("program:ml", "environment")
+    assert "CELERY_CONCURRENCY=1" in env
+
+
+def test_programs_restart_but_back_off_rather_than_looping():
+    """A lane that dies instantly and repeatedly is a broken image, not a
+    transient fault. Unbounded restarts would burn a core forever and bury the
+    original error under its own noise."""
+    cp = _parse()
+    for section in cp.sections():
+        if section.startswith("program:"):
+            assert cp.get(section, "autorestart") == "true", section
+            assert cp.getint(section, "startretries") >= 1, section
+
+
+def test_web_starts_first_because_it_runs_the_migration():
+    """A worker booting against an un-migrated schema fails in a way that
+    looks like application breakage rather than an ordering problem."""
+    cp = _parse()
+    assert cp.getint("program:web", "priority") == 1
+
+
+def test_every_program_writes_to_the_container_stdout_with_its_lane_named():
+    """Four celery workers and hypercorn on one stream are indistinguishable
+    without this. Unbuffered (`maxbytes 0`) so `docker logs` is live rather
+    than arriving in rotated chunks."""
+    cp = _parse(with_ml=True)
+    for section in cp.sections():
+        if not section.startswith("program:"):
+            continue
+        name = section.split(":", 1)[1]
+        assert cp.get(section, "stdout_logfile") == "/dev/fd/1", section
+        assert cp.getint(section, "stdout_logfile_maxbytes") == 0, section
+        assert cp.get(section, "redirect_stderr") == "true", section
+        assert f"[{name}] " in cp.get(section, "command"), section