diff --git a/backend/app/scripts/gen_supervisord.py b/backend/app/scripts/gen_supervisord.py new file mode 100644 index 0000000..2747599 --- /dev/null +++ b/backend/app/scripts/gen_supervisord.py @@ -0,0 +1,184 @@ +"""Emit a supervisord config for the single-container layout. + +Milestone 422 step 5. Writes to stdout; `entrypoint.sh all` redirects it to a +file and execs supervisord against it. + +## Why this is generated and not a checked-in .conf + +A static config would spell out each lane's `-Q` list, and that would be a +FIFTH hand-kept copy of the queue names — after `celery_app.task_routes`, and +the three collapsed in steps 1, 2 and 4 (`service_roster.ROLE_NAMES`, +`system_activity._QUEUE_NAMES`, and the Activity filter). Every one of those +had already drifted by the time it was found. + +Generating from `worker_lanes.LANES` makes a stronger guarantee than "they +match today": the processes this container runs and the lanes the application +believes in are the same list, so a lane added to `LANES` gets a process +without anyone remembering to add one, and a queue can never end up with no +consumer because a config file was missed. + +## Why supervisord + +It is one pip dependency on an image that is already Python, and it does the +four things this needs without being clever: restart a program that exits, +give each one its OWN stop timeout, signal the process GROUP rather than the +leader, and put every program's output on one stdout. + +The process-group part is not a detail. Celery's prefork pool forks children, +and a TERM delivered only to the parent leaves them running — which is how a +"graceful" shutdown turns into orphaned workers holding tasks. `stopasgroup` +and `killasgroup` are both set for every program. + +s6-overlay is the other standard answer and would work; it needs a build-time +download and a second mental model, and its advantage (correct PID-1 signal +and zombie handling) is available here from `init: true` in compose, which +puts tini in front of supervisord. Neither choice reaches the application — +nothing in FC talks to the supervisor — so this is reversible without touching +a line of product code. + +## What this does NOT start + +The `ml` lane, unless `--with-ml` is passed. Until step 6 merges the images, +torch and the ML requirements live only in `Dockerfile.ml`, so an `ml` program +in the web image would fail to import on every restart forever. Step 6 is +where that flag turns on. +""" + +from __future__ import annotations + +import argparse +import shlex +import sys + +from ..services.worker_lanes import LANES, Lane + +# One number for the whole container, and it must cover the SLOWEST lane — +# docker gives the container a single stop timeout, where compose today gives +# each service its own (90/60/180/120s). `maintenance_long` is the 180s one: +# DB backups, library audits and translation backfill. Anything less turns a +# routine restart into a SIGKILL mid-backup. +# +# Per-program values below are the old per-service ones, preserved: supervisord +# waits `stopwaitsecs` for each, and they stop in parallel, so the container's +# own timeout needs to cover the max rather than the sum. +STOP_WAIT_SECONDS: dict[str, int] = { + "worker": 90, + "scheduler": 60, + "maintenance_long": 180, + "ml": 120, +} +DEFAULT_STOP_WAIT = 60 + +# Lanes whose code is not in this image yet. See the module docstring. +_NEEDS_ML_DEPS = frozenset({"ml"}) + + +def _program(lane: Lane, *, slots: int) -> str: + """One [program:x] block. + + `stdout_logfile=/dev/fd/1` with maxbytes 0 puts the lane's output straight + on the container's stdout unbuffered, so `docker logs` shows every lane + interleaved rather than supervisord swallowing them into rotated files. + + The output is prefixed through `sed` so a line can be attributed to a lane + — four celery workers and hypercorn on one stream are otherwise + indistinguishable. The shell that the pipe requires is exactly why + `stopasgroup` matters: the signal has to reach the celery process, not the + `sh` holding the pipeline. + """ + inner = f"./entrypoint.sh {lane.entrypoint_role}" + prefixed = f"{inner} 2>&1 | sed -u 's/^/[{lane.name}] /'" + stop_wait = STOP_WAIT_SECONDS.get(lane.name, DEFAULT_STOP_WAIT) + return "\n".join([ + f"[program:{lane.name}]", + f"command=sh -c {shlex.quote(prefixed)}", + # QUOTED, and that is load-bearing. supervisord parses `environment` + # as a COMMA-separated KEY=VALUE list, so an unquoted queue list reads + # as CELERY_QUEUES=default followed by three malformed entries — and + # the lane would consume only its first queue. Silent: the worker + # starts, reports healthy, and simply never picks up `import`. + f'environment=CELERY_QUEUES="{",".join(lane.queues)}",' + f"CELERY_CONCURRENCY={slots}", + "autostart=true", + "autorestart=true", + # A lane that dies instantly and repeatedly is a broken image, not a + # transient fault. Backing off stops it burning a core in a restart + # loop while still recovering from a one-off crash. + "startretries=3", + "startsecs=5", + f"stopwaitsecs={stop_wait}", + "stopasgroup=true", + "killasgroup=true", + "stdout_logfile=/dev/fd/1", + "stdout_logfile_maxbytes=0", + "redirect_stderr=true", + "", + ]) + + +def _web_program() -> str: + """hypercorn. Started FIRST (priority) because its role runs + `alembic upgrade head`, and a worker that boots against an un-migrated + schema fails in a way that looks like application breakage.""" + prefixed = "./entrypoint.sh web 2>&1 | sed -u 's/^/[web] /'" + return "\n".join([ + "[program:web]", + f"command=sh -c {shlex.quote(prefixed)}", + "priority=1", + "autostart=true", + "autorestart=true", + "startretries=3", + "startsecs=5", + # Short: HTTP requests and the occasional file download. Matches the + # 30s the operator's production stack gives the web service. + "stopwaitsecs=30", + "stopasgroup=true", + "killasgroup=true", + "stdout_logfile=/dev/fd/1", + "stdout_logfile_maxbytes=0", + "redirect_stderr=true", + "", + ]) + + +def render(*, with_ml: bool = False) -> str: + parts = [ + "\n".join([ + "[supervisord]", + # PID 1 in the container, so it must not daemonise. + "nodaemon=true", + # supervisord's OWN log. /dev/fd/1 keeps it on the container's + # stdout beside the programs rather than in a file nobody reads. + "logfile=/dev/fd/1", + "logfile_maxbytes=0", + "loglevel=info", + "", + ]), + _web_program(), + ] + # Lanes after web, in LANES order, so the log reads in a stable sequence. + for lane in LANES: + if lane.name in _NEEDS_ML_DEPS and not with_ml: + continue + # A lane configured at zero slots still gets a PROCESS, at one slot + # with its consumers cancelled by the reconcile. Without a running + # worker there is nothing for `add_consumer` to reach, so enabling the + # lane from the UI could not work at all — the process has to exist for + # the switch to have something to switch. + parts.append(_program(lane, slots=max(1, lane.default_slots))) + return "\n".join(parts) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--with-ml", action="store_true", + help="include the ml lane (only valid once the ML deps are in this image)", + ) + args = ap.parse_args(argv) + sys.stdout.write(render(with_ml=args.with_ml)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/scripts/healthcheck_all.py b/backend/app/scripts/healthcheck_all.py new file mode 100644 index 0000000..4b8076a --- /dev/null +++ b/backend/app/scripts/healthcheck_all.py @@ -0,0 +1,83 @@ +"""Container healthcheck for the single-container layout. + +Milestone 422 step 5. Exit 0 healthy, non-zero unhealthy. + +## Why this is not just "does :8080 answer" + +In the multi-service stack every service has its OWN healthcheck, so a dead +worker turns that service unhealthy while web stays green — docker knows which +part failed. Collapsing them into one container collapses that too: a web-only +check would report a perfectly healthy container while every lane inside it +had crashed and been abandoned by supervisord after its retries. + +So this asserts both halves: hypercorn answers, AND every lane this container +was configured to run is answering the broker. + +## What it deliberately does NOT do + +It does not read the database, and it does not consult the `enabled` flag. A +DISABLED lane still has a running process with its consumers cancelled (see +the config generator), so it answers `inspect` and is healthy. Health is +"is the process alive", and whether it should be consuming is a settings +question the reconcile owns — conflating them would make turning a lane off +in the UI mark the container unhealthy. + +It also cannot distinguish "the broker is down" from "every lane is down", +and reports unhealthy either way. That is correct: a container that cannot +reach its broker is not serving, whichever half is at fault. +""" + +from __future__ import annotations + +import sys +import urllib.error +import urllib.request + +WEB_URL = "http://localhost:8080/api/health" +WEB_TIMEOUT = 5.0 + + +def _web_ok() -> tuple[bool, str]: + try: + with urllib.request.urlopen(WEB_URL, timeout=WEB_TIMEOUT) as resp: + if resp.status == 200: + return True, "" + return False, f"web returned {resp.status}" + except (urllib.error.URLError, OSError) as exc: + return False, f"web unreachable: {exc}" + + +def _lanes_ok(*, with_ml: bool) -> tuple[bool, str]: + from ..services.worker_control import inspect_lanes_sync + from ..services.worker_lanes import LANES + + expected = { + lane.name for lane in LANES + if with_ml or lane.name != "ml" + } + live = inspect_lanes_sync() + missing = sorted(n for n in expected if not live[n].present) + if missing: + return False, "lanes not answering: " + ", ".join(missing) + return True, "" + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + with_ml = "--with-ml" in argv + + ok, detail = _web_ok() + if not ok: + print(detail, file=sys.stderr) + return 1 + + ok, detail = _lanes_ok(with_ml=with_ml) + if not ok: + print(detail, file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/services/worker_lanes.py b/backend/app/services/worker_lanes.py index a0d7444..cdd639f 100644 --- a/backend/app/services/worker_lanes.py +++ b/backend/app/services/worker_lanes.py @@ -71,6 +71,12 @@ class Lane: name: str display_name: str queues: tuple[str, ...] + # Which `entrypoint.sh` role starts this lane. NOT always the lane name: + # `maintenance_long` is the plain `worker` role pointed at a different + # queue, exactly as docker-compose starts it today (`command: ["worker"]` + # with CELERY_QUEUES=maintenance_long). Recorded here so the generated + # supervisord config and the compose file cannot disagree about it. + entrypoint_role: str default_slots: int # The cap a lane STARTS with, which is not the ceiling. Set low enough # that raising slots within it is an ordinary adjustment, and raising the @@ -104,6 +110,7 @@ LANES: tuple[Lane, ...] = ( name="worker", display_name="Worker", queues=("default", "import", "thumbnail", "download"), + entrypoint_role="worker", default_slots=1, default_slots_cap=4, default_enabled=True, @@ -112,6 +119,7 @@ LANES: tuple[Lane, ...] = ( name="scheduler", display_name="Scheduler", queues=("maintenance", "scan"), + entrypoint_role="scheduler", default_slots=1, default_slots_cap=2, default_enabled=True, @@ -120,6 +128,7 @@ LANES: tuple[Lane, ...] = ( name="maintenance_long", display_name="Long maintenance", queues=("maintenance_long",), + entrypoint_role="worker", default_slots=1, default_slots_cap=2, default_enabled=True, @@ -128,6 +137,7 @@ LANES: tuple[Lane, ...] = ( name="ml", display_name="ML tagging", queues=("ml",), + entrypoint_role="ml-worker", default_slots=0, default_slots_cap=1, default_enabled=False, diff --git a/docker-compose.single.yml b/docker-compose.single.yml new file mode 100644 index 0000000..d94741c --- /dev/null +++ b/docker-compose.single.yml @@ -0,0 +1,113 @@ +# FabledCurator in three containers — the install path. +# +# docker compose -f docker-compose.single.yml up -d +# +# Milestone 422 step 5. FabledCurator runs web and every worker lane inside +# ONE container, with Postgres and Redis beside it. How much work each lane +# does is then a dial in the web UI (Settings -> Activity -> Worker lanes), +# live, with no compose edit and no restart. +# +# THE MULTI-SERVICE STACK IS NOT REPLACED. `docker-compose.yml` still runs the +# five app services separately and is the right shape for a Swarm deployment +# spread across hosts, where per-service rolling rollback and placement +# constraints matter. This file is the adopter path: one box, one command. +# +# What consolidating costs, stated here rather than discovered later: +# - Everything shares one host, so there is no spreading work across nodes. +# - Rollback is all-or-nothing; there is no rolling back `web` alone. +# - The ML lane cannot write to the library read-only any more — in the +# multi-service stack ml-worker mounts /images:ro, and one container +# cannot mount one path two ways. +# - One stop timeout for the whole container, sized to the slowest lane. +# +# FabledCurator has no authentication. Whatever can reach ${PORT} is an +# administrator, including over the stored platform session cookies. Do not +# publish this port beyond a network you trust — see "Before you expose it" +# in README.md. + +services: + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: ${DB_USER:-curator} + POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} + POSTGRES_DB: ${DB_NAME:-curator} + volumes: + - postgres_data:/var/lib/postgresql/data + # pgvector index builds and the gallery's TABLESAMPLE reads both want more + # shared memory than docker's 64MB default. + shm_size: 512m + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-curator} -d ${DB_NAME:-curator}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + fabledcurator: + image: git.fabledsword.com/bvandeusen/fabledcurator:latest + # Everything: hypercorn plus one celery process per lane, under + # supervisord, whose config is generated from the application's own lane + # table so the two cannot disagree. + command: ["all"] + # tini as PID 1, in front of supervisord. supervisord reaps its own + # children, but a container's PID 1 also inherits orphans from anywhere + # below — celery's prefork pool and gallery-dl's subprocesses both make + # them. Without this they accumulate as zombies for the life of the + # container. + init: true + # Sized to the SLOWEST lane, not the average. maintenance_long runs DB + # backups, library audits and translation backfill, and gets 180s to + # finish a chunk; the lanes stop in parallel, so this covers the max + # rather than their sum. Below this, a routine restart becomes a SIGKILL + # mid-backup — which is recoverable (the work is chunked and idempotent) + # but wastes however long it had run. + stop_grace_period: 200s + # BOTH halves: hypercorn answers AND every configured lane is answering + # the broker. A web-only check would report a healthy container while + # every lane inside it had crashed — the failure mode consolidation + # creates, since docker can no longer see the lanes as separate services. + healthcheck: + test: ["CMD", "python", "-m", "backend.app.scripts.healthcheck_all"] + interval: 30s + timeout: 15s + retries: 3 + # Covers alembic + hypercorn boot + four celery workers registering. + start_period: 90s + environment: + DB_USER: ${DB_USER:-curator} + DB_PASSWORD: ${DB_PASSWORD:-postgres} + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: ${DB_NAME:-curator} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + SECRET_KEY: ${SECRET_KEY:-change-me-before-you-expose-this} + EXTENSION_API_KEY: ${EXTENSION_API_KEY:-} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + ports: + - "${PORT:-8080}:8080" + volumes: + - ${IMAGES_DIR:-./images}:/images + # Read-only. The filesystem scan copies out of here and never writes to + # it, so a mistake cannot reach the source library. + - ${IMPORT_DIR:-./import}:/import:ro + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + restart: unless-stopped + +volumes: + redis_data: + postgres_data: diff --git a/entrypoint.sh b/entrypoint.sh index 9231cce..b03b252 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -35,12 +35,18 @@ case "$ROLE" in scheduler) QUEUES="${CELERY_QUEUES:-maintenance,scan}" - echo "[entrypoint] Starting Celery beat+worker queues=$QUEUES" + # Honours CELERY_CONCURRENCY like the `worker` role does. It was hardcoded + # to 1, which was harmless while only compose started this lane and set no + # concurrency for it — but the generated supervisord config (milestone 422 + # step 5) passes one, and a value silently ignored at boot would leave the + # lane at 1 until the reconcile sweep noticed, with nothing saying why. + CONCURRENCY="${CELERY_CONCURRENCY:-1}" + echo "[entrypoint] Starting Celery beat+worker queues=$QUEUES concurrency=$CONCURRENCY" exec celery -A backend.app.celery_app:celery worker \ --beat \ --loglevel=info \ -Q "$QUEUES" \ - --concurrency=1 + --concurrency="$CONCURRENCY" ;; ml-worker) @@ -53,6 +59,27 @@ case "$ROLE" in --concurrency=1 ;; + all) + # The single-container layout (milestone 422 step 5): hypercorn plus one + # celery process per lane, under supervisord, in one container beside + # Postgres and Redis. + # + # The config is GENERATED from services/worker_lanes.LANES rather than + # checked in, so the processes this container runs and the lanes the + # application believes in cannot disagree — see the generator's docstring + # for why a static .conf would have been a fifth copy of the queue names. + # + # supervisord is PID 1 here and never reads the database. Every lane boots + # at its LANES default; the reconcile sweep raises it to whatever the + # operator stored, within one tick. That ordering is deliberate: settings + # adjust a baseline that already works, and can never prevent a boot. + CONF="${SUPERVISOR_CONF:-/tmp/supervisord.conf}" + echo "[entrypoint] Generating $CONF from the lane table" + python -m backend.app.scripts.gen_supervisord ${FC_WITH_ML:+--with-ml} > "$CONF" + echo "[entrypoint] Starting supervisord (web + worker lanes)" + exec supervisord -c "$CONF" + ;; + shell|bash) exec /bin/bash "$@" ;; @@ -63,7 +90,7 @@ case "$ROLE" in *) echo "[entrypoint] Unknown role: $ROLE" >&2 - echo "[entrypoint] Valid roles: web | worker | scheduler | ml-worker | shell | alembic" >&2 + echo "[entrypoint] Valid roles: all | web | worker | scheduler | maintenance_long | ml | ml-worker | shell | alembic" >&2 exit 1 ;; esac diff --git a/requirements.txt b/requirements.txt index 1950b50..f54bc43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,3 +43,14 @@ py7zr>=1,<2 # #830). Handles Drive's confirm-token + virus-scan interstitial. mega.nz uses # the `megatools` binary instead (Debian apt pkg in the runtime image, not pip). gdown>=6,<7 + +# Process supervisor for the single-container layout (milestone 422 step 5). +# `entrypoint.sh all` generates its config from the lane table and execs it as +# PID 1, running hypercorn plus one celery process per lane in one container. +# Unused by the multi-service compose path, where docker supervises instead. +# +# Chosen over s6-overlay because it is a pip install on an image that is +# already Python, and gives per-program stop timeouts plus stopasgroup — +# celery's prefork pool forks children, and a TERM that reaches only the +# parent leaves them orphaned holding tasks. +supervisor>=4.2,<5 diff --git a/tests/test_gen_supervisord.py b/tests/test_gen_supervisord.py new file mode 100644 index 0000000..b92cce3 --- /dev/null +++ b/tests/test_gen_supervisord.py @@ -0,0 +1,185 @@ +"""The generated supervisord config (milestone 422 step 5). + +Asserts the config against the LANE TABLE rather than against a fixture of +expected text. A fixture would have to be updated whenever a lane changes, +which is the same hand-kept coupling generating the config exists to remove — +and it would pass while describing a container that does not match the +application's own idea of what it runs. +""" + +from __future__ import annotations + +import configparser + +from backend.app.scripts import gen_supervisord as gen +from backend.app.services.worker_lanes import LANES, LANES_BY_NAME + +# --- structure --------------------------------------------------------------- + + +def _parse(**kwargs) -> configparser.ConfigParser: + """supervisord's config is ini, so parse it rather than grepping strings. + + A substring assertion passes on a line that is present but malformed — + inside a comment, in the wrong section, or with a typo'd key that + supervisord silently ignores. + """ + cp = configparser.ConfigParser() + cp.read_string(gen.render(**kwargs)) + return cp + + +def test_it_is_valid_ini_with_a_supervisord_section(): + cp = _parse() + assert cp.has_section("supervisord") + # PID 1 in a container: daemonising would exit immediately and take the + # container with it. + assert cp.get("supervisord", "nodaemon") == "true" + + +def test_web_and_every_non_ml_lane_get_a_program(): + cp = _parse() + expected = {"program:web"} | { + f"program:{lane.name}" for lane in LANES if lane.name != "ml" + } + assert set(cp.sections()) - {"supervisord"} == expected + + +def test_the_ml_lane_is_absent_until_its_deps_are_in_the_image(): + """The web image has no torch. An `ml` program here would fail to import + on every restart, forever — startretries would give up and the lane would + be permanently dead while the container reported healthy.""" + assert not _parse().has_section("program:ml") + assert _parse(with_ml=True).has_section("program:ml") + + +# --- the coupling this generator exists to guarantee ------------------------- + + +def test_each_program_serves_exactly_its_lane_s_queues(): + """The whole point: the container's processes and the application's lane + table are one list. A queue in LANES with no program means work that + queues forever with nothing consuming it.""" + cp = _parse(with_ml=True) + for lane in LANES: + env = cp.get(f"program:{lane.name}", "environment") + # The QUOTED form. supervisord splits `environment` on commas, so an + # unquoted multi-queue value silently degrades to its first queue — + # and an assertion on the bare string passes either way, which is how + # that would have shipped. + assert f'CELERY_QUEUES="{",".join(lane.queues)}"' in env + + +def test_each_program_invokes_the_lane_s_entrypoint_role_not_its_name(): + """`maintenance_long` is the plain `worker` role pointed at a different + queue — exactly as docker-compose starts it today. Invoking + `entrypoint.sh maintenance_long` would hit the unknown-role branch and + exit 1 on every restart.""" + cp = _parse(with_ml=True) + for lane in LANES: + command = cp.get(f"program:{lane.name}", "command") + assert f"entrypoint.sh {lane.entrypoint_role}" in command + + +def test_every_entrypoint_role_a_lane_names_actually_exists(): + """Reads entrypoint.sh itself. The generator can only emit a role name; + whether the script handles it is a separate fact, and getting it wrong + fails at container start rather than here.""" + from pathlib import Path + + script = Path(__file__).resolve().parents[1] / "entrypoint.sh" + text = script.read_text() + for lane in LANES: + # Roles are `case` arms: ` worker)` possibly in an alternation. + assert f" {lane.entrypoint_role})" in text or \ + f"|{lane.entrypoint_role})" in text, \ + f"{lane.name} names entrypoint role {lane.entrypoint_role!r}, which does not exist" + + +# --- shutdown ---------------------------------------------------------------- + + +def test_every_program_signals_its_whole_process_group(): + """Celery's prefork pool forks children. A TERM delivered only to the + parent leaves them running and holding tasks — a 'graceful' shutdown that + orphans workers. The `sh -c … | sed` wrapper makes this doubly necessary: + without it the signal reaches the shell holding the pipeline, not celery.""" + cp = _parse(with_ml=True) + for section in cp.sections(): + if not section.startswith("program:"): + continue + assert cp.get(section, "stopasgroup") == "true", section + assert cp.get(section, "killasgroup") == "true", section + + +def test_the_long_maintenance_lane_keeps_its_180s_drain(): + """The per-service stop_grace_period values from the multi-service stack + are preserved per program. maintenance_long runs DB backups and library + audits; cutting its drain turns a restart into a SIGKILL mid-backup.""" + cp = _parse() + assert cp.getint("program:maintenance_long", "stopwaitsecs") == 180 + + +def test_no_program_waits_longer_than_the_compose_stop_grace_period(): + """The container gets ONE timeout and the programs stop in parallel, so it + must cover the slowest. If a lane's stopwaitsecs ever exceeds what + docker-compose.single.yml allows, docker kills the container while that + lane still believes it has time to drain.""" + import re + from pathlib import Path + + compose = (Path(__file__).resolve().parents[1] / "docker-compose.single.yml").read_text() + m = re.search(r"stop_grace_period:\s*(\d+)s", compose) + assert m, "docker-compose.single.yml has no stop_grace_period" + grace = int(m.group(1)) + + cp = _parse(with_ml=True) + for section in cp.sections(): + if section.startswith("program:"): + assert cp.getint(section, "stopwaitsecs") <= grace, section + + +# --- what runs, and how much ------------------------------------------------ + + +def test_a_zero_slot_lane_still_gets_a_running_process(): + """ML ships at 0 slots and disabled — but `add_consumer` needs something to + reach. With no process there would be nothing for the UI switch to switch, + and enabling tagging could not work at all.""" + cp = _parse(with_ml=True) + assert LANES_BY_NAME["ml"].default_slots == 0 + env = cp.get("program:ml", "environment") + assert "CELERY_CONCURRENCY=1" in env + + +def test_programs_restart_but_back_off_rather_than_looping(): + """A lane that dies instantly and repeatedly is a broken image, not a + transient fault. Unbounded restarts would burn a core forever and bury the + original error under its own noise.""" + cp = _parse() + for section in cp.sections(): + if section.startswith("program:"): + assert cp.get(section, "autorestart") == "true", section + assert cp.getint(section, "startretries") >= 1, section + + +def test_web_starts_first_because_it_runs_the_migration(): + """A worker booting against an un-migrated schema fails in a way that + looks like application breakage rather than an ordering problem.""" + cp = _parse() + assert cp.getint("program:web", "priority") == 1 + + +def test_every_program_writes_to_the_container_stdout_with_its_lane_named(): + """Four celery workers and hypercorn on one stream are indistinguishable + without this. Unbuffered (`maxbytes 0`) so `docker logs` is live rather + than arriving in rotated chunks.""" + cp = _parse(with_ml=True) + for section in cp.sections(): + if not section.startswith("program:"): + continue + name = section.split(":", 1)[1] + assert cp.get(section, "stdout_logfile") == "/dev/fd/1", section + assert cp.getint(section, "stdout_logfile_maxbytes") == 0, section + assert cp.get(section, "redirect_stderr") == "true", section + assert f"[{name}] " in cp.get(section, "command"), section