"""Emit a supervisord config for the single-container layout. Milestone 422 step 5. Writes to stdout; `entrypoint.sh all` redirects it to a file and execs supervisord against it. ## Why this is generated and not a checked-in .conf A static config would spell out each lane's `-Q` list, and that would be a FIFTH hand-kept copy of the queue names — after `celery_app.task_routes`, and the three collapsed in steps 1, 2 and 4 (`service_roster.ROLE_NAMES`, `system_activity._QUEUE_NAMES`, and the Activity filter). Every one of those had already drifted by the time it was found. Generating from `worker_lanes.LANES` makes a stronger guarantee than "they match today": the processes this container runs and the lanes the application believes in are the same list, so a lane added to `LANES` gets a process without anyone remembering to add one, and a queue can never end up with no consumer because a config file was missed. ## Why supervisord It is one pip dependency on an image that is already Python, and it does the four things this needs without being clever: restart a program that exits, give each one its OWN stop timeout, signal the process GROUP rather than the leader, and put every program's output on one stdout. The process-group part is not a detail. Celery's prefork pool forks children, and a TERM delivered only to the parent leaves them running — which is how a "graceful" shutdown turns into orphaned workers holding tasks. `stopasgroup` and `killasgroup` are both set for every program. s6-overlay is the other standard answer and would work; it needs a build-time download and a second mental model, and its advantage (correct PID-1 signal and zombie handling) is available here from `init: true` in compose, which puts tini in front of supervisord. Neither choice reaches the application — nothing in FC talks to the supervisor — so this is reversible without touching a line of product code. ## What this does NOT start The `ml` lane, unless `--with-ml` is passed. Until step 6 merges the images, torch and the ML requirements live only in `Dockerfile.ml`, so an `ml` program in the web image would fail to import on every restart forever. Step 6 is where that flag turns on. """ from __future__ import annotations import argparse import shlex import sys from ..services.worker_lanes import LANES, Lane # One number for the whole container, and it must cover the SLOWEST lane — # docker gives the container a single stop timeout, where compose today gives # each service its own (90/60/180/120s). `maintenance_long` is the 180s one: # DB backups, library audits and translation backfill. Anything less turns a # routine restart into a SIGKILL mid-backup. # # Per-program values below are the old per-service ones, preserved: supervisord # waits `stopwaitsecs` for each, and they stop in parallel, so the container's # own timeout needs to cover the max rather than the sum. STOP_WAIT_SECONDS: dict[str, int] = { "worker": 90, "scheduler": 60, "maintenance_long": 180, "ml": 120, } DEFAULT_STOP_WAIT = 60 # Lanes whose code is not in this image yet. See the module docstring. _NEEDS_ML_DEPS = frozenset({"ml"}) def _program(lane: Lane, *, slots: int) -> str: """One [program:x] block. `stdout_logfile=/dev/fd/1` with maxbytes 0 puts the lane's output straight on the container's stdout unbuffered, so `docker logs` shows every lane interleaved rather than supervisord swallowing them into rotated files. The output is prefixed through `sed` so a line can be attributed to a lane — four celery workers and hypercorn on one stream are otherwise indistinguishable. The shell that the pipe requires is exactly why `stopasgroup` matters: the signal has to reach the celery process, not the `sh` holding the pipeline. """ inner = f"./entrypoint.sh {lane.entrypoint_role}" prefixed = f"{inner} 2>&1 | sed -u 's/^/[{lane.name}] /'" stop_wait = STOP_WAIT_SECONDS.get(lane.name, DEFAULT_STOP_WAIT) return "\n".join([ f"[program:{lane.name}]", f"command=sh -c {shlex.quote(prefixed)}", # QUOTED, and that is load-bearing. supervisord parses `environment` # as a COMMA-separated KEY=VALUE list, so an unquoted queue list reads # as CELERY_QUEUES=default followed by three malformed entries — and # the lane would consume only its first queue. Silent: the worker # starts, reports healthy, and simply never picks up `import`. f'environment=CELERY_QUEUES="{",".join(lane.queues)}",' f"CELERY_CONCURRENCY={slots}", "autostart=true", "autorestart=true", # A lane that dies instantly and repeatedly is a broken image, not a # transient fault. Backing off stops it burning a core in a restart # loop while still recovering from a one-off crash. "startretries=3", "startsecs=5", f"stopwaitsecs={stop_wait}", "stopasgroup=true", "killasgroup=true", "stdout_logfile=/dev/fd/1", "stdout_logfile_maxbytes=0", "redirect_stderr=true", "", ]) def _web_program() -> str: """hypercorn. Started FIRST (priority) because its role runs `alembic upgrade head`, and a worker that boots against an un-migrated schema fails in a way that looks like application breakage.""" prefixed = "./entrypoint.sh web 2>&1 | sed -u 's/^/[web] /'" return "\n".join([ "[program:web]", f"command=sh -c {shlex.quote(prefixed)}", "priority=1", "autostart=true", "autorestart=true", "startretries=3", "startsecs=5", # Short: HTTP requests and the occasional file download. Matches the # 30s the operator's production stack gives the web service. "stopwaitsecs=30", "stopasgroup=true", "killasgroup=true", "stdout_logfile=/dev/fd/1", "stdout_logfile_maxbytes=0", "redirect_stderr=true", "", ]) def render(*, with_ml: bool = False) -> str: parts = [ "\n".join([ "[supervisord]", # PID 1 in the container, so it must not daemonise. "nodaemon=true", # supervisord's OWN log. /dev/fd/1 keeps it on the container's # stdout beside the programs rather than in a file nobody reads. "logfile=/dev/fd/1", "logfile_maxbytes=0", "loglevel=info", "", ]), _web_program(), ] # Lanes after web, in LANES order, so the log reads in a stable sequence. for lane in LANES: if lane.name in _NEEDS_ML_DEPS and not with_ml: continue # A lane configured at zero slots still gets a PROCESS, at one slot # with its consumers cancelled by the reconcile. Without a running # worker there is nothing for `add_consumer` to reach, so enabling the # lane from the UI could not work at all — the process has to exist for # the switch to have something to switch. parts.append(_program(lane, slots=max(1, lane.default_slots))) return "\n".join(parts) def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument( "--with-ml", action="store_true", help="include the ml lane (only valid once the ML deps are in this image)", ) args = ap.parse_args(argv) sys.stdout.write(render(with_ml=args.with_ml)) return 0 if __name__ == "__main__": raise SystemExit(main())