diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 0ff85aa..078c4d0 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -1555,7 +1555,15 @@ jobs: echo "smoke: every lane answered" # Say WHICH processes supervisord is running, so a lane that is # merely restart-looping is visible rather than inferred. - docker exec "$CID_ALL" supervisorctl -c "${SUPERVISOR_CONF:-/tmp/supervisord.conf}" status || true + # + # NOT `|| true` any more. Behind that, this printed + # "Error: .ini file does not include supervisorctl section" for a + # whole run and passed — supervisord was fine and supervisorctl + # could not reach it, which is the tool an operator debugging a lane + # inside the one container reaches for first. A diagnostic allowed + # to fail silently is a diagnostic that stops being true without + # telling anyone. + docker exec "$CID_ALL" supervisorctl -c "${SUPERVISOR_CONF:-/tmp/supervisord.conf}" status echo "smoke: all checks passed against $CANDIDATE, with egress blocked" diff --git a/backend/app/scripts/gen_supervisord.py b/backend/app/scripts/gen_supervisord.py index 059853e..447e66b 100644 --- a/backend/app/scripts/gen_supervisord.py +++ b/backend/app/scripts/gen_supervisord.py @@ -123,6 +123,11 @@ def _program(lane: Lane, *, slots: int) -> str: ]) +# supervisord's control socket. /tmp for the same reason the generated config +# lives there — writable by every role, and per-container by nature. +SOCKET_PATH = "/tmp/supervisor.sock" + + def _web_program() -> str: """hypercorn. Started FIRST (priority) because its role runs `alembic upgrade head`, and a worker that boots against an un-migrated @@ -161,6 +166,43 @@ def render() -> str: "loglevel=info", "", ]), + # THE CONTROL SOCKET, and it is not optional furniture. + # + # Without these three sections supervisord runs perfectly and + # `supervisorctl` cannot talk to it at all: + # + # Error: .ini file does not include supervisorctl section + # + # Which is the first thing anyone reaches for when a lane misbehaves + # in the consolidated container — `docker exec supervisorctl + # status` to see which processes are up, or `restart ml` to bounce one + # without taking the whole application down with it. Consolidation + # took away `docker ps` as the way to see the lanes; this is what + # replaces it, and shipping without it would have left an operator + # with one container, five processes inside it, and no way to ask + # about any of them. + # + # Found by the smoke's own diagnostic line on run 7322, which printed + # this error instead of a process list. It was behind `|| true`, so it + # cost nothing and said so anyway — the argument for printing evidence + # even where nothing depends on it. + # + # /tmp, like the generated config itself: writable by every role + # without assuming a volume, and per-container state that must not + # outlive the container. + "\n".join([ + "[unix_http_server]", + f"file={SOCKET_PATH}", + "chmod=0700", + "", + "[rpcinterface:supervisor]", + "supervisor.rpcinterface_factory = " + "supervisor.rpcinterface:make_main_rpcinterface", + "", + "[supervisorctl]", + f"serverurl=unix://{SOCKET_PATH}", + "", + ]), _web_program(), ] # Lanes after web, in LANES order, so the log reads in a stable sequence. diff --git a/tests/test_gen_supervisord.py b/tests/test_gen_supervisord.py index 6e6d5fa..d7c5c74 100644 --- a/tests/test_gen_supervisord.py +++ b/tests/test_gen_supervisord.py @@ -231,3 +231,33 @@ def test_every_lane_gets_its_own_celery_node_name(): ) for lane, node in names.items(): assert node == lane, f"{lane} announces itself as {node!r}" + + +def test_supervisorctl_can_reach_supervisord(): + """Consolidation took away `docker ps` as the way to see the lanes, and + this is what replaces it. + + Without the three control-socket sections supervisord runs perfectly and + `supervisorctl` cannot talk to it at all: + + Error: .ini file does not include supervisorctl section + + That is the first thing anyone reaches for when a lane misbehaves inside + the one container — `supervisorctl status` to see which processes are up, + `restart ml` to bounce one without taking the application down. Shipping + without it leaves an operator with five processes and no way to ask about + any of them. + + Asserted as the three sections AGREEING on one socket path, not merely as + three sections existing: a serverurl pointing somewhere the server does + not listen fails exactly the same way, and reads as configured. + """ + cp = _parse() + for section in ("unix_http_server", "supervisorctl", "rpcinterface:supervisor"): + assert cp.has_section(section), f"no [{section}] — supervisorctl is blind" + + listening = cp.get("unix_http_server", "file") + talking = cp.get("supervisorctl", "serverurl") + assert talking == f"unix://{listening}", ( + f"supervisorctl talks to {talking}, supervisord listens on {listening}" + )