"""The container's healthcheck. Picks the right check from the role it runs. Exit 0 healthy, non-zero unhealthy. ## Why this is in the IMAGE and not in every compose file Because the image is the only thing that knows what it is running. A deployment had to declare a healthcheck per service, which meant every compose file, stack file and README repeated the same knowledge: web -> curl /api/health worker -> celery inspect ping -d celery@$HOSTNAME all -> both, for every lane Three checks, written out by hand, once per service, in every file anyone ever wrote — and none of them wrong until a role changed. Operator, 2026-09-23: *"why isn't the healthcheck built into the image or base on what command runs if one is passed in."* There was no reason. The role is already a fact the container holds; asking the deployment to restate it is the same duplication the lane table exists to remove one level down. So `entrypoint.sh` records the role it started, the Dockerfile declares ONE `HEALTHCHECK` that runs this, and a stack file says nothing at all. Declaring one anyway still works — docker lets a service override the image's — which is the escape hatch for a deployment that genuinely wants something else. ## What each role is asked * **web** — hypercorn answers `/api/health`. No database: the endpoint is a no-DB 200 that proves the app booted and is serving after `alembic upgrade head`, which is what a rolling deploy needs to know. * **worker / scheduler / ml-worker** — THIS container's celery node answers a ping over the broker. Not "some worker answered": the node name is pinned to this container, or a healthy sibling would keep a dead one looking alive. * **all** — both halves, for every lane in the table. The failure mode consolidation creates is that docker can no longer see the lanes as separate services, so a web-only check reports a healthy container with every worker dead. * **shell / alembic / anything else** — nothing to check. These are one-shot or interactive; a liveness probe on them has no meaning, so it passes rather than inventing a verdict. ## An unrecorded role passes rather than failing If the role file is missing, the entrypoint did not run — someone used `--entrypoint` or ran a bare command. That is a debugging shape, and a healthcheck that cannot tell what it is looking at must not assert that the thing is broken (snippet #3969: an unswept read is not a verdict). It says so on stdout and exits 0. """ from __future__ import annotations import os import socket import sys import urllib.error import urllib.request # Written by entrypoint.sh at boot. /tmp because it is the one path writable # by every role without assuming a volume, and the value is per-container # state that must NOT survive into a new container. ROLE_FILE = os.environ.get("FC_ROLE_FILE", "/tmp/fc-role") WEB_URL = "http://localhost:8080/api/health" WEB_TIMEOUT = 5.0 # A broker round trip, so it gets a deadline (rule 156). Generous relative to # `inspect`'s 2s elsewhere: this runs every 30s with retries, and a transient # blip flagging a worker unhealthy would roll back a deployment that is fine. PING_TIMEOUT = 10.0 CELERY_ROLES = {"worker", "scheduler", "ml-worker"} # Roles with nothing to probe. Listed rather than treated as the default, so # an unknown role takes the "I cannot tell" path and says so. NO_CHECK_ROLES = {"shell", "bash", "alembic"} def current_role() -> str | None: """The role this container was started with, or None if nothing recorded.""" env = os.environ.get("FC_ROLE") if env: return env.strip() try: with open(ROLE_FILE) as fh: return fh.read().strip() or None except OSError: return None def _web_ok() -> tuple[bool, str]: try: with urllib.request.urlopen(WEB_URL, timeout=WEB_TIMEOUT) as resp: if resp.status == 200: return True, "" return False, f"web returned {resp.status}" except (urllib.error.URLError, OSError) as exc: return False, f"web unreachable: {exc}" def _this_node_ok() -> tuple[bool, str]: """Ping THIS container's celery node, by name. Pinned to this node deliberately. A bare `ping()` is answered by any worker on the broker, so in a stack with several replicas a dead one would go on reporting healthy for as long as a sibling was alive — the healthcheck would be measuring the cluster, not the container it is in. """ from ..celery_app import celery as celery_app node = f"{os.environ.get('CELERY_NODENAME', 'celery')}@{socket.gethostname()}" try: replies = celery_app.control.ping(destination=[node], timeout=PING_TIMEOUT) except Exception as exc: # noqa: BLE001 — a probe reports, never raises return False, f"could not reach the broker: {exc}" if not replies: return False, f"{node} did not answer a ping" return True, "" def _lanes_ok() -> tuple[bool, str]: """Every lane in the table is answering. Deliberately ignores whether a lane is ENABLED: a disabled lane still runs its process with its consumers cancelled, so it answers `inspect` and is healthy. Health is "is the process alive"; whether it should be consuming is a settings question the reconcile owns, and conflating them would make turning a lane off in the UI mark the container unhealthy. """ from ..services.worker_control import inspect_lanes_sync from ..services.worker_lanes import LANES live = inspect_lanes_sync() missing = sorted(lane.name for lane in LANES if not live[lane.name].present) if missing: return False, "lanes not answering: " + ", ".join(missing) return True, "" def main(argv: list[str] | None = None) -> int: role = current_role() if role is None: # Not a failure. See the module docstring: the entrypoint did not run, # so there is no role to check against and no basis for a verdict. print("no role recorded; nothing to check") return 0 if role in NO_CHECK_ROLES: print(f"{role}: nothing to check") return 0 checks = [] if role == "all": checks = [_web_ok, _lanes_ok] elif role == "web": checks = [_web_ok] elif role in CELERY_ROLES: checks = [_this_node_ok] else: print(f"unknown role {role!r}; nothing to check") return 0 for check in checks: ok, detail = check() if not ok: print(detail, file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())