FabledCurator/backend/app/scripts/healthcheck.py

"""The container's healthcheck. Picks the right check from the role it runs.

Exit 0 healthy, non-zero unhealthy.

## Why this is in the IMAGE and not in every compose file

Because the image is the only thing that knows what it is running. A
deployment had to declare a healthcheck per service, which meant every
compose file, stack file and README repeated the same knowledge:

    web         -> curl /api/health
    worker      -> celery inspect ping -d celery@$HOSTNAME
    all         -> both, for every lane

Three checks, written out by hand, once per service, in every file anyone
ever wrote — and none of them wrong until a role changed. Operator, 2026-09-23:
*"why isn't the healthcheck built into the image or base on what command runs
if one is passed in."* There was no reason. The role is already a fact the
container holds; asking the deployment to restate it is the same duplication
the lane table exists to remove one level down.

So `entrypoint.sh` records the role it started, the Dockerfile declares ONE
`HEALTHCHECK` that runs this, and a stack file says nothing at all. Declaring
one anyway still works — docker lets a service override the image's — which
is the escape hatch for a deployment that genuinely wants something else.

## What each role is asked

* **web** — hypercorn answers `/api/health`. No database: the endpoint is a
  no-DB 200 that proves the app booted and is serving after `alembic upgrade
  head`, which is what a rolling deploy needs to know.
* **worker / scheduler / ml-worker** — THIS container's celery node answers a
  ping over the broker. Not "some worker answered": the node name is pinned
  to this container, or a healthy sibling would keep a dead one looking alive.
* **all** — both halves, for every lane in the table. The failure mode
  consolidation creates is that docker can no longer see the lanes as
  separate services, so a web-only check reports a healthy container with
  every worker dead.
* **shell / alembic / anything else** — nothing to check. These are one-shot
  or interactive; a liveness probe on them has no meaning, so it passes
  rather than inventing a verdict.

## An unrecorded role passes rather than failing

If the role file is missing, the entrypoint did not run — someone used
`--entrypoint` or ran a bare command. That is a debugging shape, and a
healthcheck that cannot tell what it is looking at must not assert that the
thing is broken (snippet #3969: an unswept read is not a verdict). It says so
on stdout and exits 0.
"""

from __future__ import annotations

import os
import socket
import sys
import urllib.error
import urllib.request

# Written by entrypoint.sh at boot. /tmp because it is the one path writable
# by every role without assuming a volume, and the value is per-container
# state that must NOT survive into a new container.
ROLE_FILE = os.environ.get("FC_ROLE_FILE", "/tmp/fc-role")

WEB_URL = "http://localhost:8080/api/health"
WEB_TIMEOUT = 5.0

# A broker round trip, so it gets a deadline (rule 156). Generous relative to
# `inspect`'s 2s elsewhere: this runs every 30s with retries, and a transient
# blip flagging a worker unhealthy would roll back a deployment that is fine.
PING_TIMEOUT = 10.0

CELERY_ROLES = {"worker", "scheduler", "ml-worker"}
# Roles with nothing to probe. Listed rather than treated as the default, so
# an unknown role takes the "I cannot tell" path and says so.
NO_CHECK_ROLES = {"shell", "bash", "alembic"}


def current_role() -> str | None:
    """The role this container was started with, or None if nothing recorded."""
    env = os.environ.get("FC_ROLE")
    if env:
        return env.strip()
    try:
        with open(ROLE_FILE) as fh:
            return fh.read().strip() or None
    except OSError:
        return None


def _web_ok() -> tuple[bool, str]:
    try:
        with urllib.request.urlopen(WEB_URL, timeout=WEB_TIMEOUT) as resp:
            if resp.status == 200:
                return True, ""
            return False, f"web returned {resp.status}"
    except (urllib.error.URLError, OSError) as exc:
        return False, f"web unreachable: {exc}"


def _this_node_ok() -> tuple[bool, str]:
    """Ping THIS container's celery node, by name.

    Pinned to this node deliberately. A bare `ping()` is answered by any
    worker on the broker, so in a stack with several replicas a dead one
    would go on reporting healthy for as long as a sibling was alive — the
    healthcheck would be measuring the cluster, not the container it is in.
    """
    from ..celery_app import celery as celery_app

    node = f"{os.environ.get('CELERY_NODENAME', 'celery')}@{socket.gethostname()}"
    try:
        replies = celery_app.control.ping(destination=[node], timeout=PING_TIMEOUT)
    except Exception as exc:  # noqa: BLE001 — a probe reports, never raises
        return False, f"could not reach the broker: {exc}"
    if not replies:
        return False, f"{node} did not answer a ping"
    return True, ""


def _lanes_ok() -> tuple[bool, str]:
    """Every lane in the table is answering.

    Deliberately ignores whether a lane is ON: a lane at cap 0 still runs its
    process with its consumers cancelled, so it answers `inspect` and is
    healthy. Health is "is the process alive"; whether it should be consuming
    is a settings question the sizing pass owns, and conflating them would
    make turning a lane off mark the container unhealthy.

    That was not merely a risk — it was happening. Until 2026-09-23 a worker
    was attributed to its lane by the queues it was CONSUMING, and a lane with
    its consumers cancelled reports none, so it read as absent and this check
    failed. ML ships at cap 0, so a fresh install was permanently unhealthy
    and Swarm restarts an unhealthy task forever. The docstring above said the
    right thing while the code did the opposite; `worker_lanes.lane_for_node`
    is what makes it true.
    """
    from ..services.worker_control import inspect_lanes_sync
    from ..services.worker_lanes import LANES

    live = inspect_lanes_sync()
    missing = sorted(lane.name for lane in LANES if not live[lane.name].present)
    if missing:
        return False, "lanes not answering: " + ", ".join(missing)
    return True, ""


def main(argv: list[str] | None = None) -> int:
    role = current_role()

    if role is None:
        # Not a failure. See the module docstring: the entrypoint did not run,
        # so there is no role to check against and no basis for a verdict.
        print("no role recorded; nothing to check")
        return 0

    if role in NO_CHECK_ROLES:
        print(f"{role}: nothing to check")
        return 0

    checks = []
    if role == "all":
        checks = [_web_ok, _lanes_ok]
    elif role == "web":
        checks = [_web_ok]
    elif role in CELERY_ROLES:
        checks = [_this_node_ok]
    else:
        print(f"unknown role {role!r}; nothing to check")
        return 0

    for check in checks:
        ok, detail = check()
        if not ok:
            print(detail, file=sys.stderr)
            return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())