FabledCurator/tests/test_api_workers.py

"""/api/system/workers — the lane dial (milestone 422 step 2).

Exercises the real endpoint against the real database. Only `celery inspect`
is stubbed, and only to keep the suite fast: an unstubbed inspect blocks for
its full 2s timeout per call with no workers to answer, which several writes
would turn into most of the lane's runtime.
"""

import pytest
import pytest_asyncio
from sqlalchemy import select

from backend.app.api import workers as workers_api
from backend.app.models import WorkerLane
from backend.app.services import worker_control as wc
from backend.app.services import worker_lanes as wl
from backend.app.services.worker_lanes import LANES, SWEEP_PERIOD_SECONDS
from tests.doubles import RecordingSession

pytestmark = pytest.mark.integration


@pytest.fixture(autouse=True)
def a_known_machine(monkeypatch):
    """A 16-core, 64 GiB container, whatever runs the suite.

    Each lane's ceiling is derived from the container's CPU quota and memory
    limit, so without this the caps these tests may set depend on the CI
    runner's size, and a smaller runner refuses them as above the ceiling.
    """
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 16)
    monkeypatch.setattr(wl, "container_memory_bytes", lambda: 64 * wl.GIB)


@pytest_asyncio.fixture
async def no_live_workers(monkeypatch):
    """Nothing is running — which is the CI lane's actual truth, asserted
    rather than waited for. Makes every push fail, which is the interesting
    half: the setting must still be stored."""
    monkeypatch.setattr(
        wc, "inspect_lanes_sync",
        lambda: {lane.name: wc.LaneLiveState() for lane in LANES},
    )


@pytest.fixture(autouse=True)
def queued_pushes(monkeypatch):
    """Hold the background push still, and hand the test the hand-off.

    Since 2026-09-23 the endpoint stores the cap, answers, and gives the live
    push to a Quart background task — operator: *"the change should be queued
    so that it isn't blocking of the webui or the system itself."* A task that
    outlived a test's monkeypatches would reach the real broker during
    teardown and wait out its timeout there, so every test in this module
    captures the hand-off instead, and the ones that care about what the push
    DOES run it deliberately with `_run_pushes`.

    Autouse, because a test that forgets is not a test that fails — it is a
    test that leaks a 2s broker call into whichever test runs next.
    """
    scheduled: list[tuple] = []
    monkeypatch.setattr(
        workers_api, "_schedule_push",
        lambda lane, slots_cap, was_cap: scheduled.append((lane, slots_cap, was_cap)),
    )
    return scheduled


async def _run_pushes(scheduled: list[tuple]) -> list[dict]:
    """Run what the endpoint queued, in order, and clear the queue."""
    out = [
        await wc.push_lane_cap(lane, cap, was_cap=was_cap)
        for lane, cap, was_cap in scheduled
    ]
    scheduled.clear()
    return out


async def _lane_row(db, name: str) -> WorkerLane:
    return (await db.execute(
        select(WorkerLane).where(WorkerLane.name == name)
    )).scalar_one()


# --- reading -----------------------------------------------------------------


@pytest.mark.asyncio
async def test_get_lists_every_lane_with_its_ceiling(client, no_live_workers):
    resp = await client.get("/api/system/workers")
    assert resp.status_code == 200
    body = await resp.get_json()

    by_name = {lane["name"]: lane for lane in body["lanes"]}
    assert set(by_name) == {"worker", "scheduler", "maintenance_long", "ml"}
    for lane in body["lanes"]:
        assert lane["ceiling"] >= 0
        assert lane["slots_cap"] <= lane["ceiling"]
        # Nothing is running, so live state must say so rather than report
        # zeroes that read like a healthy idle lane.
        assert lane["live"]["present"] is False


@pytest.mark.asyncio
async def test_enabled_is_derived_from_the_cap_and_never_stored(
    client, no_live_workers,
):
    """The reshape of 2026-09-23. "Off" and "may use no workers" were two
    spellings of one fact, stored separately and free to disagree."""
    body = await (await client.get("/api/system/workers")).get_json()
    for lane in body["lanes"]:
        assert lane["enabled"] == (lane["slots_cap"] > 0), lane["name"]


@pytest.mark.asyncio
async def test_ml_ships_off(client, no_live_workers):
    """Rule 164's carve-out and the weak-hardware default in one row: raising
    the cap is what triggers the SigLIP download, so a fresh install must not
    find it above zero."""
    body = await (await client.get("/api/system/workers")).get_json()
    ml = next(lane for lane in body["lanes"] if lane["name"] == "ml")
    assert ml["slots_cap"] == 0
    assert ml["enabled"] is False


@pytest.mark.asyncio
async def test_the_other_lanes_ship_at_one(client, no_live_workers):
    """Operator: *"the cap defaults should be 1 and 0 for the ml-worker."*"""
    body = await (await client.get("/api/system/workers")).get_json()
    caps = {lane["name"]: lane["slots_cap"] for lane in body["lanes"]}
    assert caps == {
        "worker": 1, "scheduler": 1, "maintenance_long": 1, "ml": 0,
    }


# --- the persist / push split ------------------------------------------------


@pytest.mark.asyncio
async def test_raising_a_cap_stores_it_and_pushes_nothing(
    client, db, no_live_workers, queued_pushes,
):
    """A cap is PERMISSION, not a request. Raising it must not grow the pool
    — that would put workers on a lane with nothing to do — so the queued
    push has nothing to say to the broker.

    This asserted `applied is False` until run 7367, carried over from when
    the number meant "run this many". The code was right and the test was
    describing the control it replaced.
    """
    resp = await client.post("/api/system/workers/worker", json={"slots_cap": 3})

    assert resp.status_code == 200
    body = await resp.get_json()
    assert body["slots_cap"] == 3
    assert (await _lane_row(db, "worker")).slots_cap == 3
    assert [r["applied"] for r in await _run_pushes(queued_pushes)] == [True]


@pytest.mark.asyncio
async def test_turning_a_lane_off_is_stored_even_when_it_cannot_be_pushed(
    client, db, no_live_workers, queued_pushes,
):
    """The direction that DOES push. Consumers follow the cap immediately in
    both directions — off must take effect when it is asked for — so with
    nothing answering, the push fails.

    That is NOT a failed setting, and it is why the reply does not wait for
    it: the value is saved and the sizing pass carries it within a minute
    (lesson #4202 — a live change that does not survive, with nothing saying
    so). The push reports the failure to the log, where an operator can find
    it; the table shows the lane not answering either way.
    """
    resp = await client.post("/api/system/workers/worker", json={"slots_cap": 0})

    assert resp.status_code == 200
    assert (await _lane_row(db, "worker")).slots_cap == 0

    pushed = await _run_pushes(queued_pushes)
    assert pushed[0]["applied"] is False
    assert "not running" in pushed[0]["apply_error"]


# --- the cap is the switch ---------------------------------------------------


@pytest.mark.asyncio
async def test_a_cap_of_zero_turns_the_lane_off(client, db, no_live_workers):
    await client.post("/api/system/workers/worker", json={"slots_cap": 0})

    row = await _lane_row(db, "worker")
    assert row.slots_cap == 0
    body = await (await client.get("/api/system/workers")).get_json()
    worker = next(lane for lane in body["lanes"] if lane["name"] == "worker")
    assert worker["enabled"] is False


@pytest.mark.asyncio
async def test_raising_it_off_zero_turns_the_lane_on(client, db, no_live_workers):
    await client.post("/api/system/workers/ml", json={"slots_cap": 1})

    assert (await _lane_row(db, "ml")).slots_cap == 1
    body = await (await client.get("/api/system/workers")).get_json()
    ml = next(lane for lane in body["lanes"] if lane["name"] == "ml")
    assert ml["enabled"] is True


@pytest.mark.asyncio
async def test_the_model_fetch_fires_on_the_transition_not_on_every_write(
    client, db, no_live_workers, queued_pushes, monkeypatch,
):
    """Raising the cap off zero downloads SigLIP, once. A second nudge of the
    same dial must not re-enqueue a multi-GB download — and the trigger must
    be the TRANSITION rather than "a field was sent", which is what it tested
    before the UI stopped sending `enabled` at all."""
    monkeypatch.setattr(
        wc, "set_lane_enabled_sync", lambda lane, enabled, live=None: (True, None),
    )
    monkeypatch.setattr(
        wc, "set_lane_slots_sync", lambda lane, target, live=None: (True, None),
    )
    fired = []
    monkeypatch.setattr(wc, "_enqueue_model_fetch", lambda: fired.append(1) or True)

    first = await (await client.post(
        "/api/system/workers/ml", json={"slots_cap": 1},
    )).get_json()
    await _run_pushes(queued_pushes)
    second = await (await client.post(
        "/api/system/workers/ml", json={"slots_cap": 2},
    )).get_json()
    await _run_pushes(queued_pushes)

    # The reply promises it, the queued push performs it. Both halves are
    # checked, because the reply is what the UI says out loud.
    assert first["fetching_models"] is True
    assert second["fetching_models"] is False
    assert fired == [1]


@pytest.mark.asyncio
async def test_the_model_fetch_is_enqueued_even_if_the_lane_is_not_answering(
    client, no_live_workers, queued_pushes, monkeypatch,
):
    """Turning ML on while it is restarting must still fetch the model.

    It was gated on the consumer change having landed until 2026-09-23, on
    the reasoning that a task enqueued onto a queue nothing consumes just sits
    there. It does — and that is the right place for it to wait. Gated, this
    path stored the cap, let the sizing pass start the consumers a minute
    later, and left the lane running with no model, because nothing else ever
    asks for one.
    """
    fired = []
    monkeypatch.setattr(wc, "_enqueue_model_fetch", lambda: fired.append(1) or True)

    await client.post("/api/system/workers/ml", json={"slots_cap": 1})
    pushed = await _run_pushes(queued_pushes)

    assert pushed[0]["applied"] is False, "the fixture must leave the push failing"
    assert fired == [1]


# --- what is refused ---------------------------------------------------------


@pytest.mark.asyncio
async def test_a_cap_above_the_derived_ceiling_is_refused(
    client, db, no_live_workers,
):
    """The ceiling is the machine's, not the operator's, and it is the one
    bound they cannot lower themselves past. The detail is written to be read
    by a person — a refused control with no reason reads as a bug."""
    before = (await _lane_row(db, "ml")).slots_cap

    resp = await client.post(
        "/api/system/workers/ml", json={"slots_cap": 10_000},
    )

    assert resp.status_code == 400
    body = await resp.get_json()
    assert "container can hold" in body["detail"]
    await _refreshed(db, "ml", before)


@pytest.mark.asyncio
async def test_a_negative_cap_is_refused(client, db, no_live_workers):
    resp = await client.post("/api/system/workers/worker", json={"slots_cap": -1})
    assert resp.status_code == 400


@pytest.mark.asyncio
async def test_a_boolean_is_not_accepted_as_a_cap(client, no_live_workers):
    """`True` is an int in Python. Reading it as a cap of 1 would be a control
    that appears to work and sets something nobody asked for."""
    resp = await client.post("/api/system/workers/worker", json={"slots_cap": True})
    assert resp.status_code == 400


@pytest.mark.asyncio
async def test_the_retired_fields_are_no_longer_accepted(client, no_live_workers):
    """`slots`, `enabled` and `autoscale` are gone. A client still sending one
    must be told, not silently ignored — a POST that returns 200 having
    changed nothing is the worst of the three outcomes."""
    for field in ("slots", "enabled", "autoscale"):
        resp = await client.post(
            "/api/system/workers/worker", json={field: 2},
        )
        assert resp.status_code == 400, field


@pytest.mark.asyncio
async def test_an_unknown_lane_is_refused_and_names_the_known_ones(
    client, no_live_workers,
):
    resp = await client.post("/api/system/workers/nope", json={"slots_cap": 1})
    assert resp.status_code == 400
    body = await resp.get_json()
    assert "worker" in body["known"]


@pytest.mark.asyncio
async def test_an_empty_body_is_refused_rather_than_treated_as_a_no_op(
    client, no_live_workers,
):
    resp = await client.post("/api/system/workers/worker", json={})
    assert resp.status_code == 400


async def _refreshed(db, name: str, expected: int) -> None:
    row = await _lane_row(db, name)
    await db.refresh(row)
    assert row.slots_cap == expected, "a refused write must store nothing"


# --- the page does not ask the broker anything -------------------------------


@pytest.mark.asyncio
async def test_the_lane_read_makes_no_broker_call_at_all(client, monkeypatch):
    """Operator, 2026-09-23: *"there is a repull every time this page loads —
    is there a reason this info isn't being tracked in the background and
    stored in some way?"*

    There was: this endpoint inspected the broker on every call — four
    broadcasts on an eleven-second budget — while `size_worker_lanes` was
    already inspecting on a timer and throwing the same numbers away. The
    sweep stores them now and this reads the table.

    Asserted by making the inspect RAISE, because a version that inspected and
    was merely quick about it would pass a call-count test on a fast CI box.
    """
    monkeypatch.setattr(wc, "inspect_lanes_sync", _never_called)
    monkeypatch.setattr(wc, "_queue_depths_sync", _never_called)

    resp = await client.get("/api/system/workers")

    assert resp.status_code == 200
    assert len((await resp.get_json())["lanes"]) == len(LANES)


@pytest.mark.asyncio
async def test_a_lane_with_no_sample_yet_reads_as_unmeasured(client, monkeypatch):
    """A fresh install inside its first sweep period. `measured_at` is null and
    `present` is false — and those are DIFFERENT facts: nothing has asked yet,
    versus something asked and nothing answered. The UI says different things
    about them, so the payload must keep them apart."""
    monkeypatch.setattr(wc, "inspect_lanes_sync", _never_called)

    body = await (await client.get("/api/system/workers")).get_json()

    for lane in body["lanes"]:
        assert lane["measured_at"] is None, lane["name"]
        assert lane["live"]["present"] is False, lane["name"]


@pytest.mark.asyncio
async def test_the_stored_sample_is_what_the_page_shows(client, db, monkeypatch):
    """The whole point of the table: the sweep writes, the endpoint reads."""
    monkeypatch.setattr(wc, "inspect_lanes_sync", _never_called)
    live = {lane.name: wc.LaneLiveState() for lane in LANES}
    live["worker"] = wc.LaneLiveState(
        present=True, replicas=1, active=2, reserved=3, pools={"worker@a": 4},
    )
    await _store_sample(db, live, {"default": 7, "import": 0,
                                   "thumbnail": 0, "download": 0})

    body = await (await client.get("/api/system/workers")).get_json()
    worker = next(r for r in body["lanes"] if r["name"] == "worker")

    assert worker["live"] == {
        "present": True, "replicas": 1, "pool": 4, "active": 2, "reserved": 3,
    }
    assert worker["queue_depth"] == 7
    # depth PLUS reserved — celery prefetches, so LLEN alone under-reports.
    assert worker["pending"] == 10
    assert worker["measured_at"] is not None


@pytest.mark.asyncio
async def test_a_lane_that_stopped_answering_overwrites_its_old_reading(
    client, db, monkeypatch,
):
    """The sweep writes EVERY lane, including the ones that did not answer.

    Skipping them would leave the previous sample in place, and the page would
    go on showing a pool that is no longer there — a stale row read as a
    current one (lesson #4202: the row is the thing that has to change).
    """
    monkeypatch.setattr(wc, "inspect_lanes_sync", _never_called)
    up = {lane.name: wc.LaneLiveState() for lane in LANES}
    up["worker"] = wc.LaneLiveState(present=True, replicas=1, pools={"worker@a": 4})
    await _store_sample(db, up, {})
    await _store_sample(db, {lane.name: wc.LaneLiveState() for lane in LANES}, {})

    body = await (await client.get("/api/system/workers")).get_json()
    worker = next(r for r in body["lanes"] if r["name"] == "worker")

    assert worker["live"]["present"] is False
    assert worker["live"]["pool"] is None


@pytest.mark.asyncio
async def test_the_payload_says_how_often_it_is_measured(client, monkeypatch):
    """So the UI can explain the age of the numbers without keeping its own
    copy of the cadence, which would be free to drift from the schedule."""
    monkeypatch.setattr(wc, "inspect_lanes_sync", _never_called)
    body = await (await client.get("/api/system/workers")).get_json()
    assert body["sweep_period_seconds"] == SWEEP_PERIOD_SECONDS


def _never_called(*args, **kwargs):
    raise AssertionError("the lane read talked to the broker")


async def _store_sample(db, live, depths) -> None:
    """Write a sweep's worth of samples through the real storer.

    The production path is sync (a celery task owns a sync session); this
    suite is async, so the statements are replayed on the async session rather
    than reimplemented — the thing under test must be the shipped writer.
    """
    rec = RecordingSession()
    wc.store_lane_samples_sync(rec, live, depths)
    for stmt in rec.stmts:
        await db.execute(stmt)
    await db.commit()


# --- no database connection is held across a broker round trip ---------------


@pytest.mark.asyncio
async def test_the_lane_read_holds_no_session_while_it_inspects(monkeypatch):
    """Operator, 2026-09-23: *"something about changing the cap number is
    blocking to the website... it shouldn't be"*.

    Nothing here was slow in itself. A Postgres connection was held across a
    celery inspect whose budget is eleven seconds, on a page that polls every
    fifteen — so with a lane not answering, each poll pinned a connection for
    most of the interval. SQLAlchemy's default pool is five plus ten overflow;
    two browser tabs, the health endpoint doing the same, and a cap change
    adding more inspects exhausts it, and every OTHER request then waits on a
    connection. It surfaced as the whole site stalling rather than as one slow
    page, which is why it took a screenshot to find.

    Asserted STRUCTURALLY rather than by timing: `lane_view` must not accept a
    session at all. A timing test would be flaky, and a mock-call-order test
    would pass against a version that took the session and merely used it
    early — the property that matters is that it CANNOT.
    """
    import inspect as _inspect

    from backend.app.services.worker_control import lane_settings, lane_view

    assert "session" not in _inspect.signature(lane_view).parameters, (
        "lane_view takes a session again; the broker work must run with none held"
    )
    # And the DB half still exists, so the split did not simply lose the reads.
    assert "session" in _inspect.signature(lane_settings).parameters


@pytest.mark.asyncio
async def test_the_cap_write_holds_no_session_while_it_pushes():
    """The same property on the write path, where it was worse: a cap change
    could make three broker round trips, each with a connection held."""
    import inspect as _inspect

    from backend.app.services.worker_control import push_lane_cap, store_lane_cap

    assert "session" in _inspect.signature(store_lane_cap).parameters
    assert "session" not in _inspect.signature(push_lane_cap).parameters, (
        "push_lane_cap takes a session again; the push must run with none held"
    )


@pytest.mark.asyncio
async def test_raising_a_cap_costs_no_broker_round_trip_at_all(
    client, db, no_live_workers, queued_pushes, monkeypatch,
):
    """Raising a cap is permission, not a request — the sizing pass spends it
    — so there is nothing to tell the broker AT ALL. Asserted on the queued
    push rather than on the request, because the request no longer waits for
    it either way and would pass this vacuously."""
    calls = []
    monkeypatch.setattr(
        wc, "inspect_lanes_sync", lambda: calls.append("inspect") or {},
    )
    monkeypatch.setattr(
        wc, "set_lane_slots_sync",
        lambda *a, **k: calls.append("resize") or (True, None),
    )
    monkeypatch.setattr(
        wc, "set_lane_enabled_sync",
        lambda *a, **k: calls.append("consumers") or (True, None),
    )

    await client.post("/api/system/workers/worker", json={"slots_cap": 1})
    await _run_pushes(queued_pushes)
    calls.clear()
    resp = await client.post("/api/system/workers/worker", json={"slots_cap": 6})
    await _run_pushes(queued_pushes)

    assert resp.status_code == 200
    assert calls == [], f"raising a cap talked to the broker: {calls}"


@pytest.mark.asyncio
async def test_the_reply_never_waits_for_the_broker_in_any_direction(
    client, db, no_live_workers, queued_pushes, monkeypatch,
):
    """Operator, 2026-09-23: *"when the number is changed the change should be
    queued so that it isn't blocking of the webui or the system itself. we
    shouldn't have to wait for the validation live."*

    Raising a cap was already free. The rest were not: turning a lane off is
    four `cancel_consumer` messages, and lowering a cap reads the live pool
    first — an `inspect` on an eleven-second budget — all of it between the
    click and the response, with the stepper disabled throughout.

    Asserted by making any broker call from the request path RAISE. A timing
    assertion would be flaky, and counting calls afterwards would pass against
    a version that made them and was merely quick about it.
    """
    def boom(*args, **kwargs):
        raise AssertionError("the request path talked to the broker")

    monkeypatch.setattr(wc, "inspect_lanes_sync", boom)
    monkeypatch.setattr(wc, "set_lane_slots_sync", boom)
    monkeypatch.setattr(wc, "set_lane_enabled_sync", boom)

    # Every direction: on, up, down, off.
    for cap in (1, 4, 2, 0):
        resp = await client.post(
            "/api/system/workers/worker", json={"slots_cap": cap},
        )
        assert resp.status_code == 200, cap
        assert (await resp.get_json())["queued"] is True, cap
        assert (await _lane_row(db, "worker")).slots_cap == cap
        # The work was handed off rather than skipped — a control that answers
        # instantly by doing nothing is the failure this could become.
        assert queued_pushes, f"cap {cap} queued no push"
        queued_pushes.clear()