FabledCurator/tests/test_worker_lanes.py

"""Worker-lane definitions and the derived ceiling (milestone 422 step 1).

Pure — no DB, no broker. The ceiling reads files under /sys/fs/cgroup, so the
tests point it at temporary files instead of at whatever the CI container
happens to have.
"""

from __future__ import annotations

import pytest

from backend.app.services import worker_lanes as wl

# --- the lane definitions ----------------------------------------------------


def test_lane_names_are_unique_and_stable():
    names = [lane.name for lane in wl.LANES]
    assert len(names) == len(set(names))
    # The key every settings row and every API call uses. Renaming one orphans
    # its row silently, so the set is pinned rather than merely counted.
    assert set(names) == {"worker", "scheduler", "maintenance_long", "ml"}


def test_queue_sets_are_unique():
    """Two lanes sharing a queue set would be indistinguishable to
    `celery inspect`, which groups by exactly that (see service_roster)."""
    keys = [lane.queue_key for lane in wl.LANES]
    assert len(keys) == len(set(keys))


def test_every_routed_queue_has_a_lane_that_serves_it():
    """The check that actually matters: a queue celery routes tasks to, with
    no lane consuming it, is work that queues forever and never runs.

    Read from `celery_app`'s real routing table rather than a list written
    here, so adding a route without a lane fails this test.
    """
    from backend.app.celery_app import celery as celery_app

    routes = celery_app.conf.task_routes or {}
    routed = {spec["queue"] for spec in routes.values() if "queue" in spec}
    served = {q for lane in wl.LANES for q in lane.queues}
    assert routed - served == set(), (
        f"queues with no lane to consume them: {sorted(routed - served)}"
    )


def test_the_shipped_caps_are_one_of_each_with_ml_at_zero():
    """Operator, 2026-09-22: *"that starting value should be one of each."*
    Restated 2026-09-23 when the cap became the ONLY number: *"the cap
    defaults should be 1 and 0 for the ml-worker."*

    ML at zero is milestone 422 step 6's requirement: a cap of zero means no
    consumers, and raising it is what triggers the SigLIP download. Rule 164
    allows a runtime fetch only for a feature that is optional and clearly
    off, so a default of 1 here would make every fresh install reach
    HuggingFace.
    """
    by_name = wl.LANES_BY_NAME
    for name in ("worker", "scheduler", "maintenance_long"):
        assert by_name[name].default_slots_cap == 1
    assert by_name["ml"].default_slots_cap == 0


def test_a_lane_has_exactly_one_operator_setting():
    """The whole point of the 2026-09-23 reshape. `slots`, `enabled` and
    `autoscale` are gone: how many workers run is a measurement the sizing
    pass owns, and "off" is a cap of zero.

    Asserted against the dataclass's own fields rather than by name, so a
    second knob added later fails here instead of quietly reappearing in the
    UI beside the cap — which is exactly how three settings accumulated the
    first time."""
    import dataclasses

    settable = {
        f.name for f in dataclasses.fields(wl.Lane) if f.name.startswith("default_")
    }
    assert settable == {"default_slots_cap"}, (
        f"a lane has more than one operator default: {sorted(settable)}"
    )


def test_the_shipped_cap_is_never_the_ceiling_on_a_real_machine():
    """A cap that starts at the ceiling is a rubber stamp: there is nowhere to
    raise it to, so the nudge that tells an operator to raise it would have
    nothing to say. One-of-each leaves room on anything bigger than a
    single-core box."""
    for lane in wl.LANES:
        assert lane.default_slots_cap >= 0
        assert lane.default_slots_cap <= 1


def test_only_ml_is_memory_bound():
    """Which lane is memory-bound decides which ceiling formula applies. If
    another lane ever becomes so, it needs its own per-slot cost — the ML
    figure is about a SigLIP copy and means nothing for a thumbnailer."""
    assert [lane.name for lane in wl.LANES if lane.memory_bound] == ["ml"]


# --- reading the container's limits ------------------------------------------


def _point_memory_at(monkeypatch, tmp_path, contents: str | None, *, v2=True):
    target = tmp_path / "memory.max"
    if contents is not None:
        target.write_text(contents)
    attr = "_CGROUP_V2_MEMORY" if v2 else "_CGROUP_V1_MEMORY"
    monkeypatch.setattr(wl, attr, target)
    # Neutralise the other tier so the test controls exactly one source.
    other = "_CGROUP_V1_MEMORY" if v2 else "_CGROUP_V2_MEMORY"
    monkeypatch.setattr(wl, other, tmp_path / "absent")


def test_memory_read_from_cgroup_v2(monkeypatch, tmp_path):
    _point_memory_at(monkeypatch, tmp_path, str(8 * wl.GIB))
    assert wl.container_memory_bytes() == 8 * wl.GIB


def test_memory_read_from_cgroup_v1(monkeypatch, tmp_path):
    target = tmp_path / "limit_in_bytes"
    target.write_text(str(6 * wl.GIB))
    monkeypatch.setattr(wl, "_CGROUP_V2_MEMORY", tmp_path / "absent")
    monkeypatch.setattr(wl, "_CGROUP_V1_MEMORY", target)
    assert wl.container_memory_bytes() == 6 * wl.GIB


def test_v2_max_means_no_limit_not_zero(monkeypatch, tmp_path):
    """'max' is a container with no memory limit set. Falling through to host
    RAM is right; reading it as 0 or as an error would cap a large machine at
    the unknown-ceiling."""
    _point_memory_at(monkeypatch, tmp_path, "max")
    value = wl.container_memory_bytes()
    assert value is not None and value > wl.GIB


def test_v1_sentinel_is_recognised_as_unlimited(monkeypatch, tmp_path):
    """cgroup v1 spells 'no limit' as a PAGE_SIZE-aligned LONG_MAX rather than
    a word, so it is recognised by magnitude. Taken literally it would be
    petabytes and the ML ceiling would be nonsense."""
    target = tmp_path / "limit_in_bytes"
    target.write_text(str(9223372036854771712))
    monkeypatch.setattr(wl, "_CGROUP_V2_MEMORY", tmp_path / "absent")
    monkeypatch.setattr(wl, "_CGROUP_V1_MEMORY", target)
    value = wl.container_memory_bytes()
    assert value is not None
    assert value < wl._UNLIMITED_ABOVE


# --- the derived ceiling -----------------------------------------------------


def test_ml_ceiling_is_memory_divided_by_per_slot_cost(monkeypatch, tmp_path):
    # 2 GiB reserved for web and the other lanes, then 4 GiB per model copy.
    # Cores pinned high so the memory bound is the one being read here.
    _point_memory_at(monkeypatch, tmp_path, str(14 * wl.GIB))
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 32)
    assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == 3


def test_ml_is_bounded_by_cores_as_well_as_memory(monkeypatch, tmp_path):
    """The bug of 2026-09-23, in one assertion.

    The ML ceiling was memory ALONE. On the operator's large-memory host that
    offered ~49 slots; they took them, and each slot asks torch for
    `threads_per_slot` cores — so the lane ran ~200 threads over a box that
    has nowhere near that many. Embeds that should be seconds took 107-246s,
    and the daily CCIP sweep sharing that pool died on its 1800s soft limit.

    Memory here says 49. The cores say 8 / 4 = 2, and the smaller bound is the
    only honest one: a control must not offer a number the machine cannot
    feed.
    """
    _point_memory_at(monkeypatch, tmp_path, str(200 * wl.GIB))
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 8)

    ml = wl.LANES_BY_NAME["ml"]
    assert ml.threads_per_slot == 4
    assert wl.derived_ceiling(ml) == 2


def test_a_few_cores_still_offer_one_ml_slot_rather_than_none(
    monkeypatch, tmp_path,
):
    """The two bounds fail differently, deliberately. Too little MEMORY is
    honestly zero — the first task would OOM the container. Too few CORES is
    merely slow, which is a trade an operator may want, so it floors at one
    rather than making the lane unreachable on a small box."""
    _point_memory_at(monkeypatch, tmp_path, str(200 * wl.GIB))
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 1)
    assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == 1


def test_the_embedder_asks_for_exactly_what_the_ceiling_budgeted(monkeypatch):
    """The two halves of the same number, tied.

    `threads_per_slot` is only meaningful because `embedder.load()` calls
    `torch.set_num_threads` with it. It lived in the embedder as a private 4
    beside a comment saying "keep N_replicas x this within the cores allotted
    to ML" — a constraint stated where nothing could enforce it, and nothing
    did. If these two ever drift, the ceiling is budgeting cores for a demand
    the worker does not make, and nothing else would notice.
    """
    from backend.app.services.ml import embedder

    assert embedder._INTRA_OP_THREADS == wl.LANES_BY_NAME["ml"].threads_per_slot


def test_a_small_box_is_told_it_cannot_run_tagging(monkeypatch, tmp_path):
    """Honestly zero rather than a floor of one. A 4GB box cannot hold a model
    alongside the web process, and offering a slot that OOMs the container the
    first time it is used is precisely what the ceiling exists to prevent —
    the operator's 'I don't want it to kill their servers'."""
    _point_memory_at(monkeypatch, tmp_path, str(4 * wl.GIB))
    assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == 0


def test_unreadable_memory_limit_yields_a_low_ceiling_not_an_unlimited_one(
    monkeypatch, tmp_path
):
    """The failure DIRECTION is the point. Not knowing how much memory there
    is must never read as 'plenty' — an unswept absence is not a verdict."""
    _point_memory_at(monkeypatch, tmp_path, None)
    monkeypatch.setattr(wl.os, "sysconf", lambda _: (_ for _ in ()).throw(OSError))
    assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == wl.UNKNOWN_CEILING


def test_cpu_lanes_are_bounded_by_cores_not_memory(monkeypatch, tmp_path):
    """A CPU-bound lane on a memory-starved box must still get slots — its
    slots are processes, not model copies."""
    _point_memory_at(monkeypatch, tmp_path, str(2 * wl.GIB))
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 8)
    assert wl.derived_ceiling(wl.LANES_BY_NAME["worker"]) == 8


def test_cpu_quota_beats_host_core_count(monkeypatch, tmp_path):
    """`os.cpu_count()` reports the HOST's cores from inside a container, so a
    4-core quota on a 32-core host would otherwise offer 32 slots. The
    operator's own stack sets `cpus: '4.0'` on ml-worker, so this is real
    configuration rather than a hypothetical."""
    quota = tmp_path / "cpu.max"
    quota.write_text("400000 100000")
    monkeypatch.setattr(wl, "_CGROUP_V2_CPU", quota)
    monkeypatch.setattr(wl.os, "cpu_count", lambda: 32)
    assert wl.container_cpu_count() == 4


def test_a_single_core_box_still_gets_a_slot(monkeypatch):
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 1)
    assert wl.derived_ceiling(wl.LANES_BY_NAME["worker"]) >= wl.MIN_CEILING


def test_the_ceiling_is_computed_not_stored(monkeypatch, tmp_path):
    """The property the design rests on: the same lane yields a different
    ceiling when the container's limits change, with no row edit. A stored
    ceiling would keep authorising what the box no longer has."""
    lane = wl.LANES_BY_NAME["ml"]
    monkeypatch.setattr(wl, "container_cpu_count", lambda: 32)
    _point_memory_at(monkeypatch, tmp_path, str(34 * wl.GIB))
    big = wl.derived_ceiling(lane)
    (tmp_path / "memory.max").write_text(str(10 * wl.GIB))
    small = wl.derived_ceiling(lane)
    assert big > small


def test_ceilings_covers_every_lane():
    assert set(wl.ceilings()) == {lane.name for lane in wl.LANES}


# --- the duplicate this step collapsed ---------------------------------------


def test_role_names_is_derived_from_the_lane_table():
    """`service_roster.ROLE_NAMES` was a hand-kept second copy of 'queue set
    -> display name' and had already drifted: maintenance_long is a live lane
    with four task routes and a dedicated worker, and the roster did not know
    its name, so the System tab rendered `Worker (maintenance_long)`.

    Asserting the derivation rather than the contents — a test listing the
    names again would be a third copy.
    """
    from backend.app.services.service_roster import ROLE_NAMES, role_display_name

    assert ROLE_NAMES == {lane.queue_key: lane.display_name for lane in wl.LANES}
    assert role_display_name(("maintenance_long",)) == "Long maintenance"


def test_an_unrecognised_queue_set_still_gets_a_true_label():
    """A deployment slicing CELERY_QUEUES differently must not be handed a
    name this code invented for it."""
    from backend.app.services.service_roster import role_display_name

    assert role_display_name(("nonsense",)) == "Worker (nonsense)"


# --- the model's invariant ---------------------------------------------------


def test_the_row_carries_exactly_one_constraint_now():
    """`slots >= 0` and `slots <= slots_cap` went with the `slots` column on
    2026-09-23 — there is one number left, and the only thing that can be
    wrong with it is being negative.

    Asserted as the WHOLE set rather than as a membership check: a constraint
    left behind naming a dropped column is not a harmless leftover, it is a
    table the migration cannot have produced, and the model would then
    describe a schema no database has.
    """
    from backend.app.models import WorkerLane

    constraints = {
        c.name: str(c.sqltext) for c in WorkerLane.__table__.constraints
        if hasattr(c, "sqltext")
    }
    # The names carry the convention's `ck_worker_lane_` prefix ALREADY — the
    # model declares them bare and Base.metadata's naming_convention applies
    # it. Asserting the prefixed form is what pins the thing that actually
    # went wrong once: alembic 0088 had to rename four constraints that
    # shipped as `ck_x_ck_x_name`, because the migration pre-prefixed a name
    # the convention then prefixed again (#3275). A bare-name assertion here
    # would pass just as happily against a doubled one.
    assert constraints == {"ck_worker_lane_cap_non_negative": "slots_cap >= 0"}
    for name in constraints:
        assert not name.startswith("ck_worker_lane_ck_"), f"doubled prefix: {name}"


# --- what an optional lane tells the operator before it is enabled -----------


def test_the_ml_lane_is_marked_optional_and_declares_its_model():
    """The operator's ask, 2026-09-22: an optional lane must SAY it is
    optional, and say what enabling it downloads — which model, how big, and
    what it costs to hold — before the switch is thrown rather than after a
    multi-GB fetch has begun."""
    ml = wl.LANES_BY_NAME["ml"]
    assert ml.optional is True
    assert len(ml.models) == 1
    assert ml.models[0].repo == "google/siglip-so400m-patch14-384"
    assert ml.models[0].approx_download_bytes > 0
    assert ml.models[0].approx_resident_bytes > 0


def test_no_required_lane_claims_a_model():
    """A lane the product cannot work without must not be gated behind a
    download. If one ever needs a model, it stops being required."""
    for lane in wl.LANES:
        if lane.models:
            assert lane.optional, f"{lane.name} needs a model but is not optional"


def test_the_per_slot_ceiling_is_the_same_number_the_ui_shows():
    """DERIVED, not restated. The figure quoted to the operator before they
    enable the lane and the figure the cap enforces have to be one number, or
    the UI promises a slot the cap will then refuse."""
    ml = wl.LANES_BY_NAME["ml"]
    assert wl.ML_BYTES_PER_SLOT == ml.models[0].approx_resident_bytes


def test_estimated_numbers_are_flagged_as_estimates():
    """`measured` travels with the figures so the card can say "about". An
    estimate presented as a measurement is what decides whether someone's
    server survives — it must not be rounded into something that reads like a
    fact. Flip this to True in the same commit that records a real
    measurement."""
    assert wl.SIGLIP_MODEL.measured is False


# --- the sweep's cadence, against the thresholds that read it ----------------


def test_the_sweep_runs_often_enough_to_keep_the_roster_fresh():
    """The comparison that was never made for the GPU agent.

    Its idle lease poll backed off to a 900s ceiling while the roster called it
    stopped at 300s. Both numbers were right on their own, in different files,
    written ten weeks apart — and an idle agent was structurally guaranteed to
    read as stopped (lesson #4355).

    `size_worker_lanes` is now the ONLY writer of the celery roster, so its
    period and the staleness thresholds are in exactly that relationship. Two
    clear sweeps before a part is even doubted: one missed tick is routine,
    because the sweep rides the maintenance queue and does an inspect that can
    take eleven seconds.
    """
    from backend.app.api.system_health import (
        DOWN_AFTER_SECONDS,
        STALE_AFTER_SECONDS,
    )

    assert wl.SWEEP_PERIOD_SECONDS * 2 <= STALE_AFTER_SECONDS
    assert wl.SWEEP_PERIOD_SECONDS * 2 <= DOWN_AFTER_SECONDS


def test_the_beat_schedule_is_the_same_number_and_not_a_copy_of_it():
    """A schedule that merely happens to equal the constant is one edit away
    from disagreeing with the test above, which would then be asserting
    headroom the running system does not have."""
    from backend.app.celery_app import celery

    entry = celery.conf.beat_schedule["size-worker-lanes"]
    assert entry["schedule"] == wl.SWEEP_PERIOD_SECONDS