"""Worker-lane definitions and the derived ceiling (milestone 422 step 1). Pure — no DB, no broker. The ceiling reads files under /sys/fs/cgroup, so the tests point it at temporary files instead of at whatever the CI container happens to have. """ from __future__ import annotations import pytest from backend.app.services import worker_lanes as wl # --- the lane definitions ---------------------------------------------------- def test_lane_names_are_unique_and_stable(): names = [lane.name for lane in wl.LANES] assert len(names) == len(set(names)) # The key every settings row and every API call uses. Renaming one orphans # its row silently, so the set is pinned rather than merely counted. assert set(names) == {"worker", "scheduler", "maintenance_long", "ml"} def test_queue_sets_are_unique(): """Two lanes sharing a queue set would be indistinguishable to `celery inspect`, which groups by exactly that (see service_roster).""" keys = [lane.queue_key for lane in wl.LANES] assert len(keys) == len(set(keys)) def test_every_routed_queue_has_a_lane_that_serves_it(): """The check that actually matters: a queue celery routes tasks to, with no lane consuming it, is work that queues forever and never runs. Read from `celery_app`'s real routing table rather than a list written here, so adding a route without a lane fails this test. """ from backend.app.celery_app import celery as celery_app routes = celery_app.conf.task_routes or {} routed = {spec["queue"] for spec in routes.values() if "queue" in spec} served = {q for lane in wl.LANES for q in lane.queues} assert routed - served == set(), ( f"queues with no lane to consume them: {sorted(routed - served)}" ) def test_the_shipped_caps_are_one_of_each_with_ml_at_zero(): """Operator, 2026-09-22: *"that starting value should be one of each."* Restated 2026-09-23 when the cap became the ONLY number: *"the cap defaults should be 1 and 0 for the ml-worker."* ML at zero is milestone 422 step 6's requirement: a cap of zero means no consumers, and raising it is what triggers the SigLIP download. Rule 164 allows a runtime fetch only for a feature that is optional and clearly off, so a default of 1 here would make every fresh install reach HuggingFace. """ by_name = wl.LANES_BY_NAME for name in ("worker", "scheduler", "maintenance_long"): assert by_name[name].default_slots_cap == 1 assert by_name["ml"].default_slots_cap == 0 def test_a_lane_has_exactly_one_operator_setting(): """The whole point of the 2026-09-23 reshape. `slots`, `enabled` and `autoscale` are gone: how many workers run is a measurement the sizing pass owns, and "off" is a cap of zero. Asserted against the dataclass's own fields rather than by name, so a second knob added later fails here instead of quietly reappearing in the UI beside the cap — which is exactly how three settings accumulated the first time.""" import dataclasses settable = { f.name for f in dataclasses.fields(wl.Lane) if f.name.startswith("default_") } assert settable == {"default_slots_cap"}, ( f"a lane has more than one operator default: {sorted(settable)}" ) def test_the_shipped_cap_is_never_the_ceiling_on_a_real_machine(): """A cap that starts at the ceiling is a rubber stamp: there is nowhere to raise it to, so the nudge that tells an operator to raise it would have nothing to say. One-of-each leaves room on anything bigger than a single-core box.""" for lane in wl.LANES: assert lane.default_slots_cap >= 0 assert lane.default_slots_cap <= 1 def test_only_ml_is_memory_bound(): """Which lane is memory-bound decides which ceiling formula applies. If another lane ever becomes so, it needs its own per-slot cost — the ML figure is about a SigLIP copy and means nothing for a thumbnailer.""" assert [lane.name for lane in wl.LANES if lane.memory_bound] == ["ml"] # --- reading the container's limits ------------------------------------------ def _point_memory_at(monkeypatch, tmp_path, contents: str | None, *, v2=True): target = tmp_path / "memory.max" if contents is not None: target.write_text(contents) attr = "_CGROUP_V2_MEMORY" if v2 else "_CGROUP_V1_MEMORY" monkeypatch.setattr(wl, attr, target) # Neutralise the other tier so the test controls exactly one source. other = "_CGROUP_V1_MEMORY" if v2 else "_CGROUP_V2_MEMORY" monkeypatch.setattr(wl, other, tmp_path / "absent") def test_memory_read_from_cgroup_v2(monkeypatch, tmp_path): _point_memory_at(monkeypatch, tmp_path, str(8 * wl.GIB)) assert wl.container_memory_bytes() == 8 * wl.GIB def test_memory_read_from_cgroup_v1(monkeypatch, tmp_path): target = tmp_path / "limit_in_bytes" target.write_text(str(6 * wl.GIB)) monkeypatch.setattr(wl, "_CGROUP_V2_MEMORY", tmp_path / "absent") monkeypatch.setattr(wl, "_CGROUP_V1_MEMORY", target) assert wl.container_memory_bytes() == 6 * wl.GIB def test_v2_max_means_no_limit_not_zero(monkeypatch, tmp_path): """'max' is a container with no memory limit set. Falling through to host RAM is right; reading it as 0 or as an error would cap a large machine at the unknown-ceiling.""" _point_memory_at(monkeypatch, tmp_path, "max") value = wl.container_memory_bytes() assert value is not None and value > wl.GIB def test_v1_sentinel_is_recognised_as_unlimited(monkeypatch, tmp_path): """cgroup v1 spells 'no limit' as a PAGE_SIZE-aligned LONG_MAX rather than a word, so it is recognised by magnitude. Taken literally it would be petabytes and the ML ceiling would be nonsense.""" target = tmp_path / "limit_in_bytes" target.write_text(str(9223372036854771712)) monkeypatch.setattr(wl, "_CGROUP_V2_MEMORY", tmp_path / "absent") monkeypatch.setattr(wl, "_CGROUP_V1_MEMORY", target) value = wl.container_memory_bytes() assert value is not None assert value < wl._UNLIMITED_ABOVE # --- the derived ceiling ----------------------------------------------------- def test_ml_ceiling_is_memory_divided_by_per_slot_cost(monkeypatch, tmp_path): # 2 GiB reserved for web and the other lanes, then 4 GiB per model copy. _point_memory_at(monkeypatch, tmp_path, str(14 * wl.GIB)) assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == 3 def test_a_small_box_is_told_it_cannot_run_tagging(monkeypatch, tmp_path): """Honestly zero rather than a floor of one. A 4GB box cannot hold a model alongside the web process, and offering a slot that OOMs the container the first time it is used is precisely what the ceiling exists to prevent — the operator's 'I don't want it to kill their servers'.""" _point_memory_at(monkeypatch, tmp_path, str(4 * wl.GIB)) assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == 0 def test_unreadable_memory_limit_yields_a_low_ceiling_not_an_unlimited_one( monkeypatch, tmp_path ): """The failure DIRECTION is the point. Not knowing how much memory there is must never read as 'plenty' — an unswept absence is not a verdict.""" _point_memory_at(monkeypatch, tmp_path, None) monkeypatch.setattr(wl.os, "sysconf", lambda _: (_ for _ in ()).throw(OSError)) assert wl.derived_ceiling(wl.LANES_BY_NAME["ml"]) == wl.UNKNOWN_CEILING def test_cpu_lanes_are_bounded_by_cores_not_memory(monkeypatch, tmp_path): """A CPU-bound lane on a memory-starved box must still get slots — its slots are processes, not model copies.""" _point_memory_at(monkeypatch, tmp_path, str(2 * wl.GIB)) monkeypatch.setattr(wl, "container_cpu_count", lambda: 8) assert wl.derived_ceiling(wl.LANES_BY_NAME["worker"]) == 8 def test_cpu_quota_beats_host_core_count(monkeypatch, tmp_path): """`os.cpu_count()` reports the HOST's cores from inside a container, so a 4-core quota on a 32-core host would otherwise offer 32 slots. The operator's own stack sets `cpus: '4.0'` on ml-worker, so this is real configuration rather than a hypothetical.""" quota = tmp_path / "cpu.max" quota.write_text("400000 100000") monkeypatch.setattr(wl, "_CGROUP_V2_CPU", quota) monkeypatch.setattr(wl.os, "cpu_count", lambda: 32) assert wl.container_cpu_count() == 4 def test_a_single_core_box_still_gets_a_slot(monkeypatch): monkeypatch.setattr(wl, "container_cpu_count", lambda: 1) assert wl.derived_ceiling(wl.LANES_BY_NAME["worker"]) >= wl.MIN_CEILING def test_the_ceiling_is_computed_not_stored(monkeypatch, tmp_path): """The property the design rests on: the same lane yields a different ceiling when the container's limits change, with no row edit. A stored ceiling would keep authorising what the box no longer has.""" lane = wl.LANES_BY_NAME["ml"] _point_memory_at(monkeypatch, tmp_path, str(34 * wl.GIB)) big = wl.derived_ceiling(lane) (tmp_path / "memory.max").write_text(str(10 * wl.GIB)) small = wl.derived_ceiling(lane) assert big > small def test_ceilings_covers_every_lane(): assert set(wl.ceilings()) == {lane.name for lane in wl.LANES} # --- the duplicate this step collapsed --------------------------------------- def test_role_names_is_derived_from_the_lane_table(): """`service_roster.ROLE_NAMES` was a hand-kept second copy of 'queue set -> display name' and had already drifted: maintenance_long is a live lane with four task routes and a dedicated worker, and the roster did not know its name, so the System tab rendered `Worker (maintenance_long)`. Asserting the derivation rather than the contents — a test listing the names again would be a third copy. """ from backend.app.services.service_roster import ROLE_NAMES, role_display_name assert ROLE_NAMES == {lane.queue_key: lane.display_name for lane in wl.LANES} assert role_display_name(("maintenance_long",)) == "Long maintenance" def test_an_unrecognised_queue_set_still_gets_a_true_label(): """A deployment slicing CELERY_QUEUES differently must not be handed a name this code invented for it.""" from backend.app.services.service_roster import role_display_name assert role_display_name(("nonsense",)) == "Worker (nonsense)" # --- the model's invariant --------------------------------------------------- def test_the_row_carries_exactly_one_constraint_now(): """`slots >= 0` and `slots <= slots_cap` went with the `slots` column on 2026-09-23 — there is one number left, and the only thing that can be wrong with it is being negative. Asserted as the WHOLE set rather than as a membership check: a constraint left behind naming a dropped column is not a harmless leftover, it is a table the migration cannot have produced, and the model would then describe a schema no database has. """ from backend.app.models import WorkerLane constraints = { c.name: str(c.sqltext) for c in WorkerLane.__table__.constraints if hasattr(c, "sqltext") } # The names carry the convention's `ck_worker_lane_` prefix ALREADY — the # model declares them bare and Base.metadata's naming_convention applies # it. Asserting the prefixed form is what pins the thing that actually # went wrong once: alembic 0088 had to rename four constraints that # shipped as `ck_x_ck_x_name`, because the migration pre-prefixed a name # the convention then prefixed again (#3275). A bare-name assertion here # would pass just as happily against a doubled one. assert constraints == {"ck_worker_lane_cap_non_negative": "slots_cap >= 0"} for name in constraints: assert not name.startswith("ck_worker_lane_ck_"), f"doubled prefix: {name}" # --- what an optional lane tells the operator before it is enabled ----------- def test_the_ml_lane_is_marked_optional_and_declares_its_model(): """The operator's ask, 2026-09-22: an optional lane must SAY it is optional, and say what enabling it downloads — which model, how big, and what it costs to hold — before the switch is thrown rather than after a multi-GB fetch has begun.""" ml = wl.LANES_BY_NAME["ml"] assert ml.optional is True assert len(ml.models) == 1 assert ml.models[0].repo == "google/siglip-so400m-patch14-384" assert ml.models[0].approx_download_bytes > 0 assert ml.models[0].approx_resident_bytes > 0 def test_no_required_lane_claims_a_model(): """A lane the product cannot work without must not be gated behind a download. If one ever needs a model, it stops being required.""" for lane in wl.LANES: if lane.models: assert lane.optional, f"{lane.name} needs a model but is not optional" def test_the_per_slot_ceiling_is_the_same_number_the_ui_shows(): """DERIVED, not restated. The figure quoted to the operator before they enable the lane and the figure the cap enforces have to be one number, or the UI promises a slot the cap will then refuse.""" ml = wl.LANES_BY_NAME["ml"] assert wl.ML_BYTES_PER_SLOT == ml.models[0].approx_resident_bytes def test_estimated_numbers_are_flagged_as_estimates(): """`measured` travels with the figures so the card can say "about". An estimate presented as a measurement is what decides whether someone's server survives — it must not be rounded into something that reads like a fact. Flip this to True in the same commit that records a real measurement.""" assert wl.SIGLIP_MODEL.measured is False