"""The roster's inspect budget must exceed the inspect work. Both tests here assert a RELATION between constants rather than their values. The failure they exist for is not a wrong number — it is the budget and the work drifting apart, which is invisible in each one read on its own. """ from __future__ import annotations from backend.app.services import service_roster as sr from tests.doubles import RecordingSession def test_the_wrapper_budget_exceeds_the_work_it_waits_for(): """The bug the operator's first consolidated deploy surfaced (2026-09-23). `_inspect_celery_sync` makes INSPECT_ROUND_TRIPS broadcasts, and a broadcast with no `destination` cannot know how many replies to expect, so each waits out its full timeout rather than returning on the last reply. The sync call therefore costs round_trips x INSPECT_TIMEOUT_SECONDS. The wrapper allowed `INSPECT_TIMEOUT_SECONDS * 2`, which reads like a safety factor and is exactly that worst case with nothing left over. Live result: a TimeoutError traceback on every refresh while the two inspect calls were working perfectly, and a roster that stopped advancing. Asserted as a RELATION, not as a number. A third inspect call added to the sync function is the way this silently comes back, and the only thing that keeps the two honest is deriving one from the other. """ budget = ( sr.INSPECT_TIMEOUT_SECONDS * sr.INSPECT_ROUND_TRIPS + sr.INSPECT_SLACK_SECONDS ) work = sr.INSPECT_TIMEOUT_SECONDS * sr.INSPECT_ROUND_TRIPS assert budget > work, ( f"the wrapper allows {budget}s for {work}s of broadcasts — a budget " f"equal to the work fails under any load at all" ) assert sr.INSPECT_SLACK_SECONDS > 0 def test_the_round_trip_count_matches_the_calls_actually_made(): """INSPECT_ROUND_TRIPS is only true if someone keeps it true, so read the source rather than trusting the constant: the budget above is derived from it, and a call added without updating it puts the wrapper back under the work.""" import inspect as _inspect src = _inspect.getsource(sr._inspect_celery_sync) calls = src.count("insp.") assert calls == sr.INSPECT_ROUND_TRIPS, ( f"_inspect_celery_sync makes {calls} inspect calls but " f"INSPECT_ROUND_TRIPS says {sr.INSPECT_ROUND_TRIPS}" ) # --- a lane that is off keeps its own row ------------------------------------ def _stub_inspect(monkeypatch, active_queues): class _Insp: def __init__(self, **_): pass def active_queues(self): return active_queues def active(self): return {} class _Control: inspect = _Insp import sys import types mod = types.ModuleType("backend.app.celery_app") class _C: pass c = _C() c.control = _Control() mod.celery = c monkeypatch.setitem(sys.modules, "backend.app.celery_app", mod) def test_a_lane_with_no_consumers_keeps_the_row_it_had_while_running(monkeypatch): """The phantom, and the reason the operator had two wrong rows at once. A lane at cap 0 cancels its consumers, so it answers with an EMPTY queue list. Grouped on that, it landed under the key `celery:` and rendered as a row called `Worker ()` — reported running — while the real lane's row went stale beside it because nothing updated it any more. Keyed on the LANE's queue set now, which is the same string the row already had while the lane was consuming. So turning a lane off updates its row instead of minting a second one. """ from backend.app.services.worker_lanes import LANES_BY_NAME _stub_inspect(monkeypatch, {"ml@abc123": []}) grouped = sr._inspect_celery_sync() assert list(grouped) == [LANES_BY_NAME["ml"].queue_key] assert () not in grouped, "the empty queue set is the phantom `Worker ()`" def test_the_row_is_the_same_one_whether_the_lane_is_consuming_or_not(monkeypatch): """Stated as an identity rather than as two separate assertions: if these keys ever differ, turning a lane off silently starts a second roster row and the first goes stale — which is exactly what happened.""" ml_queues = [{"name": "ml"}] _stub_inspect(monkeypatch, {"ml@abc123": ml_queues}) on = set(sr._inspect_celery_sync()) _stub_inspect(monkeypatch, {"ml@abc123": []}) off = set(sr._inspect_celery_sync()) assert on == off def test_a_worker_this_build_did_not_name_is_still_grouped_by_its_queues( monkeypatch, ): """The fallback, and the case the roster exists to report honestly: a deployment slicing CELERY_QUEUES differently gets its raw queue list rather than a name this code invented for it.""" _stub_inspect(monkeypatch, {"celery@xyz": [{"name": "odd"}]}) assert list(sr._inspect_celery_sync()) == [("odd",)] # --- the sweep writes the roster, not the page ------------------------------- # # It used to be refreshed on the /api/system/health request path, rate-limited # to once per 20s. So the roster only advanced while someone had a browser # open: the liveness of the workers was a function of whether anyone was # looking at them. Operator, 2026-09-23: *"is there a reason this info isn't # being tracked in the background and stored in some way?"* def test_the_sync_refresh_writes_a_row_for_everything_that_answered(monkeypatch): monkeypatch.setattr( sr, "_inspect_celery_sync", lambda: {("ml",): {"hostnames": ["ml@a"], "active": 2}, ("scan",): {"hostnames": ["scheduler@a"], "active": 0}}, ) session = RecordingSession() sr.refresh_celery_roster_sync(session) assert len(session.stmts) == 2 assert session.commits == 1 def test_a_broker_that_will_not_answer_does_not_kill_the_sweep(monkeypatch): """The sizing pass runs on a timer and does three things; a roster refresh that raised would take the other two with it. Rows going stale IS the correct report about a broker nobody can reach.""" def boom(): raise RuntimeError("no broker") monkeypatch.setattr(sr, "_inspect_celery_sync", boom) session = RecordingSession() sr.refresh_celery_roster_sync(session) assert session.stmts == [] assert session.commits == 0 def test_both_refreshes_build_the_same_row(monkeypatch): """The async path (an agent lease over the API) and the sync one (the sweep) must not drift. Asserted on the shared mapping rather than by running both, because the thing that could drift is what a roster row IS — not which kind of session writes it.""" grouped = {("ml",): {"hostnames": ["ml@a", "ml@b"], "active": 3}} rows = sr._roster_rows(grouped) assert rows == [{ "key": "celery:ml", "kind": "celery", "display_name": sr.role_display_name(("ml",)), "details": { "queues": ["ml"], "hostnames": ["ml@a", "ml@b"], "replicas": 2, "active": 3, }, }]