"""Worker lanes: what each is doing, and the dial that changes it. Milestone 422 step 2. The write half of a surface `api/system_activity.py` only reads. ## Why this is a separate blueprint `system_activity` says in its own first line that it is read-only, and it answers a different question: its `/workers` is keyed on celery HOSTNAME and reports which nodes answered. That stays as it is — the existing SystemActivityTab consumes it. This is keyed on LANE, joins the stored cap to the live pool, and accepts writes. Two endpoints answering "which celery processes exist" and "how much work is each lane allowed to do" are not the same endpoint, and folding the second into the first would make a read-only module a write one. """ from __future__ import annotations from datetime import UTC, datetime from functools import partial from quart import Blueprint, current_app, jsonify, request from ..extensions import get_session from ..services.worker_control import ( LaneUpdateRefused, lane_settings, lane_view, push_lane_cap, store_lane_cap, ) from ..services.worker_lanes import LANES_BY_NAME, Lane, derived_ceiling from ._responses import error_response as _bad workers_bp = Blueprint("workers", __name__, url_prefix="/api/system/workers") @workers_bp.route("", methods=["GET"]) async def list_lanes(): """Every lane: its cap, the ceiling above it, and what is live. Response: {lanes: [...], fetched_at: iso8601} Deliberately NOT cached, unlike system_activity's 2s/5s caches. This is the surface an operator watches while dragging a stepper, and a cached reply would show them the value from before their own change and read as the control having failed. """ # The session closes BEFORE the broker work. Holding a Postgres connection # across a celery inspect is what made this page block the whole site — # see `worker_control.LaneSettings`. This endpoint polls every 15s and the # inspect budget is 11s, so each poll was pinning a connection for most of # the interval. async with get_session() as session: settings = await lane_settings(session) lanes = await lane_view(settings) return jsonify({ "lanes": lanes, "fetched_at": datetime.now(UTC).isoformat(), }) @workers_bp.route("/", methods=["POST"]) async def update_lane(name: str): """Set a lane's cap. Stores it, answers, and makes the lane follow after. ONE field, since 2026-09-23. It used to take `slots`, `slots_cap`, `enabled` and `autoscale`; how many workers are running is now a measurement the sizing pass owns, and `enabled` is `cap > 0`. ## The reply does not wait for the lane Operator, 2026-09-23: *"when the number is changed the change should be queued so that it isn't blocking of the webui or the system itself. we shouldn't have to wait for the validation live."* So the request does exactly one thing that can be slow — a row update — and hands the broker work to a background task. Turning a lane off is four `cancel_consumer` messages and a resize; lowering a cap is an `inspect` on an eleven-second budget. Both used to happen between the click and the response, with the stepper disabled the whole time. Nothing is lost by not waiting: the cap in the database is what the system obeys, the sizing pass re-reads it every minute, and the table polls, so the live columns catch up on their own. If the web process dies before the background task runs, that sweep is the backstop — which is the same guarantee the awaited version had, since a push could fail there too. Refusals still happen inline, because they are decided from the value and the machine's ceiling alone and never touch the broker: * **400** — the value is not allowed (negative, or above what this container can hold). Nothing was stored. The body carries `detail`, which is the sentence the UI shows; a refused control with no reason reads as a bug. """ lane = LANES_BY_NAME.get(name) if lane is None: return _bad("unknown_lane", detail=name, known=sorted(LANES_BY_NAME)) body = await request.get_json() if not isinstance(body, dict): return _bad("invalid_body", detail="body must be a JSON object") if "slots_cap" not in body: return _bad("invalid_body", detail="give slots_cap") value = body["slots_cap"] # Rejected rather than coerced: `True` is an int in Python, and silently # reading it as a cap of 1 would be a control that appears to work and # sets something nobody asked for. if not isinstance(value, int) or isinstance(value, bool): return _bad("invalid_body", detail="slots_cap must be an integer") # Store, close the session, THEN hand off. The session must not be held # across broker work — that is what made this page block the whole site # (see `worker_control.LaneSettings`) — and now the request does not wait # for that work either. async with get_session() as session: try: was_cap = await store_lane_cap(session, lane, value) except LaneUpdateRefused as exc: return _bad("refused", detail=str(exc)) _schedule_push(lane, value, was_cap) return jsonify({ "name": lane.name, "slots_cap": value, "ceiling": derived_ceiling(lane), "enabled": value > 0, # The value is stored; the live lane is being told separately. The UI # patches its row from this and lets the next poll bring the live # columns, rather than refetching and paying for an inspect it just # avoided. "queued": True, # Raising the cap off zero is what downloads the model (step 6), and # the background task does it. Reported here so the UI can say a # download has started rather than leaving the operator to wonder why # a lane they just turned on is busy. "fetching_models": value > 0 and was_cap == 0 and bool(lane.models), }) def _schedule_push(lane: Lane, slots_cap: int, was_cap: int) -> None: """Run the live push after the response has gone out. A seam, not an abstraction: it is one call, and it exists so the tests can hold the push still — a background task that outlived a test's patches would reach the real broker during teardown. Quart tracks the task on the app and awaits it at shutdown, so an in-flight push survives a graceful restart. `partial` rather than passing `was_cap=` through `add_background_task`, so nothing depends on how that forwards keyword arguments. """ current_app.add_background_task( partial(push_lane_cap, lane, slots_cap, was_cap=was_cap) )