feat: a saturated lane can grow itself, within the cap the operator set (4297)

Milestone 422 step 7 — the one sweep in this milestone that decides rather
than obeys, so it is off until a lane is opted in, bounded by the operator's
cap, floored at the operator's value, and it reports every decision including
the ones where it did nothing.

Growth needs BOTH halves: all slots busy AND a backlog. Depth alone means
celery is about to pick those up and growing would add idle children (#1253
is that bug in the GPU agent); saturation alone means the lane is busy with
exactly as much work as exists. The backlog is depth PLUS reserved, because
celery prefetches and LLEN reads 0 while a worker holds thirty tasks in
memory — the case an LLEN-only autoscaler misses entirely, and the reason
step 2 plumbed `reserved` through.

The two sweeps had to be taught not to fight. The reconcile drives every
lane to its stored slots every five minutes, which would have reverted each
grow on the next tick: grow, revert, grow, revert, forever. For an
autoscaling lane the stored value is now a FLOOR — restored when a lane
falls below it, never taken back above it.

The operator's "a task that runs for x concurrent time" idea stays a UI
warning rather than a trigger: a long task does not finish sooner because
the lane gained a slot, so scaling on it would spend memory to change
nothing. Read from `task_run` on our own wall clock, not celery's
`time_start`, which is the WORKER's monotonic clock and would produce a
duration that is meaningless in the direction that matters — plausible.

Caught while reading it back: the first version read the stored slots as the
CURRENT pool. The autoscaler never writes that row, so every tick would have
proposed floor+1 — resizing nothing, reporting `grew` anyway (a replica
already past the target is issued no message and reports success), and
capping the lane one slot above its floor forever while claiming otherwise.
It now reads the live pool and keeps the stored value purely as the floor,
and the tests fix the two to different numbers so an equal-fixture pass
cannot hide it again.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

tests/test_worker_control.py

@@ -321,3 +321,245 @@ def test_the_reconcile_task_is_registered_and_scheduled():
     assert name in celery.tasks
     scheduled = {e["task"] for e in celery.conf.beat_schedule.values()}
     assert name in scheduled
+
+
+# --- the autoscaler (step 7) --------------------------------------------------
+#
+# Every case below fixes the LIVE pool and the stored floor to DIFFERENT
+# numbers wherever it can. That is deliberate: the first version of this
+# function read the stored value as the current one, and with the two equal
+# — which is what a single tick of a settled system looks like — every test
+# here still passed while the autoscaler could not grow past floor+1 or shrink
+# at all. Equal fixtures cannot see that bug.
+
+
+def _autoscale_live(
+    monkeypatch, *, pools, active, reserved, depth, present=True,
+):
+    state = wc.LaneLiveState(
+        present=present,
+        replicas=len(pools),
+        hostnames=sorted(pools),
+        pools=dict(pools),
+        active=active,
+        reserved=reserved,
+        consuming=set(LANES_BY_NAME["worker"].queues),
+    )
+    monkeypatch.setattr(
+        wc, "inspect_lanes_sync",
+        lambda: {name: (state if name == "worker" else wc.LaneLiveState())
+                 for name in LANES_BY_NAME},
+    )
+    monkeypatch.setattr(
+        wc, "_queue_depths_sync",
+        lambda: {q: depth for lane in LANES_BY_NAME.values() for q in lane.queues},
+    )
+    return state
+
+
+def _worker(decisions):
+    return next(d for d in decisions if d.lane == "worker")
+
+
+def test_a_prefetched_backlog_still_triggers_growth(monkeypatch):
+    """THE case an LLEN-only implementation misses, and the reason `reserved`
+    was plumbed through in step 2. Celery prefetches, so a lane can read queue
+    depth 0 while holding thirty tasks in worker memory — an autoscaler
+    watching LLEN alone sees an idle system and never grows."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 2}, active=2, reserved=30, depth=0)
+
+    d = _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)}))
+
+    assert d.action == "grew"
+    assert d.slots == 3
+    assert control.grew == [(1, ["host-a"])]
+
+
+def test_it_keeps_climbing_past_the_floor_on_later_ticks(monkeypatch):
+    """The regression test for reading the stored value as the current one.
+
+    The row still says 2 — the autoscaler never writes it — but the live pool
+    is already 5 from earlier ticks. The target must be 6. Computing from the
+    stored 2 would propose 3, which resizes nothing (the replica is past it),
+    reports success, and pins the lane one slot above its floor forever while
+    claiming to grow on every tick."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 5}, active=5, reserved=40, depth=0)
+
+    d = _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)}))
+
+    assert d.slots == 6
+    assert control.grew == [(1, ["host-a"])]
+
+
+def test_backlog_with_free_slots_does_nothing(monkeypatch):
+    """Depth alone is not a signal — celery is about to pick those up, and
+    growing would add children that idle."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 8}, active=1, reserved=0, depth=50)
+
+    assert _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)})).action == "held"
+    assert control.grew == []
+
+
+def test_saturation_is_measured_against_every_replica(monkeypatch):
+    """`active` is summed across replicas and `pool` is one replica's size, so
+    comparing them calls two half-busy replicas of 4 saturated at 4 active.
+    Capacity is 8 here and 4 tasks are running: half the lane is idle."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(
+        monkeypatch, pools={"host-a": 4, "host-b": 4},
+        active=4, reserved=40, depth=0,
+    )
+
+    assert _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)})).action == "held"
+    assert control.grew == []
+
+
+def test_a_saturated_lane_with_no_backlog_does_nothing(monkeypatch):
+    """The long-running-task case. One slow task holding every slot with an
+    empty queue needs no extra slots — growing does not make it finish sooner,
+    which is why the operator's 'runs for x time' idea became a UI warning
+    rather than a trigger."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 2}, active=2, reserved=0, depth=0)
+
+    assert _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)})).action == "held"
+    assert control.grew == []
+
+
+def test_growth_stops_at_the_cap_and_says_so(monkeypatch):
+    """The autoscaler gets no authority the operator does not already have.
+    And it SAYS it is capped — that is the moment they would want to know they
+    set one."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 4}, active=4, reserved=99, depth=0)
+
+    d = _worker(wc.autoscale_lanes_sync({"worker": (4, 2, True)}))
+
+    assert d.action == "held"
+    assert "cap is 4" in d.reason
+    assert control.grew == []
+
+
+def test_a_cleared_backlog_returns_the_lane_to_the_operator_s_value(monkeypatch):
+    """Down toward `configured`, one step at a time."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 5}, active=0, reserved=0, depth=0)
+
+    d = _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)}))
+
+    assert d.action == "shrank"
+    assert d.slots == 4
+    assert control.shrank == [(1, ["host-a"])]
+
+
+def test_it_never_shrinks_below_what_the_operator_set(monkeypatch):
+    """The stored value is the operator's and the autoscaler must not eat it —
+    there would be nothing left to restore to."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 2}, active=0, reserved=0, depth=0)
+
+    assert _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)})).action == "held"
+    assert control.shrank == []
+
+
+def test_hysteresis_keeps_a_middling_backlog_from_flapping(monkeypatch):
+    """Between the two thresholds nothing happens. Equal thresholds would grow
+    and shrink the lane forever as one task arrives and leaves — lesson
+    #4183's churn arriving through a different door."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 5}, active=5, reserved=5, depth=0)
+
+    assert _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)})).action == "held"
+    assert control.grew == []
+    assert control.shrank == []
+
+
+def test_a_lane_that_did_not_opt_in_is_never_touched(monkeypatch):
+    """Off by default, per lane. This is the only sweep that decides rather
+    than obeys, so it acts only where someone said it may — and it does not
+    even report on a lane it was not given."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 2}, active=2, reserved=99, depth=0)
+
+    assert wc.autoscale_lanes_sync({"worker": (8, 2, False)}) == []
+    assert control.grew == []
+
+
+def test_an_absent_lane_holds_rather_than_guessing(monkeypatch):
+    """`present=False` is 'nothing answered', not 'idle'. Deciding from an
+    unswept read is snippet #3969's shape."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(
+        monkeypatch, pools={}, active=0, reserved=0, depth=0, present=False,
+    )
+
+    d = _worker(wc.autoscale_lanes_sync({"worker": (8, 2, True)}))
+
+    assert d.action == "held"
+    assert "not answering" in d.reason
+    assert control.grew == []
+
+
+def test_a_settled_lane_sends_no_control_messages(monkeypatch):
+    """The fixed point, asserted as a property rather than inferred from the
+    reported action: this runs every minute forever, so a settled system has
+    to be silent or a real correction drowns in the heartbeat."""
+    control = _stub_control(monkeypatch)
+    _autoscale_live(monkeypatch, pools={"host-a": 2}, active=1, reserved=0, depth=1)
+
+    for _ in range(5):
+        assert _worker(
+            wc.autoscale_lanes_sync({"worker": (8, 2, True)})
+        ).action == "held"
+    assert control.grew == []
+    assert control.shrank == []
+
+
+# --- the two sweeps must not fight -------------------------------------------
+
+
+def test_the_reconcile_does_not_undo_what_the_autoscaler_added(monkeypatch):
+    """Otherwise the two would fight every five minutes: grow, revert, grow,
+    revert. For an autoscaling lane the stored slots are a FLOOR."""
+    control = _stub_control(monkeypatch)
+    _stub_live(monkeypatch, "worker", pools={"host-a": 6})
+
+    wc.reconcile_lanes_sync({"worker": (2, True)}, frozenset({"worker"}))
+
+    assert control.shrank == []
+
+
+def test_the_reconcile_still_restores_an_autoscaling_lane_that_fell_below(
+    monkeypatch,
+):
+    """A floor is still a floor. A restart drops the lane to its env value and
+    this must bring it back to what the operator set."""
+    control = _stub_control(monkeypatch)
+    _stub_live(monkeypatch, "worker", pools={"host-a": 1})
+
+    wc.reconcile_lanes_sync({"worker": (4, True)}, frozenset({"worker"}))
+
+    assert control.grew == [(3, ["host-a"])]
+
+
+def test_a_non_autoscaling_lane_is_still_driven_down_to_its_value(monkeypatch):
+    """The floor applies only to lanes that opted in — otherwise turning
+    autoscale off would leave the lane stuck at whatever it had grown to."""
+    control = _stub_control(monkeypatch)
+    _stub_live(monkeypatch, "worker", pools={"host-a": 6})
+
+    wc.reconcile_lanes_sync({"worker": (2, True)}, frozenset())
+
+    assert control.shrank == [(4, ["host-a"])]
+
+
+def test_the_autoscale_task_is_registered_and_scheduled():
+    import backend.app.tasks.maintenance  # noqa: F401
+    from backend.app.celery_app import celery
+
+    name = "backend.app.tasks.maintenance.autoscale_worker_lanes"
+    assert name in celery.tasks
+    assert name in {e["task"] for e in celery.conf.beat_schedule.values()}