feat(agent): autoscale the worker count (throughput hill-climb), Auto default-on

The new per-job workload (3 detectors + several SigLIP embeds) is far more GPU-bound than the old I/O-bound CCIP pass, so the right worker count shifted and is hard to guess. Add an Auto mode (default ON) that finds it: - _control_loop samples jobs/sec + GPU util/VRAM every ~6s and hill-climbs the target: grow while throughput keeps improving and VRAM stays under budget, revert a step that doesn't help, back off under memory pressure (VRAM >= 90%), then settle and periodically re-probe (the GPU/IO balance shifts over a run). - A manual concurrency set is an override → leaves Auto; an "Auto" toggle in the control UI re-enables it. status() reports `auto`; the dial reflects the auto-chosen count (read-only) while Auto is on. - AUTO_SCALE env (default on) + compose doc. Agent py-compiled (outside CI). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
2026-06-30 18:19:15 -04:00
parent 359bc5a283
commit 6d7b17b0b5
4 changed files with 114 additions and 2 deletions
@@ -34,6 +34,10 @@ services:
      # Resume the worker automatically on container start (survive a reboot /
      # crash-restart while you're away). Set to 0 to require a manual Start.
      AUTO_START: ${AUTO_START:-1}
+      # Autoscale the worker count (throughput hill-climb that finds the sweet
+      # spot + backs off under VRAM pressure). On by default; toggle live in the
+      # control UI. Set to 0 to start in manual mode.
+      AUTO_SCALE: ${AUTO_SCALE:-1}
      # Crop embedder (SigLIP concept bag): float16 keeps VRAM low on a shared
      # desktop GPU; the model itself is announced by the server.
      SIGLIP_DTYPE: ${SIGLIP_DTYPE:-float16}
@@ -50,6 +50,13 @@ async def concurrency(request: Request):
    return JSONResponse(worker.status())


+@app.post("/auto")
+async def auto(request: Request):
+    body = await request.json()
+    worker.set_auto(bool(body.get("value", True)))
+    return JSONResponse(worker.status())
+
+
@app.get("/status")
 def status():
    s = worker.status()
@@ -83,13 +90,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 <button class=stop onclick=act('stop')>Stop</button>
 </div>
 <div class=row>
+ <label style="margin-right:1rem"><input type=checkbox id=autochk onchange="setauto(this.checked)"> Auto</label>
 workers
 <button class=step onclick=setc(-1)>−</button>
 <input id=conc type=number min=1 value=1
  style="width:3.5rem;font:700 16px system-ui;text-align:center;background:#222;color:#e8e8e8;border:1px solid #444;border-radius:6px;padding:.3rem"
  onchange="setv(this.value)">
 <button class=step onclick=setc(1)>+</button>
- <span class=cap style=color:#9aa>(more = overlap I/O, fill the GPU) max <b id=capn>8</b></span>
+ <span class=cap style=color:#9aa id=conchint>auto-tuning to fill the GPU · max <b id=capn>8</b></span>
 </div>
 <div class=row>
 <span class=stat><span class=n id=state>stopped</span><br>state</span>
@@ -113,6 +121,10 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
   body:JSON.stringify({value:v})});refresh()
 }
+ async function setauto(on){
+  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
+   body:JSON.stringify({value:on})});refresh()
+ }
 async function refresh(){
  const s=await (await fetch('/status')).json()
  CAP=s.max_concurrency||8; capn.textContent=CAP
@@ -121,6 +133,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  // Running but the queue read failed → curator is unreachable; show we're
  // riding it out rather than erroring.
  banner.style.display=(s.state==='running' && !s.queue)?'block':'none'
+  // Auto on → the dial reflects the auto-chosen count (read-only); off → manual.
+  if(document.activeElement!==autochk) autochk.checked=!!s.auto
+  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.6:1
+  conchint.textContent=s.auto?('auto-tuning to fill the GPU · max '+CAP):('manual · max '+CAP)
+  capn.textContent=CAP
  if(document.activeElement!==conc) conc.value=s.concurrency
  conc.max=CAP
  cfg.textContent=s.configured?'set':'MISSING'
@@ -18,6 +18,7 @@ class Config:
    #                            the server announces in the lease)
    auto_start: bool     # start the worker pool on boot (so a container restart
    #                      resumes processing without anyone clicking Start)
+    auto_scale: bool     # autoscale the worker count (throughput hill-climb)
    # Crop PROPOSERS (extra YOLO detectors that say where to crop). Each weight
    # spec is an ultralytics name | http(s) URL | "hf_repo::file" ("" = off).
    person_weights: str  # general COCO person detector (Western/realistic figs)
@@ -43,6 +44,7 @@ class Config:
            embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
            embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
            auto_start=os.environ.get("AUTO_START", "").lower() in ("1", "true", "yes"),
+            auto_scale=os.environ.get("AUTO_SCALE", "true").lower() in ("1", "true", "yes"),
            person_weights=os.environ.get("PERSON_WEIGHTS", "yolo11n.pt"),
            person_conf=float(os.environ.get("PERSON_CONF", "0.35")),
            anatomy_weights=os.environ.get("ANATOMY_WEIGHTS", ""),
@@ -48,6 +48,16 @@ MAX_CONCURRENCY = 32
 DEFAULT_EMBED_MODEL = "google/siglip-so400m-patch14-384"
 DEFAULT_EMBED_VERSION = "siglip-so400m-patch14-384"

+# Autoscaler (when Auto is on): a throughput hill-climb that finds the worker
+# count on its own — grows while jobs/sec keeps rising and VRAM stays under
+# budget, backs off when a step stops helping or memory gets tight, then settles
+# and periodically re-probes (the workload's GPU/IO balance shifts).
+CONTROL_INTERVAL = 6.0     # seconds between control decisions
+VRAM_HI = 0.90             # back off above this fraction of VRAM used
+UTIL_HI = 96               # GPU util% considered saturated
+TPUT_MARGIN = 0.10         # a step up must beat the baseline by this to "help"
+REPROBE_TICKS = 5          # ticks to hold after settling before re-probing up
+

 class _Slot:
    """One worker loop. `inflight` = jobs leased but not yet processed, so a
@@ -66,6 +76,9 @@ class Worker:
        self._lock = threading.Lock()
        self._running = False
        self._target = max(1, min(MAX_CONCURRENCY, cfg.concurrency))
+        self._auto = bool(cfg.auto_scale)      # autoscale worker count
+        self._ctrl_stop = threading.Event()
+        self._ctrl_thread: threading.Thread | None = None
        self._slots: list[_Slot] = []
        self.processed = 0
        self.errors = 0
@@ -85,20 +98,43 @@ class Worker:
        with self._lock:
            self._running = True
            self._reconcile_locked()
+        # (Re)start the autoscaler control loop.
+        if self._ctrl_thread is None or not self._ctrl_thread.is_alive():
+            self._ctrl_stop.clear()
+            self._ctrl_thread = threading.Thread(target=self._control_loop, daemon=True)
+            self._ctrl_thread.start()

    def stop(self):
+        self._ctrl_stop.set()
        with self._lock:
            self._running = False
            slots, self._slots = self._slots, []
        for s in slots:
            s.stop.set()          # each slot releases its inflight on exit

-    def set_concurrency(self, n: int):
+    def set_auto(self, on: bool):
        with self._lock:
+            self._auto = bool(on)
+
+    def set_concurrency(self, n: int):
+        # A manual set is an override → leave Auto.
+        with self._lock:
+            self._auto = False
            self._target = max(1, min(MAX_CONCURRENCY, int(n)))
            if self._running:
                self._reconcile_locked()

+    def _apply_step(self, delta: int) -> bool:
+        """Nudge the target by delta (bounded). Returns True if it changed."""
+        with self._lock:
+            new = max(1, min(MAX_CONCURRENCY, self._target + delta))
+            if new == self._target:
+                return False
+            self._target = new
+            if self._running:
+                self._reconcile_locked()
+            return True
+
    def _reconcile_locked(self):
        while len(self._slots) < self._target:
            slot = _Slot()
@@ -113,6 +149,7 @@ class Worker:
                "state": "running" if self._running else "stopped",
                "concurrency": self._target,
                "max_concurrency": MAX_CONCURRENCY,
+                "auto": self._auto,
                "workers": len(self._slots),
                "active": self._active,
                "processed": self.processed,
@@ -171,6 +208,58 @@ class Worker:
        a pool-shrink doesn't hang for a full backoff window."""
        slot.stop.wait(timeout=seconds)

+    # --- autoscaler --------------------------------------------------------
+    def _control_loop(self):
+        """Throughput hill-climb (Auto mode): grow the pool while jobs/sec keeps
+        improving and VRAM stays under budget; revert a step that doesn't help;
+        back off under memory pressure; settle, then periodically re-probe."""
+        import time as _t
+
+        from . import gpu as gpumod
+
+        prev_p, prev_t = self.processed, _t.monotonic()
+        base_tput = None     # throughput baseline the current climb is judged against
+        last_dir = 0         # direction of the last applied step (+1 / -1 / 0)
+        cooldown = 0         # ticks to wait (post-settle / post-backoff) before acting
+        while not self._ctrl_stop.wait(CONTROL_INTERVAL):
+            if not (self._running and self._auto):
+                prev_p, prev_t = self.processed, _t.monotonic()
+                base_tput, last_dir, cooldown = None, 0, 0
+                continue
+            now = _t.monotonic()
+            dt = max(1e-3, now - prev_t)
+            tput = (self.processed - prev_p) / dt
+            prev_p, prev_t = self.processed, now
+
+            g = gpumod.read_gpu() or {}
+            mt = g.get("mem_total_mb") or 0
+            vram = (g.get("mem_used_mb", 0) / mt) if mt else 0.0
+            util = g.get("util_pct", 0) or 0
+
+            if vram >= VRAM_HI:                      # memory pressure → always shrink
+                self._apply_step(-1)
+                base_tput, last_dir, cooldown = None, 0, 2
+                continue
+            if cooldown > 0:
+                cooldown -= 1
+                continue
+            if base_tput is None:                    # establish a baseline + probe up
+                base_tput = tput
+                last_dir = 1 if self._apply_step(1) else 0
+                if last_dir == 0:                    # already at the cap
+                    base_tput, cooldown = None, REPROBE_TICKS
+                continue
+            if last_dir > 0:
+                if tput > base_tput * (1 + TPUT_MARGIN) and util < UTIL_HI:
+                    base_tput = tput                 # the step helped → keep climbing
+                    if not self._apply_step(1):
+                        base_tput, last_dir, cooldown = None, 0, REPROBE_TICKS
+                else:                                # didn't help → revert + settle
+                    self._apply_step(-1)
+                    base_tput, last_dir, cooldown = None, 0, REPROBE_TICKS
+            else:
+                base_tput = None                     # settled → re-probe next cycle
+
    def _ensure_embedder(self, model_name: str):
        if self._embedder is not None:
            return self._embedder