feat(agent): GPU load readout + live worker-count tuning (#114)

Control UI gains what the operator asked for:
- GPU load (nvidia-smi): util %, VRAM used/total + bar, temp — so you can see how
  hard the card is working while you're at the desktop.
- Worker count is now a live − / + control (POST /concurrency), not just an env:
  the worker is a pool of independent slots (shared model, so slots add concurrent
  inference, not N× VRAM). Dial up for speed, down to free the card. Replaces
  pause/resume with Start/Stop + the worker dial.
- Graceful release on stop / pool-shrink: a slot hands its still-leased jobs back
  via client.release() so they're re-picked immediately (pairs with the server
  recovery sweep).

Not CI-tested (agent/ outside CI) — verified by running.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

agent/fc_agent/worker.py

@@ -1,8 +1,13 @@
-"""The lease → fetch → detect+embed → submit loop, with start/pause/stop control.
+"""The lease → fetch → detect+embed → submit loop, run by a pool of worker
+slots whose count is tunable live from the UI.
 
-Concurrency is 1 (one image at a time) so the GPU footprint stays small and a
-stop frees the card promptly. Stop halts leasing + finishes the current item;
-unprocessed leases expire and the server re-queues them — nothing is lost.
+Each slot is an independent loop (its own leases; the server's SKIP-LOCKED lease
+keeps them from colliding). More slots = more GPU load + throughput; the model is
+loaded once and shared, so slots add concurrent inference, not N× model VRAM.
+That's the dial the operator turns to trade desktop responsiveness for speed.
+
+Stop (or shrinking the pool) RELEASES a slot's still-leased jobs immediately so
+orphaned work is re-picked at once rather than waiting out the lease.
 """
 import threading
 import time
@@ -12,61 +17,78 @@ from .client import FcClient
 from .config import Config
 from .crops import crop_region
 
+MAX_CONCURRENCY = 8
+
+
+class _Slot:
+    """One worker loop. `inflight` = jobs leased but not yet processed, so a
+    graceful stop can hand them back."""
+    __slots__ = ("stop", "inflight")
+
+    def __init__(self):
+        self.stop = threading.Event()
+        self.inflight: list[int] = []
+
 
 class Worker:
     def __init__(self, cfg: Config):
         self.cfg = cfg
         self.client = FcClient(cfg.fc_url, cfg.token, cfg.agent_id)
-        self._state = "idle"        # idle | running | paused | stopping
         self._lock = threading.Lock()
-        self._thread: threading.Thread | None = None
+        self._running = False
+        self._target = max(1, min(MAX_CONCURRENCY, cfg.concurrency))
+        self._slots: list[_Slot] = []
         self.processed = 0
         self.errors = 0
-        self.current = None
+        self._active = 0          # slots currently mid-image
 
     # --- control -----------------------------------------------------------
     def start(self):
         with self._lock:
-            if self._state in ("running", "paused"):
-                self._state = "running"
-                return
-            self._state = "running"
-        self._thread = threading.Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def pause(self):
-        with self._lock:
-            if self._state == "running":
-                self._state = "paused"
-
-    def resume(self):
-        with self._lock:
-            if self._state == "paused":
-                self._state = "running"
+            self._running = True
+            self._reconcile_locked()
 
     def stop(self):
         with self._lock:
-            if self._state in ("running", "paused"):
-                self._state = "stopping"
+            self._running = False
+            slots, self._slots = self._slots, []
+        for s in slots:
+            s.stop.set()          # each slot releases its inflight on exit
+
+    def set_concurrency(self, n: int):
+        with self._lock:
+            self._target = max(1, min(MAX_CONCURRENCY, int(n)))
+            if self._running:
+                self._reconcile_locked()
+
+    def _reconcile_locked(self):
+        while len(self._slots) < self._target:
+            slot = _Slot()
+            self._slots.append(slot)
+            threading.Thread(target=self._loop, args=(slot,), daemon=True).start()
+        while len(self._slots) > self._target:
+            self._slots.pop().stop.set()
 
     def status(self) -> dict:
         with self._lock:
-            state = self._state
-        return {
-            "state": state, "processed": self.processed,
-            "errors": self.errors, "current": self.current,
-        }
+            return {
+                "state": "running" if self._running else "stopped",
+                "concurrency": self._target,
+                "workers": len(self._slots),
+                "active": self._active,
+                "processed": self.processed,
+                "errors": self.errors,
+            }
 
-    # --- loop --------------------------------------------------------------
-    def _run(self):
-        while True:
-            with self._lock:
-                st = self._state
-            if st == "stopping":
-                break
-            if st == "paused":
-                time.sleep(1)
-                continue
+    def _bump(self, *, processed=0, errors=0, active=0):
+        with self._lock:
+            self.processed += processed
+            self.errors += errors
+            self._active += active
+
+    # --- per-slot loop -----------------------------------------------------
+    def _loop(self, slot: _Slot):
+        while not slot.stop.is_set() and self._running:
             try:
                 jobs = self.client.lease(self.cfg.batch_size)
             except Exception:
@@ -75,18 +97,21 @@ class Worker:
             if not jobs:
                 time.sleep(self.cfg.poll_idle_seconds)
                 continue
-            ids = [j["job_id"] for j in jobs]
+            slot.inflight = [j["job_id"] for j in jobs]
             for job in jobs:
-                with self._lock:
-                    if self._state == "stopping":
-                        break
+                if slot.stop.is_set() or not self._running:
+                    break
                 self._process(job)
-                self.client.heartbeat(ids)   # keep the rest of the batch alive
-        with self._lock:
-            self._state = "idle"
+                slot.inflight = [i for i in slot.inflight if i != job["job_id"]]
+                if slot.inflight:
+                    self.client.heartbeat(slot.inflight)
+        # Graceful hand-back of anything leased but not processed.
+        if slot.inflight:
+            self.client.release(slot.inflight)
+            slot.inflight = []
 
     def _process(self, job: dict):
-        self.current = job.get("image_id")
+        self._bump(active=1)
         try:
             data = self.client.fetch_image(job["image_url"])
             if media.is_video(job.get("mime", "")):
@@ -119,9 +144,9 @@ class Worker:
                         "detector_version": dv,
                     })
             self.client.submit(job["job_id"], regions, ["figure", "face"])
-            self.processed += 1
+            self._bump(processed=1)
         except Exception as exc:   # noqa: BLE001 — report + move on
-            self.errors += 1
+            self._bump(errors=1)
             self.client.fail(job["job_id"], str(exc)[:500])
         finally:
-            self.current = None
+            self._bump(active=-1)