fix(agent): stable util-band autoscaler + live GPU meters

Two operator-reported issues with the GPU agent:

1. Worker count flopped almost every cycle, spiking the GPU. The hill-climb
   probed +1, judged it over a too-short noisy throughput window, saw no clear
   gain and reverted -1 — every tick. Replace it with a GPU-utilization-band
   controller: HOLD while smoothed util sits in a healthy band, grow only on
   clear spare capacity (util below the low mark + VRAM headroom), shrink under
   saturation or memory pressure. Util is EWMA-smoothed and decisions are spaced
   (DECIDE_EVERY samples), so a noisy nvidia-smi reading can't move the pool.
   Load stays consistent instead of probe/reverting.

2. GPU util/VRAM bars only updated on manual refresh. They rode the /status
   poll, which blocks on the curator queue call (slow when curator is busy), so
   the meters froze between refreshes. Give them a dedicated /gpu endpoint
   (local nvidia-smi only, no curator round-trip) polled every 1.5s, and drop
   the curator queue-status timeout 15s -> 5s so /status itself stays snappy.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa

agent/fc_agent/worker.py

@@ -49,15 +49,19 @@ MAX_CONCURRENCY = 32
 DEFAULT_EMBED_MODEL = "google/siglip-so400m-patch14-384"
 DEFAULT_EMBED_VERSION = "siglip-so400m-patch14-384"
 
-# Autoscaler (when Auto is on): a throughput hill-climb that finds the worker
-# count on its own — grows while jobs/sec keeps rising and VRAM stays under
-# budget, backs off when a step stops helping or memory gets tight, then settles
-# and periodically re-probes (the workload's GPU/IO balance shifts).
-CONTROL_INTERVAL = 6.0     # seconds between control decisions
-VRAM_HI = 0.90             # back off above this fraction of VRAM used
-UTIL_HI = 96               # GPU util% considered saturated
-TPUT_MARGIN = 0.10         # a step up must beat the baseline by this to "help"
-REPROBE_TICKS = 5          # ticks to hold after settling before re-probing up
+# Autoscaler (when Auto is on): a GPU-utilization-band controller. It grows the
+# pool while the GPU has spare capacity (util below the low mark + VRAM headroom)
+# and shrinks under saturation / memory pressure, then HOLDS while util sits in
+# the band — so the worker count stays steady instead of flopping. Util is EWMA-
+# smoothed and decisions are spaced out, so a single noisy nvidia-smi sample
+# can't move it.
+CONTROL_INTERVAL = 8.0     # seconds between samples
+DECIDE_EVERY = 3           # only act every Nth sample (~24s) — stability
+UTIL_LO = 70               # grow when smoothed util is below this (spare capacity)
+UTIL_HI = 92               # shrink when above this (saturated)
+VRAM_HI = 0.88             # shrink above this fraction of VRAM (memory pressure)
+VRAM_GROW_MAX = 0.80       # don't grow past this VRAM
+EWMA_ALPHA = 0.4           # util smoothing weight on the newest sample
 
 log = logging.getLogger("fc_agent.worker")
 
@@ -213,61 +217,56 @@ class Worker:
 
     # --- autoscaler --------------------------------------------------------
     def _control_loop(self):
-        """Throughput hill-climb (Auto mode): grow the pool while jobs/sec keeps
-        improving and VRAM stays under budget; revert a step that doesn't help;
-        back off under memory pressure; settle, then periodically re-probe."""
-        import time as _t
-
+        """GPU-utilization-band controller (Auto mode). Hold the worker count
+        steady while the GPU sits in a healthy util band; grow only when there's
+        clear spare capacity (smoothed util below the low mark + VRAM headroom),
+        shrink under saturation or memory pressure. Util is EWMA-smoothed and we
+        only act every DECIDE_EVERY samples, so a noisy nvidia-smi reading can't
+        make the pool flop — load stays consistent instead of probe/reverting
+        every cycle (the old hill-climb's failure mode)."""
         from . import gpu as gpumod
 
-        prev_p, prev_t = self.processed, _t.monotonic()
-        base_tput = None     # throughput baseline the current climb is judged against
-        last_dir = 0         # direction of the last applied step (+1 / -1 / 0)
-        cooldown = 0         # ticks to wait (post-settle / post-backoff) before acting
+        util_ewma = None     # smoothed GPU util%
+        tick = 0             # samples since the last decision
         while not self._ctrl_stop.wait(CONTROL_INTERVAL):
             if not (self._running and self._auto):
-                prev_p, prev_t = self.processed, _t.monotonic()
-                base_tput, last_dir, cooldown = None, 0, 0
+                util_ewma, tick = None, 0
                 continue
-            now = _t.monotonic()
-            dt = max(1e-3, now - prev_t)
-            tput = (self.processed - prev_p) / dt
-            prev_p, prev_t = self.processed, now
-            t0 = self._target
 
             g = gpumod.read_gpu() or {}
             mt = g.get("mem_total_mb") or 0
             vram = (g.get("mem_used_mb", 0) / mt) if mt else 0.0
             util = g.get("util_pct", 0) or 0
+            util_ewma = util if util_ewma is None else (
+                EWMA_ALPHA * util + (1 - EWMA_ALPHA) * util_ewma
+            )
 
-            if vram >= VRAM_HI:                      # memory pressure → always shrink
+            # Memory pressure overrides the cadence — react immediately.
+            if vram >= VRAM_HI:
+                if self._apply_step(-1):
+                    log.info(
+                        "autoscale: -1 → %d workers (vram %d%% — memory pressure)",
+                        self._target, round(vram * 100),
+                    )
+                tick = 0
+                continue
+
+            tick += 1
+            if tick < DECIDE_EVERY:                  # hold between decisions
+                continue
+            tick = 0
+
+            t0 = self._target
+            if util_ewma > UTIL_HI:                  # saturated → ease off
                 self._apply_step(-1)
-                base_tput, last_dir, cooldown = None, 0, 2
-                continue
-            if cooldown > 0:
-                cooldown -= 1
-                continue
-            if base_tput is None:                    # establish a baseline + probe up
-                base_tput = tput
-                last_dir = 1 if self._apply_step(1) else 0
-                if last_dir == 0:                    # already at the cap
-                    base_tput, cooldown = None, REPROBE_TICKS
-                continue
-            if last_dir > 0:
-                if tput > base_tput * (1 + TPUT_MARGIN) and util < UTIL_HI:
-                    base_tput = tput                 # the step helped → keep climbing
-                    if not self._apply_step(1):
-                        base_tput, last_dir, cooldown = None, 0, REPROBE_TICKS
-                else:                                # didn't help → revert + settle
-                    self._apply_step(-1)
-                    base_tput, last_dir, cooldown = None, 0, REPROBE_TICKS
-            else:
-                base_tput = None                     # settled → re-probe next cycle
+            elif util_ewma < UTIL_LO and vram < VRAM_GROW_MAX:
+                self._apply_step(+1)                 # spare capacity → grow
+            # else: util is in the band → HOLD (steady load, no flopping)
 
             if self._target != t0:
                 log.info(
-                    "autoscale: %d→%d workers (%.2f jobs/s · util %d%% · vram %d%%)",
-                    t0, self._target, tput, util, round(vram * 100),
+                    "autoscale: %d→%d workers (util~%d%% · vram %d%%)",
+                    t0, self._target, round(util_ewma), round(vram * 100),
                 )
 
     def _ensure_embedder(self, model_name: str):