Merge pull request 'Release: graceful Start/Stop (starting/stopping states) + cached GPU reads' (#163) from dev into main

2026-06-30 19:38:50 -04:00
parent 3c1e76bd44 c2e9157822
commit 2c6bf26bfc
2 changed files with 79 additions and 15 deletions
@@ -77,7 +77,8 @@ def status():
    s = worker.status()
    s["fc_url"] = cfg.fc_url
    s["configured"] = bool(cfg.token)
-    s["gpu"] = read_gpu()
+    # GPU is served by /gpu (cached); /status stays light so Start/Stop never
+    # queue behind it.
    try:
        s["queue"] = worker.client.queue_status()
    except Exception:
@@ -116,6 +117,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 .btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
  cursor:pointer;color:#fff;transition:.12s}
 .btn:hover{transform:translateY(-1px)}
+ .btn[disabled]{opacity:.45;pointer-events:none;transform:none}
+ @keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
+ .tile .n.busy{color:var(--acc);animation:pulse 1s ease-in-out infinite}
 .btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
 .btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
 .switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
@@ -171,8 +175,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 <section class=card>
  <div class=card-h>Control</div>
  <div class=controls>
-   <button class="btn start" onclick=act('start')>▶ Start</button>
-   <button class="btn stop" onclick=act('stop')>■ Stop</button>
+   <button class="btn start" id=startbtn onclick=act('start')>▶ Start</button>
+   <button class="btn stop" id=stopbtn onclick=act('stop')>■ Stop</button>
   <div class=spacer></div>
   <label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
   <div class=stepper>
@@ -211,7 +215,20 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 </div>
 <script>
 let CAP=8
- async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
+ // Optimistic transitional state on click, then apply the POST's own status
+ // response (it returns worker.status()) for instant feedback — don't wait on the
+ // separate /status poll, which can lag behind the curator queue call.
+ async function act(p){
+  pending(p==='start'?'starting':'stopping')
+  try{ applyStatus(await (await fetch('/'+p,{method:'POST'})).json()) }
+  catch{ /* leave the periodic refresh to recover the real state */ }
+  finally{ startbtn.disabled=false; stopbtn.disabled=false }
+ }
+ function pending(label){
+  state.textContent=label; state.className='n busy'
+  dot.className='dot amber'
+  startbtn.disabled=true; stopbtn.disabled=true
+ }
 function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
 async function setv(v){
  v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
@@ -224,24 +241,36 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 }
 async function refresh(){
  let s; try{ s=await (await fetch('/status')).json() }catch{ return }
+  applyStatus(s)
+ }
+ function applyStatus(s){
  CAP=s.max_concurrency||8; capn.textContent=CAP
-  const running=s.state==='running', ok=s.configured
-  state.textContent=s.state
+  const running=s.state==='running'
+  // "stopping" = stopped but in-flight jobs are still draining; resolves to
+  // "stopped" on its own once active hits 0.
+  const draining=!running && s.active>0
+  state.textContent=draining?'stopping':s.state
+  state.className='n'+(draining?' busy':'')
  active.textContent=s.active; done.textContent=s.processed
  err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
-  fc.textContent=s.fc_url; wait.textContent=s.transient||0
-  cfg.textContent=ok?'set':'MISSING'
-  // connection pill
-  dot.className='dot '+(!ok?'red':(running?(s.queue?'green':'amber'):'amber'))
-  connlbl.textContent=!ok?'no token':(running?(s.queue?'running':'running · curator unreachable'):'stopped')
-  banner.style.display=(running && !s.queue)?'block':'none'
+  wait.textContent=s.transient||0
  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
  if(document.activeElement!==autochk) autochk.checked=!!s.auto
  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
  conchint.textContent=s.auto?('auto-tuning to fill the GPU · max '+CAP):('manual · max '+CAP)
  if(document.activeElement!==conc) conc.value=s.concurrency
  conc.max=CAP
-  queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
+  // Connection pill + queue come only from the /status poll (the Start/Stop POST
+  // responses skip the slow curator call to stay snappy) — guard so an action
+  // response doesn't blank them.
+  if('configured' in s){
+   const ok=s.configured
+   fc.textContent=s.fc_url; cfg.textContent=ok?'set':'MISSING'
+   dot.className='dot '+(!ok?'red':(running?(s.queue?'green':'amber'):'amber'))
+   connlbl.textContent=!ok?'no token':(running?(s.queue?'running':'running · curator unreachable'):'stopped')
+   banner.style.display=(running && !s.queue)?'block':'none'
+   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
+  }
 }
 // GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
 // slow curator queue call can't freeze the bars (they only stale on refresh).
@@ -1,10 +1,24 @@
 """GPU load readout via nvidia-smi (present in the container thanks to the
 NVIDIA Container Toolkit's `utility` capability). Returns None if unavailable —
-the UI just shows n/a (e.g. CPU-fallback run)."""
+the UI just shows n/a (e.g. CPU-fallback run).
+
+Reads are CACHED and de-duplicated: the UI meter polls fast, /status reads it,
+and the autoscaler samples it — if each spawned its own `nvidia-smi` (slow on a
+busy GPU) those blocking subprocesses would pile up in the server's thread pool
+and make the Start/Stop buttons feel dead. So a short TTL serves recent callers
+from cache, and only ONE probe runs at a time (others get the last value)."""
 import subprocess
+import threading
+import time
+
+_TTL = 1.0                       # seconds a sample is reused before re-probing
+_lock = threading.Lock()
+_cache: dict | None = None
+_cache_t = 0.0
+_probing = False


-def read_gpu() -> dict | None:
+def _probe() -> dict | None:
    try:
        out = subprocess.run(
            [
@@ -28,3 +42,24 @@ def read_gpu() -> dict | None:
        }
    except (ValueError, IndexError):
        return None
+
+
+def read_gpu(max_age: float = _TTL) -> dict | None:
+    """Latest GPU reading, cached. Serves from cache when fresh; when stale,
+    exactly one caller re-probes while the rest get the last value — so request
+    threads never block behind more than one `nvidia-smi`."""
+    global _cache, _cache_t, _probing
+    now = time.monotonic()
+    with _lock:
+        fresh = _cache is not None and (now - _cache_t) < max_age
+        if fresh or _probing:        # fresh, or a probe is already running
+            return _cache
+        _probing = True
+    try:
+        val = _probe()
+    finally:
+        with _lock:
+            _cache = val
+            _cache_t = time.monotonic()
+            _probing = False
+    return val