fix(agent): unfreeze status view + smoothed throughput-aware autoscaler + log pane
Operator: the status tiles (state/active/processed) and the Start/Stop buttons freeze while the GPU meters stay live. Root cause: /status made an INLINE blocking curator call (queue_status) on every poll, and with curator buried under a 112k-job backlog that call stalled — freezing the whole status refresh (the GPU bars survived because /gpu is a lock-free local read). Made worse by the old util-band autoscaler, which grew workers toward the 32 cap forever because util plateaus ~50% on this IO-bound load and never hit the 70 grow threshold — piling load onto curator and the agent process. - /status is now a pure in-memory read: worker.status() is lock-free, and the curator queue snapshot is refreshed by a background poller (never inline). - Autoscaler replaced with a smoothed, throughput-aware climb that SETTLES: samples util every 2s and EWMA-smooths it (raw util swings 0↔99), then every ~24s grows by one only while each grow keeps lifting smoothed jobs/s; when a grow stops helping it backs off one and holds, re-probing occasionally. No runaway, no flopping. - GPU util bar now shows a smoothed value: the agent's own EWMA (util_smooth, exposed on /gpu) when running, else smoothed client-side — so it glides instead of bouncing 0↔99. - act() aborts a slow Start/Stop POST after 8s so the buttons can't stick; the now-always-fast /status refresh recovers state regardless. - Log pane: bound the page to the viewport (height:100vh) so the Logs card scrolls INTERNALLY instead of overflowing off-screen; cap the ring buffer at 400 lines. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
+26
-15
@@ -75,7 +75,11 @@ def gpu():
|
||||
# GPU meters poll this on their own fast cadence. It only reads local
|
||||
# nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
|
||||
# when /status is slow waiting on the (sometimes busy) curator queue call.
|
||||
return JSONResponse(read_gpu() or {})
|
||||
g = read_gpu() or {}
|
||||
us = worker.util_smooth()
|
||||
if us is not None:
|
||||
g["util_smooth"] = round(us, 1) # autoscaler's EWMA — the UI bar tracks this
|
||||
return JSONResponse(g)
|
||||
|
||||
|
||||
@app.get("/logs")
|
||||
@@ -85,15 +89,13 @@ def logs():
|
||||
|
||||
@app.get("/status")
|
||||
def status():
|
||||
# Pure in-memory read: worker.status() is lock-free and the queue snapshot is
|
||||
# kept fresh by a background poller — NO inline curator call, so this can't
|
||||
# stall the status view when curator is buried under a big backlog.
|
||||
s = worker.status()
|
||||
s["fc_url"] = cfg.fc_url
|
||||
s["configured"] = bool(cfg.token)
|
||||
# GPU is served by /gpu (cached); /status stays light so Start/Stop never
|
||||
# queue behind it.
|
||||
try:
|
||||
s["queue"] = worker.client.queue_status()
|
||||
except Exception:
|
||||
s["queue"] = None
|
||||
s["queue"] = worker.latest_queue()
|
||||
return JSONResponse(s)
|
||||
|
||||
|
||||
@@ -106,8 +108,8 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
*{box-sizing:border-box}
|
||||
body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
|
||||
background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
|
||||
.wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;min-height:100vh;
|
||||
display:flex;flex-direction:column}
|
||||
.wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;height:100vh;
|
||||
box-sizing:border-box;overflow:hidden;display:flex;flex-direction:column}
|
||||
header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
|
||||
.brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
|
||||
.logo{color:var(--acc);font-size:20px}
|
||||
@@ -163,7 +165,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
|
||||
.logs-h{display:flex;align-items:center;justify-content:space-between}
|
||||
.grow{flex:1;display:flex;flex-direction:column;min-height:0}
|
||||
.grow .logs{flex:1;min-height:120px}
|
||||
.grow .logs{flex:1;min-height:0}
|
||||
.copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
|
||||
background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
|
||||
padding:5px 11px;cursor:pointer}
|
||||
@@ -231,9 +233,12 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
// separate /status poll, which can lag behind the curator queue call.
|
||||
async function act(p){
|
||||
pending(p==='start'?'starting':'stopping')
|
||||
try{ applyStatus(await (await fetch('/'+p,{method:'POST'})).json()) }
|
||||
// Abort a slow POST after 8s so the buttons never stay stuck — the periodic
|
||||
// /status refresh (now always fast) recovers the true state either way.
|
||||
const ac=new AbortController(); const to=setTimeout(()=>ac.abort(),8000)
|
||||
try{ applyStatus(await (await fetch('/'+p,{method:'POST',signal:ac.signal})).json()) }
|
||||
catch{ /* leave the periodic refresh to recover the real state */ }
|
||||
finally{ startbtn.disabled=false; stopbtn.disabled=false }
|
||||
finally{ clearTimeout(to); startbtn.disabled=false; stopbtn.disabled=false }
|
||||
}
|
||||
function pending(label){
|
||||
state.textContent=label; state.className='n busy'
|
||||
@@ -285,13 +290,19 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
}
|
||||
// GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
|
||||
// slow curator queue call can't freeze the bars (they only stale on refresh).
|
||||
let UAVG=null // smoothed util for the bar (raw util swings 0↔99; show the trend)
|
||||
async function refreshGpu(){
|
||||
let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
|
||||
if(g && g.util_pct!=null){
|
||||
const u=g.util_pct, used=g.mem_used_mb, tot=g.mem_total_mb||1
|
||||
utillbl.textContent=u+'% · '+g.temp_c+'°C'; utilbar.style.width=u+'%'
|
||||
// Prefer the agent's own EWMA (util_smooth) when running; otherwise smooth
|
||||
// the raw reading here so a stopped agent's bar still glides, not jumps.
|
||||
const raw=g.util_pct
|
||||
UAVG = (g.util_smooth!=null) ? g.util_smooth
|
||||
: (UAVG==null ? raw : 0.25*raw + 0.75*UAVG)
|
||||
const used=g.mem_used_mb, tot=g.mem_total_mb||1
|
||||
utillbl.textContent=Math.round(UAVG)+'% · '+g.temp_c+'°C'; utilbar.style.width=Math.round(UAVG)+'%'
|
||||
vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
|
||||
} else { utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
|
||||
} else { UAVG=null; utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
|
||||
}
|
||||
async function refreshLogs(){
|
||||
try{
|
||||
|
||||
Reference in New Issue
Block a user