bvandeusen
3b34230fbd
Two operator-reported issues with the GPU agent: 1. Worker count flopped almost every cycle, spiking the GPU. The hill-climb probed +1, judged it over a too-short noisy throughput window, saw no clear gain and reverted -1 — every tick. Replace it with a GPU-utilization-band controller: HOLD while smoothed util sits in a healthy band, grow only on clear spare capacity (util below the low mark + VRAM headroom), shrink under saturation or memory pressure. Util is EWMA-smoothed and decisions are spaced (DECIDE_EVERY samples), so a noisy nvidia-smi reading can't move the pool. Load stays consistent instead of probe/reverting. 2. GPU util/VRAM bars only updated on manual refresh. They rode the /status poll, which blocks on the curator queue call (slow when curator is busy), so the meters froze between refreshes. Give them a dedicated /gpu endpoint (local nvidia-smi only, no curator round-trip) polled every 1.5s, and drop the curator queue-status timeout 15s -> 5s so /status itself stays snappy. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
274 lines
13 KiB
Python
274 lines
13 KiB
Python
"""FastAPI control surface for the agent (served on localhost).
|
||
|
||
Start / stop the worker pool, tune the worker count live (trades desktop
|
||
responsiveness for throughput), and watch GPU load + progress + the server-side
|
||
queue. Config is env-seeded; the worker count is adjustable here on the fly.
|
||
"""
|
||
from fastapi import FastAPI, Request
|
||
from fastapi.responses import HTMLResponse, JSONResponse
|
||
|
||
from . import logbuf
|
||
from .config import Config
|
||
from .gpu import read_gpu
|
||
from .worker import Worker
|
||
|
||
logbuf.install()
|
||
cfg = Config.from_env()
|
||
worker = Worker(cfg)
|
||
app = FastAPI(title="FabledCurator GPU agent")
|
||
|
||
|
||
@app.on_event("startup")
|
||
def _maybe_autostart() -> None:
|
||
# With AUTO_START set, a container restart (host reboot, or `restart:
|
||
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
||
# then ride out a still-down curator via lease backoff. Lets the agent
|
||
# survive a redeploy with nobody at the desktop to click Start.
|
||
if cfg.auto_start and cfg.token:
|
||
worker.start()
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
def index() -> str:
|
||
return _PAGE
|
||
|
||
|
||
@app.post("/start")
|
||
def start():
|
||
worker.start()
|
||
return JSONResponse(worker.status())
|
||
|
||
|
||
@app.post("/stop")
|
||
def stop():
|
||
worker.stop()
|
||
return JSONResponse(worker.status())
|
||
|
||
|
||
@app.post("/concurrency")
|
||
async def concurrency(request: Request):
|
||
body = await request.json()
|
||
worker.set_concurrency(int(body.get("value", 1)))
|
||
return JSONResponse(worker.status())
|
||
|
||
|
||
@app.post("/auto")
|
||
async def auto(request: Request):
|
||
body = await request.json()
|
||
worker.set_auto(bool(body.get("value", True)))
|
||
return JSONResponse(worker.status())
|
||
|
||
|
||
@app.get("/gpu")
|
||
def gpu():
|
||
# GPU meters poll this on their own fast cadence. It only reads local
|
||
# nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
|
||
# when /status is slow waiting on the (sometimes busy) curator queue call.
|
||
return JSONResponse(read_gpu() or {})
|
||
|
||
|
||
@app.get("/logs")
|
||
def logs():
|
||
return JSONResponse({"lines": list(logbuf.LINES)})
|
||
|
||
|
||
@app.get("/status")
|
||
def status():
|
||
s = worker.status()
|
||
s["fc_url"] = cfg.fc_url
|
||
s["configured"] = bool(cfg.token)
|
||
s["gpu"] = read_gpu()
|
||
try:
|
||
s["queue"] = worker.client.queue_status()
|
||
except Exception:
|
||
s["queue"] = None
|
||
return JSONResponse(s)
|
||
|
||
|
||
_PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||
<meta name=viewport content="width=device-width,initial-scale=1">
|
||
<title>FabledCurator · GPU agent</title>
|
||
<style>
|
||
:root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
|
||
--mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
|
||
*{box-sizing:border-box}
|
||
body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
|
||
background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
|
||
.wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;min-height:100vh;
|
||
display:flex;flex-direction:column}
|
||
header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
|
||
.brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
|
||
.logo{color:var(--acc);font-size:20px}
|
||
.brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
|
||
.conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
|
||
.dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
|
||
.dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
|
||
.dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
|
||
.meta{color:var(--mut);margin:0 0 18px;font-size:13px}
|
||
code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
|
||
font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
|
||
.card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
|
||
border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
|
||
.card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
|
||
color:var(--mut);margin-bottom:14px}
|
||
.controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
|
||
.spacer{flex:1}
|
||
.btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
|
||
cursor:pointer;color:#fff;transition:.12s}
|
||
.btn:hover{transform:translateY(-1px)}
|
||
.btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
|
||
.btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
|
||
.switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
|
||
.switch input{display:none}
|
||
.switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
|
||
.switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
|
||
background:#cdd6e0;transition:.15s}
|
||
.switch input:checked+.track{background:var(--acc)}
|
||
.switch input:checked+.track:after{transform:translateX(16px);background:#fff}
|
||
.stepper{display:inline-flex;align-items:center;gap:6px}
|
||
.step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
|
||
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
||
.step:hover{border-color:var(--acc)}
|
||
#conc{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
||
color:var(--fg);border:1px solid var(--bd);border-radius:8px}
|
||
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
||
.tiles{display:grid;grid-template-columns:repeat(5,1fr);gap:10px;margin-bottom:16px}
|
||
.tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 10px;text-align:center}
|
||
.tile .n{font:800 24px ui-monospace,monospace;line-height:1.1}
|
||
.tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
|
||
.tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
|
||
.meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
|
||
.meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
|
||
.meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
|
||
.bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
|
||
.bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
|
||
#utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
|
||
.queue{font:13px ui-monospace,monospace;color:var(--mut)}
|
||
.banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
|
||
border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
|
||
.logs-h{display:flex;align-items:center;justify-content:space-between}
|
||
.grow{flex:1;display:flex;flex-direction:column;min-height:0}
|
||
.grow .logs{flex:1;min-height:120px}
|
||
.copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
|
||
background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
|
||
padding:5px 11px;cursor:pointer}
|
||
.copybtn:hover{border-color:var(--acc)}
|
||
.logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
|
||
overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
|
||
color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
|
||
</style></head><body>
|
||
<div class=wrap>
|
||
<header>
|
||
<div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
|
||
<div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
|
||
</header>
|
||
<p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code></p>
|
||
|
||
<div id=banner class=banner style=display:none>
|
||
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
||
</div>
|
||
|
||
<section class=card>
|
||
<div class=card-h>Control</div>
|
||
<div class=controls>
|
||
<button class="btn start" onclick=act('start')>▶ Start</button>
|
||
<button class="btn stop" onclick=act('stop')>■ Stop</button>
|
||
<div class=spacer></div>
|
||
<label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
|
||
<div class=stepper>
|
||
<button class=step onclick=setc(-1)>−</button>
|
||
<input id=conc type=number min=1 value=1 onchange="setv(this.value)">
|
||
<button class=step onclick=setc(1)>+</button>
|
||
</div>
|
||
</div>
|
||
<div class=hint id=conchint>auto-tuning to fill the GPU · max <b id=capn>8</b></div>
|
||
</section>
|
||
|
||
<section class=card>
|
||
<div class=card-h>Status</div>
|
||
<div class=tiles>
|
||
<div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
|
||
<div class=tile><div class=n id=active>0</div><div class=l>active</div></div>
|
||
<div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
|
||
<div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
|
||
<div class=tile><div class=n id=wait>0</div><div class=l>waited out</div></div>
|
||
</div>
|
||
<div class=meters>
|
||
<div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
|
||
<div class=bar><i id=utilbar></i></div></div>
|
||
<div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
|
||
<div class=bar><i id=gpubar></i></div></div>
|
||
</div>
|
||
<div class=queue id=queue>queue —</div>
|
||
</section>
|
||
|
||
<section class="card grow">
|
||
<div class="card-h logs-h">Logs
|
||
<button class=copybtn id=copybtn onclick=copyLogs()>Copy</button>
|
||
</div>
|
||
<pre class=logs id=logs>waiting for activity…</pre>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
let CAP=8
|
||
async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
|
||
function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
|
||
async function setv(v){
|
||
v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
|
||
await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
|
||
body:JSON.stringify({value:v})});refresh()
|
||
}
|
||
async function setauto(on){
|
||
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
||
body:JSON.stringify({value:on})});refresh()
|
||
}
|
||
async function refresh(){
|
||
let s; try{ s=await (await fetch('/status')).json() }catch{ return }
|
||
CAP=s.max_concurrency||8; capn.textContent=CAP
|
||
const running=s.state==='running', ok=s.configured
|
||
state.textContent=s.state
|
||
active.textContent=s.active; done.textContent=s.processed
|
||
err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
|
||
fc.textContent=s.fc_url; wait.textContent=s.transient||0
|
||
cfg.textContent=ok?'set':'MISSING'
|
||
// connection pill
|
||
dot.className='dot '+(!ok?'red':(running?(s.queue?'green':'amber'):'amber'))
|
||
connlbl.textContent=!ok?'no token':(running?(s.queue?'running':'running · curator unreachable'):'stopped')
|
||
banner.style.display=(running && !s.queue)?'block':'none'
|
||
// Auto on → dial reflects the auto-chosen count (read-only); off → manual.
|
||
if(document.activeElement!==autochk) autochk.checked=!!s.auto
|
||
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
|
||
conchint.textContent=s.auto?('auto-tuning to fill the GPU · max '+CAP):('manual · max '+CAP)
|
||
if(document.activeElement!==conc) conc.value=s.concurrency
|
||
conc.max=CAP
|
||
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
||
}
|
||
// GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
|
||
// slow curator queue call can't freeze the bars (they only stale on refresh).
|
||
async function refreshGpu(){
|
||
let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
|
||
if(g && g.util_pct!=null){
|
||
const u=g.util_pct, used=g.mem_used_mb, tot=g.mem_total_mb||1
|
||
utillbl.textContent=u+'% · '+g.temp_c+'°C'; utilbar.style.width=u+'%'
|
||
vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
|
||
} else { utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
|
||
}
|
||
async function refreshLogs(){
|
||
try{
|
||
const r=await (await fetch('/logs')).json()
|
||
const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
|
||
el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
|
||
if(atBottom) el.scrollTop=el.scrollHeight
|
||
}catch{}
|
||
}
|
||
async function copyLogs(){
|
||
const txt=logs.textContent||''
|
||
try{ await navigator.clipboard.writeText(txt) }
|
||
catch{ const t=document.createElement('textarea'); t.value=txt; document.body.appendChild(t);
|
||
t.select(); try{document.execCommand('copy')}catch{}; t.remove() }
|
||
copybtn.textContent='Copied'; setTimeout(()=>{copybtn.textContent='Copy'},1200)
|
||
}
|
||
refresh(); refreshGpu(); refreshLogs()
|
||
setInterval(refresh,3000); setInterval(refreshGpu,1500); setInterval(refreshLogs,2500)
|
||
</script></body></html>"""
|