feat(agent): GPU load readout + live worker-count tuning (#114)
Control UI gains what the operator asked for: - GPU load (nvidia-smi): util %, VRAM used/total + bar, temp — so you can see how hard the card is working while you're at the desktop. - Worker count is now a live − / + control (POST /concurrency), not just an env: the worker is a pool of independent slots (shared model, so slots add concurrent inference, not N× VRAM). Dial up for speed, down to free the card. Replaces pause/resume with Start/Stop + the worker dial. - Graceful release on stop / pool-shrink: a slot hands its still-leased jobs back via client.release() so they're re-picked immediately (pairs with the server recovery sweep). Not CI-tested (agent/ outside CI) — verified by running. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
+46
-30
@@ -1,13 +1,14 @@
|
||||
"""FastAPI control surface for the agent (served on localhost).
|
||||
|
||||
Start / pause / resume / stop the worker, set nothing else here (config is env),
|
||||
and watch progress + the server-side queue. The container exposes this on a
|
||||
localhost port; stopping the worker frees the GPU.
|
||||
Start / stop the worker pool, tune the worker count live (trades desktop
|
||||
responsiveness for throughput), and watch GPU load + progress + the server-side
|
||||
queue. Config is env-seeded; the worker count is adjustable here on the fly.
|
||||
"""
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from .config import Config
|
||||
from .gpu import read_gpu
|
||||
from .worker import Worker
|
||||
|
||||
cfg = Config.from_env()
|
||||
@@ -26,29 +27,25 @@ def start():
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/pause")
|
||||
def pause():
|
||||
worker.pause()
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/resume")
|
||||
def resume():
|
||||
worker.resume()
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/stop")
|
||||
def stop():
|
||||
worker.stop()
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/concurrency")
|
||||
async def concurrency(request: Request):
|
||||
body = await request.json()
|
||||
worker.set_concurrency(int(body.get("value", 1)))
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.get("/status")
|
||||
def status():
|
||||
s = worker.status()
|
||||
s["fc_url"] = cfg.fc_url
|
||||
s["configured"] = bool(cfg.token)
|
||||
s["gpu"] = read_gpu()
|
||||
try:
|
||||
s["queue"] = worker.client.queue_status()
|
||||
except Exception:
|
||||
@@ -59,35 +56,54 @@ def status():
|
||||
_PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<title>FabledCurator GPU agent</title>
|
||||
<style>
|
||||
body{font:14px system-ui;margin:2rem;max-width:640px;background:#14171a;color:#e8e8e8}
|
||||
body{font:14px system-ui;margin:2rem;max-width:680px;background:#14171a;color:#e8e8e8}
|
||||
h1{font-size:18px} button{font:14px system-ui;padding:.5rem 1rem;border:0;border-radius:6px;
|
||||
margin-right:.5rem;cursor:pointer;color:#fff} .start{background:#2e7d32}.pause{background:#b26a00}
|
||||
.stop{background:#b3261e} .stat{display:inline-block;margin-right:1.5rem}
|
||||
margin-right:.5rem;cursor:pointer;color:#fff} .start{background:#2e7d32}.stop{background:#b3261e}
|
||||
.step{background:#33373b;padding:.4rem .7rem;font-weight:700}
|
||||
.stat{display:inline-block;margin-right:1.5rem;vertical-align:top}
|
||||
.n{font-size:22px;font-weight:700} code{background:#222;padding:2px 6px;border-radius:4px}
|
||||
.q{margin-top:1rem;color:#9aa}
|
||||
.q,.gpu{margin-top:1rem;color:#9aa} .bar{height:8px;border-radius:4px;background:#222;overflow:hidden;
|
||||
max-width:320px;margin-top:4px} .bar>i{display:block;height:100%;background:#3f7d3f}
|
||||
.row{margin:.8rem 0}
|
||||
</style></head><body>
|
||||
<h1>FabledCurator GPU agent</h1>
|
||||
<p>FC: <code id=fc>—</code> · token <code id=cfg>—</code></p>
|
||||
<p>
|
||||
<div class=row>
|
||||
<button class=start onclick=act('start')>Start</button>
|
||||
<button class=pause onclick=act('pause')>Pause</button>
|
||||
<button class=pause onclick=act('resume')>Resume</button>
|
||||
<button class=stop onclick=act('stop')>Stop</button>
|
||||
</p>
|
||||
<p>
|
||||
<span class=stat><span class=n id=state>idle</span><br>state</span>
|
||||
</div>
|
||||
<div class=row>
|
||||
workers
|
||||
<button class=step onclick=setc(-1)>−</button>
|
||||
<b id=conc style=margin:0+.5rem>1</b>
|
||||
<button class=step onclick=setc(1)>+</button>
|
||||
<span class=cap style=color:#9aa>(more = faster + more GPU)</span>
|
||||
</div>
|
||||
<div class=row>
|
||||
<span class=stat><span class=n id=state>stopped</span><br>state</span>
|
||||
<span class=stat><span class=n id=active>0</span><br>active now</span>
|
||||
<span class=stat><span class=n id=done>0</span><br>processed</span>
|
||||
<span class=stat><span class=n id=err>0</span><br>errors</span>
|
||||
<span class=stat><span class=n id=cur>—</span><br>current image</span>
|
||||
</p>
|
||||
</div>
|
||||
<div class=gpu id=gpu>GPU — …</div>
|
||||
<div class=bar><i id=gpubar style=width:0%></i></div>
|
||||
<div class=q id=queue></div>
|
||||
<script>
|
||||
async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
|
||||
async function setc(d){
|
||||
const v=Math.max(1,Math.min(8,parseInt(conc.textContent||'1')+d))
|
||||
await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:v})});refresh()
|
||||
}
|
||||
async function refresh(){
|
||||
const s=await (await fetch('/status')).json()
|
||||
state.textContent=s.state; done.textContent=s.processed; err.textContent=s.errors
|
||||
cur.textContent=s.current??'—'; fc.textContent=s.fc_url
|
||||
state.textContent=s.state; active.textContent=s.active; done.textContent=s.processed
|
||||
err.textContent=s.errors; conc.textContent=s.concurrency; fc.textContent=s.fc_url
|
||||
cfg.textContent=s.configured?'set':'MISSING'
|
||||
if(s.gpu){
|
||||
gpu.textContent=`GPU — ${s.gpu.util_pct}% util · VRAM ${s.gpu.mem_used_mb}/${s.gpu.mem_total_mb} MB · ${s.gpu.temp_c}°C`
|
||||
gpubar.style.width=Math.round(100*s.gpu.mem_used_mb/s.gpu.mem_total_mb)+'%'
|
||||
} else { gpu.textContent='GPU — n/a (CPU fallback?)'; gpubar.style.width='0%' }
|
||||
queue.textContent=s.queue?`queue — pending ${s.queue.pending} · in flight ${s.queue.leased} · done ${s.queue.done} · errored ${s.queue.error}`:'queue — unreachable'
|
||||
}
|
||||
refresh(); setInterval(refresh,3000)
|
||||
|
||||
Reference in New Issue
Block a user