feat(agent): in-UI log console + a real styling pass on the control page

- logbuf.py: bounded in-memory log ring buffer + a logging.Handler on the root logger; GET /logs serves it; the control page polls it into a console pane — so runs are monitorable without `docker logs`. worker now logs autoscale moves (one line per change, with jobs/s + util + VRAM) and job failures (job + image + reason); detectors already log load/disable. - Restyled the whole control page: a proper dark layout with a header + live connection pill, cards (Control / Status / Logs), a styled Auto switch + worker stepper, status tiles, separate GPU-util and VRAM meters, and the log console. No longer feels like an afterthought; all the existing control hooks are preserved. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
2026-06-30 18:34:22 -04:00
parent 6d7b17b0b5
commit c1b099e5a3
3 changed files with 202 additions and 54 deletions
@@ -7,10 +7,12 @@ queue. Config is env-seeded; the worker count is adjustable here on the fly.
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse, JSONResponse

+from . import logbuf
 from .config import Config
 from .gpu import read_gpu
 from .worker import Worker

+logbuf.install()
 cfg = Config.from_env()
 worker = Worker(cfg)
 app = FastAPI(title="FabledCurator GPU agent")
@@ -57,6 +59,11 @@ async def auto(request: Request):
    return JSONResponse(worker.status())


+@app.get("/logs")
+def logs():
+    return JSONResponse({"lines": list(logbuf.LINES)})
+
+
@app.get("/status")
 def status():
    s = worker.status()
@@ -71,51 +78,123 @@ def status():


 _PAGE = """<!doctype html><html><head><meta charset=utf-8>
-<title>FabledCurator GPU agent</title>
+<meta name=viewport content="width=device-width,initial-scale=1">
+<title>FabledCurator · GPU agent</title>
 <style>
- body{font:14px system-ui;margin:2rem;max-width:680px;background:#14171a;color:#e8e8e8}
- h1{font-size:18px} button{font:14px system-ui;padding:.5rem 1rem;border:0;border-radius:6px;
- margin-right:.5rem;cursor:pointer;color:#fff} .start{background:#2e7d32}.stop{background:#b3261e}
- .step{background:#33373b;padding:.4rem .7rem;font-weight:700}
- .stat{display:inline-block;margin-right:1.5rem;vertical-align:top}
- .n{font-size:22px;font-weight:700} code{background:#222;padding:2px 6px;border-radius:4px}
- .q,.gpu{margin-top:1rem;color:#9aa} .bar{height:8px;border-radius:4px;background:#222;overflow:hidden;
- max-width:320px;margin-top:4px} .bar>i{display:block;height:100%;background:#3f7d3f}
- .row{margin:.8rem 0}
+ :root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
+  --mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
+ *{box-sizing:border-box}
+ body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
+  background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
+ .wrap{max-width:780px;margin:0 auto;padding:28px 20px 48px}
+ header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
+ .brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
+ .logo{color:var(--acc);font-size:20px}
+ .brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
+ .conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
+ .dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
+ .dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
+ .dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
+ .meta{color:var(--mut);margin:0 0 18px;font-size:13px}
+ code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
+  font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
+ .card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
+  border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
+ .card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
+  color:var(--mut);margin-bottom:14px}
+ .controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
+ .spacer{flex:1}
+ .btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
+  cursor:pointer;color:#fff;transition:.12s}
+ .btn:hover{transform:translateY(-1px)}
+ .btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
+ .btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
+ .switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
+ .switch input{display:none}
+ .switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
+ .switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
+  background:#cdd6e0;transition:.15s}
+ .switch input:checked+.track{background:var(--acc)}
+ .switch input:checked+.track:after{transform:translateX(16px);background:#fff}
+ .stepper{display:inline-flex;align-items:center;gap:6px}
+ .step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
+  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
+ .step:hover{border-color:var(--acc)}
+ #conc{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
+  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
+ .hint{color:var(--mut);font-size:12px;margin-top:12px}
+ .tiles{display:grid;grid-template-columns:repeat(5,1fr);gap:10px;margin-bottom:16px}
+ .tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 10px;text-align:center}
+ .tile .n{font:800 24px ui-monospace,monospace;line-height:1.1}
+ .tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
+ .tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
+ .meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
+ .meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
+ .meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
+ .bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
+ .bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
+ #utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
+ .queue{font:13px ui-monospace,monospace;color:var(--mut)}
+ .banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
+  border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
+ .logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
+  height:230px;overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
+  color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
 </style></head><body>
-<h1>FabledCurator GPU agent</h1>
-<p>FC: <code id=fc>—</code> · token <code id=cfg>—</code></p>
-<div class=row>
- <button class=start onclick=act('start')>Start</button>
- <button class=stop onclick=act('stop')>Stop</button>
+<div class=wrap>
+ <header>
+  <div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
+  <div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
+ </header>
+ <p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code></p>
+
+ <div id=banner class=banner style=display:none>
+  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
+ </div>
+
+ <section class=card>
+  <div class=card-h>Control</div>
+  <div class=controls>
+   <button class="btn start" onclick=act('start')>▶ Start</button>
+   <button class="btn stop" onclick=act('stop')>■ Stop</button>
+   <div class=spacer></div>
+   <label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
+   <div class=stepper>
+    <button class=step onclick=setc(-1)>−</button>
+    <input id=conc type=number min=1 value=1 onchange="setv(this.value)">
+    <button class=step onclick=setc(1)>+</button>
+   </div>
+  </div>
+  <div class=hint id=conchint>auto-tuning to fill the GPU · max <b id=capn>8</b></div>
+ </section>
+
+ <section class=card>
+  <div class=card-h>Status</div>
+  <div class=tiles>
+   <div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
+   <div class=tile><div class=n id=active>0</div><div class=l>active</div></div>
+   <div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
+   <div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
+   <div class=tile><div class=n id=wait>0</div><div class=l>waited out</div></div>
+  </div>
+  <div class=meters>
+   <div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
+    <div class=bar><i id=utilbar></i></div></div>
+   <div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
+    <div class=bar><i id=gpubar></i></div></div>
+  </div>
+  <div class=queue id=queue>queue —</div>
+ </section>
+
+ <section class=card>
+  <div class=card-h>Logs</div>
+  <pre class=logs id=logs>waiting for activity…</pre>
+ </section>
 </div>
-<div class=row>
- <label style="margin-right:1rem"><input type=checkbox id=autochk onchange="setauto(this.checked)"> Auto</label>
- workers
- <button class=step onclick=setc(-1)>−</button>
- <input id=conc type=number min=1 value=1
-  style="width:3.5rem;font:700 16px system-ui;text-align:center;background:#222;color:#e8e8e8;border:1px solid #444;border-radius:6px;padding:.3rem"
-  onchange="setv(this.value)">
- <button class=step onclick=setc(1)>+</button>
- <span class=cap style=color:#9aa id=conchint>auto-tuning to fill the GPU · max <b id=capn>8</b></span>
-</div>
-<div class=row>
- <span class=stat><span class=n id=state>stopped</span><br>state</span>
- <span class=stat><span class=n id=active>0</span><br>active now</span>
- <span class=stat><span class=n id=done>0</span><br>processed</span>
- <span class=stat><span class=n id=err>0</span><br>errors</span>
- <span class=stat><span class=n id=wait>0</span><br>waited out</span>
-</div>
-<div id=banner style="display:none;margin:.6rem 0;padding:.5rem .8rem;border-radius:6px;background:#5a4a17;color:#ffe28a">
- curator unreachable — holding work + retrying, will resume on its own (no restart needed)
-</div>
-<div class=gpu id=gpu>GPU — …</div>
-<div class=bar><i id=gpubar style=width:0%></i></div>
-<div class=q id=queue></div>
 <script>
 let CAP=8
 async function act(p){await fetch('/'+p,{method:'POST'});refresh()}
- function setc(d){ setv((parseInt(conc.value||'1'))+d) }
+ function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
 async function setv(v){
  v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
  await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -126,26 +205,39 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   body:JSON.stringify({value:on})});refresh()
 }
 async function refresh(){
-  const s=await (await fetch('/status')).json()
+  let s; try{ s=await (await fetch('/status')).json() }catch{ return }
  CAP=s.max_concurrency||8; capn.textContent=CAP
-  state.textContent=s.state; active.textContent=s.active; done.textContent=s.processed
-  err.textContent=s.errors; fc.textContent=s.fc_url; wait.textContent=s.transient||0
-  // Running but the queue read failed → curator is unreachable; show we're
-  // riding it out rather than erroring.
-  banner.style.display=(s.state==='running' && !s.queue)?'block':'none'
-  // Auto on → the dial reflects the auto-chosen count (read-only); off → manual.
+  const running=s.state==='running', ok=s.configured
+  state.textContent=s.state
+  active.textContent=s.active; done.textContent=s.processed
+  err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
+  fc.textContent=s.fc_url; wait.textContent=s.transient||0
+  cfg.textContent=ok?'set':'MISSING'
+  // connection pill
+  dot.className='dot '+(!ok?'red':(running?(s.queue?'green':'amber'):'amber'))
+  connlbl.textContent=!ok?'no token':(running?(s.queue?'running':'running · curator unreachable'):'stopped')
+  banner.style.display=(running && !s.queue)?'block':'none'
+  // Auto on → dial reflects the auto-chosen count (read-only); off → manual.
  if(document.activeElement!==autochk) autochk.checked=!!s.auto
-  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.6:1
+  conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
  conchint.textContent=s.auto?('auto-tuning to fill the GPU · max '+CAP):('manual · max '+CAP)
-  capn.textContent=CAP
  if(document.activeElement!==conc) conc.value=s.concurrency
  conc.max=CAP
-  cfg.textContent=s.configured?'set':'MISSING'
  if(s.gpu){
-   gpu.textContent=`GPU — ${s.gpu.util_pct}% util · VRAM ${s.gpu.mem_used_mb}/${s.gpu.mem_total_mb} MB · ${s.gpu.temp_c}°C`
-   gpubar.style.width=Math.round(100*s.gpu.mem_used_mb/s.gpu.mem_total_mb)+'%'
-  } else { gpu.textContent='GPU — n/a (CPU fallback?)'; gpubar.style.width='0%' }
-  queue.textContent=s.queue?`queue — pending ${s.queue.pending} · in flight ${s.queue.leased} · done ${s.queue.done} · errored ${s.queue.error}`:'queue — unreachable'
+   const u=s.gpu.util_pct, used=s.gpu.mem_used_mb, tot=s.gpu.mem_total_mb||1
+   utillbl.textContent=u+'% · '+s.gpu.temp_c+'°C'; utilbar.style.width=u+'%'
+   vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
+  } else { utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
+  queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
 }
- refresh(); setInterval(refresh,3000)
+ async function refreshLogs(){
+  try{
+   const r=await (await fetch('/logs')).json()
+   const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
+   el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
+   if(atBottom) el.scrollTop=el.scrollHeight
+  }catch{}
+ }
+ refresh(); refreshLogs()
+ setInterval(refresh,3000); setInterval(refreshLogs,2500)
 </script></body></html>"""
@@ -0,0 +1,40 @@
+"""In-memory log ring buffer so the control UI can show recent agent logs
+(detector loads, job errors, autoscaler decisions, outage back-offs) without
+needing `docker logs`. A bounded deque holds the last N formatted lines; a
+logging.Handler appends to it; the UI polls /logs."""
+import logging
+from collections import deque
+
+LINES: deque[str] = deque(maxlen=600)
+
+
+class RingHandler(logging.Handler):
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            LINES.append(self.format(record))
+        except Exception:
+            pass
+
+
+_installed = False
+
+
+def install(level: int = logging.INFO) -> None:
+    """Attach the ring handler to the root logger once. fc_agent module loggers
+    propagate to root, so their records land here."""
+    global _installed
+    if _installed:
+        return
+    _installed = True
+    h = RingHandler()
+    h.setFormatter(
+        logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
+    )
+    root = logging.getLogger()
+    root.addHandler(h)
+    if root.level == logging.NOTSET or root.level > level:
+        root.setLevel(level)
+    # Keep the buffer signal-rich: drop the per-request access spam + ultralytics
+    # banner noise to WARNING.
+    logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+    logging.getLogger("ultralytics").setLevel(logging.WARNING)
@@ -9,6 +9,7 @@ That's the dial the operator turns to trade desktop responsiveness for speed.
 Stop (or shrinking the pool) RELEASES a slot's still-leased jobs immediately so
 orphaned work is re-picked at once rather than waiting out the lease.
 """
+import logging
 import threading

 import numpy as np
@@ -58,6 +59,8 @@ UTIL_HI = 96               # GPU util% considered saturated
 TPUT_MARGIN = 0.10         # a step up must beat the baseline by this to "help"
 REPROBE_TICKS = 5          # ticks to hold after settling before re-probing up

+log = logging.getLogger("fc_agent.worker")
+

 class _Slot:
    """One worker loop. `inflight` = jobs leased but not yet processed, so a
@@ -230,6 +233,7 @@ class Worker:
            dt = max(1e-3, now - prev_t)
            tput = (self.processed - prev_p) / dt
            prev_p, prev_t = self.processed, now
+            t0 = self._target

            g = gpumod.read_gpu() or {}
            mt = g.get("mem_total_mb") or 0
@@ -260,6 +264,12 @@ class Worker:
            else:
                base_tput = None                     # settled → re-probe next cycle

+            if self._target != t0:
+                log.info(
+                    "autoscale: %d→%d workers (%.2f jobs/s · util %d%% · vram %d%%)",
+                    t0, self._target, tput, util, round(vram * 100),
+                )
+
    def _ensure_embedder(self, model_name: str):
        if self._embedder is not None:
            return self._embedder
@@ -400,15 +410,21 @@ class Worker:
                # effort; no-ops if the server is still down, then the server's
                # orphan-recovery reclaims it) and signal the loop to wait.
                self._bump(transient=1)
+                log.info("curator unreachable — released job %s, backing off",
+                         job.get("job_id"))
                self.client.release([job["job_id"]])
                return False
            # A job-specific HTTP fault (404 image gone, 400) → fail it so it
            # doesn't re-lease forever.
            self._bump(errors=1)
+            log.warning("job %s (image %s) failed: %s",
+                        job.get("job_id"), job.get("image_id"), str(exc)[:200])
            self.client.fail(job["job_id"], str(exc)[:500])
            return True
        except Exception as exc:   # noqa: BLE001 — a genuine job fault: report it
            self._bump(errors=1)
+            log.warning("job %s (image %s) failed: %s",
+                        job.get("job_id"), job.get("image_id"), str(exc)[:200])
            self.client.fail(job["job_id"], str(exc)[:500])
            return True
        finally: