feat: a GPU agent on the CPU shows as degraded, in the System view and on its own page (4410)

torch and onnxruntime both fall back to the CPU without raising, so the agent that ran CPU-bound for weeks after a driver update leased and checked in like a healthy one. - The agent sends its startup accel report on every lease and heartbeat. - The server keeps a bounded copy on the roster row. A running agent with a runtime off the GPU becomes `degraded`, with a sentence naming the runtime and the reason. - The top nav shows it amber. - The agent page carries a banner, and its pill reads "CPU only". Also: the bandwidth field gets the page's − / + stepper, and both number fields drop the browser's spin arrows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-24 19:09:07 -04:00
parent e39ec1c550
commit cc53d8db7b
10 changed files with 210 additions and 10 deletions
@@ -95,6 +95,23 @@ def _cuda_device_error(load=ctypes.CDLL) -> str | None:
    return None if count.value > 0 else "no CUDA device visible"


+def summary() -> dict | None:
+    """The report as FabledCurator stores it: each runtime's device, and why
+    when it is not the GPU. Sent on every lease and heartbeat, so the System
+    view can call a running agent that fell back to the CPU "degraded" rather
+    than "running" — the 2026-09-24 fallback went unseen for weeks because
+    only this agent's own log said so. None before report() has run."""
+    if not LAST:
+        return None
+    out = {}
+    for name, s in LAST.items():
+        entry = {"device": s.get("device")}
+        if s.get("error"):
+            entry["error"] = str(s["error"])[:200]
+        out[name] = entry
+    return out
+
+
 def report() -> dict:
    """Check both runtimes, log the result, and keep it for /status."""
    LAST.clear()
@@ -192,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
 .step:hover{border-color:var(--acc)}
 #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
+  color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
+ /* The browser's own spin arrows, hidden: the − / + beside each field are the
+    control, styled like the rest of the page (operator, 2026-09-24). */
+ #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
+ #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
 .unit{color:var(--mut);font-size:12px;font-weight:600}
 .hint{color:var(--mut);font-size:12px;margin-top:12px}
 .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
@@ -231,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
 </div>
+ <div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
 <div id=banner class=banner style=display:none>
  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
 </div>
@@ -248,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
    <button class=step onclick=setc(1)>+</button>
   </div>
   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
+    <button class=step onclick=stepbw(-1)>−</button>
    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
+    <button class=step onclick=stepbw(1)>+</button>
    <span class=unit>MB/s</span>
   </div>
  </div>
@@ -316,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
   body:JSON.stringify({value:on})});refresh()
 }
+ function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
+ // Runtimes that did NOT get the GPU, from the startup report. Both fall back
+ // to the CPU without raising, so this banner and the pill are the only place
+ // on this page a slow, CPU-bound agent announces itself.
+ function cpuOnly(s){
+  const a=s.accel||{}
+  return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
+ }
 async function setbw(v){
  v=Math.max(0,parseFloat(v)||0); bw.value=v
  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -386,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   // unreachable curator; grey when stopped; red with no token.
   let dc='dot', lbl='stopped'
   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
+   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
+    if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
   dot.className=dc; connlbl.textContent=lbl
   banner.style.display=(st==='running' && !s.queue)?'block':'none'
+   const slow=cpuOnly(s)
+   accelbanner.style.display=slow.length?'block':'none'
+   accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
+    +slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
+    +'. After a driver update, regenerate the CDI spec (agent README).'):''
   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
  }
 }
@@ -7,6 +7,8 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry

+from . import accel
+

 class FcClient:
    def __init__(self, base_url: str, token: str, agent_id: str):
@@ -72,7 +74,10 @@ class FcClient:
    def lease(self, batch_size: int) -> list[dict]:
        r = self.s.post(
            f"{self.base}/api/gpu/jobs/lease",
-            json={"agent_id": self.agent_id, "batch_size": batch_size},
+            json={
+                "agent_id": self.agent_id, "batch_size": batch_size,
+                "accel": accel.summary(),
+            },
            timeout=30,
        )
        r.raise_for_status()
@@ -90,7 +95,9 @@ class FcClient:
        })

    def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        self._post_quiet(
+            "/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
+        )

    def fail(self, job_id: int, error: str) -> None:
        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})