feat: a GPU agent on the CPU shows as degraded, in the System view and on its own page (4410)

torch and onnxruntime both fall back to the CPU without raising, so the agent that ran CPU-bound for weeks after a driver update leased and checked in like a healthy one. - The agent sends its startup accel report on every lease and heartbeat. - The server keeps a bounded copy on the roster row. A running agent with a runtime off the GPU becomes `degraded`, with a sentence naming the runtime and the reason. - The top nav shows it amber. - The agent page carries a banner, and its pill reads "CPU only". Also: the bandwidth field gets the page's − / + stepper, and both number fields drop the browser's spin arrows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-24 19:09:07 -04:00
parent e39ec1c550
commit cc53d8db7b
10 changed files with 210 additions and 10 deletions
@@ -95,6 +95,23 @@ def _cuda_device_error(load=ctypes.CDLL) -> str | None:
    return None if count.value > 0 else "no CUDA device visible"
 def summary() -> dict | None:
    """The report as FabledCurator stores it: each runtime's device, and why
    when it is not the GPU. Sent on every lease and heartbeat, so the System
    view can call a running agent that fell back to the CPU "degraded" rather
    than "running" — the 2026-09-24 fallback went unseen for weeks because
    only this agent's own log said so. None before report() has run."""
    if not LAST:
        return None
    out = {}
    for name, s in LAST.items():
        entry = {"device": s.get("device")}
        if s.get("error"):
            entry["error"] = str(s["error"])[:200]
        out[name] = entry
    return out
 def report() -> dict:
    """Check both runtimes, log the result, and keep it for /status."""
    LAST.clear()
@@ -192,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
 .step:hover{border-color:var(--acc)}
 #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
+  color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
 /* The browser's own spin arrows, hidden: the − / + beside each field are the
    control, styled like the rest of the page (operator, 2026-09-24). */
 #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
 #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
 .unit{color:var(--mut);font-size:12px;font-weight:600}
 .hint{color:var(--mut);font-size:12px;margin-top:12px}
 .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
@@ -231,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
 </div>
 <div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
 <div id=banner class=banner style=display:none>
  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
 </div>
@@ -248,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
    <button class=step onclick=setc(1)>+</button>
   </div>
   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
    <button class=step onclick=stepbw(-1)>−</button>
    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
    <button class=step onclick=stepbw(1)>+</button>
    <span class=unit>MB/s</span>
   </div>
  </div>
@@ -316,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
   body:JSON.stringify({value:on})});refresh()
 }
 function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
 // Runtimes that did NOT get the GPU, from the startup report. Both fall back
 // to the CPU without raising, so this banner and the pill are the only place
 // on this page a slow, CPU-bound agent announces itself.
 function cpuOnly(s){
  const a=s.accel||{}
  return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
 }
 async function setbw(v){
  v=Math.max(0,parseFloat(v)||0); bw.value=v
  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -386,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   // unreachable curator; grey when stopped; red with no token.
   let dc='dot', lbl='stopped'
   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
+   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
    if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
   dot.className=dc; connlbl.textContent=lbl
   banner.style.display=(st==='running' && !s.queue)?'block':'none'
   const slow=cpuOnly(s)
   accelbanner.style.display=slow.length?'block':'none'
   accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
    +slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
    +'. After a driver update, regenerate the CDI spec (agent README).'):''
   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
  }
 }
@@ -7,6 +7,8 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 from . import accel
 class FcClient:
    def __init__(self, base_url: str, token: str, agent_id: str):
@@ -72,7 +74,10 @@ class FcClient:
    def lease(self, batch_size: int) -> list[dict]:
        r = self.s.post(
            f"{self.base}/api/gpu/jobs/lease",
-            json={"agent_id": self.agent_id, "batch_size": batch_size},
+            json={
                "agent_id": self.agent_id, "batch_size": batch_size,
                "accel": accel.summary(),
            },
            timeout=30,
        )
        r.raise_for_status()
@@ -90,7 +95,9 @@ class FcClient:
        })
    def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        self._post_quiet(
            "/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
        )
    def fail(self, job_id: int, error: str) -> None:
        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):
 # --- Agent (bearer token): lease / submit / heartbeat / fail ------------
 def _accel_detail(body: dict) -> dict:
    """The agent's own report of which runtime got the GPU, kept on its roster
    row so the System view can call a CPU-bound agent degraded (#4410).
    Only a dict of {runtime: {device, error?}} is kept, and each value is
    reduced to those two short strings: this is written on every lease, by a
    client the server does not control. An agent that sends nothing (an
    older build) simply has no `accel`, which reads as not-yet-reported.
    """
    raw = body.get("accel")
    if not isinstance(raw, dict):
        return {}
    accel = {}
    for name, entry in list(raw.items())[:4]:
        if not isinstance(entry, dict):
            continue
        clean = {"device": str(entry.get("device") or "")[:16]}
        if entry.get("error"):
            clean["error"] = str(entry["error"])[:200]
        accel[str(name)[:16]] = clean
    return {"accel": accel} if accel else {}
@gpu_bp.route("/jobs/lease", methods=["POST"])
 async def lease():
    body = await request.get_json(silent=True) or {}
@@ -267,7 +290,10 @@ async def lease():
            key=f"agent:{agent_id}",
            kind="agent",
            display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
+            details={
                "agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
                **_accel_detail(body),
            },
        )
        ml = await MLSettings.load(session)
        # image rows for url/mime in one shot
@@ -347,7 +373,10 @@ async def heartbeat():
            key=f"agent:{agent_id}",
            kind="agent",
            display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
+            details={
                "agent_id": agent_id, "last_call": "heartbeat", "extended": n,
                **_accel_detail(body),
            },
        )
        await session.commit()
    return jsonify({"extended": n})
@@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
 PROBE_TIMEOUT_SECONDS = 2.0
 _OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
 # Checking in, but working at a fraction of its speed: a GPU agent whose
 # runtimes fell back to the CPU (#4410). Below stale — a part that may have
 # stopped is the more urgent question — and above unknown, because this one
 # IS known to be wrong.
 _DEGRADED = "degraded"
 # Worst-first, so an overall verdict is just the max.
-_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
+_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
 def _age_state(age_seconds: float) -> str:
@@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
    return f"{name} has not checked in for {ago} — treat it as stopped"
 def _cpu_runtimes(details: dict) -> list[str]:
    """The runtimes an agent reported as NOT on the GPU, with why.
    Both torch and onnxruntime fall back to the CPU without raising, so an
    agent in that state leases, works and checks in exactly like a healthy
    one. On 2026-09-24 one had been doing so since a driver update left a
    stale CDI spec; the only sign was a line in the agent's own log.
    """
    accel = details.get("accel")
    if not isinstance(accel, dict):
        return []
    out = []
    for name, entry in sorted(accel.items()):
        if not isinstance(entry, dict) or entry.get("device") == "cuda":
            continue
        why = entry.get("error") or entry.get("device") or "unknown"
        out.append(f"{name} ({why})")
    return out
 def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
    """A roster row's state and its sentence, degraded included."""
    if state == _OK:
        cpu = _cpu_runtimes(details)
        if cpu:
            return _DEGRADED, (
                f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
                "After a driver update, regenerate the agent host's CDI spec "
                "(agent README)."
            )
    return state, _describe_learned(name, state, age, details)
 async def _probe_postgres(session) -> dict:
    started = time.monotonic()
    try:
@@ -175,13 +213,15 @@ async def system_health():
            ).scalars().all()
            for row in rows:
                age = (now - row.last_seen_at).total_seconds()
-                state = _age_state(age)
+                state, detail = _learned_state(
                    row.display_name, _age_state(age), age, row.details or {},
                )
                parts.append({
                    "key": row.key,
                    "kind": row.kind,
                    "name": row.display_name,
                    "state": state,
-                    "detail": _describe_learned(row.display_name, state, age, row.details or {}),
+                    "detail": detail,
                    "last_seen_at": row.last_seen_at.isoformat(),
                    "first_seen_at": row.first_seen_at.isoformat(),
                    **{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
@@ -153,6 +153,14 @@ const health = computed(() => {
      label: (worst?.detail || 'A part has stopped') + suffix,
    }
  }
  // Running but slow — a GPU agent that fell back to the CPU (#4410). Worth
  // the amber dot: nothing else anywhere says so.
  if (overall === 'degraded') {
    return {
      icon: 'mdi-speedometer-slow', color: 'warning',
      label: (worst?.detail || 'A part is running degraded') + suffix,
    }
  }
  if (overall === 'stale') {
    return {
      icon: 'mdi-alert', color: 'warning',
@@ -368,6 +368,7 @@ function step(lane, delta) {
 .fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
 .fc-sys__dot--ok      { background: rgb(var(--v-theme-success)); }
 .fc-sys__dot--stale   { background: rgb(var(--v-theme-warning)); }
 .fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); }
 .fc-sys__dot--down    { background: rgb(var(--v-theme-error)); }
 .fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
@@ -24,7 +24,8 @@ export function queueKey(queues) {
 }
 // Worst first. A stopped datastore is why someone opened this tab.
-export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
+// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410).
 export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 }
 export function kindLabel(kind) {
  if (kind === 'celery') return 'worker lane'
@@ -103,3 +103,20 @@ def test_a_missing_runtime_is_reported_not_raised():
    assert accel.torch_status(imp)["device"] == "unavailable"
    assert accel.onnx_status(imp)["device"] == "unavailable"
 def test_summary_is_what_the_server_stores(monkeypatch):
    """Device plus a bounded reason, per runtime — sent on every lease."""
    monkeypatch.setattr(accel, "LAST", {
        "torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"},
        "onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500},
    })
    assert accel.summary() == {
        "torch": {"device": "cuda"},
        "onnx": {"device": "cpu", "error": "e" * 200},
    }
 def test_summary_before_the_report_is_none(monkeypatch):
    monkeypatch.setattr(accel, "LAST", {})
    assert accel.summary() is None
@@ -0,0 +1,59 @@
 """A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410).
 Both runtimes fall back without raising, so a CPU-bound agent leases, works
 and checks in exactly like a healthy one. On 2026-09-24 one had been doing so
 since a driver update left a stale CDI spec; the only sign was a line in the
 agent's own log. The agent now sends its startup report on every lease and
 heartbeat, and the System view derives the state from it.
 """
 from __future__ import annotations
 from backend.app.api.gpu import _accel_detail
 from backend.app.api.system_health import _SEVERITY, _learned_state
 GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}}
 CPU = {
    "torch": {"device": "cpu"},
    "onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"},
 }
 def test_a_running_agent_on_the_gpu_is_ok():
    state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU})
    assert state == "ok"
    assert detail == "GPU agent is running"
 def test_a_running_agent_on_the_cpu_is_degraded_and_says_why():
    state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU})
    assert state == "degraded"
    assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail
    assert "torch (cpu)" in detail
 def test_an_agent_that_never_reported_is_not_called_degraded():
    """An older agent build sends no `accel`: that is not-yet-known, not slow."""
    assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok"
 def test_stopped_outranks_degraded():
    """Whether a quiet agent is still running is the more urgent question."""
    state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU})
    assert state == "down"
    assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"]
 def test_the_lease_keeps_only_a_bounded_report():
    """Written on every lease by a client the server does not control."""
    body = {"accel": {
        "torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"},
        "onnx": "not a dict",
    }}
    kept = _accel_detail(body)["accel"]
    assert kept == {"torch": {"device": "cpu", "error": "x" * 200}}
 def test_a_lease_without_a_report_adds_nothing():
    assert _accel_detail({}) == {}
    assert _accel_detail({"accel": None}) == {}