2026-09-25 10:02:40 -04:00
10 changed files with 210 additions and 10 deletions
@@ -95,6 +95,23 @@ def _cuda_device_error(load=ctypes.CDLL) -> str | None:
    return None if count.value > 0 else "no CUDA device visible"


+def summary() -> dict | None:
+    """The report as FabledCurator stores it: each runtime's device, and why
+    when it is not the GPU. Sent on every lease and heartbeat, so the System
+    view can call a running agent that fell back to the CPU "degraded" rather
+    than "running" — the 2026-09-24 fallback went unseen for weeks because
+    only this agent's own log said so. None before report() has run."""
+    if not LAST:
+        return None
+    out = {}
+    for name, s in LAST.items():
+        entry = {"device": s.get("device")}
+        if s.get("error"):
+            entry["error"] = str(s["error"])[:200]
+        out[name] = entry
+    return out
+
+
 def report() -> dict:
    """Check both runtimes, log the result, and keep it for /status."""
    LAST.clear()
@@ -192,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
 .step:hover{border-color:var(--acc)}
 #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
-  color:var(--fg);border:1px solid var(--bd);border-radius:8px}
+  color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
+ /* The browser's own spin arrows, hidden: the − / + beside each field are the
+    control, styled like the rest of the page (operator, 2026-09-24). */
+ #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
+ #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
 .unit{color:var(--mut);font-size:12px;font-weight:600}
 .hint{color:var(--mut);font-size:12px;margin-top:12px}
 .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
@@ -231,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
 <div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
  a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
 </div>
+ <div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
 <div id=banner class=banner style=display:none>
  curator unreachable — holding work + retrying, resumes on its own (no restart needed)
 </div>
@@ -248,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
    <button class=step onclick=setc(1)>+</button>
   </div>
   <div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
+    <button class=step onclick=stepbw(-1)>−</button>
    <input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
+    <button class=step onclick=stepbw(1)>+</button>
    <span class=unit>MB/s</span>
   </div>
  </div>
@@ -316,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
  await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
   body:JSON.stringify({value:on})});refresh()
 }
+ function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
+ // Runtimes that did NOT get the GPU, from the startup report. Both fall back
+ // to the CPU without raising, so this banner and the pill are the only place
+ // on this page a slow, CPU-bound agent announces itself.
+ function cpuOnly(s){
+  const a=s.accel||{}
+  return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
+ }
 async function setbw(v){
  v=Math.max(0,parseFloat(v)||0); bw.value=v
  await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
@@ -386,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
   // unreachable curator; grey when stopped; red with no token.
   let dc='dot', lbl='stopped'
   if(!ok){ dc='dot red'; lbl='no token' }
-   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
+   else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
+    if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
   else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
   else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
   dot.className=dc; connlbl.textContent=lbl
   banner.style.display=(st==='running' && !s.queue)?'block':'none'
+   const slow=cpuOnly(s)
+   accelbanner.style.display=slow.length?'block':'none'
+   accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
+    +slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
+    +'. After a driver update, regenerate the CDI spec (agent README).'):''
   queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
  }
 }
@@ -7,6 +7,8 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry

+from . import accel
+

 class FcClient:
    def __init__(self, base_url: str, token: str, agent_id: str):
@@ -72,7 +74,10 @@ class FcClient:
    def lease(self, batch_size: int) -> list[dict]:
        r = self.s.post(
            f"{self.base}/api/gpu/jobs/lease",
-            json={"agent_id": self.agent_id, "batch_size": batch_size},
+            json={
+                "agent_id": self.agent_id, "batch_size": batch_size,
+                "accel": accel.summary(),
+            },
            timeout=30,
        )
        r.raise_for_status()
@@ -90,7 +95,9 @@ class FcClient:
        })

    def heartbeat(self, job_ids: list[int]) -> None:
-        self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
+        self._post_quiet(
+            "/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
+        )

    def fail(self, job_id: int, error: str) -> None:
        self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):

 # --- Agent (bearer token): lease / submit / heartbeat / fail ------------

+
+def _accel_detail(body: dict) -> dict:
+    """The agent's own report of which runtime got the GPU, kept on its roster
+    row so the System view can call a CPU-bound agent degraded (#4410).
+
+    Only a dict of {runtime: {device, error?}} is kept, and each value is
+    reduced to those two short strings: this is written on every lease, by a
+    client the server does not control. An agent that sends nothing (an
+    older build) simply has no `accel`, which reads as not-yet-reported.
+    """
+    raw = body.get("accel")
+    if not isinstance(raw, dict):
+        return {}
+    accel = {}
+    for name, entry in list(raw.items())[:4]:
+        if not isinstance(entry, dict):
+            continue
+        clean = {"device": str(entry.get("device") or "")[:16]}
+        if entry.get("error"):
+            clean["error"] = str(entry["error"])[:200]
+        accel[str(name)[:16]] = clean
+    return {"accel": accel} if accel else {}
+
@gpu_bp.route("/jobs/lease", methods=["POST"])
 async def lease():
    body = await request.get_json(silent=True) or {}
@@ -267,7 +290,10 @@ async def lease():
            key=f"agent:{agent_id}",
            kind="agent",
            display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
+            details={
+                "agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
+                **_accel_detail(body),
+            },
        )
        ml = await MLSettings.load(session)
        # image rows for url/mime in one shot
@@ -347,7 +373,10 @@ async def heartbeat():
            key=f"agent:{agent_id}",
            kind="agent",
            display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
-            details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
+            details={
+                "agent_id": agent_id, "last_call": "heartbeat", "extended": n,
+                **_accel_detail(body),
+            },
        )
        await session.commit()
    return jsonify({"extended": n})
@@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
 PROBE_TIMEOUT_SECONDS = 2.0

 _OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
+# Checking in, but working at a fraction of its speed: a GPU agent whose
+# runtimes fell back to the CPU (#4410). Below stale — a part that may have
+# stopped is the more urgent question — and above unknown, because this one
+# IS known to be wrong.
+_DEGRADED = "degraded"

 # Worst-first, so an overall verdict is just the max.
-_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
+_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}


 def _age_state(age_seconds: float) -> str:
@@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
    return f"{name} has not checked in for {ago} — treat it as stopped"


+def _cpu_runtimes(details: dict) -> list[str]:
+    """The runtimes an agent reported as NOT on the GPU, with why.
+
+    Both torch and onnxruntime fall back to the CPU without raising, so an
+    agent in that state leases, works and checks in exactly like a healthy
+    one. On 2026-09-24 one had been doing so since a driver update left a
+    stale CDI spec; the only sign was a line in the agent's own log.
+    """
+    accel = details.get("accel")
+    if not isinstance(accel, dict):
+        return []
+    out = []
+    for name, entry in sorted(accel.items()):
+        if not isinstance(entry, dict) or entry.get("device") == "cuda":
+            continue
+        why = entry.get("error") or entry.get("device") or "unknown"
+        out.append(f"{name} ({why})")
+    return out
+
+
+def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
+    """A roster row's state and its sentence, degraded included."""
+    if state == _OK:
+        cpu = _cpu_runtimes(details)
+        if cpu:
+            return _DEGRADED, (
+                f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
+                "After a driver update, regenerate the agent host's CDI spec "
+                "(agent README)."
+            )
+    return state, _describe_learned(name, state, age, details)
+
+
 async def _probe_postgres(session) -> dict:
    started = time.monotonic()
    try:
@@ -175,13 +213,15 @@ async def system_health():
            ).scalars().all()
            for row in rows:
                age = (now - row.last_seen_at).total_seconds()
-                state = _age_state(age)
+                state, detail = _learned_state(
+                    row.display_name, _age_state(age), age, row.details or {},
+                )
                parts.append({
                    "key": row.key,
                    "kind": row.kind,
                    "name": row.display_name,
                    "state": state,
-                    "detail": _describe_learned(row.display_name, state, age, row.details or {}),
+                    "detail": detail,
                    "last_seen_at": row.last_seen_at.isoformat(),
                    "first_seen_at": row.first_seen_at.isoformat(),
                    **{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
@@ -153,6 +153,14 @@ const health = computed(() => {
      label: (worst?.detail || 'A part has stopped') + suffix,
    }
  }
+  // Running but slow — a GPU agent that fell back to the CPU (#4410). Worth
+  // the amber dot: nothing else anywhere says so.
+  if (overall === 'degraded') {
+    return {
+      icon: 'mdi-speedometer-slow', color: 'warning',
+      label: (worst?.detail || 'A part is running degraded') + suffix,
+    }
+  }
  if (overall === 'stale') {
    return {
      icon: 'mdi-alert', color: 'warning',
@@ -368,6 +368,7 @@ function step(lane, delta) {
 .fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
 .fc-sys__dot--ok      { background: rgb(var(--v-theme-success)); }
 .fc-sys__dot--stale   { background: rgb(var(--v-theme-warning)); }
+.fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); }
 .fc-sys__dot--down    { background: rgb(var(--v-theme-error)); }
 .fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }

@@ -24,7 +24,8 @@ export function queueKey(queues) {
 }

 // Worst first. A stopped datastore is why someone opened this tab.
-export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
+// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410).
+export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 }

 export function kindLabel(kind) {
  if (kind === 'celery') return 'worker lane'
@@ -103,3 +103,20 @@ def test_a_missing_runtime_is_reported_not_raised():

    assert accel.torch_status(imp)["device"] == "unavailable"
    assert accel.onnx_status(imp)["device"] == "unavailable"
+
+
+def test_summary_is_what_the_server_stores(monkeypatch):
+    """Device plus a bounded reason, per runtime — sent on every lease."""
+    monkeypatch.setattr(accel, "LAST", {
+        "torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"},
+        "onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500},
+    })
+    assert accel.summary() == {
+        "torch": {"device": "cuda"},
+        "onnx": {"device": "cpu", "error": "e" * 200},
+    }
+
+
+def test_summary_before_the_report_is_none(monkeypatch):
+    monkeypatch.setattr(accel, "LAST", {})
+    assert accel.summary() is None
@@ -0,0 +1,59 @@
+"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410).
+
+Both runtimes fall back without raising, so a CPU-bound agent leases, works
+and checks in exactly like a healthy one. On 2026-09-24 one had been doing so
+since a driver update left a stale CDI spec; the only sign was a line in the
+agent's own log. The agent now sends its startup report on every lease and
+heartbeat, and the System view derives the state from it.
+"""
+
+from __future__ import annotations
+
+from backend.app.api.gpu import _accel_detail
+from backend.app.api.system_health import _SEVERITY, _learned_state
+
+GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}}
+CPU = {
+    "torch": {"device": "cpu"},
+    "onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"},
+}
+
+
+def test_a_running_agent_on_the_gpu_is_ok():
+    state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU})
+    assert state == "ok"
+    assert detail == "GPU agent is running"
+
+
+def test_a_running_agent_on_the_cpu_is_degraded_and_says_why():
+    state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU})
+    assert state == "degraded"
+    assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail
+    assert "torch (cpu)" in detail
+
+
+def test_an_agent_that_never_reported_is_not_called_degraded():
+    """An older agent build sends no `accel`: that is not-yet-known, not slow."""
+    assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok"
+
+
+def test_stopped_outranks_degraded():
+    """Whether a quiet agent is still running is the more urgent question."""
+    state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU})
+    assert state == "down"
+    assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"]
+
+
+def test_the_lease_keeps_only_a_bounded_report():
+    """Written on every lease by a client the server does not control."""
+    body = {"accel": {
+        "torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"},
+        "onnx": "not a dict",
+    }}
+    kept = _accel_detail(body)["accel"]
+    assert kept == {"torch": {"device": "cpu", "error": "x" * 200}}
+
+
+def test_a_lease_without_a_report_adds_nothing():
+    assert _accel_detail({}) == {}
+    assert _accel_detail({"accel": None}) == {}