diff --git a/agent/fc_agent/accel.py b/agent/fc_agent/accel.py index 5f29fc5..601bcad 100644 --- a/agent/fc_agent/accel.py +++ b/agent/fc_agent/accel.py @@ -95,6 +95,23 @@ def _cuda_device_error(load=ctypes.CDLL) -> str | None: return None if count.value > 0 else "no CUDA device visible" +def summary() -> dict | None: + """The report as FabledCurator stores it: each runtime's device, and why + when it is not the GPU. Sent on every lease and heartbeat, so the System + view can call a running agent that fell back to the CPU "degraded" rather + than "running" — the 2026-09-24 fallback went unseen for weeks because + only this agent's own log said so. None before report() has run.""" + if not LAST: + return None + out = {} + for name, s in LAST.items(): + entry = {"device": s.get("device")} + if s.get("error"): + entry["error"] = str(s["error"])[:200] + out[name] = entry + return out + + def report() -> dict: """Check both runtimes, log the result, and keep it for /status.""" LAST.clear() diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index 877bea5..cc8a040 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -192,7 +192,11 @@ _PAGE = """ width:30px;height:32px;font:700 16px system-ui;cursor:pointer} .step:hover{border-color:var(--acc)} #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a; - color:var(--fg);border:1px solid var(--bd);border-radius:8px} + color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield} + /* The browser's own spin arrows, hidden: the − / + beside each field are the + control, styled like the rest of the page (operator, 2026-09-24). */ + #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button, + #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0} .unit{color:var(--mut);font-size:12px;font-weight:600} .hint{color:var(--mut);font-size:12px;margin-top:12px} .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px} @@ -231,6 +235,7 @@ _PAGE = """ + @@ -248,7 +253,9 @@ _PAGE = """
+ + MB/s
@@ -316,6 +323,14 @@ _PAGE = """ await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'}, body:JSON.stringify({value:on})});refresh() } + function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) } + // Runtimes that did NOT get the GPU, from the startup report. Both fall back + // to the CPU without raising, so this banner and the pill are the only place + // on this page a slow, CPU-bound agent announces itself. + function cpuOnly(s){ + const a=s.accel||{} + return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda') + } async function setbw(v){ v=Math.max(0,parseFloat(v)||0); bw.value=v await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'}, @@ -386,11 +401,17 @@ _PAGE = """ // unreachable curator; grey when stopped; red with no token. let dc='dot', lbl='stopped' if(!ok){ dc='dot red'; lbl='no token' } - else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' } + else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' + if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } } else if(st==='starting'){ dc='dot amber'; lbl='starting…' } else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' } dot.className=dc; connlbl.textContent=lbl banner.style.display=(st==='running' && !s.queue)?'block':'none' + const slow=cpuOnly(s) + accelbanner.style.display=slow.length?'block':'none' + accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: ' + +slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ') + +'. After a driver update, regenerate the CDI spec (agent README).'):'' queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable' } } diff --git a/agent/fc_agent/client.py b/agent/fc_agent/client.py index b2b8cea..582290a 100644 --- a/agent/fc_agent/client.py +++ b/agent/fc_agent/client.py @@ -7,6 +7,8 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from . import accel + class FcClient: def __init__(self, base_url: str, token: str, agent_id: str): @@ -72,7 +74,10 @@ class FcClient: def lease(self, batch_size: int) -> list[dict]: r = self.s.post( f"{self.base}/api/gpu/jobs/lease", - json={"agent_id": self.agent_id, "batch_size": batch_size}, + json={ + "agent_id": self.agent_id, "batch_size": batch_size, + "accel": accel.summary(), + }, timeout=30, ) r.raise_for_status() @@ -90,7 +95,9 @@ class FcClient: }) def heartbeat(self, job_ids: list[int]) -> None: - self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids}) + self._post_quiet( + "/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()}, + ) def fail(self, job_id: int, error: str) -> None: self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error}) diff --git a/backend/app/api/gpu.py b/backend/app/api/gpu.py index f0723fb..7ae69fe 100644 --- a/backend/app/api/gpu.py +++ b/backend/app/api/gpu.py @@ -245,6 +245,29 @@ async def errors_recover(image_id: int): # --- Agent (bearer token): lease / submit / heartbeat / fail ------------ + +def _accel_detail(body: dict) -> dict: + """The agent's own report of which runtime got the GPU, kept on its roster + row so the System view can call a CPU-bound agent degraded (#4410). + + Only a dict of {runtime: {device, error?}} is kept, and each value is + reduced to those two short strings: this is written on every lease, by a + client the server does not control. An agent that sends nothing (an + older build) simply has no `accel`, which reads as not-yet-reported. + """ + raw = body.get("accel") + if not isinstance(raw, dict): + return {} + accel = {} + for name, entry in list(raw.items())[:4]: + if not isinstance(entry, dict): + continue + clean = {"device": str(entry.get("device") or "")[:16]} + if entry.get("error"): + clean["error"] = str(entry["error"])[:200] + accel[str(name)[:16]] = clean + return {"accel": accel} if accel else {} + @gpu_bp.route("/jobs/lease", methods=["POST"]) async def lease(): body = await request.get_json(silent=True) or {} @@ -267,7 +290,10 @@ async def lease(): key=f"agent:{agent_id}", kind="agent", display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})", - details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)}, + details={ + "agent_id": agent_id, "last_call": "lease", "leased": len(jobs), + **_accel_detail(body), + }, ) ml = await MLSettings.load(session) # image rows for url/mime in one shot @@ -347,7 +373,10 @@ async def heartbeat(): key=f"agent:{agent_id}", kind="agent", display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})", - details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n}, + details={ + "agent_id": agent_id, "last_call": "heartbeat", "extended": n, + **_accel_detail(body), + }, ) await session.commit() return jsonify({"extended": n}) diff --git a/backend/app/api/system_health.py b/backend/app/api/system_health.py index 92dde1a..6fffe84 100644 --- a/backend/app/api/system_health.py +++ b/backend/app/api/system_health.py @@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, ( PROBE_TIMEOUT_SECONDS = 2.0 _OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown" +# Checking in, but working at a fraction of its speed: a GPU agent whose +# runtimes fell back to the CPU (#4410). Below stale — a part that may have +# stopped is the more urgent question — and above unknown, because this one +# IS known to be wrong. +_DEGRADED = "degraded" # Worst-first, so an overall verdict is just the max. -_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3} +_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4} def _age_state(age_seconds: float) -> str: @@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str: return f"{name} has not checked in for {ago} — treat it as stopped" +def _cpu_runtimes(details: dict) -> list[str]: + """The runtimes an agent reported as NOT on the GPU, with why. + + Both torch and onnxruntime fall back to the CPU without raising, so an + agent in that state leases, works and checks in exactly like a healthy + one. On 2026-09-24 one had been doing so since a driver update left a + stale CDI spec; the only sign was a line in the agent's own log. + """ + accel = details.get("accel") + if not isinstance(accel, dict): + return [] + out = [] + for name, entry in sorted(accel.items()): + if not isinstance(entry, dict) or entry.get("device") == "cuda": + continue + why = entry.get("error") or entry.get("device") or "unknown" + out.append(f"{name} ({why})") + return out + + +def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]: + """A roster row's state and its sentence, degraded included.""" + if state == _OK: + cpu = _cpu_runtimes(details) + if cpu: + return _DEGRADED, ( + f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. " + "After a driver update, regenerate the agent host's CDI spec " + "(agent README)." + ) + return state, _describe_learned(name, state, age, details) + + async def _probe_postgres(session) -> dict: started = time.monotonic() try: @@ -175,13 +213,15 @@ async def system_health(): ).scalars().all() for row in rows: age = (now - row.last_seen_at).total_seconds() - state = _age_state(age) + state, detail = _learned_state( + row.display_name, _age_state(age), age, row.details or {}, + ) parts.append({ "key": row.key, "kind": row.kind, "name": row.display_name, "state": state, - "detail": _describe_learned(row.display_name, state, age, row.details or {}), + "detail": detail, "last_seen_at": row.last_seen_at.isoformat(), "first_seen_at": row.first_seen_at.isoformat(), **{k: v for k, v in (row.details or {}).items() if k != "agent_id"}, diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index 31a9645..e1249ca 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -153,6 +153,14 @@ const health = computed(() => { label: (worst?.detail || 'A part has stopped') + suffix, } } + // Running but slow — a GPU agent that fell back to the CPU (#4410). Worth + // the amber dot: nothing else anywhere says so. + if (overall === 'degraded') { + return { + icon: 'mdi-speedometer-slow', color: 'warning', + label: (worst?.detail || 'A part is running degraded') + suffix, + } + } if (overall === 'stale') { return { icon: 'mdi-alert', color: 'warning', diff --git a/frontend/src/components/settings/SystemHealthTab.vue b/frontend/src/components/settings/SystemHealthTab.vue index 9a365fb..e1b30c4 100644 --- a/frontend/src/components/settings/SystemHealthTab.vue +++ b/frontend/src/components/settings/SystemHealthTab.vue @@ -368,6 +368,7 @@ function step(lane, delta) { .fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; } .fc-sys__dot--ok { background: rgb(var(--v-theme-success)); } .fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); } +.fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); } .fc-sys__dot--down { background: rgb(var(--v-theme-error)); } .fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); } diff --git a/frontend/src/utils/systemParts.js b/frontend/src/utils/systemParts.js index 61656c1..8df1dc5 100644 --- a/frontend/src/utils/systemParts.js +++ b/frontend/src/utils/systemParts.js @@ -24,7 +24,8 @@ export function queueKey(queues) { } // Worst first. A stopped datastore is why someone opened this tab. -export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 } +// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410). +export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 } export function kindLabel(kind) { if (kind === 'celery') return 'worker lane' diff --git a/tests/test_agent_accel.py b/tests/test_agent_accel.py index af7721e..46629cd 100644 --- a/tests/test_agent_accel.py +++ b/tests/test_agent_accel.py @@ -103,3 +103,20 @@ def test_a_missing_runtime_is_reported_not_raised(): assert accel.torch_status(imp)["device"] == "unavailable" assert accel.onnx_status(imp)["device"] == "unavailable" + + +def test_summary_is_what_the_server_stores(monkeypatch): + """Device plus a bounded reason, per runtime — sent on every lease.""" + monkeypatch.setattr(accel, "LAST", { + "torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"}, + "onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500}, + }) + assert accel.summary() == { + "torch": {"device": "cuda"}, + "onnx": {"device": "cpu", "error": "e" * 200}, + } + + +def test_summary_before_the_report_is_none(monkeypatch): + monkeypatch.setattr(accel, "LAST", {}) + assert accel.summary() is None diff --git a/tests/test_agent_degraded.py b/tests/test_agent_degraded.py new file mode 100644 index 0000000..0bb33d8 --- /dev/null +++ b/tests/test_agent_degraded.py @@ -0,0 +1,59 @@ +"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410). + +Both runtimes fall back without raising, so a CPU-bound agent leases, works +and checks in exactly like a healthy one. On 2026-09-24 one had been doing so +since a driver update left a stale CDI spec; the only sign was a line in the +agent's own log. The agent now sends its startup report on every lease and +heartbeat, and the System view derives the state from it. +""" + +from __future__ import annotations + +from backend.app.api.gpu import _accel_detail +from backend.app.api.system_health import _SEVERITY, _learned_state + +GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}} +CPU = { + "torch": {"device": "cpu"}, + "onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"}, +} + + +def test_a_running_agent_on_the_gpu_is_ok(): + state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU}) + assert state == "ok" + assert detail == "GPU agent is running" + + +def test_a_running_agent_on_the_cpu_is_degraded_and_says_why(): + state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU}) + assert state == "degraded" + assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail + assert "torch (cpu)" in detail + + +def test_an_agent_that_never_reported_is_not_called_degraded(): + """An older agent build sends no `accel`: that is not-yet-known, not slow.""" + assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok" + + +def test_stopped_outranks_degraded(): + """Whether a quiet agent is still running is the more urgent question.""" + state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU}) + assert state == "down" + assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"] + + +def test_the_lease_keeps_only_a_bounded_report(): + """Written on every lease by a client the server does not control.""" + body = {"accel": { + "torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"}, + "onnx": "not a dict", + }} + kept = _accel_detail(body)["accel"] + assert kept == {"torch": {"device": "cpu", "error": "x" * 200}} + + +def test_a_lease_without_a_report_adds_nothing(): + assert _accel_detail({}) == {} + assert _accel_detail({"accel": None}) == {}