feat: a GPU agent on the CPU shows as degraded, in the System view and on its own page (4410)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Successful in 2m22s
CI and images / sign-extension (push) Successful in 4s
CI and images / build-web (push) Successful in 2m46s
CI and images / smoke-web (push) Successful in 50s
CI and images / build-agent (push) Successful in 6m35s
CI and images / promote (push) Successful in 2s

torch and onnxruntime both fall back to the CPU without raising, so the agent
that ran CPU-bound for weeks after a driver update leased and checked in like
a healthy one.

- The agent sends its startup accel report on every lease and heartbeat.
- The server keeps a bounded copy on the roster row. A running agent with a
  runtime off the GPU becomes `degraded`, with a sentence naming the runtime
  and the reason.
- The top nav shows it amber.
- The agent page carries a banner, and its pill reads "CPU only".

Also: the bandwidth field gets the page's − / + stepper, and both number
fields drop the browser's spin arrows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
2026-09-24 19:09:07 -04:00
co-authored by Claude Opus 5.5
parent e39ec1c550
commit cc53d8db7b
10 changed files with 210 additions and 10 deletions
+17
View File
@@ -103,3 +103,20 @@ def test_a_missing_runtime_is_reported_not_raised():
assert accel.torch_status(imp)["device"] == "unavailable"
assert accel.onnx_status(imp)["device"] == "unavailable"
def test_summary_is_what_the_server_stores(monkeypatch):
"""Device plus a bounded reason, per runtime — sent on every lease."""
monkeypatch.setattr(accel, "LAST", {
"torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"},
"onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500},
})
assert accel.summary() == {
"torch": {"device": "cuda"},
"onnx": {"device": "cpu", "error": "e" * 200},
}
def test_summary_before_the_report_is_none(monkeypatch):
monkeypatch.setattr(accel, "LAST", {})
assert accel.summary() is None
+59
View File
@@ -0,0 +1,59 @@
"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410).
Both runtimes fall back without raising, so a CPU-bound agent leases, works
and checks in exactly like a healthy one. On 2026-09-24 one had been doing so
since a driver update left a stale CDI spec; the only sign was a line in the
agent's own log. The agent now sends its startup report on every lease and
heartbeat, and the System view derives the state from it.
"""
from __future__ import annotations
from backend.app.api.gpu import _accel_detail
from backend.app.api.system_health import _SEVERITY, _learned_state
GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}}
CPU = {
"torch": {"device": "cpu"},
"onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"},
}
def test_a_running_agent_on_the_gpu_is_ok():
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU})
assert state == "ok"
assert detail == "GPU agent is running"
def test_a_running_agent_on_the_cpu_is_degraded_and_says_why():
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU})
assert state == "degraded"
assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail
assert "torch (cpu)" in detail
def test_an_agent_that_never_reported_is_not_called_degraded():
"""An older agent build sends no `accel`: that is not-yet-known, not slow."""
assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok"
def test_stopped_outranks_degraded():
"""Whether a quiet agent is still running is the more urgent question."""
state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU})
assert state == "down"
assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"]
def test_the_lease_keeps_only_a_bounded_report():
"""Written on every lease by a client the server does not control."""
body = {"accel": {
"torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"},
"onnx": "not a dict",
}}
kept = _accel_detail(body)["accel"]
assert kept == {"torch": {"device": "cpu", "error": "x" * 200}}
def test_a_lease_without_a_report_adds_nothing():
assert _accel_detail({}) == {}
assert _accel_detail({"accel": None}) == {}