feat: a GPU agent on the CPU shows as degraded, in the System view and on its own page (4410)
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Successful in 2m22s
CI and images / sign-extension (push) Successful in 4s
CI and images / build-web (push) Successful in 2m46s
CI and images / smoke-web (push) Successful in 50s
CI and images / build-agent (push) Successful in 6m35s
CI and images / promote (push) Successful in 2s
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 27s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Successful in 2m22s
CI and images / sign-extension (push) Successful in 4s
CI and images / build-web (push) Successful in 2m46s
CI and images / smoke-web (push) Successful in 50s
CI and images / build-agent (push) Successful in 6m35s
CI and images / promote (push) Successful in 2s
torch and onnxruntime both fall back to the CPU without raising, so the agent that ran CPU-bound for weeks after a driver update leased and checked in like a healthy one. - The agent sends its startup accel report on every lease and heartbeat. - The server keeps a bounded copy on the roster row. A running agent with a runtime off the GPU becomes `degraded`, with a sentence naming the runtime and the reason. - The top nav shows it amber. - The agent page carries a banner, and its pill reads "CPU only". Also: the bandwidth field gets the page's − / + stepper, and both number fields drop the browser's spin arrows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
This commit is contained in:
@@ -103,3 +103,20 @@ def test_a_missing_runtime_is_reported_not_raised():
|
||||
|
||||
assert accel.torch_status(imp)["device"] == "unavailable"
|
||||
assert accel.onnx_status(imp)["device"] == "unavailable"
|
||||
|
||||
|
||||
def test_summary_is_what_the_server_stores(monkeypatch):
|
||||
"""Device plus a bounded reason, per runtime — sent on every lease."""
|
||||
monkeypatch.setattr(accel, "LAST", {
|
||||
"torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"},
|
||||
"onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500},
|
||||
})
|
||||
assert accel.summary() == {
|
||||
"torch": {"device": "cuda"},
|
||||
"onnx": {"device": "cpu", "error": "e" * 200},
|
||||
}
|
||||
|
||||
|
||||
def test_summary_before_the_report_is_none(monkeypatch):
|
||||
monkeypatch.setattr(accel, "LAST", {})
|
||||
assert accel.summary() is None
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410).
|
||||
|
||||
Both runtimes fall back without raising, so a CPU-bound agent leases, works
|
||||
and checks in exactly like a healthy one. On 2026-09-24 one had been doing so
|
||||
since a driver update left a stale CDI spec; the only sign was a line in the
|
||||
agent's own log. The agent now sends its startup report on every lease and
|
||||
heartbeat, and the System view derives the state from it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.app.api.gpu import _accel_detail
|
||||
from backend.app.api.system_health import _SEVERITY, _learned_state
|
||||
|
||||
GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}}
|
||||
CPU = {
|
||||
"torch": {"device": "cpu"},
|
||||
"onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_running_agent_on_the_gpu_is_ok():
|
||||
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU})
|
||||
assert state == "ok"
|
||||
assert detail == "GPU agent is running"
|
||||
|
||||
|
||||
def test_a_running_agent_on_the_cpu_is_degraded_and_says_why():
|
||||
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU})
|
||||
assert state == "degraded"
|
||||
assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail
|
||||
assert "torch (cpu)" in detail
|
||||
|
||||
|
||||
def test_an_agent_that_never_reported_is_not_called_degraded():
|
||||
"""An older agent build sends no `accel`: that is not-yet-known, not slow."""
|
||||
assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok"
|
||||
|
||||
|
||||
def test_stopped_outranks_degraded():
|
||||
"""Whether a quiet agent is still running is the more urgent question."""
|
||||
state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU})
|
||||
assert state == "down"
|
||||
assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"]
|
||||
|
||||
|
||||
def test_the_lease_keeps_only_a_bounded_report():
|
||||
"""Written on every lease by a client the server does not control."""
|
||||
body = {"accel": {
|
||||
"torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"},
|
||||
"onnx": "not a dict",
|
||||
}}
|
||||
kept = _accel_detail(body)["accel"]
|
||||
assert kept == {"torch": {"device": "cpu", "error": "x" * 200}}
|
||||
|
||||
|
||||
def test_a_lease_without_a_report_adds_nothing():
|
||||
assert _accel_detail({}) == {}
|
||||
assert _accel_detail({"accel": None}) == {}
|
||||
Reference in New Issue
Block a user