"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410). Both runtimes fall back without raising, so a CPU-bound agent leases, works and checks in exactly like a healthy one. On 2026-09-24 one had been doing so since a driver update left a stale CDI spec; the only sign was a line in the agent's own log. The agent now sends its startup report on every lease and heartbeat, and the System view derives the state from it. """ from __future__ import annotations from backend.app.api.gpu import _accel_detail from backend.app.api.system_health import _SEVERITY, _learned_state GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}} CPU = { "torch": {"device": "cpu"}, "onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"}, } def test_a_running_agent_on_the_gpu_is_ok(): state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU}) assert state == "ok" assert detail == "GPU agent is running" def test_a_running_agent_on_the_cpu_is_degraded_and_says_why(): state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU}) assert state == "degraded" assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail assert "torch (cpu)" in detail def test_an_agent_that_never_reported_is_not_called_degraded(): """An older agent build sends no `accel`: that is not-yet-known, not slow.""" assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok" def test_stopped_outranks_degraded(): """Whether a quiet agent is still running is the more urgent question.""" state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU}) assert state == "down" assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"] def test_the_lease_keeps_only_a_bounded_report(): """Written on every lease by a client the server does not control.""" body = {"accel": { "torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"}, "onnx": "not a dict", }} kept = _accel_detail(body)["accel"] assert kept == {"torch": {"device": "cpu", "error": "x" * 200}} def test_a_lease_without_a_report_adds_nothing(): assert _accel_detail({}) == {} assert _accel_detail({"accel": None}) == {}