fix: the agent's ONNX check asks CUDA for a device instead of trusting that the libraries loaded (1451)

It reported "onnx on GPU" beside torch failing cuInit with "CUDA unknown
error". Every library resolved, but no device could be used. The check now
calls cudaGetDeviceCount and reports the CUDA error when there is one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

agent/fc_agent/accel.py

@@ -69,11 +69,32 @@ def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
     except OSError as e:
         out["device"] = "cpu"
         out["error"] = str(e)
-    else:
+        return out
-        out["device"] = "cuda"
+    # Loading proves the libraries resolve, NOT that a GPU can be used: on
+    # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
+    # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
+    # runtime for a device initialises the driver the provider would use.
+    error = _cuda_device_error(load)
+    out["device"] = "cpu" if error else "cuda"
+    if error:
+        out["error"] = error
     return out
 
 
+def _cuda_device_error(load=ctypes.CDLL) -> str | None:
+    """None when the CUDA runtime can reach a device, else why it cannot."""
+    try:
+        cudart = load("libcudart.so.13")
+    except OSError as e:
+        return str(e)
+    count = ctypes.c_int(0)
+    rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
+    if rc != 0:
+        cudart.cudaGetErrorString.restype = ctypes.c_char_p
+        return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
+    return None if count.value > 0 else "no CUDA device visible"
+
+
 def report() -> dict:
     """Check both runtimes, log the result, and keep it for /status."""
     LAST.clear()

tests/test_agent_accel.py

@@ -28,15 +28,45 @@ def _imp(mod):
     return lambda name: mod
 
 
-def test_onnx_on_gpu_when_the_cuda_provider_loads():
+class _Cudart:
+    """libcudart as ctypes sees it: cudaGetDeviceCount writes through a pointer."""
+
+    def __init__(self, rc=0, count=1):
+        self.rc = rc
+        self.count = count
+        self.cudaGetErrorString = lambda rc: b"unknown error"
+
+    def cudaGetDeviceCount(self, ref):
+        ref._obj.value = self.count
+        return self.rc
+
+
+def _loader(calls, cudart):
+    def load(path, mode=0):
+        calls.append(path)
+        return cudart if path == "libcudart.so.13" else None
+    return load
+
+
+GPU_BUILD = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
+
+def test_onnx_on_gpu_when_the_provider_loads_and_a_device_answers():
     calls = []
     s = accel.onnx_status(
-        _imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"], lambda: calls.append("preload"))),
+        _imp(_ort(GPU_BUILD, lambda: calls.append("preload"))),
-        load=lambda path, mode=0: calls.append(path),
+        load=_loader(calls, _Cudart()),
     )
     assert s["device"] == "cuda"
     assert calls[0] == "preload"
-    assert calls[-1].endswith("capi/libonnxruntime_providers_cuda.so")
+    assert any(c.endswith("capi/libonnxruntime_providers_cuda.so") for c in calls)
+
+
+def test_onnx_libraries_loading_is_not_a_gpu_when_cuda_cannot_initialise():
+    """The 2026-09-24 case: every library resolved, cuInit failed."""
+    s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=_loader([], _Cudart(rc=999)))
+    assert s["device"] == "cpu"
+    assert "unknown error" in s["error"]
 
 
 def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason():
@@ -44,7 +74,7 @@ def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason():
         if path.endswith("providers_cuda.so"):
             raise OSError("libcudart.so.13: cannot open shared object file")
 
-    s = accel.onnx_status(_imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"])), load=load)
+    s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=load)
     assert s["device"] == "cpu"
     assert "libcudart.so.13" in s["error"]