agent/fc_agent/accel.py

@@ -69,11 +69,32 @@ def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
     except OSError as e:
         out["device"] = "cpu"
         out["error"] = str(e)
-    else:
-        out["device"] = "cuda"
+        return out
+    # Loading proves the libraries resolve, NOT that a GPU can be used: on
+    # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
+    # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
+    # runtime for a device initialises the driver the provider would use.
+    error = _cuda_device_error(load)
+    out["device"] = "cpu" if error else "cuda"
+    if error:
+        out["error"] = error
     return out
 
 
+def _cuda_device_error(load=ctypes.CDLL) -> str | None:
+    """None when the CUDA runtime can reach a device, else why it cannot."""
+    try:
+        cudart = load("libcudart.so.13")
+    except OSError as e:
+        return str(e)
+    count = ctypes.c_int(0)
+    rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
+    if rc != 0:
+        cudart.cudaGetErrorString.restype = ctypes.c_char_p
+        return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
+    return None if count.value > 0 else "no CUDA device visible"
+
+
 def report() -> dict:
     """Check both runtimes, log the result, and keep it for /status."""
     LAST.clear()

tests/test_agent_accel.py

@@ -28,15 +28,45 @@ def _imp(mod):
     return lambda name: mod
 
 
-def test_onnx_on_gpu_when_the_cuda_provider_loads():
+class _Cudart:
+    """libcudart as ctypes sees it: cudaGetDeviceCount writes through a pointer."""
+
+    def __init__(self, rc=0, count=1):
+        self.rc = rc
+        self.count = count
+        self.cudaGetErrorString = lambda rc: b"unknown error"
+
+    def cudaGetDeviceCount(self, ref):
+        ref._obj.value = self.count
+        return self.rc
+
+
+def _loader(calls, cudart):
+    def load(path, mode=0):
+        calls.append(path)
+        return cudart if path == "libcudart.so.13" else None
+    return load
+
+
+GPU_BUILD = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+
+
+def test_onnx_on_gpu_when_the_provider_loads_and_a_device_answers():
     calls = []
     s = accel.onnx_status(
-        _imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"], lambda: calls.append("preload"))),
-        load=lambda path, mode=0: calls.append(path),
+        _imp(_ort(GPU_BUILD, lambda: calls.append("preload"))),
+        load=_loader(calls, _Cudart()),
     )
     assert s["device"] == "cuda"
     assert calls[0] == "preload"
-    assert calls[-1].endswith("capi/libonnxruntime_providers_cuda.so")
+    assert any(c.endswith("capi/libonnxruntime_providers_cuda.so") for c in calls)
+
+
+def test_onnx_libraries_loading_is_not_a_gpu_when_cuda_cannot_initialise():
+    """The 2026-09-24 case: every library resolved, cuInit failed."""
+    s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=_loader([], _Cudart(rc=999)))
+    assert s["device"] == "cpu"
+    assert "unknown error" in s["error"]
 
 
 def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason():
@@ -44,7 +74,7 @@ def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason():
         if path.endswith("providers_cuda.so"):
             raise OSError("libcudart.so.13: cannot open shared object file")
 
-    s = accel.onnx_status(_imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"])), load=load)
+    s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=load)
     assert s["device"] == "cpu"
     assert "libcudart.so.13" in s["error"]