diff --git a/agent/fc_agent/accel.py b/agent/fc_agent/accel.py index 3bf309d..5f29fc5 100644 --- a/agent/fc_agent/accel.py +++ b/agent/fc_agent/accel.py @@ -69,11 +69,32 @@ def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict: except OSError as e: out["device"] = "cpu" out["error"] = str(e) - else: - out["device"] = "cuda" + return out + # Loading proves the libraries resolve, NOT that a GPU can be used: on + # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with + # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA + # runtime for a device initialises the driver the provider would use. + error = _cuda_device_error(load) + out["device"] = "cpu" if error else "cuda" + if error: + out["error"] = error return out +def _cuda_device_error(load=ctypes.CDLL) -> str | None: + """None when the CUDA runtime can reach a device, else why it cannot.""" + try: + cudart = load("libcudart.so.13") + except OSError as e: + return str(e) + count = ctypes.c_int(0) + rc = cudart.cudaGetDeviceCount(ctypes.byref(count)) + if rc != 0: + cudart.cudaGetErrorString.restype = ctypes.c_char_p + return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})" + return None if count.value > 0 else "no CUDA device visible" + + def report() -> dict: """Check both runtimes, log the result, and keep it for /status.""" LAST.clear() diff --git a/tests/test_agent_accel.py b/tests/test_agent_accel.py index cc509a2..af7721e 100644 --- a/tests/test_agent_accel.py +++ b/tests/test_agent_accel.py @@ -28,15 +28,45 @@ def _imp(mod): return lambda name: mod -def test_onnx_on_gpu_when_the_cuda_provider_loads(): +class _Cudart: + """libcudart as ctypes sees it: cudaGetDeviceCount writes through a pointer.""" + + def __init__(self, rc=0, count=1): + self.rc = rc + self.count = count + self.cudaGetErrorString = lambda rc: b"unknown error" + + def cudaGetDeviceCount(self, ref): + ref._obj.value = self.count + return self.rc + + +def _loader(calls, cudart): + def load(path, mode=0): + calls.append(path) + return cudart if path == "libcudart.so.13" else None + return load + + +GPU_BUILD = ["CUDAExecutionProvider", "CPUExecutionProvider"] + + +def test_onnx_on_gpu_when_the_provider_loads_and_a_device_answers(): calls = [] s = accel.onnx_status( - _imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"], lambda: calls.append("preload"))), - load=lambda path, mode=0: calls.append(path), + _imp(_ort(GPU_BUILD, lambda: calls.append("preload"))), + load=_loader(calls, _Cudart()), ) assert s["device"] == "cuda" assert calls[0] == "preload" - assert calls[-1].endswith("capi/libonnxruntime_providers_cuda.so") + assert any(c.endswith("capi/libonnxruntime_providers_cuda.so") for c in calls) + + +def test_onnx_libraries_loading_is_not_a_gpu_when_cuda_cannot_initialise(): + """The 2026-09-24 case: every library resolved, cuInit failed.""" + s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=_loader([], _Cudart(rc=999))) + assert s["device"] == "cpu" + assert "unknown error" in s["error"] def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason(): @@ -44,7 +74,7 @@ def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason(): if path.endswith("providers_cuda.so"): raise OSError("libcudart.so.13: cannot open shared object file") - s = accel.onnx_status(_imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"])), load=load) + s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=load) assert s["device"] == "cpu" assert "libcudart.so.13" in s["error"]