"""Which accelerator each runtime actually got — reported once, at startup. The agent has two GPU runtimes and both fall back to the CPU without raising: torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils detector + CCIP models) when its CUDA provider cannot load its libraries. A fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base (#1451), and whether the ONNX half was on the GPU could not be answered from anything the agent had ever logged. Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT 1.21+) loads them from there, so the provider resolves them by soname. Stdlib-only at import, so the unit suite can import it — torch and onnxruntime are imported inside the functions. """ from __future__ import annotations import ctypes import importlib import logging from pathlib import Path log = logging.getLogger("fc_agent.accel") # Filled by report(); /status carries it so the page can show it too. LAST: dict = {} def torch_status(imp=importlib.import_module) -> dict: try: torch = imp("torch") except Exception as e: return {"device": "unavailable", "error": str(e)} out = {"version": torch.__version__, "cuda_build": torch.version.cuda} if torch.cuda.is_available(): out["device"] = "cuda" out["gpu"] = torch.cuda.get_device_name(0) else: out["device"] = "cpu" return out def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict: try: ort = imp("onnxruntime") except Exception as e: return {"device": "unavailable", "error": str(e)} out = {"version": ort.__version__, "providers": list(ort.get_available_providers())} if "CUDAExecutionProvider" not in out["providers"]: out["device"] = "cpu" return out preload = getattr(ort, "preload_dlls", None) if preload is not None: try: preload() except Exception as e: out["preload_error"] = str(e) # "Available" only means the build HAS the provider. Loading its library is # what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are # missing, and the one a session would otherwise fail silently on. capi = Path(ort.__file__).parent / "capi" try: load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL) load(str(capi / "libonnxruntime_providers_cuda.so")) except OSError as e: out["device"] = "cpu" out["error"] = str(e) return out # Loading proves the libraries resolve, NOT that a GPU can be used: on # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA # runtime for a device initialises the driver the provider would use. error = _cuda_device_error(load) out["device"] = "cpu" if error else "cuda" if error: out["error"] = error return out def _cuda_device_error(load=ctypes.CDLL) -> str | None: """None when the CUDA runtime can reach a device, else why it cannot.""" try: cudart = load("libcudart.so.13") except OSError as e: return str(e) count = ctypes.c_int(0) rc = cudart.cudaGetDeviceCount(ctypes.byref(count)) if rc != 0: cudart.cudaGetErrorString.restype = ctypes.c_char_p return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})" return None if count.value > 0 else "no CUDA device visible" def summary() -> dict | None: """The report as FabledCurator stores it: each runtime's device, and why when it is not the GPU. Sent on every lease and heartbeat, so the System view can call a running agent that fell back to the CPU "degraded" rather than "running" — the 2026-09-24 fallback went unseen for weeks because only this agent's own log said so. None before report() has run.""" if not LAST: return None out = {} for name, s in LAST.items(): entry = {"device": s.get("device")} if s.get("error"): entry["error"] = str(s["error"])[:200] out[name] = entry return out def report() -> dict: """Check both runtimes, log the result, and keep it for /status.""" LAST.clear() LAST.update(torch=torch_status(), onnx=onnx_status()) for name, s in LAST.items(): if s.get("device") == "cuda": log.info("accel: %s on GPU (%s)", name, s) else: log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s) return dict(LAST)