From 7a09dc3cdadbee2b9a178fb6cd903afd2c28b723 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 24 Sep 2026 17:45:39 -0400 Subject: [PATCH] build: the agent installs one CUDA-13 stack instead of two, the web image drops ML packages it never imported, and Redis moves to 8 (1451, 1452) Agent: - The image ran PyPI's CUDA-13 torch 2.14 and onnxruntime-gpu 1.30 on a CUDA 12.9 cudnn-runtime base. requirements.txt had silently replaced the Dockerfile's torch 2.6+cu124, because ultralytics pulls torchvision, which pulls its own torch. That left ~3 GB of base libraries and a ~3 GB torch nothing loaded: 10 GB compressed. - Now: an nvidia/cuda 13.0.3 `base` image, with torch and torchvision installed together from cu130. CUDA and cuDNN come from the nvidia-* pip packages; onnxruntime-gpu declares its [cuda,cudnn] extras. - fc_agent/accel.py preloads those libraries for onnxruntime. It then logs, and reports in /status, whether torch and the ONNX CUDA provider actually got the GPU, since both fall back to the CPU silently. Web image: - Drop opencv-python-headless and onnxruntime, plus the opencv-only apt libs. Both have been listed since the scaffold and nothing in backend/ imports them. - torch/torchvision move to 2.14/0.29, and the unexplained caps are lifted (rule 154). Redis: 8-alpine in both compose files and both CI service containers. That gives an AGPLv3 licence option, where 7.4 was RSAL/SSPL only. The client moves to >=8.1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- .forgejo/workflows/build.yml | 8 ++-- Dockerfile | 10 ++--- agent/Dockerfile | 33 +++++++++----- agent/README.md | 3 +- agent/fc_agent/accel.py | 86 ++++++++++++++++++++++++++++++++++++ agent/fc_agent/app.py | 6 ++- agent/requirements.txt | 10 +++-- docker-compose.single.yml | 2 +- docker-compose.yml | 2 +- requirements-ml.txt | 22 ++++----- requirements.txt | 4 +- tests/test_agent_accel.py | 75 +++++++++++++++++++++++++++++++ 12 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 agent/fc_agent/accel.py create mode 100644 tests/test_agent_accel.py diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 3a6d09c..0b1aed6 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -419,7 +419,7 @@ jobs: --health-timeout 5s --health-retries 10 redis: - image: redis:7-alpine + image: redis:8-alpine options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -434,7 +434,7 @@ jobs: docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' echo "=== end landscape ===" PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1) + RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") @@ -1593,7 +1593,7 @@ jobs: --health-timeout 5s --health-retries 10 redis: - image: redis:7-alpine + image: redis:8-alpine options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -1619,7 +1619,7 @@ jobs: # in a container against a mounted docker socket, so the services are # SIBLINGS reachable by IP, not by hostname. PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1) + RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") diff --git a/Dockerfile b/Dockerfile index 633c34b..36eb7f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,11 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libwebp7 \ libpng16-16 \ ca-certificates \ - # opencv-python-headless (via requirements-ml.txt) links these even in its - # headless build. Came from Dockerfile.ml when the images merged - # (milestone 422 step 6). - libgl1 \ - libglib2.0-0 \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -61,7 +56,8 @@ RUN pip install -r requirements.txt # # torch 2.12.1+cpu wheel 192.3 MB # torchvision 0.27.1+cpu 1.8 MB -# transformers / onnxruntime / opencv / sklearn and friends +# transformers / onnxruntime / opencv / sklearn and friends (opencv and +# onnxruntime since dropped, #1451 — nothing here imported them) # 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB # largest newly-pushed layer 222.07 MB # @@ -85,7 +81,7 @@ RUN pip install -r requirements.txt # CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the # GPU agent is a separate service with its own image. RUN pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.12,<3.0" "torchvision>=0.27,<0.28" + "torch>=2.14" "torchvision>=0.29" RUN pip install -r requirements-ml.txt # Where the model lands. Deliberately NOT a VOLUME instruction: that mints an diff --git a/agent/Dockerfile b/agent/Dockerfile index 1f95b02..2d0323f 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -1,10 +1,21 @@ # FabledCurator GPU agent — runs on the desktop with the GPU. -# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs -# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open -# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12. -# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are -# built against (CUDA 13 has only nascent ONNX Runtime support). -FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04 +# +# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the +# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only +# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES / +# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was +# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then +# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision, +# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image +# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB +# torch nothing loaded: 10 GB compressed. +# +# 13.0 because that is the line both wheels are built for (torch's cu130 index, +# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that +# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and +# onnxruntime actually got the GPU, since both fall back to the CPU silently. +# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12. +FROM nvidia/cuda:13.0.3-base-ubuntu24.04 # PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally # managed (PEP 668), so a global `pip install` errors without this. It's a @@ -16,10 +27,12 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /app -# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN -# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first -# + separately so the GPU build of torch is deterministic and layer-cached. -RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 +# torch AND torchvision from the cu130 index, together and first. Installing +# torch alone is what let the next step swap it out: ultralytics needs +# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to +# match. With both present, requirements.txt finds them satisfied. +RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \ + torch torchvision COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt COPY fc_agent ./fc_agent diff --git a/agent/README.md b/agent/README.md index ec0119b..8084e23 100644 --- a/agent/README.md +++ b/agent/README.md @@ -15,7 +15,8 @@ sudo pacman -S nvidia-container-toolkit sudo nvidia-ctk runtime configure --runtime=docker sudo systemctl restart docker # verify: -docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi +docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi +# the header's CUDA version must be 13.0 or later (driver 580+) ``` ## 1. Get a token diff --git a/agent/fc_agent/accel.py b/agent/fc_agent/accel.py new file mode 100644 index 0000000..3bf309d --- /dev/null +++ b/agent/fc_agent/accel.py @@ -0,0 +1,86 @@ +"""Which accelerator each runtime actually got — reported once, at startup. + +The agent has two GPU runtimes and both fall back to the CPU without raising: +torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils +detector + CCIP models) when its CUDA provider cannot load its libraries. A +fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the +image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base +(#1451), and whether the ONNX half was on the GPU could not be answered from +anything the agent had ever logged. + +Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA +provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image +they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT +1.21+) loads them from there, so the provider resolves them by soname. + +Stdlib-only at import, so the unit suite can import it — torch and onnxruntime +are imported inside the functions. +""" + +from __future__ import annotations + +import ctypes +import importlib +import logging +from pathlib import Path + +log = logging.getLogger("fc_agent.accel") + +# Filled by report(); /status carries it so the page can show it too. +LAST: dict = {} + + +def torch_status(imp=importlib.import_module) -> dict: + try: + torch = imp("torch") + except Exception as e: + return {"device": "unavailable", "error": str(e)} + out = {"version": torch.__version__, "cuda_build": torch.version.cuda} + if torch.cuda.is_available(): + out["device"] = "cuda" + out["gpu"] = torch.cuda.get_device_name(0) + else: + out["device"] = "cpu" + return out + + +def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict: + try: + ort = imp("onnxruntime") + except Exception as e: + return {"device": "unavailable", "error": str(e)} + out = {"version": ort.__version__, "providers": list(ort.get_available_providers())} + if "CUDAExecutionProvider" not in out["providers"]: + out["device"] = "cpu" + return out + preload = getattr(ort, "preload_dlls", None) + if preload is not None: + try: + preload() + except Exception as e: + out["preload_error"] = str(e) + # "Available" only means the build HAS the provider. Loading its library is + # what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are + # missing, and the one a session would otherwise fail silently on. + capi = Path(ort.__file__).parent / "capi" + try: + load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL) + load(str(capi / "libonnxruntime_providers_cuda.so")) + except OSError as e: + out["device"] = "cpu" + out["error"] = str(e) + else: + out["device"] = "cuda" + return out + + +def report() -> dict: + """Check both runtimes, log the result, and keep it for /status.""" + LAST.clear() + LAST.update(torch=torch_status(), onnx=onnx_status()) + for name, s in LAST.items(): + if s.get("device") == "cuda": + log.info("accel: %s on GPU (%s)", name, s) + else: + log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s) + return dict(LAST) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index f9a5d8a..877bea5 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -11,7 +11,7 @@ import logging from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse, JSONResponse -from . import logbuf +from . import accel, logbuf from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version from .config import Config from .gpu import read_gpu @@ -47,6 +47,9 @@ async def _no_store(request, call_next): @app.on_event("startup") def _maybe_autostart() -> None: + # Before the worker: the report also preloads the CUDA libraries the ONNX + # models need, and it says in the log which runtimes landed on the GPU. + accel.report() # With AUTO_START set, a container restart (host reboot, or `restart: # unless-stopped` after a crash) resumes the worker on its own — the slots # then ride out a still-down curator via lease backoff. Lets the agent @@ -137,6 +140,7 @@ def status(): s["version"] = FC_VERSION or None s["channel"] = FC_CHANNEL or None s["revision"] = FC_REVISION or None + s["accel"] = accel.LAST or None return JSONResponse(s) diff --git a/agent/requirements.txt b/agent/requirements.txt index b71fdb0..934de2e 100644 --- a/agent/requirements.txt +++ b/agent/requirements.txt @@ -1,10 +1,12 @@ # CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace). dghs-imgutils>=0.4 # GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow -# server-side fallback run. -onnxruntime-gpu -# The crop EMBEDDER (concept bag). torch is installed separately in the -# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic; +# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its +# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on +# torch happening to install the same ones. +onnxruntime-gpu[cuda,cudnn] +# The crop EMBEDDER (concept bag). torch + torchvision are installed separately +# in the Dockerfile from the cu130 wheel index, so pip never swaps them out; # transformers loads whatever SigLIP-family model the server announces. transformers>=4.45 # Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic diff --git a/docker-compose.single.yml b/docker-compose.single.yml index 20a6c23..7654447 100644 --- a/docker-compose.single.yml +++ b/docker-compose.single.yml @@ -29,7 +29,7 @@ services: redis: - image: redis:7-alpine + image: redis:8-alpine volumes: - redis_data:/data healthcheck: diff --git a/docker-compose.yml b/docker-compose.yml index 671dca0..ab8f520 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ x-celery-healthcheck: &celery_healthcheck services: redis: - image: redis:7-alpine + image: redis:8-alpine volumes: - redis_data:/data healthcheck: diff --git a/requirements-ml.txt b/requirements-ml.txt index 52e20e5..fe6965d 100644 --- a/requirements-ml.txt +++ b/requirements-ml.txt @@ -8,21 +8,21 @@ # so Dockerfile uses the +cpu wheels from # https://download.pytorch.org/whl/cpu instead. # -# IMPORTANT: torchvision 0.27 declares requires_python "!=3.14.1,>=3.10" — -# Python 3.14.1 specifically is excluded due to a known incompatibility. -# The python-ci runner pulls python:3.14-bookworm (latest patch); if that -# resolves to 3.14.1 the install will fail. Pin a specific Python patch in -# the runner image (CI-Runner/CI-python/Dockerfile) if this becomes a -# blocker. 3.14.0 and 3.14.2+ are fine. +# torchvision declares requires_python "!=3.14.1" (0.27 through 0.29). The +# image's python:3.14-slim is past that patch, so it only bites a build pinned +# to exactly 3.14.1. +# +# No caps below: rule 154 wants a named breakage for one, and none of the +# `=5.8,<6.0 -onnxruntime>=1.26,<2.0 -huggingface-hub>=1.14,<2.0 -opencv-python-headless>=4.13,<5.0 +transformers>=5.8 +huggingface-hub>=1.14 # scikit-learn powers the tag-eval (#1130) head-vs-centroid comparison: logistic # regression + cross-validated precision/recall/AP. Battle-tested metrics matter # because that eval's whole purpose is producing trustworthy numbers. numpy is # left to resolve transitively (torch/transformers/sklearn all pull it) to avoid # pinning against their constraints. -scikit-learn>=1.7,<2.0 +scikit-learn>=1.7 diff --git a/requirements.txt b/requirements.txt index f54bc43..60ab750 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,9 @@ pgvector>=0.5,<0.6 # Task queue celery>=5.6,<5.7 -redis>=7.4,<8.0 +# Uncapped (rule 154). 8.0 only changed type hints. kombu's `redis` extra +# still says <6.5, but celery is installed without that extra, so it never applies. +redis>=8.1 # Crypto for credential storage (lands in FC-3, but pinned now for stability) cryptography>=49,<50 diff --git a/tests/test_agent_accel.py b/tests/test_agent_accel.py new file mode 100644 index 0000000..cc509a2 --- /dev/null +++ b/tests/test_agent_accel.py @@ -0,0 +1,75 @@ +"""The agent's startup report of which runtime landed on the GPU (#1451). + +Both runtimes fall back to the CPU without raising, so the report is the only +thing that says so. These pin the distinction it exists for: onnxruntime +listing the CUDA provider is not the same as the provider being able to load +its libraries. +""" + +from __future__ import annotations + +import types + +from agent.fc_agent import accel + + +def _ort(providers, preload=None): + mod = types.SimpleNamespace( + __version__="1.30.0", + __file__="/site/onnxruntime/__init__.py", + get_available_providers=lambda: providers, + ) + if preload is not None: + mod.preload_dlls = preload + return mod + + +def _imp(mod): + return lambda name: mod + + +def test_onnx_on_gpu_when_the_cuda_provider_loads(): + calls = [] + s = accel.onnx_status( + _imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"], lambda: calls.append("preload"))), + load=lambda path, mode=0: calls.append(path), + ) + assert s["device"] == "cuda" + assert calls[0] == "preload" + assert calls[-1].endswith("capi/libonnxruntime_providers_cuda.so") + + +def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason(): + def load(path, mode=0): + if path.endswith("providers_cuda.so"): + raise OSError("libcudart.so.13: cannot open shared object file") + + s = accel.onnx_status(_imp(_ort(["CUDAExecutionProvider", "CPUExecutionProvider"])), load=load) + assert s["device"] == "cpu" + assert "libcudart.so.13" in s["error"] + + +def test_onnx_cpu_build_never_tries_the_cuda_library(): + def load(path, mode=0): + raise AssertionError("a CPU build has no CUDA provider to load") + + s = accel.onnx_status(_imp(_ort(["CPUExecutionProvider"])), load=load) + assert s["device"] == "cpu" + + +def test_torch_reports_cpu_when_cuda_is_unavailable(): + torch = types.SimpleNamespace( + __version__="2.14.0+cu130", + version=types.SimpleNamespace(cuda="13.0"), + cuda=types.SimpleNamespace(is_available=lambda: False), + ) + s = accel.torch_status(_imp(torch)) + assert s == {"version": "2.14.0+cu130", "cuda_build": "13.0", "device": "cpu"} + + +def test_a_missing_runtime_is_reported_not_raised(): + def imp(name): + raise ImportError(f"No module named {name!r}") + + assert accel.torch_status(imp)["device"] == "unavailable" + assert accel.onnx_status(imp)["device"] == "unavailable"