bvandeusen
3b34230fbd
Two operator-reported issues with the GPU agent: 1. Worker count flopped almost every cycle, spiking the GPU. The hill-climb probed +1, judged it over a too-short noisy throughput window, saw no clear gain and reverted -1 — every tick. Replace it with a GPU-utilization-band controller: HOLD while smoothed util sits in a healthy band, grow only on clear spare capacity (util below the low mark + VRAM headroom), shrink under saturation or memory pressure. Util is EWMA-smoothed and decisions are spaced (DECIDE_EVERY samples), so a noisy nvidia-smi reading can't move the pool. Load stays consistent instead of probe/reverting. 2. GPU util/VRAM bars only updated on manual refresh. They rode the /status poll, which blocks on the curator queue call (slow when curator is busy), so the meters froze between refreshes. Give them a dedicated /gpu endpoint (local nvidia-smi only, no curator round-trip) polled every 1.5s, and drop the curator queue-status timeout 15s -> 5s so /status itself stays snappy. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
101 lines
3.6 KiB
Python
101 lines
3.6 KiB
Python
"""HTTP client for the FabledCurator GPU-job API.
|
|
|
|
The agent's ONLY contact with FC — lease/submit/heartbeat/fail + fetch image
|
|
bytes, all over HTTP with the bearer token. No DB/Redis.
|
|
"""
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
|
|
|
|
class FcClient:
|
|
def __init__(self, base_url: str, token: str, agent_id: str):
|
|
self.base = base_url.rstrip("/")
|
|
self.agent_id = agent_id
|
|
self.s = requests.Session()
|
|
self.s.headers["Authorization"] = f"Bearer {token}"
|
|
# Many worker threads share this Session; the default pool (10) would
|
|
# throttle them + spam "connection pool is full". Size it for the cap.
|
|
adapter = HTTPAdapter(pool_connections=64, pool_maxsize=64)
|
|
self.s.mount("http://", adapter)
|
|
self.s.mount("https://", adapter)
|
|
|
|
def lease(self, batch_size: int) -> list[dict]:
|
|
r = self.s.post(
|
|
f"{self.base}/api/gpu/jobs/lease",
|
|
json={"agent_id": self.agent_id, "batch_size": batch_size},
|
|
timeout=30,
|
|
)
|
|
r.raise_for_status()
|
|
return r.json().get("jobs", [])
|
|
|
|
def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict:
|
|
r = self.s.post(
|
|
f"{self.base}/api/gpu/jobs/submit",
|
|
json={
|
|
"agent_id": self.agent_id, "job_id": job_id,
|
|
"regions": regions, "replace_kinds": replace_kinds,
|
|
},
|
|
timeout=120,
|
|
)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict:
|
|
"""Post a whole-image SigLIP embedding (the 'embed' task) → image_record."""
|
|
r = self.s.post(
|
|
f"{self.base}/api/gpu/jobs/submit_embedding",
|
|
json={
|
|
"agent_id": self.agent_id, "job_id": job_id,
|
|
"embedding": embedding, "embedding_version": version,
|
|
},
|
|
timeout=120,
|
|
)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
def heartbeat(self, job_ids: list[int]) -> None:
|
|
try:
|
|
self.s.post(
|
|
f"{self.base}/api/gpu/jobs/heartbeat",
|
|
json={"agent_id": self.agent_id, "job_ids": job_ids},
|
|
timeout=30,
|
|
)
|
|
except requests.RequestException:
|
|
pass
|
|
|
|
def fail(self, job_id: int, error: str) -> None:
|
|
try:
|
|
self.s.post(
|
|
f"{self.base}/api/gpu/jobs/fail",
|
|
json={"agent_id": self.agent_id, "job_id": job_id, "error": error},
|
|
timeout=30,
|
|
)
|
|
except requests.RequestException:
|
|
pass
|
|
|
|
def release(self, job_ids: list[int]) -> None:
|
|
# Graceful hand-back on stop so orphaned work is re-leased at once.
|
|
if not job_ids:
|
|
return
|
|
try:
|
|
self.s.post(
|
|
f"{self.base}/api/gpu/jobs/release",
|
|
json={"agent_id": self.agent_id, "job_ids": job_ids},
|
|
timeout=30,
|
|
)
|
|
except requests.RequestException:
|
|
pass
|
|
|
|
def fetch_image(self, image_url: str) -> bytes:
|
|
# image_url is a server-relative path ("/images/...").
|
|
r = self.s.get(f"{self.base}{image_url}", timeout=180)
|
|
r.raise_for_status()
|
|
return r.content
|
|
|
|
def queue_status(self) -> dict:
|
|
# Short timeout: this backs the UI /status poll, so a busy curator must
|
|
# not hang the page for long (the GPU meters poll /gpu separately).
|
|
r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
|
|
r.raise_for_status()
|
|
return r.json()
|