diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index f52435a..3a225e1 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -15,7 +15,7 @@ from .worker import Worker # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-06-30.5 · stop+poll+ci" +VERSION = "2026-06-30.6 · submit-retry+py310" logbuf.install() cfg = Config.from_env() diff --git a/agent/fc_agent/client.py b/agent/fc_agent/client.py index fb648a5..f226a08 100644 --- a/agent/fc_agent/client.py +++ b/agent/fc_agent/client.py @@ -5,19 +5,44 @@ bytes, all over HTTP with the bearer token. No DB/Redis. """ import requests from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry class FcClient: def __init__(self, base_url: str, token: str, agent_id: str): self.base = base_url.rstrip("/") self.agent_id = agent_id - self.s = requests.Session() - self.s.headers["Authorization"] = f"Bearer {token}" - # Many worker threads share this Session; the default pool (10) would + # Main session: NO in-request retry — lease/fetch are cheap to redo and + # the worker loop already backs off + re-leases on failure. (Auto-retrying + # a lease could double-claim a batch if a response is lost.) + self.s = self._session(token) + # Submit session: retry in-place, because by submit time the GPU work is + # already DONE — a momentary blip (dropped connection, gateway 5xx during + # a curator redeploy) must not throw that work away and force a full + # re-download + recompute on another agent. A duplicate submit after a + # lost response is harmless: the job is already closed, so it just returns + # 409 lease_invalid (a no-op). Idempotent enough to retry POST safely. + retry = Retry( + total=3, connect=3, read=3, status=3, + backoff_factor=0.5, # ~0.5s, 1s, 2s between tries + status_forcelist=(500, 502, 503, 504), # transient server/gateway + allowed_methods=frozenset({"POST"}), + raise_on_status=False, # let raise_for_status decide + ) + self._submit_s = self._session(token, retry) + + @staticmethod + def _session(token: str, retry: Retry | None = None) -> requests.Session: + s = requests.Session() + s.headers["Authorization"] = f"Bearer {token}" + # Many worker threads share a Session; the default pool (10) would # throttle them + spam "connection pool is full". Size it for the cap. - adapter = HTTPAdapter(pool_connections=64, pool_maxsize=64) - self.s.mount("http://", adapter) - self.s.mount("https://", adapter) + adapter = HTTPAdapter( + pool_connections=64, pool_maxsize=64, max_retries=retry or 0 + ) + s.mount("http://", adapter) + s.mount("https://", adapter) + return s def lease(self, batch_size: int) -> list[dict]: r = self.s.post( @@ -29,7 +54,7 @@ class FcClient: return r.json().get("jobs", []) def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict: - r = self.s.post( + r = self._submit_s.post( f"{self.base}/api/gpu/jobs/submit", json={ "agent_id": self.agent_id, "job_id": job_id, @@ -42,7 +67,7 @@ class FcClient: def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict: """Post a whole-image SigLIP embedding (the 'embed' task) → image_record.""" - r = self.s.post( + r = self._submit_s.post( f"{self.base}/api/gpu/jobs/submit_embedding", json={ "agent_id": self.agent_id, "job_id": job_id, diff --git a/agent/fc_agent/config.py b/agent/fc_agent/config.py index 059896d..42a7c75 100644 --- a/agent/fc_agent/config.py +++ b/agent/fc_agent/config.py @@ -1,4 +1,9 @@ """Agent config, all from env (the control container is configured at run).""" +# Lazy annotations so the `from_env(cls) -> Config` self-reference is a string, +# not evaluated at class-definition time — otherwise it NameErrors on the agent's +# Python 3.10 (CI lints on 3.14, where PEP 649 hides this). +from __future__ import annotations + import os from dataclasses import dataclass diff --git a/agent/ruff.toml b/agent/ruff.toml new file mode 100644 index 0000000..c796be5 --- /dev/null +++ b/agent/ruff.toml @@ -0,0 +1,7 @@ +# The agent runs on the CUDA base image's Python 3.10 — NOT the 3.14 that CI's +# ci-python image and the repo-root ruff.toml target. Pin the agent to py310 so +# ruff enforces 3.10 compatibility and never auto-applies a 3.11+/3.14-only fix +# (e.g. unquoting a self-referential annotation, which PEP 649 makes safe on 3.14 +# but NameErrors on 3.10). Inherit the root lint rules. +extend = "../ruff.toml" +target-version = "py310"