diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index ec2d474..2a4baf8 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -17,7 +17,7 @@ from .worker import Worker # Bump on every agent change. The page embeds this and /status reports it; the UI # warns to reload when they differ — so a stale browser-cached page can't be # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-07-01.4 · gentler downloads, failure-aware scaling" +VERSION = "2026-07-01.5 · name the real fetch-failure reason" logbuf.install() cfg = Config.from_env() diff --git a/agent/fc_agent/worker.py b/agent/fc_agent/worker.py index 3ceff61..7ccd7b3 100644 --- a/agent/fc_agent/worker.py +++ b/agent/fc_agent/worker.py @@ -61,6 +61,19 @@ def _is_transient(exc: requests.RequestException) -> bool: return resp.status_code >= 500 or resp.status_code in (401, 403, 408, 409, 429) +def _transient_reason(exc: requests.RequestException) -> str: + """A SPECIFIC label for a transient failure — so the log distinguishes a + stalled/slow transfer from an actually-unreachable curator, which need + different fixes. `HTTP ` for a 5xx/auth/conflict, else the exception + class: ReadTimeout (transfer stalled >60s between bytes — curator up, this + file/stream is slow), ConnectTimeout (curator didn't accept in 10s → web + workers/pool exhausted or down), ConnectionError (reset mid-transfer).""" + resp = getattr(exc, "response", None) + if resp is not None: + return f"HTTP {resp.status_code}" + return type(exc).__name__ + + # Pipeline sizing. Downloaders are I/O-bound, but every download streams a full # original (large videos included) THROUGH curator's single Python file-serving # path — so the ceiling is deliberately modest: too many concurrent large-file @@ -420,7 +433,12 @@ class Worker: self._bump(transient=1) self.client.release([jid]) self._unhold(jid) - log.info("curator unreachable — released job %s, backing off", jid) + # Name the ACTUAL failure — "curator unreachable" was + # printed for every transient, hiding whether a single + # file's transfer stalled (ReadTimeout, curator fine) or + # curator itself is down (ConnectTimeout/ConnectionError). + log.info("fetch failed job %s (image %s, %s) — released, backing off", + jid, job.get("image_id"), _transient_reason(exc)) self._release_owned(owned) owned = [] if not stop_evt.wait(backoff): @@ -690,7 +708,8 @@ class Worker: # while we worked. NOT the job's fault — hand it back (best # effort; then the server's orphan-recovery reclaims it if down). self._bump(transient=1) - log.info("curator unreachable — released job %s (post-GPU)", jid) + log.info("submit failed job %s (%s) — released, re-lease later", + jid, _transient_reason(exc)) self.client.release([jid]) self._unhold(jid) return False