Merge pull request 'fix(agent): log the real fetch/submit failure reason (diagnose #1225)' (#180) from dev into main

agent/fc_agent/app.py

@@ -17,7 +17,7 @@ from .worker import Worker
 # Bump on every agent change. The page embeds this and /status reports it; the UI
 # warns to reload when they differ — so a stale browser-cached page can't be
 # mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
-VERSION = "2026-07-01.4 · gentler downloads, failure-aware scaling"
+VERSION = "2026-07-01.5 · name the real fetch-failure reason"
 
 logbuf.install()
 cfg = Config.from_env()

agent/fc_agent/worker.py

@@ -61,6 +61,19 @@ def _is_transient(exc: requests.RequestException) -> bool:
     return resp.status_code >= 500 or resp.status_code in (401, 403, 408, 409, 429)
 
 
+def _transient_reason(exc: requests.RequestException) -> str:
+    """A SPECIFIC label for a transient failure — so the log distinguishes a
+    stalled/slow transfer from an actually-unreachable curator, which need
+    different fixes. `HTTP <code>` for a 5xx/auth/conflict, else the exception
+    class: ReadTimeout (transfer stalled >60s between bytes — curator up, this
+    file/stream is slow), ConnectTimeout (curator didn't accept in 10s → web
+    workers/pool exhausted or down), ConnectionError (reset mid-transfer)."""
+    resp = getattr(exc, "response", None)
+    if resp is not None:
+        return f"HTTP {resp.status_code}"
+    return type(exc).__name__
+
+
 # Pipeline sizing. Downloaders are I/O-bound, but every download streams a full
 # original (large videos included) THROUGH curator's single Python file-serving
 # path — so the ceiling is deliberately modest: too many concurrent large-file
@@ -420,7 +433,12 @@ class Worker:
                         self._bump(transient=1)
                         self.client.release([jid])
                         self._unhold(jid)
-                        log.info("curator unreachable — released job %s, backing off", jid)
+                        # Name the ACTUAL failure — "curator unreachable" was
+                        # printed for every transient, hiding whether a single
+                        # file's transfer stalled (ReadTimeout, curator fine) or
+                        # curator itself is down (ConnectTimeout/ConnectionError).
+                        log.info("fetch failed job %s (image %s, %s) — released, backing off",
+                                 jid, job.get("image_id"), _transient_reason(exc))
                         self._release_owned(owned)
                         owned = []
                         if not stop_evt.wait(backoff):
@@ -690,7 +708,8 @@ class Worker:
                 # while we worked. NOT the job's fault — hand it back (best
                 # effort; then the server's orphan-recovery reclaims it if down).
                 self._bump(transient=1)
-                log.info("curator unreachable — released job %s (post-GPU)", jid)
+                log.info("submit failed job %s (%s) — released, re-lease later",
+                         jid, _transient_reason(exc))
                 self.client.release([jid])
                 self._unhold(jid)
                 return False