|
|
|
@@ -61,6 +61,19 @@ def _is_transient(exc: requests.RequestException) -> bool:
|
|
|
|
|
return resp.status_code >= 500 or resp.status_code in (401, 403, 408, 409, 429)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _transient_reason(exc: requests.RequestException) -> str:
|
|
|
|
|
"""A SPECIFIC label for a transient failure — so the log distinguishes a
|
|
|
|
|
stalled/slow transfer from an actually-unreachable curator, which need
|
|
|
|
|
different fixes. `HTTP <code>` for a 5xx/auth/conflict, else the exception
|
|
|
|
|
class: ReadTimeout (transfer stalled >60s between bytes — curator up, this
|
|
|
|
|
file/stream is slow), ConnectTimeout (curator didn't accept in 10s → web
|
|
|
|
|
workers/pool exhausted or down), ConnectionError (reset mid-transfer)."""
|
|
|
|
|
resp = getattr(exc, "response", None)
|
|
|
|
|
if resp is not None:
|
|
|
|
|
return f"HTTP {resp.status_code}"
|
|
|
|
|
return type(exc).__name__
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Pipeline sizing. Downloaders are I/O-bound, but every download streams a full
|
|
|
|
|
# original (large videos included) THROUGH curator's single Python file-serving
|
|
|
|
|
# path — so the ceiling is deliberately modest: too many concurrent large-file
|
|
|
|
@@ -420,7 +433,12 @@ class Worker:
|
|
|
|
|
self._bump(transient=1)
|
|
|
|
|
self.client.release([jid])
|
|
|
|
|
self._unhold(jid)
|
|
|
|
|
log.info("curator unreachable — released job %s, backing off", jid)
|
|
|
|
|
# Name the ACTUAL failure — "curator unreachable" was
|
|
|
|
|
# printed for every transient, hiding whether a single
|
|
|
|
|
# file's transfer stalled (ReadTimeout, curator fine) or
|
|
|
|
|
# curator itself is down (ConnectTimeout/ConnectionError).
|
|
|
|
|
log.info("fetch failed job %s (image %s, %s) — released, backing off",
|
|
|
|
|
jid, job.get("image_id"), _transient_reason(exc))
|
|
|
|
|
self._release_owned(owned)
|
|
|
|
|
owned = []
|
|
|
|
|
if not stop_evt.wait(backoff):
|
|
|
|
@@ -690,7 +708,8 @@ class Worker:
|
|
|
|
|
# while we worked. NOT the job's fault — hand it back (best
|
|
|
|
|
# effort; then the server's orphan-recovery reclaims it if down).
|
|
|
|
|
self._bump(transient=1)
|
|
|
|
|
log.info("curator unreachable — released job %s (post-GPU)", jid)
|
|
|
|
|
log.info("submit failed job %s (%s) — released, re-lease later",
|
|
|
|
|
jid, _transient_reason(exc))
|
|
|
|
|
self.client.release([jid])
|
|
|
|
|
self._unhold(jid)
|
|
|
|
|
return False
|
|
|
|
|