feat(agent): store ffmpeg's actual failure reason in the job's error field
sample_frames_from_url now returns (frames, reason) — reason carries the SPECIFIC cause on failure (ffmpeg's stderr tail, e.g. "moov atom not found", or the timeout) instead of only logging it agent-side. The worker folds it into the failure it reports, so curator's GpuJob.error reads e.g. no frames sampled from video — ffmpeg exit 183: moov atom not found ... instead of the bare "(unprocessable)". The errored-jobs list becomes self-describing: after a retry sweep, surviving errors name their real defect without needing the agent log. Return-value plumbing (not shared state) so concurrent downloaders stay isolated. Agent VERSION → 2026-07-02.2. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,7 @@ log = logging.getLogger("fc_agent.app")
|
|||||||
# Bump on every agent change. The page embeds this and /status reports it; the UI
|
# Bump on every agent change. The page embeds this and /status reports it; the UI
|
||||||
# warns to reload when they differ — so a stale browser-cached page can't be
|
# warns to reload when they differ — so a stale browser-cached page can't be
|
||||||
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
|
# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.)
|
||||||
VERSION = "2026-07-02.1 · short videos sample again (select, not fps) + ffmpeg errors logged"
|
VERSION = "2026-07-02.2 · job.error now carries ffmpeg's actual failure reason"
|
||||||
|
|
||||||
logbuf.install()
|
logbuf.install()
|
||||||
cfg = Config.from_env()
|
cfg = Config.from_env()
|
||||||
|
|||||||
+18
-11
@@ -125,7 +125,7 @@ def _terminate(proc: subprocess.Popen) -> None:
|
|||||||
def sample_frames_from_url(
|
def sample_frames_from_url(
|
||||||
url: str, interval_seconds: float, max_frames: int,
|
url: str, interval_seconds: float, max_frames: int,
|
||||||
*, headers: str = "", timeout: float = 1200.0, should_stop=None,
|
*, headers: str = "", timeout: float = 1200.0, should_stop=None,
|
||||||
) -> list[tuple[float, Image.Image]]:
|
) -> tuple[list[tuple[float, Image.Image]], str | None]:
|
||||||
"""Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
|
"""Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
|
||||||
only the video index + up to max_frames worth of content, so the agent never
|
only the video index + up to max_frames worth of content, so the agent never
|
||||||
downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
|
downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
|
||||||
@@ -133,7 +133,12 @@ def sample_frames_from_url(
|
|||||||
the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
|
the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
|
||||||
run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
|
run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
|
||||||
subprocess at once — otherwise a downloader stuck in a long decode keeps the
|
subprocess at once — otherwise a downloader stuck in a long decode keeps the
|
||||||
agent "working" long after Stop. Empty on failure / stop / timeout."""
|
agent "working" long after Stop.
|
||||||
|
|
||||||
|
Returns (frames, reason): frames is empty on failure/stop/timeout, and
|
||||||
|
`reason` then carries the SPECIFIC cause (ffmpeg's stderr tail / timeout) so
|
||||||
|
the caller can put it in the job's error — a bare "no frames" hid a filter
|
||||||
|
bug as "unprocessable" for weeks. None reason on success."""
|
||||||
interval = max(0.5, float(interval_seconds or 4.0))
|
interval = max(0.5, float(interval_seconds or 4.0))
|
||||||
cap = max(1, int(max_frames or 64))
|
cap = max(1, int(max_frames or 64))
|
||||||
hdr = ["-headers", headers] if headers else []
|
hdr = ["-headers", headers] if headers else []
|
||||||
@@ -174,17 +179,19 @@ def sample_frames_from_url(
|
|||||||
stopped = should_stop and should_stop()
|
stopped = should_stop and should_stop()
|
||||||
if stopped or (time.monotonic() - start > timeout):
|
if stopped or (time.monotonic() - start > timeout):
|
||||||
_terminate(proc)
|
_terminate(proc)
|
||||||
if not stopped:
|
if stopped:
|
||||||
log.warning("ffmpeg timed out after %.0fs: %s",
|
return [], "stopped"
|
||||||
timeout, url)
|
log.warning("ffmpeg timed out after %.0fs: %s",
|
||||||
return []
|
timeout, url)
|
||||||
except (OSError, ValueError):
|
return [], f"ffmpeg timed out after {timeout:.0f}s"
|
||||||
return []
|
except (OSError, ValueError) as exc:
|
||||||
|
return [], f"ffmpeg not runnable: {exc}"
|
||||||
frames = _collect_frames(tmp, interval, cap)
|
frames = _collect_frames(tmp, interval, cap)
|
||||||
if not frames:
|
if not frames:
|
||||||
log.warning("ffmpeg produced no frames (exit %s) for %s — stderr: %s",
|
reason = f"ffmpeg exit {proc.returncode}: {_tail(errpath)}"
|
||||||
proc.returncode, url, _tail(errpath))
|
log.warning("ffmpeg produced no frames for %s — %s", url, reason)
|
||||||
return frames
|
return [], reason
|
||||||
|
return frames, None
|
||||||
|
|
||||||
|
|
||||||
def _tail(path: str, limit: int = 300) -> str:
|
def _tail(path: str, limit: int = 300) -> str:
|
||||||
|
|||||||
@@ -632,7 +632,7 @@ class Worker:
|
|||||||
# mid-download was the failure loop). Environment-agnostic + resilient.
|
# mid-download was the failure loop). Environment-agnostic + resilient.
|
||||||
url = f"{self.cfg.fc_url}{job['image_url']}"
|
url = f"{self.cfg.fc_url}{job['image_url']}"
|
||||||
with self._timed("decode"):
|
with self._timed("decode"):
|
||||||
frames = media.sample_frames_from_url(
|
frames, ffmpeg_err = media.sample_frames_from_url(
|
||||||
url, job.get("frame_interval_seconds", 4.0),
|
url, job.get("frame_interval_seconds", 4.0),
|
||||||
job.get("max_frames", 64),
|
job.get("max_frames", 64),
|
||||||
headers=self._auth_header, timeout=self.cfg.ffmpeg_timeout,
|
headers=self._auth_header, timeout=self.cfg.ffmpeg_timeout,
|
||||||
@@ -644,11 +644,14 @@ class Worker:
|
|||||||
if self._stopped(stop_evt):
|
if self._stopped(stop_evt):
|
||||||
raise requests.ConnectionError("stopped during video sampling")
|
raise requests.ConnectionError("stopped during video sampling")
|
||||||
# Else couldn't sample. If curator is up, the file is unprocessable
|
# Else couldn't sample. If curator is up, the file is unprocessable
|
||||||
# → a job fault (fail it, don't re-lease forever). If curator is
|
# → a job fault: fail it WITH ffmpeg's reason, so the job's stored
|
||||||
# unreachable, it's transient → let the loop back off + retry
|
# error says e.g. "moov atom not found" instead of a bare
|
||||||
# (survives a redeploy). ConnectionError is caught as transient.
|
# "unprocessable". If curator is unreachable, it's transient → let
|
||||||
|
# the loop back off + retry (ConnectionError is caught as such).
|
||||||
if self.client.is_reachable():
|
if self.client.is_reachable():
|
||||||
raise RuntimeError("no frames sampled from video (unprocessable)")
|
raise RuntimeError(
|
||||||
|
f"no frames sampled from video — {ffmpeg_err or 'unknown reason'}"
|
||||||
|
)
|
||||||
raise requests.ConnectionError("curator unreachable during video sampling")
|
raise requests.ConnectionError("curator unreachable during video sampling")
|
||||||
# Temporal dedup: a near-static video re-runs the whole detect+embed
|
# Temporal dedup: a near-static video re-runs the whole detect+embed
|
||||||
# chain on ~identical frames — drop near-dups HERE (CPU) pre-GPU.
|
# chain on ~identical frames — drop near-dups HERE (CPU) pre-GPU.
|
||||||
|
|||||||
Reference in New Issue
Block a user