fix(agent): stream videos via ffmpeg-from-URL instead of downloading the whole file
The failing "poison" jobs were 800MB+ 4K VR videos: the agent pulled the ENTIRE file into memory (r.content) just to sample a few frames, which buffered ~1GB in RAM and — on any slow/contended media store — got cut off mid-download (ChunkedEncodingError), failed, and re-leased forever. Measured the media read at ~4–6 MB/s (raw off the share, curator out of the path), so no serving-layer tweak helps; the file simply shouldn't be fully downloaded. Environment-agnostic fix (works for any deployment, completes even when slow): - media.sample_frames_from_url(): point ffmpeg straight at curator's /images URL. It Range-reads only the video index + up to max_frames of content — never the whole file — and reconnect flags resume a dropped transfer instead of failing. Generous, env-tunable timeout (FFMPEG_TIMEOUT, default 1200s) = completion over speed. Removes the bytes-based sample_frames (dead once videos stream). - worker._download_decode: videos now stream (no fetch_image, no RAM blowup); stills still download+decode. On an ffmpeg miss, probe curator liveness (client.is_reachable) → fail the job if curator is up (unprocessable file, stops the infinite re-lease) vs release if curator is down (transient, survives a redeploy). Auth header passed so it works whether or not /images is gated. Build marker 2026-07-01.6. Refs issue #1225. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
+35
-19
@@ -85,32 +85,48 @@ def load_image(data: bytes) -> Image.Image:
|
||||
return to_rgb(Image.open(io.BytesIO(data)))
|
||||
|
||||
|
||||
def sample_frames(
|
||||
data: bytes, interval_seconds: float, max_frames: int
|
||||
# ffmpeg reconnect flags — resume a dropped HTTP transfer (a slow/contended media
|
||||
# store can cut a long stream) instead of failing the whole job. Relies only on
|
||||
# HTTP + Range, which every FC deployment serves → environment-agnostic.
|
||||
_RECONNECT = [
|
||||
"-reconnect", "1", "-reconnect_streamed", "1",
|
||||
"-reconnect_on_network_error", "1", "-reconnect_delay_max", "5",
|
||||
]
|
||||
|
||||
|
||||
def _collect_frames(
|
||||
tmp: str, interval: float, cap: int
|
||||
) -> list[tuple[float, Image.Image]]:
|
||||
"""Extract up to max_frames frames at one-every-interval_seconds via ffmpeg.
|
||||
Returns [(timestamp_seconds, frame)]. Empty on failure (caller falls back)."""
|
||||
out: list[tuple[float, Image.Image]] = []
|
||||
names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
|
||||
for i, name in enumerate(names[:cap]):
|
||||
with Image.open(os.path.join(tmp, name)) as im:
|
||||
out.append((round(i * interval, 2), to_rgb(im)))
|
||||
return out
|
||||
|
||||
|
||||
def sample_frames_from_url(
|
||||
url: str, interval_seconds: float, max_frames: int,
|
||||
*, headers: str = "", timeout: float = 1200.0,
|
||||
) -> list[tuple[float, Image.Image]]:
|
||||
"""Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
|
||||
only the video index + up to max_frames worth of content, so the agent never
|
||||
downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
|
||||
RAM and get cut off mid-download). Reconnect flags resume a dropped transfer;
|
||||
the generous timeout lets it finish however slow the link is — completion over
|
||||
speed, and environment-agnostic (any HTTP+Range source). Empty on failure."""
|
||||
interval = max(0.5, float(interval_seconds or 4.0))
|
||||
cap = max(1, int(max_frames or 64))
|
||||
hdr = ["-headers", headers] if headers else []
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = os.path.join(tmp, "in")
|
||||
with open(src, "wb") as fh:
|
||||
fh.write(data)
|
||||
pattern = os.path.join(tmp, "f_%05d.jpg")
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-nostdin", "-loglevel", "error", "-i", src,
|
||||
"-vf", f"fps=1/{interval}", "-frames:v", str(cap),
|
||||
"-q:v", "3", pattern,
|
||||
],
|
||||
check=True, timeout=600,
|
||||
["ffmpeg", "-nostdin", "-loglevel", "error", *_RECONNECT, *hdr,
|
||||
"-i", url, "-vf", f"fps=1/{interval}", "-frames:v", str(cap),
|
||||
"-q:v", "3", pattern],
|
||||
check=True, timeout=timeout,
|
||||
)
|
||||
except (subprocess.SubprocessError, FileNotFoundError):
|
||||
return []
|
||||
out: list[tuple[float, Image.Image]] = []
|
||||
names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
|
||||
for i, name in enumerate(names[:cap]):
|
||||
with Image.open(os.path.join(tmp, name)) as im:
|
||||
out.append((round(i * interval, 2), to_rgb(im)))
|
||||
return out
|
||||
return _collect_frames(tmp, interval, cap)
|
||||
|
||||
Reference in New Issue
Block a user