feat(ml): video tagging + embedding via frame sampling

Videos now route through a 10-frame sampling branch (configurable via VIDEO_ML_FRAMES env) instead of the previous unsupported_format early return. WD14 predictions are aggregated by max-confidence per (name, category) across frames so sparse signals aren't diluted; SigLIP embeddings are mean-pooled for a representative shot. Also generates a fallback thumbnail when the record is missing one, and removes the video_filter from backfill so videos get enqueued. Celery soft/hard limits bumped to 240s/360s to accommodate 10x inference. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-20 18:39:33 -04:00
parent ac9b5cbc06
commit bc6b0a4a58
2 changed files with 136 additions and 17 deletions
@@ -556,6 +556,72 @@ def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:
    return generate_video_thumbnail(str(video_path_p), str(thumb_path), time_position=time_position)


+def _probe_video_duration(video_path: str) -> float | None:
+    """Return duration in seconds via ffprobe, or None on failure."""
+    try:
+        result = subprocess.run(
+            [
+                "ffprobe", "-v", "error",
+                "-show_entries", "format=duration",
+                "-of", "default=noprint_wrappers=1:nokey=1",
+                str(video_path),
+            ],
+            capture_output=True, text=True, timeout=30, check=True,
+        )
+        return float(result.stdout.strip())
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError):
+        return None
+
+
+def extract_video_frames(video_path: str, count: int = 10) -> list[str]:
+    """Extract `count` evenly-spaced JPEG frames from a video into tempfiles.
+
+    Frames are sampled between 10% and 90% of the video's duration so title
+    cards and black-fade endings don't dominate the sample. Returns a list of
+    absolute paths; the caller is responsible for deleting the containing
+    temp directory when done. Returns [] if duration probe fails or no frames
+    could be extracted.
+    """
+    duration = _probe_video_duration(video_path)
+    if duration is None or duration <= 0:
+        return []
+
+    if count <= 1:
+        fractions = [0.5]
+    else:
+        fractions = [0.1 + 0.8 * (i / (count - 1)) for i in range(count)]
+
+    tmpdir = tempfile.mkdtemp(prefix="vframes_")
+    frame_paths: list[str] = []
+    for i, frac in enumerate(fractions):
+        t = duration * frac
+        out = os.path.join(tmpdir, f"frame_{i:02d}.jpg")
+        try:
+            subprocess.run(
+                [
+                    "ffmpeg", "-y",
+                    "-ss", f"{t:.3f}",
+                    "-i", str(video_path),
+                    "-vframes", "1",
+                    out,
+                ],
+                check=True,
+                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+                timeout=FFMPEG_THUMB_TIMEOUT,
+            )
+            if os.path.exists(out) and os.path.getsize(out) > 0:
+                frame_paths.append(out)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
+            continue
+
+    if not frame_paths:
+        try:
+            os.rmdir(tmpdir)
+        except OSError:
+            pass
+    return frame_paths
+
+
 def transcode_video_to_mp4(input_path: str, output_path: str = None) -> str | None:
    """
    Transcode a video to H.264 MP4 for universal browser playback.