diag(ml): per-stage timing + file size in video inference logs

Adds per-call instrumentation to _run_video_inference so we can see
where the cost goes on slow videos: frame extraction (ffmpeg seeks),
WD14, or SigLIP. Also captures file size to correlate slowness with
file characteristics.

Output shape:
  tag_and_embed: video 7384 timings extract=12.3s wd14=24.1s
  siglip=82.6s frames=10/10 file_size=412.5MB

Investigating a soft-time-limit storm where the deployed code has
soft_time_limit=900 but workers are firing limits at ~120s intervals.
This patch is pure observability — no behavior change.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-26 11:09:41 -04:00
parent 4cf77aac83
commit 641a52d1ad
+29
View File
@@ -44,26 +44,55 @@ def _run_video_inference(image, wd14, siglip) -> tuple[list[dict], np.ndarray] |
distance behavior since the pgvector index normalizes as needed. distance behavior since the pgvector index normalizes as needed.
Returns (predictions, embedding) or None if frame extraction fails. Returns (predictions, embedding) or None if frame extraction fails.
Emits per-stage timing on every call so we can see which phase is the
cost driver on slow videos: frame extraction (ffmpeg seeks), WD14
inference, or SigLIP inference. Also captures file size so we can
correlate slowness with file characteristics.
""" """
import shutil import shutil
from app.utils.image_importer import extract_video_frames from app.utils.image_importer import extract_video_frames
try:
file_size_mb = os.path.getsize(image.filepath) / (1024 * 1024)
except OSError:
file_size_mb = -1.0
t_extract_start = time.time()
frame_paths = extract_video_frames(image.filepath, count=VIDEO_ML_FRAMES) frame_paths = extract_video_frames(image.filepath, count=VIDEO_ML_FRAMES)
t_extract = time.time() - t_extract_start
if not frame_paths: if not frame_paths:
log.info(
f"tag_and_embed: video {image.id} extract_video_frames returned 0 frames "
f"(extract={t_extract:.2f}s file_size={file_size_mb:.1f}MB)"
)
return None return None
tmpdir = os.path.dirname(frame_paths[0]) tmpdir = os.path.dirname(frame_paths[0])
try: try:
best: dict[tuple[str, str], dict] = {} best: dict[tuple[str, str], dict] = {}
embeddings: list[np.ndarray] = [] embeddings: list[np.ndarray] = []
wd14_total = 0.0
siglip_total = 0.0
for fp in frame_paths: for fp in frame_paths:
t0 = time.time()
raw = wd14.infer_filtered(fp, min_any=WD14_STORE_FLOOR) raw = wd14.infer_filtered(fp, min_any=WD14_STORE_FLOOR)
wd14_total += time.time() - t0
for pred in raw: for pred in raw:
key = (pred['name'], pred['category']) key = (pred['name'], pred['category'])
prev = best.get(key) prev = best.get(key)
if prev is None or pred['confidence'] > prev['confidence']: if prev is None or pred['confidence'] > prev['confidence']:
best[key] = pred best[key] = pred
t0 = time.time()
embeddings.append(siglip.infer(fp)) embeddings.append(siglip.infer(fp))
siglip_total += time.time() - t0
log.info(
f"tag_and_embed: video {image.id} timings extract={t_extract:.2f}s "
f"wd14={wd14_total:.2f}s siglip={siglip_total:.2f}s "
f"frames={len(frame_paths)}/{VIDEO_ML_FRAMES} "
f"file_size={file_size_mb:.1f}MB"
)
if not embeddings: if not embeddings:
return None return None