diag(ml): per-stage timing + file size in video inference logs
Adds per-call instrumentation to _run_video_inference so we can see where the cost goes on slow videos: frame extraction (ffmpeg seeks), WD14, or SigLIP. Also captures file size to correlate slowness with file characteristics. Output shape: tag_and_embed: video 7384 timings extract=12.3s wd14=24.1s siglip=82.6s frames=10/10 file_size=412.5MB Investigating a soft-time-limit storm where the deployed code has soft_time_limit=900 but workers are firing limits at ~120s intervals. This patch is pure observability — no behavior change. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -44,26 +44,55 @@ def _run_video_inference(image, wd14, siglip) -> tuple[list[dict], np.ndarray] |
|
||||
distance behavior since the pgvector index normalizes as needed.
|
||||
|
||||
Returns (predictions, embedding) or None if frame extraction fails.
|
||||
|
||||
Emits per-stage timing on every call so we can see which phase is the
|
||||
cost driver on slow videos: frame extraction (ffmpeg seeks), WD14
|
||||
inference, or SigLIP inference. Also captures file size so we can
|
||||
correlate slowness with file characteristics.
|
||||
"""
|
||||
import shutil
|
||||
from app.utils.image_importer import extract_video_frames
|
||||
|
||||
try:
|
||||
file_size_mb = os.path.getsize(image.filepath) / (1024 * 1024)
|
||||
except OSError:
|
||||
file_size_mb = -1.0
|
||||
|
||||
t_extract_start = time.time()
|
||||
frame_paths = extract_video_frames(image.filepath, count=VIDEO_ML_FRAMES)
|
||||
t_extract = time.time() - t_extract_start
|
||||
if not frame_paths:
|
||||
log.info(
|
||||
f"tag_and_embed: video {image.id} extract_video_frames returned 0 frames "
|
||||
f"(extract={t_extract:.2f}s file_size={file_size_mb:.1f}MB)"
|
||||
)
|
||||
return None
|
||||
|
||||
tmpdir = os.path.dirname(frame_paths[0])
|
||||
try:
|
||||
best: dict[tuple[str, str], dict] = {}
|
||||
embeddings: list[np.ndarray] = []
|
||||
wd14_total = 0.0
|
||||
siglip_total = 0.0
|
||||
for fp in frame_paths:
|
||||
t0 = time.time()
|
||||
raw = wd14.infer_filtered(fp, min_any=WD14_STORE_FLOOR)
|
||||
wd14_total += time.time() - t0
|
||||
for pred in raw:
|
||||
key = (pred['name'], pred['category'])
|
||||
prev = best.get(key)
|
||||
if prev is None or pred['confidence'] > prev['confidence']:
|
||||
best[key] = pred
|
||||
t0 = time.time()
|
||||
embeddings.append(siglip.infer(fp))
|
||||
siglip_total += time.time() - t0
|
||||
|
||||
log.info(
|
||||
f"tag_and_embed: video {image.id} timings extract={t_extract:.2f}s "
|
||||
f"wd14={wd14_total:.2f}s siglip={siglip_total:.2f}s "
|
||||
f"frames={len(frame_paths)}/{VIDEO_ML_FRAMES} "
|
||||
f"file_size={file_size_mb:.1f}MB"
|
||||
)
|
||||
|
||||
if not embeddings:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user