diag(ml): per-stage timing + file size in video inference logs
Adds per-call instrumentation to _run_video_inference so we can see where the cost goes on slow videos: frame extraction (ffmpeg seeks), WD14, or SigLIP. Also captures file size to correlate slowness with file characteristics. Output shape: tag_and_embed: video 7384 timings extract=12.3s wd14=24.1s siglip=82.6s frames=10/10 file_size=412.5MB Investigating a soft-time-limit storm where the deployed code has soft_time_limit=900 but workers are firing limits at ~120s intervals. This patch is pure observability — no behavior change. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -44,26 +44,55 @@ def _run_video_inference(image, wd14, siglip) -> tuple[list[dict], np.ndarray] |
|
|||||||
distance behavior since the pgvector index normalizes as needed.
|
distance behavior since the pgvector index normalizes as needed.
|
||||||
|
|
||||||
Returns (predictions, embedding) or None if frame extraction fails.
|
Returns (predictions, embedding) or None if frame extraction fails.
|
||||||
|
|
||||||
|
Emits per-stage timing on every call so we can see which phase is the
|
||||||
|
cost driver on slow videos: frame extraction (ffmpeg seeks), WD14
|
||||||
|
inference, or SigLIP inference. Also captures file size so we can
|
||||||
|
correlate slowness with file characteristics.
|
||||||
"""
|
"""
|
||||||
import shutil
|
import shutil
|
||||||
from app.utils.image_importer import extract_video_frames
|
from app.utils.image_importer import extract_video_frames
|
||||||
|
|
||||||
|
try:
|
||||||
|
file_size_mb = os.path.getsize(image.filepath) / (1024 * 1024)
|
||||||
|
except OSError:
|
||||||
|
file_size_mb = -1.0
|
||||||
|
|
||||||
|
t_extract_start = time.time()
|
||||||
frame_paths = extract_video_frames(image.filepath, count=VIDEO_ML_FRAMES)
|
frame_paths = extract_video_frames(image.filepath, count=VIDEO_ML_FRAMES)
|
||||||
|
t_extract = time.time() - t_extract_start
|
||||||
if not frame_paths:
|
if not frame_paths:
|
||||||
|
log.info(
|
||||||
|
f"tag_and_embed: video {image.id} extract_video_frames returned 0 frames "
|
||||||
|
f"(extract={t_extract:.2f}s file_size={file_size_mb:.1f}MB)"
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
tmpdir = os.path.dirname(frame_paths[0])
|
tmpdir = os.path.dirname(frame_paths[0])
|
||||||
try:
|
try:
|
||||||
best: dict[tuple[str, str], dict] = {}
|
best: dict[tuple[str, str], dict] = {}
|
||||||
embeddings: list[np.ndarray] = []
|
embeddings: list[np.ndarray] = []
|
||||||
|
wd14_total = 0.0
|
||||||
|
siglip_total = 0.0
|
||||||
for fp in frame_paths:
|
for fp in frame_paths:
|
||||||
|
t0 = time.time()
|
||||||
raw = wd14.infer_filtered(fp, min_any=WD14_STORE_FLOOR)
|
raw = wd14.infer_filtered(fp, min_any=WD14_STORE_FLOOR)
|
||||||
|
wd14_total += time.time() - t0
|
||||||
for pred in raw:
|
for pred in raw:
|
||||||
key = (pred['name'], pred['category'])
|
key = (pred['name'], pred['category'])
|
||||||
prev = best.get(key)
|
prev = best.get(key)
|
||||||
if prev is None or pred['confidence'] > prev['confidence']:
|
if prev is None or pred['confidence'] > prev['confidence']:
|
||||||
best[key] = pred
|
best[key] = pred
|
||||||
|
t0 = time.time()
|
||||||
embeddings.append(siglip.infer(fp))
|
embeddings.append(siglip.infer(fp))
|
||||||
|
siglip_total += time.time() - t0
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
f"tag_and_embed: video {image.id} timings extract={t_extract:.2f}s "
|
||||||
|
f"wd14={wd14_total:.2f}s siglip={siglip_total:.2f}s "
|
||||||
|
f"frames={len(frame_paths)}/{VIDEO_ML_FRAMES} "
|
||||||
|
f"file_size={file_size_mb:.1f}MB"
|
||||||
|
)
|
||||||
|
|
||||||
if not embeddings:
|
if not embeddings:
|
||||||
return None
|
return None
|
||||||
|
|||||||
Reference in New Issue
Block a user