fix(ml): bump tag_and_embed time limits + handle SoftTimeLimitExceeded

The deployed worker hit soft_time_limit=120s on a video that took longer
than that to process — 10-frame SigLIP on CPU is ~10-15s/frame plus WD14,
so 120s (and even the prior 240s in source) was tight for any non-trivial
video.

- soft_time_limit: 240 -> 900 (15min), time_limit: 360 -> 1200 (20min).
  Gives videos comfortable headroom on CPU.
- Catch SoftTimeLimitExceeded explicitly so the task ends with a clean
  'soft_time_limit' status instead of bubbling up + retrying through
  self.retry(). Retrying a CPU-bound timeout is pointless — same input,
  same wall.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-26 09:45:12 -04:00
parent ce560d09a1
commit 09b6146656
+13 -1
View File
@@ -5,6 +5,7 @@ import os
import time
import numpy as np
from celery.exceptions import SoftTimeLimitExceeded
from sqlalchemy import and_, func
from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -85,7 +86,7 @@ def _ensure_video_thumb(image) -> None:
@celery.task(bind=True, name='app.tasks.ml.tag_and_embed',
max_retries=2, default_retry_delay=60,
soft_time_limit=240, time_limit=360)
soft_time_limit=900, time_limit=1200)
def tag_and_embed(self, image_id: int):
"""Run WD14 + SigLIP on one image (or sampled video frames) and persist results."""
from app.ml import wd14, siglip # lazy import so web process never loads torch
@@ -154,6 +155,17 @@ def tag_and_embed(self, image_id: int):
kind = 'video' if is_video else 'image'
log.info(f"tag_and_embed: {kind} {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)")
return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf, 'kind': kind}
except SoftTimeLimitExceeded:
# Retrying a soft-time-limit hit just walks into the same wall on
# the next attempt — these are usually slow videos that exceed the
# CPU budget. Roll back, log loudly, return a terminal status so
# backfill stops re-enqueueing it on the next sweep.
db.session.rollback()
log.error(
f"tag_and_embed: soft time limit exceeded on image {image_id} "
f"(is_video={is_video}); not retrying",
)
return {'status': 'soft_time_limit', 'image_id': image_id, 'is_video': is_video}
except Exception as e:
db.session.rollback()
log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True)