diff --git a/app/tasks/ml.py b/app/tasks/ml.py index 77bd9e5..61d8cf6 100644 --- a/app/tasks/ml.py +++ b/app/tasks/ml.py @@ -5,6 +5,7 @@ import os import time import numpy as np +from celery.exceptions import SoftTimeLimitExceeded from sqlalchemy import and_, func from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -85,7 +86,7 @@ def _ensure_video_thumb(image) -> None: @celery.task(bind=True, name='app.tasks.ml.tag_and_embed', max_retries=2, default_retry_delay=60, - soft_time_limit=240, time_limit=360) + soft_time_limit=900, time_limit=1200) def tag_and_embed(self, image_id: int): """Run WD14 + SigLIP on one image (or sampled video frames) and persist results.""" from app.ml import wd14, siglip # lazy import so web process never loads torch @@ -154,6 +155,17 @@ def tag_and_embed(self, image_id: int): kind = 'video' if is_video else 'image' log.info(f"tag_and_embed: {kind} {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)") return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf, 'kind': kind} + except SoftTimeLimitExceeded: + # Retrying a soft-time-limit hit just walks into the same wall on + # the next attempt — these are usually slow videos that exceed the + # CPU budget. Roll back, log loudly, return a terminal status so + # backfill stops re-enqueueing it on the next sweep. + db.session.rollback() + log.error( + f"tag_and_embed: soft time limit exceeded on image {image_id} " + f"(is_video={is_video}); not retrying", + ) + return {'status': 'soft_time_limit', 'image_id': image_id, 'is_video': is_video} except Exception as e: db.session.rollback() log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True)