From 09b614665632ce722c55bd36f61033381fe6e078 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 26 Apr 2026 09:45:12 -0400 Subject: [PATCH] fix(ml): bump tag_and_embed time limits + handle SoftTimeLimitExceeded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deployed worker hit soft_time_limit=120s on a video that took longer than that to process — 10-frame SigLIP on CPU is ~10-15s/frame plus WD14, so 120s (and even the prior 240s in source) was tight for any non-trivial video. - soft_time_limit: 240 -> 900 (15min), time_limit: 360 -> 1200 (20min). Gives videos comfortable headroom on CPU. - Catch SoftTimeLimitExceeded explicitly so the task ends with a clean 'soft_time_limit' status instead of bubbling up + retrying through self.retry(). Retrying a CPU-bound timeout is pointless — same input, same wall. Co-Authored-By: Claude Opus 4.7 --- app/tasks/ml.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/tasks/ml.py b/app/tasks/ml.py index 77bd9e5..61d8cf6 100644 --- a/app/tasks/ml.py +++ b/app/tasks/ml.py @@ -5,6 +5,7 @@ import os import time import numpy as np +from celery.exceptions import SoftTimeLimitExceeded from sqlalchemy import and_, func from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -85,7 +86,7 @@ def _ensure_video_thumb(image) -> None: @celery.task(bind=True, name='app.tasks.ml.tag_and_embed', max_retries=2, default_retry_delay=60, - soft_time_limit=240, time_limit=360) + soft_time_limit=900, time_limit=1200) def tag_and_embed(self, image_id: int): """Run WD14 + SigLIP on one image (or sampled video frames) and persist results.""" from app.ml import wd14, siglip # lazy import so web process never loads torch @@ -154,6 +155,17 @@ def tag_and_embed(self, image_id: int): kind = 'video' if is_video else 'image' log.info(f"tag_and_embed: {kind} {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)") return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf, 'kind': kind} + except SoftTimeLimitExceeded: + # Retrying a soft-time-limit hit just walks into the same wall on + # the next attempt — these are usually slow videos that exceed the + # CPU budget. Roll back, log loudly, return a terminal status so + # backfill stops re-enqueueing it on the next sweep. + db.session.rollback() + log.error( + f"tag_and_embed: soft time limit exceeded on image {image_id} " + f"(is_video={is_video}); not retrying", + ) + return {'status': 'soft_time_limit', 'image_id': image_id, 'is_video': is_video} except Exception as e: db.session.rollback() log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True)