fix(ml): bump tag_and_embed time limits + handle SoftTimeLimitExceeded
The deployed worker hit soft_time_limit=120s on a video that took longer than that to process — 10-frame SigLIP on CPU is ~10-15s/frame plus WD14, so 120s (and even the prior 240s in source) was tight for any non-trivial video. - soft_time_limit: 240 -> 900 (15min), time_limit: 360 -> 1200 (20min). Gives videos comfortable headroom on CPU. - Catch SoftTimeLimitExceeded explicitly so the task ends with a clean 'soft_time_limit' status instead of bubbling up + retrying through self.retry(). Retrying a CPU-bound timeout is pointless — same input, same wall. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+13
-1
@@ -5,6 +5,7 @@ import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from sqlalchemy import and_, func
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
@@ -85,7 +86,7 @@ def _ensure_video_thumb(image) -> None:
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.ml.tag_and_embed',
|
||||
max_retries=2, default_retry_delay=60,
|
||||
soft_time_limit=240, time_limit=360)
|
||||
soft_time_limit=900, time_limit=1200)
|
||||
def tag_and_embed(self, image_id: int):
|
||||
"""Run WD14 + SigLIP on one image (or sampled video frames) and persist results."""
|
||||
from app.ml import wd14, siglip # lazy import so web process never loads torch
|
||||
@@ -154,6 +155,17 @@ def tag_and_embed(self, image_id: int):
|
||||
kind = 'video' if is_video else 'image'
|
||||
log.info(f"tag_and_embed: {kind} {image_id} done in {t_inf:.2f}s ({len(raw)} predictions)")
|
||||
return {'status': 'ok', 'predictions': len(raw), 'duration_s': t_inf, 'kind': kind}
|
||||
except SoftTimeLimitExceeded:
|
||||
# Retrying a soft-time-limit hit just walks into the same wall on
|
||||
# the next attempt — these are usually slow videos that exceed the
|
||||
# CPU budget. Roll back, log loudly, return a terminal status so
|
||||
# backfill stops re-enqueueing it on the next sweep.
|
||||
db.session.rollback()
|
||||
log.error(
|
||||
f"tag_and_embed: soft time limit exceeded on image {image_id} "
|
||||
f"(is_video={is_video}); not retrying",
|
||||
)
|
||||
return {'status': 'soft_time_limit', 'image_id': image_id, 'is_video': is_video}
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"tag_and_embed failed for image {image_id}: {e}", exc_info=True)
|
||||
|
||||
Reference in New Issue
Block a user