feat(beat): self-gating schedules for ML backfill, centroids, auto-accept
Promotes three previously-manual maintenance tasks to Celery Beat schedules
so the user doesn't have to remember to run them:
- ml.backfill daily
- apply_auto_accept_predictions daily
- recompute_all_centroids weekly
Cadences are env-overridable (ML_BACKFILL_EVERY_SECONDS,
AUTO_ACCEPT_EVERY_SECONDS, CENTROIDS_EVERY_SECONDS).
Each task self-gates so a scheduled run is a no-op when there's nothing
to do:
- ml.backfill: already self-gating — its first paginated query returns
zero rows when no image is missing predictions/embeddings, the loop
breaks, and the task returns. No code change.
- apply_auto_accept_predictions: adds a fast-path NOT EXISTS query that
returns immediately when no WD14 prediction at/above the threshold
exists for an unattached, non-rejected (image, tag) pair. The full
walk only fires when fresh predictions have landed since the last run.
Returns {'skipped_no_candidates': True} on the no-op path.
- recompute_all_centroids: tightens the aggregate query to LEFT JOIN
tag_reference_embedding and skip tags whose stored reference_count
already matches current image_tags membership count. Without this gate
the daily-scheduled sweep would re-enqueue a recompute for every
eligible tag every run, contending with tag_and_embed on the ml queue.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
+42
-9
@@ -319,17 +319,25 @@ def recompute_centroid(self, tag_name: str):
|
||||
@celery.task(bind=True, name='app.tasks.ml.recompute_all_centroids',
|
||||
soft_time_limit=None, time_limit=None)
|
||||
def recompute_all_centroids(self):
|
||||
"""Enqueue recompute_centroid for every eligible tag with enough reference images.
|
||||
"""Enqueue recompute_centroid for every eligible tag whose member count
|
||||
has changed since its centroid was last computed.
|
||||
|
||||
Uses a single aggregate query to find tags with >= min_reference_images applied
|
||||
images, then enqueues one recompute_centroid task per tag on the ml queue.
|
||||
Without the count-delta gate, the daily-scheduled call would re-enqueue
|
||||
a recompute for every eligible tag every run, even if no images had
|
||||
been attached or detached — wasteful both in work and in ml-queue
|
||||
contention with `tag_and_embed`.
|
||||
|
||||
Joins `tag_reference_embedding` on `(tag_name, model_version)` so the
|
||||
aggregate `HAVING` can compare current count against stored
|
||||
`reference_count`. NULL stored count (no centroid yet) always
|
||||
qualifies. Equal counts skip.
|
||||
"""
|
||||
from app.ml.siglip import MODEL_VERSION as SIGLIP_VER
|
||||
from app.services.tag_suggestions import ELIGIBLE_CENTROID_KINDS
|
||||
|
||||
min_refs_row = TagSuggestionConfig.query.filter_by(key='min_reference_images').first()
|
||||
min_refs = int(min_refs_row.value) if min_refs_row else 5
|
||||
|
||||
# Build an IS NULL / IN filter that covers ELIGIBLE_CENTROID_KINDS including None.
|
||||
kinds_not_null = [k for k in ELIGIBLE_CENTROID_KINDS if k is not None]
|
||||
allow_null = None in ELIGIBLE_CENTROID_KINDS
|
||||
|
||||
@@ -337,18 +345,43 @@ def recompute_all_centroids(self):
|
||||
if allow_null:
|
||||
kind_filter = kind_filter | Tag.kind.is_(None)
|
||||
|
||||
# LEFT JOIN to TagReferenceEmbedding for the SigLIP version so absent
|
||||
# rows surface as NULL stored_count (always !=, always recompute).
|
||||
rows = (
|
||||
db.session.query(Tag.name, func.count(image_tags.c.image_id).label('n'))
|
||||
db.session.query(
|
||||
Tag.name,
|
||||
func.count(image_tags.c.image_id).label('current_count'),
|
||||
TagReferenceEmbedding.reference_count.label('stored_count'),
|
||||
)
|
||||
.join(image_tags, image_tags.c.tag_id == Tag.id)
|
||||
.outerjoin(
|
||||
TagReferenceEmbedding,
|
||||
and_(
|
||||
TagReferenceEmbedding.tag_name == Tag.name,
|
||||
TagReferenceEmbedding.model_version == SIGLIP_VER,
|
||||
),
|
||||
)
|
||||
.filter(kind_filter)
|
||||
.group_by(Tag.name)
|
||||
.group_by(Tag.name, TagReferenceEmbedding.reference_count)
|
||||
.having(func.count(image_tags.c.image_id) >= min_refs)
|
||||
.all()
|
||||
)
|
||||
|
||||
enqueued = 0
|
||||
for tag_name, n in rows:
|
||||
skipped_unchanged = 0
|
||||
for tag_name, current_count, stored_count in rows:
|
||||
if stored_count is not None and stored_count == current_count:
|
||||
skipped_unchanged += 1
|
||||
continue
|
||||
recompute_centroid.apply_async(args=[tag_name], queue='ml')
|
||||
enqueued += 1
|
||||
log.info(f"recompute_all_centroids: enqueued {enqueued} tags (min_refs={min_refs})")
|
||||
return {'status': 'ok', 'enqueued': enqueued, 'min_refs': min_refs}
|
||||
log.info(
|
||||
"recompute_all_centroids: enqueued=%d skipped_unchanged=%d min_refs=%d",
|
||||
enqueued, skipped_unchanged, min_refs,
|
||||
)
|
||||
return {
|
||||
'status': 'ok',
|
||||
'enqueued': enqueued,
|
||||
'skipped_unchanged': skipped_unchanged,
|
||||
'min_refs': min_refs,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user