feat(maintenance): library-wide apply of auto-accept predictions
Adds a Celery task + Settings button that walks every image and applies general-category WD14 predictions at or above the auto-accept threshold, without needing the user to open each modal. Same side effects as the modal's per-image auto-accept flow (creates kind='user' tag if needed, attaches to image, writes SuggestionFeedback decision='accepted'). Lazy-imports app.services.tag_suggestions so the maintenance worker doesn't pay its overhead unless this task fires. Amortizes _config() and _existing_tag_names() across the loop. Commits per-batch (100 images) to keep transactions short and let later batches see freshly created Tag rows. Skips integrity-flagged images (get_suggestions already returns empty for those). Idempotent — re-running just no-ops on already-applied images via the existing 'tag not in img.tags' guard. Trigger: Settings -> Maintenance -> "Apply auto-accepts library-wide" (confirm-gated since it walks the entire library). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -75,6 +75,7 @@ def make_celery(app=None):
|
|||||||
'app.tasks.maintenance.sync_character_fandoms_to_images': {'queue': 'maintenance'},
|
'app.tasks.maintenance.sync_character_fandoms_to_images': {'queue': 'maintenance'},
|
||||||
'app.tasks.maintenance.verify_media_integrity': {'queue': 'maintenance'},
|
'app.tasks.maintenance.verify_media_integrity': {'queue': 'maintenance'},
|
||||||
'app.tasks.maintenance.verify_unverified_images': {'queue': 'maintenance'},
|
'app.tasks.maintenance.verify_unverified_images': {'queue': 'maintenance'},
|
||||||
|
'app.tasks.maintenance.apply_auto_accept_predictions': {'queue': 'maintenance'},
|
||||||
|
|
||||||
# Import tasks - handled by worker (heavy processing)
|
# Import tasks - handled by worker (heavy processing)
|
||||||
'app.tasks.import_file.*': {'queue': 'import'},
|
'app.tasks.import_file.*': {'queue': 'import'},
|
||||||
|
|||||||
+10
@@ -655,6 +655,16 @@ def trigger_sync_character_fandoms_to_images():
|
|||||||
return redirect(url_for('main.settings', tab='maintenance'))
|
return redirect(url_for('main.settings', tab='maintenance'))
|
||||||
|
|
||||||
|
|
||||||
|
@main.post('/settings/maintenance/apply-auto-accept-predictions')
|
||||||
|
def trigger_apply_auto_accept_predictions():
|
||||||
|
"""Sweep every image, apply general WD14 predictions above the auto-accept
|
||||||
|
threshold without requiring the user to open each modal. One-shot —
|
||||||
|
intended to be run after a backfill drains."""
|
||||||
|
from app.tasks.maintenance import apply_auto_accept_predictions
|
||||||
|
apply_auto_accept_predictions.apply_async(queue='maintenance')
|
||||||
|
return redirect(url_for('main.settings', tab='maintenance'))
|
||||||
|
|
||||||
|
|
||||||
@main.post('/settings/maintenance/verify-images')
|
@main.post('/settings/maintenance/verify-images')
|
||||||
def trigger_verify_unverified_images():
|
def trigger_verify_unverified_images():
|
||||||
"""Sweep ImageRecords and enqueue per-image structural verification.
|
"""Sweep ImageRecords and enqueue per-image structural verification.
|
||||||
|
|||||||
+101
-1
@@ -12,7 +12,7 @@ from celery import shared_task
|
|||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
from app import db
|
from app import db
|
||||||
from app.models import ImageRecord, ImportTask, Tag
|
from app.models import ImageRecord, ImportTask, SuggestionFeedback, Tag
|
||||||
from app.services.integrity import verify_path
|
from app.services.integrity import verify_path
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -336,3 +336,103 @@ def sweep_blocklisted_tag_from_images(name: str) -> dict:
|
|||||||
'image_tags_deleted': rowcount,
|
'image_tags_deleted': rowcount,
|
||||||
'tag_deleted': True,
|
'tag_deleted': True,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(
|
||||||
|
name='app.tasks.maintenance.apply_auto_accept_predictions',
|
||||||
|
soft_time_limit=1800,
|
||||||
|
time_limit=2700,
|
||||||
|
)
|
||||||
|
def apply_auto_accept_predictions(batch_size: int = 100) -> dict:
|
||||||
|
"""Walk every image and apply general-category WD14 predictions whose
|
||||||
|
confidence is at or above the auto_accept_general_threshold config.
|
||||||
|
|
||||||
|
Mirrors the side effects of the modal's per-image auto-accept flow
|
||||||
|
(`get_image_suggestions`): for each candidate, find-or-create a
|
||||||
|
kind='user' Tag, attach it to the image, and log a SuggestionFeedback
|
||||||
|
row with decision='accepted'. Idempotent — re-running just skips
|
||||||
|
images whose existing tags + per-image rejections already cover the
|
||||||
|
candidates.
|
||||||
|
|
||||||
|
Designed for one-shot library-wide application after a backfill
|
||||||
|
completes, so the user doesn't have to open every modal to land
|
||||||
|
high-confidence WD14 tags. Skips integrity-flagged rows (the suggestion
|
||||||
|
service already returns empty for those) and amortizes the config
|
||||||
|
fetch + Tag-table scan across the loop.
|
||||||
|
"""
|
||||||
|
# Lazy-import the suggestion service so the maintenance worker doesn't
|
||||||
|
# pay its overhead on unrelated tasks.
|
||||||
|
from app.services.tag_suggestions import (
|
||||||
|
_config, _existing_tag_names, get_suggestions,
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = _config()
|
||||||
|
existing_all = _existing_tag_names()
|
||||||
|
|
||||||
|
scanned = 0
|
||||||
|
images_with_applies = 0
|
||||||
|
tags_applied = 0
|
||||||
|
last_id = 0
|
||||||
|
|
||||||
|
while True:
|
||||||
|
rows = (
|
||||||
|
ImageRecord.query
|
||||||
|
.filter(ImageRecord.id > last_id)
|
||||||
|
.filter(ImageRecord.integrity_status.in_(('ok', 'unknown')))
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(batch_size)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
|
||||||
|
for img in rows:
|
||||||
|
scanned += 1
|
||||||
|
grouped = get_suggestions(img.id, cfg=cfg, existing_all=existing_all)
|
||||||
|
candidates = grouped.get('auto_accept_candidates') or []
|
||||||
|
if not candidates:
|
||||||
|
continue
|
||||||
|
|
||||||
|
applied_for_image = 0
|
||||||
|
for cand in candidates:
|
||||||
|
name = cand['name']
|
||||||
|
tag = Tag.query.filter_by(kind='user', name=name).first()
|
||||||
|
if tag is None:
|
||||||
|
tag = Tag(kind='user', name=name)
|
||||||
|
db.session.add(tag)
|
||||||
|
db.session.flush()
|
||||||
|
if tag not in img.tags:
|
||||||
|
img.tags.append(tag)
|
||||||
|
applied_for_image += 1
|
||||||
|
db.session.add(SuggestionFeedback(
|
||||||
|
image_id=img.id,
|
||||||
|
tag_name=name,
|
||||||
|
suggestion_source=cand.get('source', 'wd14'),
|
||||||
|
confidence=cand.get('confidence', 0.0),
|
||||||
|
decision='accepted',
|
||||||
|
))
|
||||||
|
|
||||||
|
if applied_for_image:
|
||||||
|
images_with_applies += 1
|
||||||
|
tags_applied += applied_for_image
|
||||||
|
|
||||||
|
# Commit once per batch — keeps transactions short and lets the
|
||||||
|
# next batch see freshly-created Tag rows.
|
||||||
|
db.session.commit()
|
||||||
|
last_id = rows[-1].id
|
||||||
|
log.info(
|
||||||
|
"apply_auto_accept_predictions: progress scanned=%d "
|
||||||
|
"images_with_applies=%d tags_applied=%d last_id=%d",
|
||||||
|
scanned, images_with_applies, tags_applied, last_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
"apply_auto_accept_predictions: done scanned=%d "
|
||||||
|
"images_with_applies=%d tags_applied=%d",
|
||||||
|
scanned, images_with_applies, tags_applied,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
'scanned': scanned,
|
||||||
|
'images_with_applies': images_with_applies,
|
||||||
|
'tags_applied': tags_applied,
|
||||||
|
}
|
||||||
|
|||||||
@@ -494,6 +494,24 @@
|
|||||||
</form>
|
</form>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section class="settings-section">
|
||||||
|
<h3>Apply auto-accepted predictions to all images</h3>
|
||||||
|
<p class="settings-hint">
|
||||||
|
Walks every image and applies general-category WD14 predictions
|
||||||
|
whose confidence is at or above the auto-accept threshold. Same
|
||||||
|
side-effects as opening each modal individually (creates the
|
||||||
|
kind='user' tag if needed, attaches it to the image, logs a
|
||||||
|
SuggestionFeedback rejection-protected record). Idempotent — safe
|
||||||
|
to re-run. Useful after a fresh ML backfill so high-confidence
|
||||||
|
tags land without needing to view each image. Skips images flagged
|
||||||
|
as integrity-corrupt.
|
||||||
|
</p>
|
||||||
|
<form action="{{ url_for('main.trigger_apply_auto_accept_predictions') }}" method="post"
|
||||||
|
onsubmit="return confirm('Apply every above-threshold general-category WD14 prediction to its image? This walks the entire library.');">
|
||||||
|
<button type="submit" class="btn secondary-btn">Apply auto-accepts library-wide</button>
|
||||||
|
</form>
|
||||||
|
</section>
|
||||||
|
|
||||||
<section class="settings-section">
|
<section class="settings-section">
|
||||||
<h3>Auto-accept threshold (general tags)</h3>
|
<h3>Auto-accept threshold (general tags)</h3>
|
||||||
<p class="settings-hint">
|
<p class="settings-hint">
|
||||||
|
|||||||
Reference in New Issue
Block a user