bvandeusen
485387ff0b
Heads + CCIP are the tag source and head auto-apply is the earned propagation.
The Camie tagger ran only to feed the allowlist bulk-apply (its ImagePrediction
rows had no other consumer), and the allowlist was a SECOND, un-earned auto-apply
path firing in parallel with heads on every accept — exactly the un-earned spray
the v2 pivot replaced. Retire both.
Behavior change: accepting a suggestion now applies the tag to THAT image only
(source='ml_accepted', a head-training positive) — it no longer allowlists +
fans the tag across the library via Camie. Propagation is heads' earned
auto-apply. (Loses instant cold-start propagation for booru-vocab tags; that was
un-earned and bypassed the precision gate.)
- tag_and_embed is now EMBED-ONLY (no Camie load/infer, no ImagePrediction
writes); backfill enqueues it for images with no embedding.
- Removed: services/ml/tagger.py, apply_allowlist_tags + helpers + daily beat +
every enqueue caller (accept/alias/merge/per-image), api/allowlist.py +
blueprint, ImagePrediction + TagAllowlist models/tables (migration 0067),
AllowlistTable.vue + allowlist store, the accept coverage-projection payload.
- AllowlistService gutted to accept/dismiss/undismiss/reject (the rejection store
the rail still needs); accept returns nothing, API returns {accepted, tag_id}.
- tag merge no longer repoints/triggers the allowlist; _keep_as_alias now keys on
ML-applied image_tag sources (incl. head_auto) instead of the allowlist.
- UI: MLBackfillCard relabelled to embedding-only; accept toast simplified;
MaintenancePanel drops the allowlist tile.
Left for a follow-up hygiene pass (now-inert, harmless): the dead settings
columns (tagger_store_floor, tagger_model_version, suggestion_threshold_*,
video_min_tag_frames), image_record.tagger_model_version, MLThresholdSliders
trim, and the Camie model download in download_models.py.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
247 lines
12 KiB
Python
247 lines
12 KiB
Python
"""Celery configuration with separate queue lanes.
|
|
|
|
Queues:
|
|
import — filesystem import path (FC-2)
|
|
ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
|
|
thumbnail — image and video thumbnail generation (FC-2)
|
|
download — gallery-dl tasks (FC-3)
|
|
scan — periodic source checks (FC-3) — kept separate so long imports
|
|
don't starve the scheduler
|
|
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
|
default — anything not explicitly routed
|
|
"""
|
|
|
|
from celery import Celery
|
|
|
|
from .config import get_config
|
|
|
|
|
|
def make_celery() -> Celery:
|
|
cfg = get_config()
|
|
app = Celery(
|
|
"fabledcurator",
|
|
broker=cfg.celery_broker_url,
|
|
backend=cfg.celery_result_backend,
|
|
include=[
|
|
"backend.app.tasks.smoke",
|
|
"backend.app.tasks.scan",
|
|
"backend.app.tasks.import_file",
|
|
"backend.app.tasks.thumbnail",
|
|
"backend.app.tasks.maintenance",
|
|
"backend.app.tasks.ml",
|
|
"backend.app.tasks.download",
|
|
"backend.app.tasks.external",
|
|
"backend.app.tasks.backup",
|
|
"backend.app.tasks.admin",
|
|
"backend.app.tasks.library_audit",
|
|
],
|
|
)
|
|
app.conf.update(
|
|
task_default_queue="default",
|
|
task_routes={
|
|
"backend.app.tasks.import_file.*": {"queue": "import"},
|
|
"backend.app.tasks.ml.*": {"queue": "ml"},
|
|
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
|
"backend.app.tasks.download.*": {"queue": "download"},
|
|
# External file-host fetches are downloads — same lane (they can run
|
|
# long, but the download worker already tolerates long backfills).
|
|
"backend.app.tasks.external.*": {"queue": "download"},
|
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
|
# `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
|
|
# (concurrency-1 on the scheduler). The long one-shots (DB backups,
|
|
# library audits, admin maintenance: normalize/re-extract/cascade-
|
|
# delete) run on a SEPARATE `maintenance_long` lane + worker so they
|
|
# can never starve the quick self-healing sweeps (operator-flagged
|
|
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
|
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
|
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
|
},
|
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
|
task_acks_late=True,
|
|
worker_prefetch_multiplier=1,
|
|
# Broker resilience (2026-06-24): a swarm overlay-network blip after a
|
|
# redeploy left Redis healthy but transiently unreachable, and a worker
|
|
# starting in that window crash-looped on the initial broker connect
|
|
# (kombu OperationalError) instead of waiting it out — needing a manual
|
|
# Redis reset to recover. Retry the broker FOREVER (None) on startup and
|
|
# at runtime so a transient outage self-heals when routing returns,
|
|
# rather than the worker exiting.
|
|
broker_connection_retry_on_startup=True,
|
|
broker_connection_retry=True,
|
|
broker_connection_max_retries=None,
|
|
# Redis-transport socket options (apply to the BROKER connection): a
|
|
# short connect timeout + TCP keepalive so a dead/blocked socket is
|
|
# noticed and retried, and a periodic health check that proactively
|
|
# reconnects a live worker through a network hiccup.
|
|
broker_transport_options={
|
|
"socket_connect_timeout": 5,
|
|
"socket_timeout": 30,
|
|
"socket_keepalive": True,
|
|
"retry_on_timeout": True,
|
|
"health_check_interval": 30,
|
|
},
|
|
# Same hardening for the Redis RESULT backend (separate connection pool).
|
|
redis_socket_connect_timeout=5,
|
|
redis_socket_timeout=30,
|
|
redis_socket_keepalive=True,
|
|
redis_retry_on_timeout=True,
|
|
redis_backend_health_check_interval=30,
|
|
beat_schedule={
|
|
"recover-interrupted-tasks": {
|
|
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
|
"schedule": 300.0, # every 5 minutes
|
|
},
|
|
"cleanup-old-tasks": {
|
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"ml-backfill-daily": {
|
|
"task": "backend.app.tasks.ml.backfill",
|
|
"schedule": 86400.0,
|
|
},
|
|
"train-heads-nightly": {
|
|
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
|
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
|
},
|
|
"apply-head-tags-daily": {
|
|
"task": "backend.app.tasks.ml.scheduled_apply_head_tags",
|
|
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
|
},
|
|
"recover-orphaned-gpu-jobs": {
|
|
"task": "backend.app.tasks.ml.recover_orphaned_gpu_jobs",
|
|
"schedule": 60.0, # quick pickup of work a dead agent orphaned
|
|
},
|
|
"enqueue-ccip-backfill-hourly": {
|
|
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
|
"schedule": 3600.0, # auto-feed new images (+ retry errored) so
|
|
"args": ("ccip",), # the queue keeps moving without the button
|
|
},
|
|
"enqueue-siglip-backfill-daily": {
|
|
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
|
"schedule": 86400.0, # drain the concept-crop back-catalogue +
|
|
"args": ("siglip",), # retry failed embeds, no button needed
|
|
},
|
|
"enqueue-embed-backfill-daily": {
|
|
"task": "backend.app.tasks.ml.enqueue_gpu_backfill",
|
|
"schedule": 86400.0, # whole-image re-embed under the current
|
|
"args": ("embed",), # model (an operator swap) drains via agent
|
|
},
|
|
"ccip-auto-apply-daily": {
|
|
"task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
|
"schedule": 86400.0, # no-op unless ccip_auto_apply_enabled
|
|
},
|
|
"snapshot-head-metrics-daily": {
|
|
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
|
"schedule": 86400.0,
|
|
},
|
|
"integrity-verify-weekly": {
|
|
"task": "backend.app.tasks.maintenance.verify_integrity",
|
|
"schedule": 604800.0, # weekly
|
|
},
|
|
"fc3d-tick-due-sources": {
|
|
"task": "backend.app.tasks.scan.tick_due_sources",
|
|
"schedule": 60.0, # every minute
|
|
},
|
|
"fc3d-cleanup-download-events": {
|
|
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"recover-stalled-download-events": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_download_events",
|
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
|
},
|
|
"recover-stalled-task-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
|
},
|
|
"prune-task-runs": {
|
|
"task": "backend.app.tasks.maintenance.prune_task_runs",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"vacuum-analyze": {
|
|
"task": "backend.app.tasks.maintenance.vacuum_analyze",
|
|
"schedule": 604800.0, # weekly — reclaim dead-tuple bloat + refresh stats
|
|
},
|
|
"fc3h-backup-db-nightly": {
|
|
"task": "backend.app.tasks.backup.backup_db_nightly",
|
|
"schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour
|
|
},
|
|
"fc3h-prune-backups": {
|
|
"task": "backend.app.tasks.backup.prune_backups",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
# Audit 2026-06-02 — three new per-entity recovery sweeps.
|
|
# Each runs every 5 min like the other recover_stalled_*
|
|
# sweeps; each is a no-op when nothing is stuck.
|
|
"recover-stalled-backup-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_backup_runs",
|
|
"schedule": 300.0,
|
|
},
|
|
"recover-stalled-library-audit-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
|
|
"schedule": 300.0,
|
|
},
|
|
"recover-stalled-tag-eval-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_tag_eval_runs",
|
|
"schedule": 300.0,
|
|
},
|
|
"recover-stalled-head-training-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_head_training_runs",
|
|
"schedule": 300.0,
|
|
},
|
|
"recover-stalled-head-auto-apply-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs",
|
|
"schedule": 300.0,
|
|
},
|
|
"recover-stalled-import-batches": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
|
|
"schedule": 300.0,
|
|
},
|
|
# Audit 2026-06-02 — daily retention for two entities
|
|
# whose terminal rows otherwise accumulate forever.
|
|
"prune-library-audit-runs": {
|
|
"task": "backend.app.tasks.maintenance.prune_library_audit_runs",
|
|
"schedule": 86400.0,
|
|
},
|
|
"prune-import-batches": {
|
|
"task": "backend.app.tasks.maintenance.prune_import_batches",
|
|
"schedule": 86400.0,
|
|
},
|
|
# Audit 2026-06-02 — backfill_thumbnails's docstring claimed
|
|
# "periodic Beat" but the entry was never registered, so the
|
|
# library got no self-healing thumbnail repair; only the
|
|
# manual admin-UI button fired it. Daily cadence is gentle
|
|
# (the task is idempotent and only enqueues regen for rows
|
|
# whose stored thumbnails are missing or corrupt).
|
|
"backfill-thumbnails-daily": {
|
|
"task": "backend.app.tasks.thumbnail.backfill_thumbnails",
|
|
"schedule": 86400.0,
|
|
},
|
|
# External file-host downloads (#830): a steady sweep catches links
|
|
# the post-download hook missed (worker down, etc.); recovery re-tries
|
|
# dead links daily; retention prunes long-dead rows.
|
|
"extdl-sweep": {
|
|
"task": "backend.app.tasks.external.sweep_external_links",
|
|
"schedule": 600.0, # every 10 min
|
|
},
|
|
"extdl-recover-daily": {
|
|
"task": "backend.app.tasks.external.recover_external_links",
|
|
"schedule": 86400.0,
|
|
},
|
|
"extdl-prune-daily": {
|
|
"task": "backend.app.tasks.external.prune_external_links",
|
|
"schedule": 86400.0,
|
|
},
|
|
},
|
|
timezone="UTC",
|
|
)
|
|
# FC-3i: register task_run signal handlers (side-effect import).
|
|
from . import celery_signals # noqa: F401
|
|
|
|
return app
|
|
|
|
|
|
celery = make_celery()
|