feat(fc2a): add maintenance tasks (recovery + cleanup) with beat schedule
recover_interrupted_tasks runs every 5 minutes, finds ImportTask rows stuck in 'processing' for >30 minutes (well above any legitimate import duration), and re-queues them. cleanup_old_tasks runs daily and deletes finished tasks older than 7 days so the task table stays an operational view rather than an archive. Both thresholds match ImageRepo's precedent. The 30-min stuck threshold is documented inline so a future reader can adjust it intentionally rather than mistaking it for a 'magic number'. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -44,6 +44,17 @@ def make_celery() -> Celery:
|
|||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
worker_prefetch_multiplier=1,
|
worker_prefetch_multiplier=1,
|
||||||
broker_connection_retry_on_startup=True,
|
broker_connection_retry_on_startup=True,
|
||||||
|
beat_schedule={
|
||||||
|
"recover-interrupted-tasks": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
||||||
|
"schedule": 300.0, # every 5 minutes
|
||||||
|
},
|
||||||
|
"cleanup-old-tasks": {
|
||||||
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||||
|
"schedule": 86400.0, # daily
|
||||||
|
},
|
||||||
|
},
|
||||||
|
timezone="UTC",
|
||||||
)
|
)
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks."""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, delete, select, update
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..config import get_config
|
||||||
|
from ..models import ImportTask
|
||||||
|
|
||||||
|
|
||||||
|
STUCK_THRESHOLD_MINUTES = 30
|
||||||
|
OLD_TASK_DAYS = 7
|
||||||
|
|
||||||
|
|
||||||
|
def _sync_session_factory():
|
||||||
|
cfg = get_config()
|
||||||
|
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
||||||
|
return sessionmaker(engine, expire_on_commit=False)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||||
|
def recover_interrupted_tasks() -> int:
|
||||||
|
"""Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
|
||||||
|
|
||||||
|
Why 30 min: large videos can legitimately take many minutes to import;
|
||||||
|
30 is a safe gate that catches actual crashes (which leave the row stuck
|
||||||
|
forever) without resetting slow-but-still-running jobs.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
stuck_ids = session.execute(
|
||||||
|
select(ImportTask.id)
|
||||||
|
.where(ImportTask.status == "processing")
|
||||||
|
.where(ImportTask.started_at < cutoff)
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
if not stuck_ids:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
session.execute(
|
||||||
|
update(ImportTask)
|
||||||
|
.where(ImportTask.id.in_(stuck_ids))
|
||||||
|
.values(status="queued", started_at=None, error="recovered from stuck state")
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
from .import_file import import_media_file
|
||||||
|
for tid in stuck_ids:
|
||||||
|
import_media_file.delay(tid)
|
||||||
|
|
||||||
|
return len(stuck_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||||
|
def cleanup_old_tasks() -> int:
|
||||||
|
"""Delete completed/skipped/failed ImportTask rows older than 7 days.
|
||||||
|
|
||||||
|
Why 7 days: long enough to debug an issue an operator only notices days
|
||||||
|
later; short enough that the task table stays a useful operational view
|
||||||
|
rather than an archive. Matches IR's default.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=OLD_TASK_DAYS)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
delete(ImportTask)
|
||||||
|
.where(ImportTask.status.in_(["complete", "skipped", "failed"]))
|
||||||
|
.where(ImportTask.finished_at < cutoff)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return result.rowcount or 0
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
"""Unit tests for the maintenance tasks. Eager-mode Celery so the task
|
||||||
|
function runs synchronously inside the same DB transaction as the test.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from backend.app.celery_app import celery
|
||||||
|
from backend.app.models import ImportBatch, ImportTask
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def eager():
|
||||||
|
celery.conf.task_always_eager = True
|
||||||
|
yield
|
||||||
|
celery.conf.task_always_eager = False
|
||||||
|
|
||||||
|
|
||||||
|
def _make_batch(session) -> int:
|
||||||
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||||
|
session.add(batch)
|
||||||
|
session.flush()
|
||||||
|
return batch.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_interrupted_only_old(db_sync):
|
||||||
|
batch_id = _make_batch(db_sync)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
fresh = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/a.jpg", task_type="media",
|
||||||
|
status="processing", started_at=now - timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
stale = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/b.jpg", task_type="media",
|
||||||
|
status="processing", started_at=now - timedelta(hours=2),
|
||||||
|
)
|
||||||
|
db_sync.add_all([fresh, stale])
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
from backend.app.tasks.maintenance import recover_interrupted_tasks
|
||||||
|
recovered = recover_interrupted_tasks.apply().get()
|
||||||
|
assert recovered == 1
|
||||||
|
|
||||||
|
db_sync.refresh(fresh)
|
||||||
|
db_sync.refresh(stale)
|
||||||
|
assert fresh.status == "processing"
|
||||||
|
assert stale.status == "queued"
|
||||||
|
assert stale.started_at is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_old_deletes_finished_old(db_sync):
|
||||||
|
batch_id = _make_batch(db_sync)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
old_complete = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/a.jpg", task_type="media",
|
||||||
|
status="complete", finished_at=now - timedelta(days=10),
|
||||||
|
)
|
||||||
|
recent_complete = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/b.jpg", task_type="media",
|
||||||
|
status="complete", finished_at=now - timedelta(days=2),
|
||||||
|
)
|
||||||
|
old_pending = ImportTask(
|
||||||
|
batch_id=batch_id, source_path="/import/c.jpg", task_type="media",
|
||||||
|
status="pending",
|
||||||
|
)
|
||||||
|
db_sync.add_all([old_complete, recent_complete, old_pending])
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
from backend.app.tasks.maintenance import cleanup_old_tasks
|
||||||
|
deleted = cleanup_old_tasks.apply().get()
|
||||||
|
assert deleted == 1
|
||||||
|
|
||||||
|
remaining = {t.source_path for t in db_sync.query(ImportTask).all()}
|
||||||
|
assert remaining == {"/import/b.jpg", "/import/c.jpg"}
|
||||||
Reference in New Issue
Block a user