fc3d: tick_due_sources Celery task on the scan queue

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-21 16:01:59 -04:00
parent 87fb534722
commit 9d5793f0ce
2 changed files with 201 additions and 4 deletions
+69 -4
View File
@@ -1,16 +1,24 @@
"""scan_directory task: walks the configured import path, creates an
ImportBatch, enumerates files into ImportTasks, and enqueues
import_media_file for each.
"""scan tasks:
* ``scan_directory`` — walks the configured import path, creates an
ImportBatch, enumerates files into ImportTasks, and enqueues
import_media_file for each.
* ``tick_due_sources`` (FC-3d) — periodic Beat-fired scan that fires
``download_source.delay()`` for sources due for a check.
"""
import asyncio
from datetime import UTC, datetime
from pathlib import Path
from sqlalchemy import create_engine, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import sessionmaker
from ..celery_app import celery
from ..config import get_config
from ..models import ImportBatch, ImportSettings, ImportTask
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
from ..services.scheduler_service import select_due_sources
def _iter_import_files(import_root: Path):
@@ -94,3 +102,60 @@ def scan_directory(self, triggered_by: str = "manual",
backfill_phash.delay()
return batch_id
# --- FC-3d: periodic source-check tick ------------------------------------
def _async_session_factory():
cfg = get_config()
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
async def _tick_due_sources_async() -> dict:
factory, engine = _async_session_factory()
try:
async with factory() as session:
due = await select_due_sources(session)
if not due:
return {
"due": 0, "queued": 0, "skipped_in_flight": 0,
"tick_at": datetime.now(UTC).isoformat(),
}
due_ids = [s.id for s in due]
in_flight_rows = (await session.execute(
select(DownloadEvent.source_id).where(
DownloadEvent.source_id.in_(due_ids),
DownloadEvent.status.in_(["pending", "running"]),
)
)).scalars().all()
in_flight = set(in_flight_rows)
# Local import: keep top of module light, avoid circular import
# at celery task discovery time.
from .download import download_source
queued = 0
for s in due:
if s.id in in_flight:
continue
session.add(DownloadEvent(source_id=s.id, status="pending"))
await session.commit()
download_source.delay(s.id)
queued += 1
return {
"due": len(due),
"queued": queued,
"skipped_in_flight": len(in_flight),
"tick_at": datetime.now(UTC).isoformat(),
}
finally:
await engine.dispose()
@celery.task(name="backend.app.tasks.scan.tick_due_sources", bind=True)
def tick_due_sources(self) -> dict:
"""FC-3d: scan for sources due for a periodic check and fire
``download_source.delay()`` for each."""
return asyncio.run(_tick_due_sources_async())