fc3d: tick_due_sources Celery task on the scan queue
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,16 +1,24 @@
|
||||
"""scan_directory task: walks the configured import path, creates an
|
||||
ImportBatch, enumerates files into ImportTasks, and enqueues
|
||||
import_media_file for each.
|
||||
"""scan tasks:
|
||||
|
||||
* ``scan_directory`` — walks the configured import path, creates an
|
||||
ImportBatch, enumerates files into ImportTasks, and enqueues
|
||||
import_media_file for each.
|
||||
* ``tick_due_sources`` (FC-3d) — periodic Beat-fired scan that fires
|
||||
``download_source.delay()`` for sources due for a check.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..config import get_config
|
||||
from ..models import ImportBatch, ImportSettings, ImportTask
|
||||
from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask
|
||||
from ..services.scheduler_service import select_due_sources
|
||||
|
||||
|
||||
def _iter_import_files(import_root: Path):
|
||||
@@ -94,3 +102,60 @@ def scan_directory(self, triggered_by: str = "manual",
|
||||
backfill_phash.delay()
|
||||
|
||||
return batch_id
|
||||
|
||||
|
||||
# --- FC-3d: periodic source-check tick ------------------------------------
|
||||
|
||||
|
||||
def _async_session_factory():
|
||||
cfg = get_config()
|
||||
engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True)
|
||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine
|
||||
|
||||
|
||||
async def _tick_due_sources_async() -> dict:
|
||||
factory, engine = _async_session_factory()
|
||||
try:
|
||||
async with factory() as session:
|
||||
due = await select_due_sources(session)
|
||||
if not due:
|
||||
return {
|
||||
"due": 0, "queued": 0, "skipped_in_flight": 0,
|
||||
"tick_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
due_ids = [s.id for s in due]
|
||||
in_flight_rows = (await session.execute(
|
||||
select(DownloadEvent.source_id).where(
|
||||
DownloadEvent.source_id.in_(due_ids),
|
||||
DownloadEvent.status.in_(["pending", "running"]),
|
||||
)
|
||||
)).scalars().all()
|
||||
in_flight = set(in_flight_rows)
|
||||
|
||||
# Local import: keep top of module light, avoid circular import
|
||||
# at celery task discovery time.
|
||||
from .download import download_source
|
||||
|
||||
queued = 0
|
||||
for s in due:
|
||||
if s.id in in_flight:
|
||||
continue
|
||||
session.add(DownloadEvent(source_id=s.id, status="pending"))
|
||||
await session.commit()
|
||||
download_source.delay(s.id)
|
||||
queued += 1
|
||||
return {
|
||||
"due": len(due),
|
||||
"queued": queued,
|
||||
"skipped_in_flight": len(in_flight),
|
||||
"tick_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.scan.tick_due_sources", bind=True)
|
||||
def tick_due_sources(self) -> dict:
|
||||
"""FC-3d: scan for sources due for a periodic check and fire
|
||||
``download_source.delay()`` for each."""
|
||||
return asyncio.run(_tick_due_sources_async())
|
||||
|
||||
Reference in New Issue
Block a user