bvandeusen
d3bb8c509a
scan_directory walks ImportSettings.import_scan_path, creates an ImportBatch, enumerates supported files into ImportTasks, and enqueues import_media_file per task. import_media_file moves the task through its state machine (pending → queued → processing → complete/skipped/failed), updates ImportBatch counters atomically (UPDATE ... SET col = col + 1), enqueues a thumbnail task on success, and marks the batch complete when the last task drains. generate_thumbnail runs on its own queue (thumbnail) so big imports don't starve thumbnail throughput; failure here is logged and does not fail the import. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
79 lines
2.4 KiB
Python
79 lines
2.4 KiB
Python
"""scan_directory task: walks the configured import path, creates an
|
|
ImportBatch, enumerates files into ImportTasks, and enqueues
|
|
import_media_file for each.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from ..celery_app import celery
|
|
from ..config import get_config
|
|
from ..models import ImportBatch, ImportSettings, ImportTask
|
|
from ..services.importer import is_supported
|
|
|
|
|
|
def _sync_session_factory():
|
|
cfg = get_config()
|
|
engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True)
|
|
return sessionmaker(engine, expire_on_commit=False)
|
|
|
|
|
|
@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True)
|
|
def scan_directory(self, triggered_by: str = "manual") -> int:
|
|
"""Walks the import root and creates ImportTasks. Returns the batch id."""
|
|
SessionLocal = _sync_session_factory()
|
|
with SessionLocal() as session:
|
|
settings = session.execute(
|
|
select(ImportSettings).where(ImportSettings.id == 1)
|
|
).scalar_one()
|
|
import_root = Path(settings.import_scan_path)
|
|
|
|
batch = ImportBatch(
|
|
triggered_by=triggered_by,
|
|
source_path=str(import_root),
|
|
scan_mode="quick", # FC-2a is quick only; FC-2d adds 'deep'
|
|
status="running",
|
|
)
|
|
session.add(batch)
|
|
session.flush()
|
|
batch_id = batch.id
|
|
|
|
# Walk and enumerate.
|
|
files_seen = 0
|
|
for entry in import_root.rglob("*"):
|
|
if not entry.is_file():
|
|
continue
|
|
if not is_supported(entry):
|
|
continue
|
|
try:
|
|
size = entry.stat().st_size
|
|
except OSError:
|
|
size = None
|
|
task = ImportTask(
|
|
batch_id=batch_id,
|
|
source_path=str(entry),
|
|
task_type="media",
|
|
status="pending",
|
|
size_bytes=size,
|
|
)
|
|
session.add(task)
|
|
files_seen += 1
|
|
|
|
batch.total_files = files_seen
|
|
session.commit()
|
|
|
|
# Now enqueue import_media_file for each pending task.
|
|
for task in session.execute(
|
|
select(ImportTask).where(ImportTask.batch_id == batch_id)
|
|
).scalars():
|
|
task.status = "queued"
|
|
session.add(task)
|
|
from .import_file import import_media_file
|
|
|
|
import_media_file.delay(task.id)
|
|
session.commit()
|
|
|
|
return batch_id
|