bvandeusen
171c486939
The `select(ImportSettings).where(id == 1)).scalar_one()` singleton load was repeated 15× across services, API, and 5 task modules. Added async load() + sync load_sync() classmethods on the model and migrated all 15 full-row sites (callers already imported ImportSettings, so no new imports; dropped download's now-orphaned select import). Left maintenance.py's deliberate column-select (import_scan_path only) as-is. Rest of the service layer was already adequately DRY — the Record/to_dict pattern is only 2 instances and the savepoint find-or-create recovery is correctly per-entity, so neither was forced into a shared abstraction. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
77 lines
2.7 KiB
Python
77 lines
2.7 KiB
Python
"""download_source Celery task — runs DownloadService for one source."""
|
|
|
|
import asyncio
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
|
|
|
from ..celery_app import celery
|
|
from ..models import ImportSettings
|
|
from ..services.credential_crypto import CredentialCrypto
|
|
from ..services.credential_service import CredentialService
|
|
from ..services.download_service import DownloadService
|
|
from ..services.gallery_dl import GalleryDLService
|
|
from ..services.importer import Importer
|
|
from ..services.thumbnailer import Thumbnailer
|
|
from ._async_session import async_session_factory
|
|
from .import_file import _sync_session_factory
|
|
|
|
IMAGES_ROOT = Path("/images")
|
|
_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
|
|
|
|
|
@celery.task(
|
|
name="backend.app.tasks.download.download_source",
|
|
bind=True,
|
|
acks_late=True,
|
|
autoretry_for=(OperationalError, DBAPIError, OSError),
|
|
retry_backoff=10,
|
|
retry_backoff_max=120,
|
|
retry_jitter=True,
|
|
max_retries=3,
|
|
soft_time_limit=900,
|
|
time_limit=1200,
|
|
)
|
|
def download_source(self, source_id: int) -> int:
|
|
"""Returns the DownloadEvent.id."""
|
|
|
|
async def _run():
|
|
async_factory, async_engine = async_session_factory()
|
|
SyncFactory = _sync_session_factory()
|
|
try:
|
|
with SyncFactory() as sync_session:
|
|
settings = ImportSettings.load_sync(sync_session)
|
|
rate_limit = settings.download_rate_limit_seconds
|
|
validate_files = settings.download_validate_files
|
|
|
|
gdl = GalleryDLService(
|
|
images_root=IMAGES_ROOT,
|
|
rate_limit=rate_limit,
|
|
validate_files=validate_files,
|
|
)
|
|
crypto = CredentialCrypto(_KEY_PATH)
|
|
|
|
async with async_factory() as async_session:
|
|
cred_service = CredentialService(async_session, crypto)
|
|
with SyncFactory() as sync_session:
|
|
sync_settings = ImportSettings.load_sync(sync_session)
|
|
importer = Importer(
|
|
session=sync_session,
|
|
images_root=IMAGES_ROOT,
|
|
import_root=IMAGES_ROOT,
|
|
thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
|
|
settings=sync_settings,
|
|
)
|
|
svc = DownloadService(
|
|
async_session=async_session,
|
|
sync_session=sync_session,
|
|
gdl=gdl,
|
|
importer=importer,
|
|
cred_service=cred_service,
|
|
)
|
|
return await svc.download_source(source_id)
|
|
finally:
|
|
await async_engine.dispose()
|
|
|
|
return asyncio.run(_run())
|