GallerySubscriber/backend/app/tasks/celery_app.py

"""Celery application configuration."""

import logging
from celery import Celery
from celery.schedules import crontab

from app.config import get_settings

logger = logging.getLogger(__name__)
settings = get_settings()

# Log Redis configuration at import time (helps diagnose DB mismatch issues)
_redis_db = settings.redis_url.split('/')[-1] if '/' in settings.redis_url else '0'
logger.info(f"Celery using Redis: {settings.redis_url} (database: {_redis_db})")


def _get_parallel_limit_from_db():
    """Try to get parallel_limit from database, fall back to environment.

    This is called once at Celery startup. Changes require worker restart.
    """
    try:
        import asyncio
        from sqlalchemy import select
        from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker

        async def _fetch():
            engine = create_async_engine(settings.async_database_url, echo=False)
            session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
            try:
                async with session_factory() as session:
                    from app.models.setting import Setting, DEFAULT_SETTINGS
                    result = await session.execute(
                        select(Setting).where(Setting.key == "download.parallel_limit")
                    )
                    setting = result.scalar()
                    if setting:
                        return setting.value
                    return DEFAULT_SETTINGS.get("download.parallel_limit", settings.download_parallel_limit)
            finally:
                await engine.dispose()

        return asyncio.run(_fetch())
    except Exception as e:
        logger.warning(f"Could not read parallel_limit from database, using environment: {e}")
        return settings.download_parallel_limit


# Get parallel limit (database takes precedence over environment)
parallel_limit = _get_parallel_limit_from_db()
logger.info(f"Celery worker concurrency set to: {parallel_limit}")

celery_app = Celery(
    "gallery_subscriber",
    broker=settings.redis_url,
    backend=settings.redis_url,
    include=["app.tasks.downloads", "app.tasks.maintenance"],
)

# Celery configuration
celery_app.conf.update(
    task_serializer="json",
    accept_content=["json", "pickle"],  # Accept pickle for backward compatibility during upgrades
    result_serializer="json",
    timezone="UTC",
    enable_utc=True,
    task_track_started=True,
    task_time_limit=3600,  # 1 hour max per task
    worker_prefetch_multiplier=1,  # Process one task at a time
    worker_concurrency=parallel_limit,
    # Task routing - send tasks to specific queues
    task_routes={
        # Lightweight scheduling/maintenance tasks go to 'scheduling' queue
        "app.tasks.downloads.scheduled_check": {"queue": "scheduling"},
        "app.tasks.downloads.cleanup_old_downloads": {"queue": "scheduling"},
        "tasks.update_storage_stats": {"queue": "scheduling"},
        "tasks.reset_orphaned_jobs": {"queue": "scheduling"},
        "tasks.requeue_stale_jobs": {"queue": "scheduling"},
        # Heavy download tasks go to 'downloads' queue (default)
        "app.tasks.downloads.download_source": {"queue": "downloads"},
        "app.tasks.downloads.process_download": {"queue": "downloads"},
    },
    # Default queue for any unrouted tasks
    task_default_queue="downloads",
)

# Celery Beat schedule for periodic tasks
# The scheduler runs periodically to check if any sources are due for download.
# The actual schedule_interval (e.g., 8 hours) is enforced inside the task
# by comparing each source's last_check timestamp against the DB setting.
# 15 minutes is a good balance - responsive enough without excessive overhead.
celery_app.conf.beat_schedule = {
    "check-sources-scheduler": {
        "task": "app.tasks.downloads.scheduled_check",
        "schedule": 900,  # Run every 15 minutes to check for due sources
    },
    "cleanup-old-downloads-daily": {
        "task": "app.tasks.downloads.cleanup_old_downloads",
        "schedule": crontab(hour=3, minute=0),  # Daily at 3 AM
    },
    "update-storage-stats": {
        "task": "tasks.update_storage_stats",
        "schedule": crontab(minute="*/30"),  # Every 30 minutes
    },
    "reset-orphaned-jobs": {
        "task": "tasks.reset_orphaned_jobs",
        "schedule": crontab(minute="*/15"),  # Every 15 minutes
    },
    "requeue-stale-jobs": {
        "task": "tasks.requeue_stale_jobs",
        "schedule": crontab(minute="*/15"),  # Every 15 minutes, check for stale queued jobs
    },
}


# Worker startup signal - only reset orphaned jobs on scheduler worker
from celery.signals import worker_ready

@worker_ready.connect
def on_worker_ready(sender, **kwargs):
    """Called when a worker is ready to accept tasks.

    Only the scheduler worker resets orphaned jobs to avoid duplicate resets
    when multiple download workers start.
    """
    # Check if this worker is consuming from the 'scheduling' queue
    # sender.app.amqp.queues contains the queues this worker consumes from
    try:
        queues = [q.name for q in sender.app.amqp.queues.consume_from.values()]
        is_scheduler = 'scheduling' in queues
    except Exception:
        # Fallback: check hostname or other indicators
        is_scheduler = False

    # Verify Redis connectivity and log configuration
    try:
        import redis
        redis_url = settings.redis_url
        redis_db = redis_url.split('/')[-1] if '/' in redis_url else '0'
        r = redis.from_url(redis_url)
        r.ping()
        logger.info(f"Redis connectivity verified: {redis_url} (DB: {redis_db})")

        # Check for other workers' markers to detect DB mismatches
        marker_key = f"gallerysubscriber:worker:{'scheduler' if is_scheduler else 'download'}"
        r.setex(marker_key, 3600, redis_url)  # Expires in 1 hour

        # Check if scheduler marker exists (helps detect if scheduler is on different DB)
        scheduler_marker = r.get("gallerysubscriber:worker:scheduler")
        download_marker = r.get("gallerysubscriber:worker:download")
        if scheduler_marker and download_marker:
            if scheduler_marker != download_marker:
                logger.error(f"REDIS DB MISMATCH DETECTED! Scheduler: {scheduler_marker.decode()}, Worker: {download_marker.decode()}")
    except Exception as e:
        logger.error(f"Redis connectivity check failed: {e}")

    if is_scheduler:
        logger.info("Scheduler worker starting - resetting orphaned jobs...")
        try:
            from app.tasks.maintenance import reset_all_running_jobs_sync
            result = reset_all_running_jobs_sync()
            logger.info(f"Scheduler startup: Reset {result.get('reset_count', 0)} orphaned running jobs")
        except Exception as e:
            logger.error(f"Failed to reset orphaned jobs on scheduler startup: {e}")

        # Re-queue any jobs stuck in 'queued' status (Celery tasks may have been lost)
        logger.info("Scheduler startup: Re-queuing any stale queued jobs...")
        try:
            from app.tasks.maintenance import requeue_all_queued_jobs_sync
            result = requeue_all_queued_jobs_sync()
            logger.info(f"Scheduler startup: Re-queued {result.get('requeued_count', 0)} queued jobs")
        except Exception as e:
            logger.error(f"Failed to re-queue jobs on scheduler startup: {e}")

        # Trigger an immediate scheduled_check on startup (don't wait for beat timer)
        logger.info("Scheduler startup: Triggering immediate scheduled_check...")
        try:
            from app.tasks.downloads import scheduled_check
            scheduled_check.delay()
            logger.info("Scheduler startup: scheduled_check task queued")
        except Exception as e:
            logger.error(f"Failed to queue initial scheduled_check: {e}")
    else:
        logger.info("Download worker starting - skipping orphaned job reset (handled by scheduler)")