changes to task scheduling and readability change on downloads view

backend/app/tasks/celery_app.py

@@ -9,6 +9,10 @@ from app.config import get_settings
 logger = logging.getLogger(__name__)
 settings = get_settings()
 
+# Log Redis configuration at import time (helps diagnose DB mismatch issues)
+_redis_db = settings.redis_url.split('/')[-1] if '/' in settings.redis_url else '0'
+logger.info(f"Celery using Redis: {settings.redis_url} (database: {_redis_db})")
+
 
 def _get_parallel_limit_from_db():
     """Try to get parallel_limit from database, fall back to environment.
@@ -56,7 +60,7 @@ celery_app = Celery(
 # Celery configuration
 celery_app.conf.update(
     task_serializer="json",
-    accept_content=["json"],
+    accept_content=["json", "pickle"],  # Accept pickle for backward compatibility during upgrades
     result_serializer="json",
     timezone="UTC",
     enable_utc=True,
@@ -64,16 +68,31 @@ celery_app.conf.update(
     task_time_limit=3600,  # 1 hour max per task
     worker_prefetch_multiplier=1,  # Process one task at a time
     worker_concurrency=parallel_limit,
+    # Task routing - send tasks to specific queues
+    task_routes={
+        # Lightweight scheduling/maintenance tasks go to 'scheduling' queue
+        "app.tasks.downloads.scheduled_check": {"queue": "scheduling"},
+        "app.tasks.downloads.cleanup_old_downloads": {"queue": "scheduling"},
+        "tasks.update_storage_stats": {"queue": "scheduling"},
+        "tasks.reset_orphaned_jobs": {"queue": "scheduling"},
+        "tasks.requeue_stale_jobs": {"queue": "scheduling"},
+        # Heavy download tasks go to 'downloads' queue (default)
+        "app.tasks.downloads.download_source": {"queue": "downloads"},
+        "app.tasks.downloads.process_download": {"queue": "downloads"},
+    },
+    # Default queue for any unrouted tasks
+    task_default_queue="downloads",
 )
 
 # Celery Beat schedule for periodic tasks
-# Note: The scheduler runs frequently (every 60s) but only triggers downloads
-# for sources whose individual check_interval has elapsed. Per-source intervals
-# are stored in the database and can be configured via the UI.
+# The scheduler runs periodically to check if any sources are due for download.
+# The actual schedule_interval (e.g., 8 hours) is enforced inside the task
+# by comparing each source's last_check timestamp against the DB setting.
+# 15 minutes is a good balance - responsive enough without excessive overhead.
 celery_app.conf.beat_schedule = {
     "check-sources-scheduler": {
         "task": "app.tasks.downloads.scheduled_check",
-        "schedule": 60,  # Run every 60 seconds to check for due sources
+        "schedule": 900,  # Run every 15 minutes to check for due sources
     },
     "cleanup-old-downloads-daily": {
         "task": "app.tasks.downloads.cleanup_old_downloads",
@@ -87,28 +106,79 @@ celery_app.conf.beat_schedule = {
         "task": "tasks.reset_orphaned_jobs",
         "schedule": crontab(minute="*/15"),  # Every 15 minutes
     },
+    "requeue-stale-jobs": {
+        "task": "tasks.requeue_stale_jobs",
+        "schedule": crontab(minute="*/15"),  # Every 15 minutes, check for stale queued jobs
+    },
 }
 
 
-# Worker startup signal - reset any orphaned running jobs
-@celery_app.on_after_configure.connect
-def setup_worker_startup(sender, **kwargs):
-    """Called after Celery is configured but before workers start processing."""
-    pass  # Placeholder - actual reset happens in worker_ready signal
-
-
+# Worker startup signal - only reset orphaned jobs on scheduler worker
 from celery.signals import worker_ready
 
 @worker_ready.connect
 def on_worker_ready(sender, **kwargs):
     """Called when a worker is ready to accept tasks.
 
-    Resets any jobs stuck in 'running' status from previous runs.
-    This handles the case where workers crashed or were restarted.
+    Only the scheduler worker resets orphaned jobs to avoid duplicate resets
+    when multiple download workers start.
     """
+    # Check if this worker is consuming from the 'scheduling' queue
+    # sender.app.amqp.queues contains the queues this worker consumes from
     try:
-        from app.tasks.maintenance import reset_all_running_jobs_sync
-        result = reset_all_running_jobs_sync()
-        logger.info(f"Worker startup: {result}")
+        queues = [q.name for q in sender.app.amqp.queues.consume_from.values()]
+        is_scheduler = 'scheduling' in queues
+    except Exception:
+        # Fallback: check hostname or other indicators
+        is_scheduler = False
+
+    # Verify Redis connectivity and log configuration
+    try:
+        import redis
+        redis_url = settings.redis_url
+        redis_db = redis_url.split('/')[-1] if '/' in redis_url else '0'
+        r = redis.from_url(redis_url)
+        r.ping()
+        logger.info(f"Redis connectivity verified: {redis_url} (DB: {redis_db})")
+
+        # Check for other workers' markers to detect DB mismatches
+        marker_key = f"gallerysubscriber:worker:{'scheduler' if is_scheduler else 'download'}"
+        r.setex(marker_key, 3600, redis_url)  # Expires in 1 hour
+
+        # Check if scheduler marker exists (helps detect if scheduler is on different DB)
+        scheduler_marker = r.get("gallerysubscriber:worker:scheduler")
+        download_marker = r.get("gallerysubscriber:worker:download")
+        if scheduler_marker and download_marker:
+            if scheduler_marker != download_marker:
+                logger.error(f"REDIS DB MISMATCH DETECTED! Scheduler: {scheduler_marker.decode()}, Worker: {download_marker.decode()}")
     except Exception as e:
-        logger.error(f"Failed to reset orphaned jobs on worker startup: {e}")
+        logger.error(f"Redis connectivity check failed: {e}")
+
+    if is_scheduler:
+        logger.info("Scheduler worker starting - resetting orphaned jobs...")
+        try:
+            from app.tasks.maintenance import reset_all_running_jobs_sync
+            result = reset_all_running_jobs_sync()
+            logger.info(f"Scheduler startup: Reset {result.get('reset_count', 0)} orphaned running jobs")
+        except Exception as e:
+            logger.error(f"Failed to reset orphaned jobs on scheduler startup: {e}")
+
+        # Re-queue any jobs stuck in 'queued' status (Celery tasks may have been lost)
+        logger.info("Scheduler startup: Re-queuing any stale queued jobs...")
+        try:
+            from app.tasks.maintenance import requeue_all_queued_jobs_sync
+            result = requeue_all_queued_jobs_sync()
+            logger.info(f"Scheduler startup: Re-queued {result.get('requeued_count', 0)} queued jobs")
+        except Exception as e:
+            logger.error(f"Failed to re-queue jobs on scheduler startup: {e}")
+
+        # Trigger an immediate scheduled_check on startup (don't wait for beat timer)
+        logger.info("Scheduler startup: Triggering immediate scheduled_check...")
+        try:
+            from app.tasks.downloads import scheduled_check
+            scheduled_check.delay()
+            logger.info("Scheduler startup: scheduled_check task queued")
+        except Exception as e:
+            logger.error(f"Failed to queue initial scheduled_check: {e}")
+    else:
+        logger.info("Download worker starting - skipping orphaned job reset (handled by scheduler)")