changes to task scheduling and readability change on downloads view

backend/app/tasks/maintenance.py

@@ -23,6 +23,11 @@ settings = get_settings()
 # This should be longer than the task_time_limit (1 hour) to account for legitimate long runs
 STALE_RUNNING_THRESHOLD_MINUTES = 90
 
+# Maximum time a job can be in "queued" state before considered lost
+# If a queued job hasn't been picked up in this time, the Celery task was probably lost
+# (e.g., due to Redis DB mismatch or Redis restart)
+STALE_QUEUED_THRESHOLD_MINUTES = 30
+
 
 def get_async_session():
     """Get async session factory for Celery tasks."""
@@ -242,3 +247,164 @@ def reset_all_running_jobs_sync() -> dict:
     logger.info("Resetting all running jobs on worker startup...")
     result = asyncio.run(_reset_all_running_jobs_async())
     return result
+
+
+async def _requeue_stale_queued_jobs_async(threshold_minutes: int = None) -> dict:
+    """Re-queue Celery tasks for downloads stuck in 'queued' status.
+
+    Downloads can get stuck in 'queued' status if the Celery task was lost
+    (e.g., sent to wrong Redis DB, Redis restart, etc.). This function
+    detects these stale queued jobs and re-sends the Celery tasks.
+
+    Only one download per source is re-queued to prevent duplicate work.
+
+    Args:
+        threshold_minutes: Jobs queued longer than this are considered stale.
+                          If None, uses STALE_QUEUED_THRESHOLD_MINUTES.
+
+    Returns:
+        Dict with count of re-queued jobs
+    """
+    if threshold_minutes is None:
+        threshold_minutes = STALE_QUEUED_THRESHOLD_MINUTES
+
+    # Import here to avoid circular imports
+    from app.tasks.downloads import download_source
+
+    session_factory, engine = get_async_session()
+
+    try:
+        async with session_factory() as session:
+            now = utcnow()
+            cutoff = now - timedelta(minutes=threshold_minutes)
+
+            # Find queued jobs created before the cutoff, ordered by creation time
+            # so we process older jobs first
+            query = select(Download).where(
+                Download.status == DownloadStatus.QUEUED,
+                Download.created_at < cutoff
+            ).order_by(Download.created_at.asc())
+
+            result = await session.execute(query)
+            stale_jobs = result.scalars().all()
+
+            # Track which sources we've already queued to avoid duplicates
+            queued_source_ids = set()
+            requeued_count = 0
+            skipped_duplicates = 0
+
+            for job in stale_jobs:
+                # Skip if we've already queued a job for this source
+                if job.source_id in queued_source_ids:
+                    # Mark duplicate as skipped so it won't be checked again
+                    job.status = DownloadStatus.SKIPPED
+                    job.error_message = "Duplicate queued job - another job for this source was re-queued"
+                    skipped_duplicates += 1
+                    logger.info(f"Marked duplicate job {job.id} as SKIPPED (source {job.source_id} already re-queued)")
+                    continue
+
+                try:
+                    # Re-send the Celery task
+                    download_source.delay(job.source_id, download_id=job.id)
+                    queued_source_ids.add(job.source_id)
+                    requeued_count += 1
+                    logger.info(f"Re-queued stale job {job.id} (source_id: {job.source_id}, queued since: {job.created_at})")
+                except Exception as e:
+                    logger.error(f"Failed to re-queue job {job.id}: {e}")
+
+            # Commit the status changes for skipped duplicates
+            if skipped_duplicates > 0:
+                await session.commit()
+
+            if requeued_count > 0:
+                logger.info(f"Re-queued {requeued_count} stale queued jobs (marked {skipped_duplicates} duplicates as SKIPPED)")
+            else:
+                logger.debug("No stale queued jobs found")
+
+            return {"requeued_count": requeued_count, "skipped_duplicates": skipped_duplicates, "threshold_minutes": threshold_minutes}
+
+    finally:
+        await _cleanup_engine(engine)
+
+
+async def _requeue_all_queued_jobs_async() -> dict:
+    """Re-queue Celery tasks for ALL downloads in 'queued' status.
+
+    This is useful on scheduler startup to recover from Redis DB mismatch
+    or other issues that caused Celery tasks to be lost.
+
+    Only one download per source is re-queued to prevent duplicate work.
+
+    Returns:
+        Dict with count of re-queued jobs
+    """
+    # Import here to avoid circular imports
+    from app.tasks.downloads import download_source
+
+    session_factory, engine = get_async_session()
+
+    try:
+        async with session_factory() as session:
+            # Find all queued jobs, ordered by creation time (oldest first)
+            query = select(Download).where(
+                Download.status == DownloadStatus.QUEUED
+            ).order_by(Download.created_at.asc())
+            result = await session.execute(query)
+            queued_jobs = result.scalars().all()
+
+            # Track which sources we've already queued to avoid duplicates
+            queued_source_ids = set()
+            requeued_count = 0
+            skipped_duplicates = 0
+
+            for job in queued_jobs:
+                # Skip if we've already queued a job for this source
+                if job.source_id in queued_source_ids:
+                    # Mark duplicate as skipped so it won't be checked again
+                    job.status = DownloadStatus.SKIPPED
+                    job.error_message = "Duplicate queued job - another job for this source was re-queued"
+                    skipped_duplicates += 1
+                    logger.info(f"Marked duplicate job {job.id} as SKIPPED (source {job.source_id} already re-queued)")
+                    continue
+
+                try:
+                    download_source.delay(job.source_id, download_id=job.id)
+                    queued_source_ids.add(job.source_id)
+                    requeued_count += 1
+                    logger.info(f"Re-queued job {job.id} (source_id: {job.source_id}) on startup")
+                except Exception as e:
+                    logger.error(f"Failed to re-queue job {job.id}: {e}")
+
+            # Commit the status changes for skipped duplicates
+            if skipped_duplicates > 0:
+                await session.commit()
+
+            if requeued_count > 0:
+                logger.info(f"Re-queued {requeued_count} queued jobs on startup (marked {skipped_duplicates} duplicates as SKIPPED)")
+
+            return {"requeued_count": requeued_count, "skipped_duplicates": skipped_duplicates}
+
+    finally:
+        await _cleanup_engine(engine)
+
+
+@celery_app.task(name="tasks.requeue_stale_jobs")
+def requeue_stale_jobs(threshold_minutes: int = None) -> dict:
+    """Celery task to re-queue stale queued jobs.
+
+    Jobs that have been in 'queued' status for longer than the threshold
+    are considered lost (Celery task missing) and re-queued.
+    """
+    logger.info("Checking for stale queued jobs...")
+    result = asyncio.run(_requeue_stale_queued_jobs_async(threshold_minutes))
+    return result
+
+
+def requeue_all_queued_jobs_sync() -> dict:
+    """Synchronous function to re-queue all queued jobs.
+
+    Called on scheduler startup to recover lost Celery tasks.
+    """
+    logger.info("Re-queuing all queued jobs on scheduler startup...")
+    result = asyncio.run(_requeue_all_queued_jobs_async())
+    return result