ongoing polish in queue handling

backend/app/tasks/celery_app.py

@@ -1,12 +1,51 @@
 """Celery application configuration."""
 
+import logging
 from celery import Celery
 from celery.schedules import crontab
 
 from app.config import get_settings
 
+logger = logging.getLogger(__name__)
 settings = get_settings()
 
+
+def _get_parallel_limit_from_db():
+    """Try to get parallel_limit from database, fall back to environment.
+
+    This is called once at Celery startup. Changes require worker restart.
+    """
+    try:
+        import asyncio
+        from sqlalchemy import select
+        from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
+
+        async def _fetch():
+            engine = create_async_engine(settings.async_database_url, echo=False)
+            session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+            try:
+                async with session_factory() as session:
+                    from app.models.setting import Setting, DEFAULT_SETTINGS
+                    result = await session.execute(
+                        select(Setting).where(Setting.key == "download.parallel_limit")
+                    )
+                    setting = result.scalar()
+                    if setting:
+                        return setting.value
+                    return DEFAULT_SETTINGS.get("download.parallel_limit", settings.download_parallel_limit)
+            finally:
+                await engine.dispose()
+
+        return asyncio.run(_fetch())
+    except Exception as e:
+        logger.warning(f"Could not read parallel_limit from database, using environment: {e}")
+        return settings.download_parallel_limit
+
+
+# Get parallel limit (database takes precedence over environment)
+parallel_limit = _get_parallel_limit_from_db()
+logger.info(f"Celery worker concurrency set to: {parallel_limit}")
+
 celery_app = Celery(
     "gallery_subscriber",
     broker=settings.redis_url,
@@ -24,7 +63,7 @@ celery_app.conf.update(
     task_track_started=True,
     task_time_limit=3600,  # 1 hour max per task
     worker_prefetch_multiplier=1,  # Process one task at a time
-    worker_concurrency=settings.download_parallel_limit,
+    worker_concurrency=parallel_limit,
 )
 
 # Celery Beat schedule for periodic tasks
@@ -44,4 +83,32 @@ celery_app.conf.beat_schedule = {
         "task": "tasks.update_storage_stats",
         "schedule": crontab(minute="*/30"),  # Every 30 minutes
     },
+    "reset-orphaned-jobs": {
+        "task": "tasks.reset_orphaned_jobs",
+        "schedule": crontab(minute="*/15"),  # Every 15 minutes
+    },
 }
+
+
+# Worker startup signal - reset any orphaned running jobs
+@celery_app.on_after_configure.connect
+def setup_worker_startup(sender, **kwargs):
+    """Called after Celery is configured but before workers start processing."""
+    pass  # Placeholder - actual reset happens in worker_ready signal
+
+
+from celery.signals import worker_ready
+
+@worker_ready.connect
+def on_worker_ready(sender, **kwargs):
+    """Called when a worker is ready to accept tasks.
+
+    Resets any jobs stuck in 'running' status from previous runs.
+    This handles the case where workers crashed or were restarted.
+    """
+    try:
+        from app.tasks.maintenance import reset_all_running_jobs_sync
+        result = reset_all_running_jobs_sync()
+        logger.info(f"Worker startup: {result}")
+    except Exception as e:
+        logger.error(f"Failed to reset orphaned jobs on worker startup: {e}")