ongoing polish in queue handling

backend/app/tasks/downloads.py

@@ -15,6 +15,7 @@ from app.models.base import utcnow
 from app.models.source import Source
 from app.models.subscription import Subscription
 from app.models.download import Download, DownloadStatus, ErrorType as DBErrorType
+from app.models.setting import Setting, DEFAULT_SETTINGS
 from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult
 from app.services.credential_manager import CredentialManager
 from app.events import publish_event
@@ -41,6 +42,46 @@ async def _cleanup_engine(engine):
     await engine.dispose()
 
 
+async def _get_db_setting(session: AsyncSession, key: str, default=None):
+    """Get a setting value from the database, falling back to defaults.
+
+    Args:
+        session: Database session
+        key: Setting key (e.g., 'download.rate_limit')
+        default: Fallback if not in database or DEFAULT_SETTINGS
+
+    Returns:
+        The setting value
+    """
+    result = await session.execute(
+        select(Setting).where(Setting.key == key)
+    )
+    setting = result.scalar()
+
+    if setting:
+        return setting.value
+
+    # Fall back to DEFAULT_SETTINGS, then to provided default
+    return DEFAULT_SETTINGS.get(key, default)
+
+
+def _get_db_setting_sync(key: str, default=None):
+    """Synchronously get a setting value from the database.
+
+    This is used for getting settings in Celery task retry logic.
+    Creates a fresh connection for each call.
+    """
+    async def _fetch():
+        session_factory, engine = get_async_session()
+        try:
+            async with session_factory() as session:
+                return await _get_db_setting(session, key, default)
+        finally:
+            await _cleanup_engine(engine)
+
+    return asyncio.run(_fetch())
+
+
 async def _download_source_async(source_id: int, download_id: int = None) -> dict:
     """Async implementation of source download.
 
@@ -139,6 +180,9 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
                 if not cookies_path and source_platform in ["patreon", "subscribestar"]:
                     logger.warning(f"No credentials for {source_platform}, download may fail")
 
+            # Get rate limit from database settings
+            db_rate_limit = await _get_db_setting(session, "download.rate_limit", settings.download_rate_limit)
+
         # Session is now closed - DB connection released
     finally:
         await _cleanup_engine(engine)
@@ -149,7 +193,7 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
     dl_error = None
 
     try:
-        gdl_service = GalleryDLService()
+        gdl_service = GalleryDLService(rate_limit=db_rate_limit)
         dl_result = await gdl_service.download(
             url=source_url,
             subscription_name=subscription_name,
@@ -271,9 +315,16 @@ async def _scheduled_check_async() -> dict:
         async with session_factory() as session:
             now = utcnow()
 
+            # Get global schedule interval from database settings
+            # This is the primary interval that controls how often sources are checked
+            global_interval = await _get_db_setting(
+                session, "download.schedule_interval", 3600  # Default 1 hour
+            )
+            logger.debug(f"Using global schedule interval: {global_interval} seconds")
+
             # Find sources due for checking:
             # - enabled = True
-            # - last_check is NULL OR (now - last_check) > check_interval
+            # - last_check is NULL OR (now - last_check) > global_interval
             # Order by subscription priority, then by when last checked
             query = (
                 select(Source)
@@ -298,14 +349,15 @@ async def _scheduled_check_async() -> dict:
             result = await session.execute(query)
             sources = result.scalars().all()
 
-            # Filter by check_interval
+            # Filter by global schedule interval from Settings
+            # All sources use the same interval - no per-source overrides
             due_sources = []
             for source in sources:
                 if source.last_check is None:
                     due_sources.append(source)
                 else:
                     time_since_check = (now - source.last_check).total_seconds()
-                    if time_since_check >= source.check_interval:
+                    if time_since_check >= global_interval:
                         due_sources.append(source)
 
             logger.info(f"Found {len(due_sources)} sources due for checking")
@@ -403,7 +455,7 @@ async def _cleanup_old_downloads_async() -> dict:
 
 # Celery tasks that wrap async functions
 
-@celery_app.task(bind=True, max_retries=3)
+@celery_app.task(bind=True)
 def download_source(self, source_id: int, download_id: int = None):
     """Download new content from a single source.
 
@@ -415,8 +467,14 @@ def download_source(self, source_id: int, download_id: int = None):
         return asyncio.run(_download_source_async(source_id, download_id=download_id))
     except Exception as e:
         logger.exception(f"Task failed for source {source_id}: {e}")
-        # Retry with exponential backoff
-        raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries))
+        # Get retry count from database settings
+        max_retries = _get_db_setting_sync("download.retry_count", 3)
+        if self.request.retries < max_retries:
+            # Retry with exponential backoff
+            raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries), max_retries=max_retries)
+        else:
+            logger.error(f"Max retries ({max_retries}) exceeded for source {source_id}")
+            raise
 
 
 @celery_app.task
@@ -437,7 +495,7 @@ def cleanup_old_downloads():
     return asyncio.run(_cleanup_old_downloads_async())
 
 
-@celery_app.task(bind=True, max_retries=2)
+@celery_app.task(bind=True)
 def process_download(self, download_id: int):
     """Process a single download record (for retries).
 
@@ -469,4 +527,10 @@ def process_download(self, download_id: int):
         return asyncio.run(_process())
     except Exception as e:
         logger.exception(f"Task failed for download {download_id}: {e}")
-        raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries))
+        # Get retry count from database settings
+        max_retries = _get_db_setting_sync("download.retry_count", 3)
+        if self.request.retries < max_retries:
+            raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries), max_retries=max_retries)
+        else:
+            logger.error(f"Max retries ({max_retries}) exceeded for download {download_id}")
+            raise