changes to task scheduling and readability change on downloads view
This commit is contained in:
Bryan Van Deusen
@@ -59,10 +59,13 @@ async def _get_db_setting(session: AsyncSession, key: str, default=None):
|
||||
setting = result.scalar()
|
||||
|
||||
if setting:
|
||||
logger.debug(f"DB setting '{key}' found: {setting.value} (type: {type(setting.value).__name__})")
|
||||
return setting.value
|
||||
|
||||
# Fall back to DEFAULT_SETTINGS, then to provided default
|
||||
return DEFAULT_SETTINGS.get(key, default)
|
||||
fallback = DEFAULT_SETTINGS.get(key, default)
|
||||
logger.debug(f"DB setting '{key}' not found, using fallback: {fallback}")
|
||||
return fallback
|
||||
|
||||
|
||||
def _get_db_setting_sync(key: str, default=None):
|
||||
@@ -138,6 +141,28 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
)
|
||||
download = result.scalar()
|
||||
if download:
|
||||
# Guard against race conditions: only proceed if still QUEUED
|
||||
if download.status != DownloadStatus.QUEUED:
|
||||
logger.info(f"Download {download_id} is no longer QUEUED (status: {download.status}), skipping")
|
||||
return {"source_id": source_id, "download_id": download_id, "status": "skipped", "reason": "already_processed"}
|
||||
|
||||
# Also check if source already has another RUNNING download
|
||||
running_check = await session.execute(
|
||||
select(Download.id).where(
|
||||
Download.source_id == source_id,
|
||||
Download.status == DownloadStatus.RUNNING,
|
||||
Download.id != download_id
|
||||
)
|
||||
)
|
||||
running_download_id = running_check.scalar()
|
||||
if running_download_id:
|
||||
# Mark this download as skipped since another is already running
|
||||
download.status = DownloadStatus.SKIPPED
|
||||
download.error_message = f"Skipped - another download ({running_download_id}) already running for this source"
|
||||
await session.commit()
|
||||
logger.info(f"Source {source_id} already has a running download, marked {download_id} as SKIPPED")
|
||||
return {"source_id": source_id, "download_id": download_id, "status": "skipped", "reason": "source_already_running"}
|
||||
|
||||
download.status = DownloadStatus.RUNNING
|
||||
download.started_at = utcnow()
|
||||
else:
|
||||
@@ -145,6 +170,17 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
download = None
|
||||
|
||||
if not download_id or download is None:
|
||||
# Check if source already has an active download before creating new one
|
||||
active_check = await session.execute(
|
||||
select(Download.id).where(
|
||||
Download.source_id == source_id,
|
||||
Download.status.in_([DownloadStatus.QUEUED, DownloadStatus.RUNNING])
|
||||
)
|
||||
)
|
||||
if active_check.scalar():
|
||||
logger.info(f"Source {source_id} already has an active download, skipping new download creation")
|
||||
return {"source_id": source_id, "status": "skipped", "reason": "source_already_active"}
|
||||
|
||||
download = Download(
|
||||
source_id=source_id,
|
||||
url=source_url,
|
||||
@@ -317,10 +353,12 @@ async def _scheduled_check_async() -> dict:
|
||||
|
||||
# Get global schedule interval from database settings
|
||||
# This is the primary interval that controls how often sources are checked
|
||||
global_interval = await _get_db_setting(
|
||||
session, "download.schedule_interval", 3600 # Default 1 hour
|
||||
raw_interval = await _get_db_setting(
|
||||
session, "download.schedule_interval", 28800 # Default 8 hours
|
||||
)
|
||||
logger.debug(f"Using global schedule interval: {global_interval} seconds")
|
||||
# Ensure we have an integer value (JSONB might return different types)
|
||||
global_interval = int(raw_interval) if raw_interval is not None else 28800
|
||||
logger.info(f"Schedule check: using interval {global_interval}s (raw value: {raw_interval}, type: {type(raw_interval).__name__})")
|
||||
|
||||
# Find sources due for checking:
|
||||
# - enabled = True
|
||||
@@ -352,15 +390,21 @@ async def _scheduled_check_async() -> dict:
|
||||
# Filter by global schedule interval from Settings
|
||||
# All sources use the same interval - no per-source overrides
|
||||
due_sources = []
|
||||
not_due_count = 0
|
||||
for source in sources:
|
||||
if source.last_check is None:
|
||||
due_sources.append(source)
|
||||
logger.debug(f"Source {source.id} ({source.platform}) - never checked, marking due")
|
||||
else:
|
||||
time_since_check = (now - source.last_check).total_seconds()
|
||||
if time_since_check >= global_interval:
|
||||
due_sources.append(source)
|
||||
logger.debug(f"Source {source.id} ({source.platform}) - last check {time_since_check:.0f}s ago >= {global_interval}s, marking due")
|
||||
else:
|
||||
not_due_count += 1
|
||||
logger.debug(f"Source {source.id} ({source.platform}) - last check {time_since_check:.0f}s ago < {global_interval}s, not due yet")
|
||||
|
||||
logger.info(f"Found {len(due_sources)} sources due for checking")
|
||||
logger.info(f"Schedule check: {len(due_sources)} sources due, {not_due_count} sources not due (interval: {global_interval}s)")
|
||||
|
||||
# Get source IDs that already have a queued or running download
|
||||
due_source_ids = [s.id for s in due_sources]
|
||||
@@ -483,7 +527,14 @@ def scheduled_check():
|
||||
|
||||
This is called periodically by Celery Beat.
|
||||
"""
|
||||
return asyncio.run(_scheduled_check_async())
|
||||
logger.info("=== SCHEDULED_CHECK TASK STARTING ===")
|
||||
try:
|
||||
result = asyncio.run(_scheduled_check_async())
|
||||
logger.info(f"=== SCHEDULED_CHECK TASK COMPLETE: {result} ===")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.exception(f"=== SCHEDULED_CHECK TASK FAILED: {e} ===")
|
||||
raise
|
||||
|
||||
|
||||
@celery_app.task
|
||||
|
||||
Reference in New Issue
Block a user