Bryan Van Deusen
44 lines
1.3 KiB
Python
44 lines
1.3 KiB
Python
"""Celery application configuration."""
|
|
|
|
from celery import Celery
|
|
from celery.schedules import crontab
|
|
|
|
from app.config import get_settings
|
|
|
|
settings = get_settings()
|
|
|
|
celery_app = Celery(
|
|
"gallery_subscriber",
|
|
broker=settings.redis_url,
|
|
backend=settings.redis_url,
|
|
include=["app.tasks.downloads"],
|
|
)
|
|
|
|
# Celery configuration
|
|
celery_app.conf.update(
|
|
task_serializer="json",
|
|
accept_content=["json"],
|
|
result_serializer="json",
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
task_track_started=True,
|
|
task_time_limit=3600, # 1 hour max per task
|
|
worker_prefetch_multiplier=1, # Process one task at a time
|
|
worker_concurrency=settings.download_parallel_limit,
|
|
)
|
|
|
|
# Celery Beat schedule for periodic tasks
|
|
# Note: The scheduler runs frequently (every 60s) but only triggers downloads
|
|
# for sources whose individual check_interval has elapsed. Per-source intervals
|
|
# are stored in the database and can be configured via the UI.
|
|
celery_app.conf.beat_schedule = {
|
|
"check-sources-scheduler": {
|
|
"task": "app.tasks.downloads.scheduled_check",
|
|
"schedule": 60, # Run every 60 seconds to check for due sources
|
|
},
|
|
"cleanup-old-downloads-daily": {
|
|
"task": "app.tasks.downloads.cleanup_old_downloads",
|
|
"schedule": crontab(hour=3, minute=0), # Daily at 3 AM
|
|
},
|
|
}
|