diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 0576d85..eae20e6 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -61,7 +61,33 @@ def make_celery() -> Celery: # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, worker_prefetch_multiplier=1, + # Broker resilience (2026-06-24): a swarm overlay-network blip after a + # redeploy left Redis healthy but transiently unreachable, and a worker + # starting in that window crash-looped on the initial broker connect + # (kombu OperationalError) instead of waiting it out — needing a manual + # Redis reset to recover. Retry the broker FOREVER (None) on startup and + # at runtime so a transient outage self-heals when routing returns, + # rather than the worker exiting. broker_connection_retry_on_startup=True, + broker_connection_retry=True, + broker_connection_max_retries=None, + # Redis-transport socket options (apply to the BROKER connection): a + # short connect timeout + TCP keepalive so a dead/blocked socket is + # noticed and retried, and a periodic health check that proactively + # reconnects a live worker through a network hiccup. + broker_transport_options={ + "socket_connect_timeout": 5, + "socket_timeout": 30, + "socket_keepalive": True, + "retry_on_timeout": True, + "health_check_interval": 30, + }, + # Same hardening for the Redis RESULT backend (separate connection pool). + redis_socket_connect_timeout=5, + redis_socket_timeout=30, + redis_socket_keepalive=True, + redis_retry_on_timeout=True, + redis_backend_health_check_interval=30, beat_schedule={ "recover-interrupted-tasks": { "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", diff --git a/frontend/src/components/modal/SuggestionsPanel.vue b/frontend/src/components/modal/SuggestionsPanel.vue index 8b0ce45..0e40779 100644 --- a/frontend/src/components/modal/SuggestionsPanel.vue +++ b/frontend/src/components/modal/SuggestionsPanel.vue @@ -12,7 +12,11 @@ No suggestions above threshold. - +