bvandeusen
b85327a79d
A swarm overlay-network blip after the :latest redeploy left Redis healthy but transiently unreachable; a worker starting in that window crash-looped on the initial broker connect (kombu OperationalError) and needed a manual Redis reset to recover. Retry the broker forever on startup + at runtime (broker_connection_max_retries =None), add redis-transport socket options to the broker (short connect timeout, TCP keepalive, retry_on_timeout, periodic health check), and mirror the same on the Redis result backend. Now a transient outage self-heals when overlay routing returns instead of the worker exiting. Test pins the key resilience settings. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XCUHUGQLrBrkgyk1t49kpX
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
"""Smoke test — celery runs the ping task in eager mode (no broker required)."""
|
|
|
|
import pytest
|
|
|
|
from backend.app.celery_app import celery
|
|
from backend.app.tasks.smoke import ping
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def celery_eager():
|
|
celery.conf.task_always_eager = True
|
|
celery.conf.task_eager_propagates = True
|
|
yield
|
|
celery.conf.task_always_eager = False
|
|
|
|
|
|
def test_ping_returns_pong():
|
|
result = ping.delay()
|
|
assert result.get(timeout=1) == "pong"
|
|
|
|
|
|
def test_ping_with_value():
|
|
result = ping.delay("hello")
|
|
assert result.get(timeout=1) == "hello"
|
|
|
|
|
|
def test_broker_resilience_configured():
|
|
# Workers must ride out a transient broker outage (swarm overlay blip after
|
|
# a redeploy left Redis briefly unreachable) instead of crash-looping on the
|
|
# initial connect and needing a manual Redis reset (2026-06-24).
|
|
conf = celery.conf
|
|
assert conf.broker_connection_retry_on_startup is True
|
|
assert conf.broker_connection_max_retries is None # retry forever
|
|
opts = conf.broker_transport_options
|
|
assert opts["socket_keepalive"] is True
|
|
assert opts["socket_connect_timeout"] == 5
|
|
assert opts["health_check_interval"] == 30
|
|
assert conf.redis_retry_on_timeout is True
|