feat(ops): graceful shutdown — worker stop-grace + Interpreter drain resilience
Deploys (docker SIGTERM→SIGKILL, default 10s) were killing Celery jobs mid-flight. Give in-flight work room to drain and make interrupted work resume cleanly instead of stalling. - docker-compose.yml: stop_grace_period per lane (web 30s / worker 90s / scheduler 60s / maintenance-long 180s / ml-worker 120s) so warm shutdown can actually drain before SIGKILL. - celery_app.py: task_reject_on_worker_lost=True — a task killed past the grace window is re-queued (safe: idempotent + chunked, recovery sweeps re-drive stragglers). - interpreter_client.py: map 429/5xx (502/503/504) → InterpreterUnavailable and parse Retry-After (delta-seconds or HTTP-date); a draining Interpreter behind a reverse proxy no longer raises an opaque HTTPError. - translation.py: thread retry_after out of _translate_batch; retranslate_posts resumes after the Retry-After hint (or 60s default, capped 900s) on an interrupt with _reset_done=True, self-terminating via the health gate. - tests: 429/5xx mapping + Retry-After parse; interrupt-resume + default backoff. No migration. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -6,9 +6,10 @@ from backend.app.services import interpreter_client as ic
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, status_code=200, payload=None):
|
||||
def __init__(self, status_code=200, payload=None, headers=None):
|
||||
self.status_code = status_code
|
||||
self._payload = payload or {}
|
||||
self.headers = headers or {}
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
@@ -66,6 +67,38 @@ def test_translate_connection_error_raises_unavailable(monkeypatch):
|
||||
ic.translate(["x"], base_url="http://i.lan")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [429, 500, 502, 503, 504])
|
||||
def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code):
|
||||
# A gracefully-draining service behind a reverse proxy returns 429/502/503/504
|
||||
# — all mean "retry later", not an opaque error.
|
||||
monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(code, {}))
|
||||
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
||||
ic.translate(["x"], base_url="http://i.lan")
|
||||
assert ei.value.retry_after is None
|
||||
|
||||
|
||||
def test_translate_honours_retry_after_seconds(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ic.requests, "post",
|
||||
lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}),
|
||||
)
|
||||
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
||||
ic.translate(["x"], base_url="http://i.lan")
|
||||
assert ei.value.retry_after == 42
|
||||
|
||||
|
||||
def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch):
|
||||
# HTTP-date form; a past date → non-negative clamp to 0 (deterministic).
|
||||
monkeypatch.setattr(
|
||||
ic.requests, "post",
|
||||
lambda *a, **k: _Resp(
|
||||
503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}),
|
||||
)
|
||||
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
||||
ic.translate(["x"], base_url="http://i.lan")
|
||||
assert ei.value.retry_after == 0.0
|
||||
|
||||
|
||||
def test_translate_400_raises_bad_request(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ic.requests, "post", lambda *a, **k: _Resp(400, {"error": "bad target"})
|
||||
|
||||
Reference in New Issue
Block a user