feat(ops): graceful shutdown — worker stop-grace + Interpreter drain resilience

Deploys (docker SIGTERM→SIGKILL, default 10s) were killing Celery jobs
mid-flight. Give in-flight work room to drain and make interrupted work
resume cleanly instead of stalling.

- docker-compose.yml: stop_grace_period per lane (web 30s / worker 90s /
  scheduler 60s / maintenance-long 180s / ml-worker 120s) so warm shutdown
  can actually drain before SIGKILL.
- celery_app.py: task_reject_on_worker_lost=True — a task killed past the
  grace window is re-queued (safe: idempotent + chunked, recovery sweeps
  re-drive stragglers).
- interpreter_client.py: map 429/5xx (502/503/504) → InterpreterUnavailable
  and parse Retry-After (delta-seconds or HTTP-date); a draining Interpreter
  behind a reverse proxy no longer raises an opaque HTTPError.
- translation.py: thread retry_after out of _translate_batch; retranslate_posts
  resumes after the Retry-After hint (or 60s default, capped 900s) on an
  interrupt with _reset_done=True, self-terminating via the health gate.
- tests: 429/5xx mapping + Retry-After parse; interrupt-resume + default backoff.

No migration.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_interpreter_client.py

@@ -6,9 +6,10 @@ from backend.app.services import interpreter_client as ic
 
 
 class _Resp:
-    def __init__(self, status_code=200, payload=None):
+    def __init__(self, status_code=200, payload=None, headers=None):
         self.status_code = status_code
         self._payload = payload or {}
+        self.headers = headers or {}
 
     def json(self):
         return self._payload
@@ -66,6 +67,38 @@ def test_translate_connection_error_raises_unavailable(monkeypatch):
         ic.translate(["x"], base_url="http://i.lan")
 
 
+@pytest.mark.parametrize("code", [429, 500, 502, 503, 504])
+def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code):
+    # A gracefully-draining service behind a reverse proxy returns 429/502/503/504
+    # — all mean "retry later", not an opaque error.
+    monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(code, {}))
+    with pytest.raises(ic.InterpreterUnavailable) as ei:
+        ic.translate(["x"], base_url="http://i.lan")
+    assert ei.value.retry_after is None
+
+
+def test_translate_honours_retry_after_seconds(monkeypatch):
+    monkeypatch.setattr(
+        ic.requests, "post",
+        lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}),
+    )
+    with pytest.raises(ic.InterpreterUnavailable) as ei:
+        ic.translate(["x"], base_url="http://i.lan")
+    assert ei.value.retry_after == 42
+
+
+def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch):
+    # HTTP-date form; a past date → non-negative clamp to 0 (deterministic).
+    monkeypatch.setattr(
+        ic.requests, "post",
+        lambda *a, **k: _Resp(
+            503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}),
+    )
+    with pytest.raises(ic.InterpreterUnavailable) as ei:
+        ic.translate(["x"], base_url="http://i.lan")
+    assert ei.value.retry_after == 0.0
+
+
 def test_translate_400_raises_bad_request(monkeypatch):
     monkeypatch.setattr(
         ic.requests, "post", lambda *a, **k: _Resp(400, {"error": "bad target"})

tests/test_translate_posts.py

@@ -4,6 +4,7 @@ import pytest
 from sqlalchemy import select
 
 from backend.app.models import Artist, ImportSettings, Post
+from backend.app.services import interpreter_client as ic
 from backend.app.tasks.translation import retranslate_posts, translate_posts
 
 pytestmark = pytest.mark.integration
@@ -239,3 +240,61 @@ def test_retranslate_unhealthy_does_not_reset(db_sync, monkeypatch):
     db_sync.refresh(p)
     assert p.translated_source_lang == "ja"
     assert p.post_title_translated == "OLD"
+
+
+def test_retranslate_interrupt_resumes_after_retry_after(db_sync, monkeypatch):
+    # Interpreter drains mid-chunk (health passed, then translate 503s with a
+    # Retry-After): the bulk re-translate re-enqueues itself after the hinted
+    # backoff, with _reset_done=True so it never re-wipes fresh work — instead of
+    # stalling until the daily beat.
+    _patch(monkeypatch, db_sync)
+    monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
+
+    def _drain(texts, **k):
+        raise ic.InterpreterUnavailable("draining", retry_after=42)
+
+    monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain)
+    calls = []
+    monkeypatch.setattr(
+        "backend.app.tasks.translation.retranslate_posts.apply_async",
+        lambda *a, **k: calls.append((a, k)),
+    )
+    a = _artist(db_sync)
+    p = _post(db_sync, a.id, title="ねこ", ext="p1")
+    _mark_translated(p)
+    _enable(db_sync)
+    db_sync.commit()
+
+    assert "interrupted" in retranslate_posts(artist_ids=[a.id])
+    assert len(calls) == 1                             # resume re-enqueued
+    args, kw = calls[0]
+    assert args[1]["_reset_done"] is True
+    assert args[1]["artist_ids"] == [a.id]
+    assert kw["countdown"] == 42                       # honoured the Retry-After
+    db_sync.refresh(p)
+    assert p.translated_source_lang is None            # reset happened, not re-run
+
+
+def test_retranslate_interrupt_default_backoff_without_hint(db_sync, monkeypatch):
+    # No Retry-After header → fall back to the default backoff (60s).
+    _patch(monkeypatch, db_sync)
+    monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
+
+    def _drain(texts, **k):
+        raise ic.InterpreterUnavailable("draining")
+
+    monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain)
+    calls = []
+    monkeypatch.setattr(
+        "backend.app.tasks.translation.retranslate_posts.apply_async",
+        lambda *a, **k: calls.append((a, k)),
+    )
+    a = _artist(db_sync)
+    p = _post(db_sync, a.id, title="ねこ", ext="p1")
+    _mark_translated(p)
+    _enable(db_sync)
+    db_sync.commit()
+
+    retranslate_posts(artist_ids=[a.id])
+    assert len(calls) == 1
+    assert calls[0][1]["countdown"] == 60