feat(ops): graceful shutdown — worker stop-grace + Interpreter drain resilience

Deploys (docker SIGTERM→SIGKILL, default 10s) were killing Celery jobs
mid-flight. Give in-flight work room to drain and make interrupted work
resume cleanly instead of stalling.

- docker-compose.yml: stop_grace_period per lane (web 30s / worker 90s /
  scheduler 60s / maintenance-long 180s / ml-worker 120s) so warm shutdown
  can actually drain before SIGKILL.
- celery_app.py: task_reject_on_worker_lost=True — a task killed past the
  grace window is re-queued (safe: idempotent + chunked, recovery sweeps
  re-drive stragglers).
- interpreter_client.py: map 429/5xx (502/503/504) → InterpreterUnavailable
  and parse Retry-After (delta-seconds or HTTP-date); a draining Interpreter
  behind a reverse proxy no longer raises an opaque HTTPError.
- translation.py: thread retry_after out of _translate_batch; retranslate_posts
  resumes after the Retry-After hint (or 60s default, capped 900s) on an
  interrupt with _reset_done=True, self-terminating via the health gate.
- tests: 429/5xx mapping + Retry-After parse; interrupt-resume + default backoff.

No migration.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/services/interpreter_client.py

@@ -8,12 +8,22 @@ behind a reverse proxy). Full API contract: Scribe note #1347.
 """
 from __future__ import annotations
 
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
 import requests
 
 
 class InterpreterUnavailable(Exception):
-    """The translation engine is down / unreachable (HTTP 503 or a connection
-    error) — retry later, don't drop the item."""
+    """The translation engine is down / unreachable / draining — a connection
+    error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
+    the service restarts). Retry later, don't drop the item. ``retry_after`` holds
+    the server's Retry-After hint in seconds when it sent one, so the caller can
+    back off exactly as long as it's asked to."""
+
+    def __init__(self, message: str, *, retry_after: float | None = None):
+        super().__init__(message)
+        self.retry_after = retry_after
 
 
 class InterpreterBadRequest(Exception):
@@ -24,6 +34,29 @@ def _url(base_url: str, path: str) -> str:
     return f"{base_url.rstrip('/')}{path}"
 
 
+def _parse_retry_after(resp) -> float | None:
+    """Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
+    non-negative seconds, or None if absent/unparseable — so a gracefully-
+    draining Interpreter can tell Curator exactly how long to wait before it
+    tries again."""
+    raw = (resp.headers.get("Retry-After") or "").strip()
+    if not raw:
+        return None
+    try:
+        return max(0.0, float(int(raw)))       # delta-seconds form
+    except ValueError:
+        pass
+    try:                                        # HTTP-date form
+        when = parsedate_to_datetime(raw)
+    except (TypeError, ValueError):
+        return None
+    if when is None:
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=timezone.utc)
+    return max(0.0, (when - datetime.now(timezone.utc)).total_seconds())
+
+
 def health(base_url: str, *, timeout: float = 5.0) -> bool:
     """True iff the Interpreter LLM engine is up. Any error (unset URL, network,
     non-200, engine down) → False, so the sweep just no-ops rather than raising."""
@@ -53,8 +86,9 @@ def translate(
     Interpreter batch metadata is aggregate (first item only) — fine for one
     post's ``[title, description]`` batch since they share a language. Passthrough
     items (already target-language / emoji-only) come back UNCHANGED in their
-    slot. Raises InterpreterUnavailable on 503 / connection error (retry later),
-    InterpreterBadRequest on 400. (Scribe note #1347.)
+    slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
+    later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
+    #1347.)
     """
     if not texts:
         return {"translations": [], "detected_lang": None,
@@ -68,10 +102,18 @@ def translate(
         )
     except requests.RequestException as e:
         raise InterpreterUnavailable(str(e)) from e
-    if r.status_code == 503:
-        raise InterpreterUnavailable("interpreter engine unavailable")
     if r.status_code == 400:
         raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
+    # A gracefully-draining service — often behind a reverse proxy — returns 429
+    # or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
+    # one as "unavailable, retry later" and honour any Retry-After it sends, so a
+    # rolling Interpreter restart interrupts the sweep cleanly (instead of raising
+    # an opaque HTTPError) and resumes exactly when the service says it's ready.
+    if r.status_code == 429 or r.status_code >= 500:
+        raise InterpreterUnavailable(
+            f"interpreter unavailable (HTTP {r.status_code})",
+            retry_after=_parse_retry_after(r),
+        )
     r.raise_for_status()
     body = r.json() or {}
     out = body.get("translatedText")