c64261593d
Deploys (docker SIGTERM→SIGKILL, default 10s) were killing Celery jobs mid-flight. Give in-flight work room to drain and make interrupted work resume cleanly instead of stalling. - docker-compose.yml: stop_grace_period per lane (web 30s / worker 90s / scheduler 60s / maintenance-long 180s / ml-worker 120s) so warm shutdown can actually drain before SIGKILL. - celery_app.py: task_reject_on_worker_lost=True — a task killed past the grace window is re-queued (safe: idempotent + chunked, recovery sweeps re-drive stragglers). - interpreter_client.py: map 429/5xx (502/503/504) → InterpreterUnavailable and parse Retry-After (delta-seconds or HTTP-date); a draining Interpreter behind a reverse proxy no longer raises an opaque HTTPError. - translation.py: thread retry_after out of _translate_batch; retranslate_posts resumes after the Retry-After hint (or 60s default, capped 900s) on an interrupt with _reset_done=True, self-terminating via the health gate. - tests: 429/5xx mapping + Retry-After parse; interrupt-resume + default backoff. No migration. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
131 lines
5.2 KiB
Python
131 lines
5.2 KiB
Python
"""Interpreter translation client (milestone 143) — a thin SYNC wrapper over the
|
|
self-hosted Interpreter LAN service (LibreTranslate-compatible `/v1/translate`).
|
|
|
|
Sync (requests) because the only caller is the sync celery translate sweep, and
|
|
it mirrors FC's other platform clients. The service knows nothing about Curator —
|
|
all Curator logic stays here. base_url is operator-configured (empty until set;
|
|
behind a reverse proxy). Full API contract: Scribe note #1347.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
import requests
|
|
|
|
|
|
class InterpreterUnavailable(Exception):
|
|
"""The translation engine is down / unreachable / draining — a connection
|
|
error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
|
|
the service restarts). Retry later, don't drop the item. ``retry_after`` holds
|
|
the server's Retry-After hint in seconds when it sent one, so the caller can
|
|
back off exactly as long as it's asked to."""
|
|
|
|
def __init__(self, message: str, *, retry_after: float | None = None):
|
|
super().__init__(message)
|
|
self.retry_after = retry_after
|
|
|
|
|
|
class InterpreterBadRequest(Exception):
|
|
"""Bad request params (HTTP 400)."""
|
|
|
|
|
|
def _url(base_url: str, path: str) -> str:
|
|
return f"{base_url.rstrip('/')}{path}"
|
|
|
|
|
|
def _parse_retry_after(resp) -> float | None:
|
|
"""Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
|
|
non-negative seconds, or None if absent/unparseable — so a gracefully-
|
|
draining Interpreter can tell Curator exactly how long to wait before it
|
|
tries again."""
|
|
raw = (resp.headers.get("Retry-After") or "").strip()
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return max(0.0, float(int(raw))) # delta-seconds form
|
|
except ValueError:
|
|
pass
|
|
try: # HTTP-date form
|
|
when = parsedate_to_datetime(raw)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if when is None:
|
|
return None
|
|
if when.tzinfo is None:
|
|
when = when.replace(tzinfo=timezone.utc)
|
|
return max(0.0, (when - datetime.now(timezone.utc)).total_seconds())
|
|
|
|
|
|
def health(base_url: str, *, timeout: float = 5.0) -> bool:
|
|
"""True iff the Interpreter LLM engine is up. Any error (unset URL, network,
|
|
non-200, engine down) → False, so the sweep just no-ops rather than raising."""
|
|
if not base_url:
|
|
return False
|
|
try:
|
|
r = requests.get(_url(base_url, "/v1/health"), timeout=timeout)
|
|
except requests.RequestException:
|
|
return False
|
|
if r.status_code != 200:
|
|
return False
|
|
engines = (r.json() or {}).get("engines") or {}
|
|
return bool(engines.get("llm"))
|
|
|
|
|
|
def translate(
|
|
texts: list[str], *, base_url: str, target: str = "en",
|
|
source: str = "auto", timeout: float = 120.0,
|
|
) -> dict:
|
|
"""Translate a batch. Returns::
|
|
|
|
{"translations": [str, ...], # SAME order & length as `texts`
|
|
"detected_lang": str | None, # aggregate: describes the FIRST item
|
|
"engine": str | None,
|
|
"engine_version": str | None}
|
|
|
|
Interpreter batch metadata is aggregate (first item only) — fine for one
|
|
post's ``[title, description]`` batch since they share a language. Passthrough
|
|
items (already target-language / emoji-only) come back UNCHANGED in their
|
|
slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
|
|
later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
|
|
#1347.)
|
|
"""
|
|
if not texts:
|
|
return {"translations": [], "detected_lang": None,
|
|
"engine": None, "engine_version": None}
|
|
try:
|
|
r = requests.post(
|
|
_url(base_url, "/v1/translate"),
|
|
json={"q": list(texts), "source": source,
|
|
"target": target, "engine": "auto"},
|
|
timeout=timeout,
|
|
)
|
|
except requests.RequestException as e:
|
|
raise InterpreterUnavailable(str(e)) from e
|
|
if r.status_code == 400:
|
|
raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
|
|
# A gracefully-draining service — often behind a reverse proxy — returns 429
|
|
# or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
|
|
# one as "unavailable, retry later" and honour any Retry-After it sends, so a
|
|
# rolling Interpreter restart interrupts the sweep cleanly (instead of raising
|
|
# an opaque HTTPError) and resumes exactly when the service says it's ready.
|
|
if r.status_code == 429 or r.status_code >= 500:
|
|
raise InterpreterUnavailable(
|
|
f"interpreter unavailable (HTTP {r.status_code})",
|
|
retry_after=_parse_retry_after(r),
|
|
)
|
|
r.raise_for_status()
|
|
body = r.json() or {}
|
|
out = body.get("translatedText")
|
|
# q was an array → translatedText is an array (same order & length). Guard a
|
|
# scalar reply just in case (shouldn't happen for an array q).
|
|
if not isinstance(out, list):
|
|
out = [out]
|
|
interp = body.get("interpreter") or {}
|
|
return {
|
|
"translations": out,
|
|
"detected_lang": (body.get("detectedLanguage") or {}).get("language"),
|
|
"engine": interp.get("engine"),
|
|
"engine_version": interp.get("engine_version"),
|
|
}
|