bvandeusen
7a4de7278d
tasks/translation.py — translate_posts: picks untranslated posts (title OR description non-empty), per-post [title, description] batch via the Interpreter client, stores translations + detected lang + engine_version; passthrough / already-target posts are marked handled with no stored translation. 503 or a connection error interrupts (retry next cycle), 400 stops (fix config), per-post commit keeps progress; wall-clock bounded. Wired into celery (maintenance_long lane) + a daily beat. No-op unless enabled + base URL set + healthy. GET /settings/translation/status + POST .../run for the Settings card. Task tests (stubbed client, monkeypatched session). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
102 lines
4.4 KiB
Python
102 lines
4.4 KiB
Python
"""Post-text translation Celery task (milestone 143).
|
|
|
|
Backfills non-English post title/description to the target language via the
|
|
self-hosted Interpreter LAN service, storing the result + engine_version so
|
|
viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync),
|
|
same pattern as the other backfill sweeps. No-op unless translation is enabled,
|
|
a base URL is set, and the service is healthy.
|
|
"""
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
|
|
from celery.exceptions import SoftTimeLimitExceeded
|
|
from sqlalchemy import or_, select
|
|
|
|
from ..celery_app import celery
|
|
from ..models import ImportSettings, Post
|
|
from ..services import interpreter_client as ic
|
|
from ..utils.text import html_to_plain
|
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
# Bound one run's work so it commits progress rather than dying at the time
|
|
# limit; the rest resumes next cycle (idempotent — only untranslated posts are
|
|
# picked, so an interrupted run just re-runs = rule-89 recovery).
|
|
_MAX_POSTS_PER_RUN = 300
|
|
|
|
|
|
@celery.task(
|
|
name="backend.app.tasks.translation.translate_posts",
|
|
soft_time_limit=1800, time_limit=2100,
|
|
)
|
|
def translate_posts() -> str:
|
|
"""Translate untranslated non-English post title/description (default target
|
|
en) via Interpreter. Per-post [title, description] batch → per-post detected
|
|
language. Passthrough / already-target posts are marked handled (source ==
|
|
target) with no stored translation. 503 or a connection error interrupts the
|
|
run (retry next cycle); 400 stops it (fix config); posts done so far stay
|
|
committed. No-op unless configured + healthy. Returns a summary string."""
|
|
SessionLocal = _sync_session_factory()
|
|
with SessionLocal() as session:
|
|
cfg = ImportSettings.load_sync(session)
|
|
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
|
|
return "disabled"
|
|
base_url = cfg.interpreter_base_url.strip()
|
|
target = (cfg.translation_target_lang or "en").strip() or "en"
|
|
if not ic.health(base_url):
|
|
return "interpreter unavailable"
|
|
|
|
posts = session.execute(
|
|
select(Post)
|
|
.where(Post.translated_source_lang.is_(None))
|
|
.where(or_(
|
|
Post.post_title.is_not(None), Post.description.is_not(None),
|
|
))
|
|
.limit(_MAX_POSTS_PER_RUN)
|
|
).scalars().all()
|
|
|
|
translated = 0
|
|
try:
|
|
for post in posts:
|
|
translated += _translate_one(session, post, base_url, target)
|
|
session.commit() # per-post → an interrupted run keeps progress
|
|
except ic.InterpreterUnavailable:
|
|
session.rollback()
|
|
return f"interrupted (service down) — translated={translated}"
|
|
except ic.InterpreterBadRequest as e:
|
|
session.rollback()
|
|
log.warning("translate_posts bad request: %s", e)
|
|
return f"stopped (bad request) — translated={translated}"
|
|
except SoftTimeLimitExceeded:
|
|
return f"time limit — translated={translated}"
|
|
return f"translated={translated} scanned={len(posts)}"
|
|
|
|
|
|
def _translate_one(session, post, base_url: str, target: str) -> int:
|
|
"""Translate one post's title/description in place. Returns 1 if it stored a
|
|
translation, 0 for passthrough/empty (which still marks the post handled)."""
|
|
title = (post.post_title or "").strip()
|
|
desc = (html_to_plain(post.description) if post.description else "") or ""
|
|
desc = desc.strip()
|
|
fields: list[tuple[str, str]] = []
|
|
if title:
|
|
fields.append(("title", title))
|
|
if desc:
|
|
fields.append(("desc", desc))
|
|
if not fields:
|
|
post.translated_source_lang = target # nothing to translate → handled
|
|
return 0
|
|
res = ic.translate([t for _, t in fields], base_url=base_url, target=target)
|
|
detected = res["detected_lang"] or target
|
|
if detected == target or res["engine"] == "none":
|
|
post.translated_source_lang = target # already target language → handled
|
|
return 0
|
|
by_field = {f: res["translations"][i] for i, (f, _) in enumerate(fields)}
|
|
post.post_title_translated = by_field.get("title")
|
|
post.description_translated = by_field.get("desc")
|
|
post.translated_source_lang = detected
|
|
post.translation_engine_version = res["engine_version"]
|
|
post.translated_at = datetime.now(UTC)
|
|
return 1
|