FabledCurator/backend/app/tasks/translation.py

"""Post-text translation Celery tasks (milestone 143 + re-translate m146).

Backfills non-English post title/description to the target language via the
self-hosted Interpreter LAN service, storing the result + engine_version so
viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync),
same pattern as the other backfill sweeps. No-op unless translation is enabled,
a base URL is set, and the service is healthy.

Two entry points:
- ``translate_posts`` — the daily untranslated sweep (translated_source_lang IS
  NULL only); also the Settings "Translate now" button.
- ``retranslate_posts`` — after a model change, resets the stored translation
  columns for a scoped set of posts (all, or a given set of artists) so the
  untranslated selection re-runs them, then chases the tail until drained
  (run-until-done). The Interpreter cache keys on engine_version: a changed
  model genuinely re-translates, an unchanged one is ~1ms cache-fast.
"""
import logging
from datetime import UTC, datetime

from celery.exceptions import SoftTimeLimitExceeded
from sqlalchemy import func, or_, select, update

from ..celery_app import celery
from ..models import ImportSettings, Post
from ..services import interpreter_client as ic
from ..utils.text import html_to_plain
from ._sync_engine import sync_session_factory as _sync_session_factory

log = logging.getLogger(__name__)

# Bound one run's work so it commits progress rather than dying at the time
# limit; the rest resumes next cycle (idempotent — only untranslated posts are
# picked, so an interrupted run just re-runs = rule-89 recovery).
_MAX_POSTS_PER_RUN = 300
# Short gap between run-until-done chunks so a big re-translate finishes on its
# own without monopolising the maintenance_long worker in one uninterrupted run.
_RETRANSLATE_COUNTDOWN = 5


@celery.task(
    name="backend.app.tasks.translation.translate_posts",
    soft_time_limit=1800, time_limit=2100,
)
def translate_posts() -> str:
    """Translate untranslated non-English post title/description (default target
    en) via Interpreter. Per-post [title, description] batch → per-post detected
    language. Passthrough / already-target posts are marked handled (source ==
    target) with no stored translation. 503 or a connection error interrupts the
    run (retry next cycle); 400 stops it (fix config); posts done so far stay
    committed. No-op unless configured + healthy. Returns a summary string."""
    SessionLocal = _sync_session_factory()
    with SessionLocal() as session:
        ready = _translation_config(session)
        if isinstance(ready, str):
            return ready
        base_url, target = ready
        posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
        status, translated = _translate_batch(session, posts, base_url, target)
    return _summary(status, translated, len(posts))


@celery.task(
    name="backend.app.tasks.translation.retranslate_posts",
    soft_time_limit=1800, time_limit=2100,
)
def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
    """Re-translate posts after a model change. On the first call resets the
    stored translation columns to NULL for the scoped posts — all artists when
    ``artist_ids`` is falsy, else ``WHERE artist_id IN artist_ids`` — so the
    normal untranslated selection re-runs them through Interpreter. Then handles
    one bounded chunk and re-enqueues itself until the scoped backlog is drained
    (run-until-done; ``_reset_done`` guards against wiping fresh work on the
    follow-up chunks). No reset happens unless the service is configured +
    healthy, so translations are never cleared when they can't be rebuilt."""
    SessionLocal = _sync_session_factory()
    with SessionLocal() as session:
        ready = _translation_config(session)
        if isinstance(ready, str):
            return ready
        base_url, target = ready

        if not _reset_done:
            n = _reset_translations(session, artist_ids)
            log.info(
                "retranslate_posts: reset %d post(s) (artist_ids=%s)",
                n, artist_ids,
            )

        posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
        status, translated = _translate_batch(session, posts, base_url, target)

        # run-until-done: only chase the tail when the chunk finished cleanly. A
        # service interruption / bad request stops here and leaves the rest NULL
        # for the daily beat or a fresh trigger to resume (rule-89 recovery).
        # Termination is guaranteed: every handled post leaves the NULL set, so
        # the remaining count strictly decreases each chunk.
        if status == "ok":
            remaining = _count_untranslated(session, artist_ids)
            if remaining:
                retranslate_posts.apply_async(
                    (),
                    {"artist_ids": artist_ids, "_reset_done": True},
                    countdown=_RETRANSLATE_COUNTDOWN,
                )
                log.info(
                    "retranslate_posts: %d remaining → re-enqueued", remaining,
                )
    return _summary(status, translated, len(posts))


def _translation_config(session):
    """Resolve (base_url, target) if translation is enabled + healthy, else a
    short status string ("disabled" / "interpreter unavailable") for the caller
    to return. Centralises the guard so translate/retranslate agree exactly."""
    cfg = ImportSettings.load_sync(session)
    if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
        return "disabled"
    base_url = cfg.interpreter_base_url.strip()
    target = (cfg.translation_target_lang or "en").strip() or "en"
    if not ic.health(base_url):
        return "interpreter unavailable"
    return base_url, target


def _untranslated_filter(stmt, artist_ids):
    """Add the untranslated-post predicate (+ optional artist scope) to a
    select/count over Post. Untranslated = translated_source_lang IS NULL with
    some text to translate."""
    stmt = stmt.where(
        Post.translated_source_lang.is_(None)
    ).where(or_(
        Post.post_title.is_not(None), Post.description.is_not(None),
    ))
    if artist_ids:
        stmt = stmt.where(Post.artist_id.in_(artist_ids))
    return stmt


def _select_untranslated(session, artist_ids, limit):
    return session.execute(
        _untranslated_filter(select(Post), artist_ids).limit(limit)
    ).scalars().all()


def _count_untranslated(session, artist_ids) -> int:
    return int(session.execute(
        _untranslated_filter(select(func.count(Post.id)), artist_ids)
    ).scalar_one())


def _reset_translations(session, artist_ids) -> int:
    """Clear the stored translation columns for scoped, already-handled posts so
    the untranslated sweep re-runs them. Only touches rows that were translated
    (translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
    Returns the row count reset (commits it)."""
    stmt = (
        update(Post)
        .where(Post.translated_source_lang.is_not(None))
        .values(
            post_title_translated=None,
            description_translated=None,
            translated_source_lang=None,
            translation_engine_version=None,
            translated_at=None,
        )
    )
    if artist_ids:
        stmt = stmt.where(Post.artist_id.in_(artist_ids))
    n = session.execute(stmt).rowcount
    session.commit()
    return n


def _translate_batch(session, posts, base_url: str, target: str):
    """Translate each post in the chunk with a per-post commit (an interrupted
    run keeps progress). Returns (status, translated) where status is one of
    "ok" / "interrupted" (503/conn) / "stopped" (400) / "timeout"."""
    translated = 0
    for post in posts:
        try:
            translated += _translate_one(session, post, base_url, target)
            session.commit()
        except ic.InterpreterUnavailable:
            session.rollback()
            return "interrupted", translated
        except ic.InterpreterBadRequest as e:
            session.rollback()
            log.warning("translate bad request: %s", e)
            return "stopped", translated
        except SoftTimeLimitExceeded:
            return "timeout", translated
    return "ok", translated


def _summary(status: str, translated: int, scanned: int) -> str:
    if status == "interrupted":
        return f"interrupted (service down) — translated={translated}"
    if status == "stopped":
        return f"stopped (bad request) — translated={translated}"
    if status == "timeout":
        return f"time limit — translated={translated}"
    return f"translated={translated} scanned={scanned}"


def _translate_one(session, post, base_url: str, target: str) -> int:
    """Translate one post's title/description in place. Returns 1 if it stored a
    translation, 0 for passthrough/empty (which still marks the post handled)."""
    title = (post.post_title or "").strip()
    desc = (html_to_plain(post.description) if post.description else "") or ""
    desc = desc.strip()
    fields: list[tuple[str, str]] = []
    if title:
        fields.append(("title", title))
    if desc:
        fields.append(("desc", desc))
    if not fields:
        post.translated_source_lang = target   # nothing to translate → handled
        return 0
    res = ic.translate([t for _, t in fields], base_url=base_url, target=target)
    detected = res["detected_lang"] or target
    if detected == target or res["engine"] == "none":
        post.translated_source_lang = target   # already target language → handled
        return 0
    by_field = {f: res["translations"][i] for i, (f, _) in enumerate(fields)}
    post.post_title_translated = by_field.get("title")
    post.description_translated = by_field.get("desc")
    post.translated_source_lang = detected
    post.translation_engine_version = res["engine_version"]
    post.translated_at = datetime.now(UTC)
    return 1