feat(translation): acceptance gate on Interpreter detection confidence (#1376) #208

Merged
bvandeusen merged 2 commits from dev into main 2026-07-09 17:58:22 -04:00
4 changed files with 147 additions and 3 deletions
+48 -3
View File
@@ -6,6 +6,11 @@ viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync),
same pattern as the other backfill sweeps. No-op unless translation is enabled,
a base URL is set, and the service is healthy.
Curator does NO language detection of its own: each field's translation is
accepted or rejected purely on Interpreter's reported engine + detected language
+ confidence (see ``_accept``) — a short English title Interpreter mis-labels as
a European language scores below the latin-script floor and is kept as-is.
Two entry points:
- ``translate_posts`` — the periodic untranslated sweep (every 8h;
translated_source_lang IS NULL only), one bounded chunk per fire. The Settings
@@ -45,6 +50,25 @@ _RETRANSLATE_COUNTDOWN = 5
_INTERRUPT_BACKOFF = 60
_INTERRUPT_BACKOFF_MAX = 900
# --- Translation acceptance gate (Scribe rule 133; contract note #1347) -------
# Curator does NO language detection of its own — it accepts or rejects a
# translation purely on Interpreter's reported detection. CJK is script-detected
# (kana/hangul/kanji) and reliably high-confidence, so ja/ko/zh are trusted
# outright (pure-kanji Japanese can even come back labelled zh ~0.75, still
# correctly translated). Latin-script detection is a real statistical
# probability, so a short English title Interpreter mis-labels as a European
# language scores low; require it to clear this floor, else keep the original.
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
# real langdetect confidence: genuine German detected at 1.0, while a correctly-
# detected but ambiguous latin string dipped to 0.86 — so the floor sits at 0.80,
# below that band, to keep legitimate ambiguous non-English while still rejecting
# genuinely-unsure guesses. Real langdetect also fixed the original mis-flag at
# the source (short English titles now detect as English → passthrough), so this
# floor is a safety net, not the primary fix. Re-tune via the "Test translation"
# box (send fresh text — cache hits report 1.0).
_CJK_LANGS = frozenset({"ja", "ko", "zh"})
_MIN_LATIN_CONFIDENCE = 0.80
def _interrupt_backoff(retry_after) -> int:
"""Seconds to wait before resuming after an Interpreter interruption: the
@@ -272,18 +296,39 @@ def _summary(status: str, translated: int, scanned: int) -> str:
return f"translated={translated} scanned={scanned}"
def _accept(language: str, confidence) -> bool:
"""Should curator store Interpreter's translation, or keep the original?
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
133): CJK languages are trusted regardless of the reported number; a
latin-script detection must clear ``_MIN_LATIN_CONFIDENCE``. A missing
confidence fails OPEN (accept), so a service that omits the field never causes
translations to be silently dropped."""
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
return True
if confidence is None:
return True
return confidence >= _MIN_LATIN_CONFIDENCE
def _translate_field(text: str, base_url: str, target: str):
"""Translate ONE field independently. Returns (translated, source_lang,
engine_version), or (None, None, None) when there's nothing to store — empty
text, already the target language, or a passthrough (engine "none"). Per-field
(not aggregate first-item) detection so a non-English description still gets
translated when the title is already English (mixed-language posts)."""
text, already the target language, a passthrough (engine "none"), or a
latin-script detection the acceptance gate rejects as too low-confidence (kept
as the original). Per-field (not aggregate first-item) detection so a
non-English description still gets translated when the title is already
English (mixed-language posts)."""
if not text:
return None, None, None
res = ic.translate([text], base_url=base_url, target=target)
detected = res["detected_lang"] or target
if detected == target or res["engine"] == "none":
return None, None, None
# Trust only Interpreter's own detection (Scribe rule 133): keep the original
# when a latin-script detection doesn't clear the confidence floor, so a short
# English title mis-labelled as e.g. German isn't rewritten into the archive.
if not _accept(detected, res.get("detected_confidence")):
return None, None, None
return res["translations"][0], detected, res["engine_version"]
+3
View File
@@ -38,6 +38,9 @@ def test_translate_maps_batch_in_order(monkeypatch):
assert out["engine_version"] == "ollama:x:12b"
assert captured["json"]["q"] == ["ねこが可愛い", "金髪ギャル!"]
assert captured["json"]["target"] == "en"
# Source MUST stay "auto": an explicit source makes Interpreter skip detection
# and report confidence 1.0, which would defeat the acceptance gate.
assert captured["json"]["source"] == "auto"
def test_translate_passthrough_unchanged(monkeypatch):
+62
View File
@@ -252,6 +252,68 @@ def test_translate_posts_beat_single_chunk_no_tail_chase(db_sync, monkeypatch):
assert calls == [] # no self-re-enqueue
def _mock_lang(monkeypatch, *, lang, conf, ver="v1"):
# Interpreter is up and returns a translation with the given detected language
# + confidence, so the acceptance gate can be exercised end-to-end.
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
monkeypatch.setattr(
"backend.app.tasks.translation.ic.translate",
lambda texts, **k: {
"translations": [f"EN:{t}" for t in texts],
"detected_lang": lang, "detected_confidence": conf,
"engine": "llm", "engine_version": ver,
},
)
def test_translate_posts_rejects_low_confidence_latin(db_sync, monkeypatch):
# A short English title Interpreter mis-labels as German with LOW confidence is
# NOT stored — the original is kept, and the post is marked handled (source ==
# target) so the sweep won't revisit it.
_patch(monkeypatch, db_sync)
_mock_lang(monkeypatch, lang="de", conf=0.42)
a = _artist(db_sync)
p = _post(db_sync, a.id, title="Nami Heroines - WIP Part 1", ext="p1")
_enable(db_sync)
db_sync.commit()
translate_posts()
db_sync.refresh(p)
assert p.post_title_translated is None # mis-flag rejected, kept
assert p.translated_source_lang == "en" # marked handled (== target)
def test_translate_posts_accepts_high_confidence_latin(db_sync, monkeypatch):
# A confident latin-script detection (real German) clears the floor → stored.
_patch(monkeypatch, db_sync)
_mock_lang(monkeypatch, lang="de", conf=0.98)
a = _artist(db_sync)
p = _post(db_sync, a.id, title="Das Mädchen mit dem Perlenohrring", ext="p2")
_enable(db_sync)
db_sync.commit()
translate_posts()
db_sync.refresh(p)
assert p.post_title_translated == "EN:Das Mädchen mit dem Perlenohrring"
assert p.translated_source_lang == "de"
def test_translate_posts_accepts_low_confidence_cjk(db_sync, monkeypatch):
# CJK is script-detected and trusted regardless of the reported number — a
# zh 0.75 (e.g. pure-kanji Japanese) is still translated, not rejected.
_patch(monkeypatch, db_sync)
_mock_lang(monkeypatch, lang="zh", conf=0.75)
a = _artist(db_sync)
p = _post(db_sync, a.id, title="猫が可愛い", ext="p3")
_enable(db_sync)
db_sync.commit()
translate_posts()
db_sync.refresh(p)
assert p.post_title_translated == "EN:猫が可愛い"
assert p.translated_source_lang == "zh"
def _mock_new_model(monkeypatch, *, ver="v2"):
"""Interpreter is up and translates with a NEW engine version."""
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
+34
View File
@@ -0,0 +1,34 @@
"""Translation acceptance gate (#1376) — pure unit tests for ``_accept``, which
decides whether curator stores Interpreter's translation or keeps the original.
Curator does no detection of its own; it only thresholds Interpreter's reported
detected language + confidence (Scribe rule 133). No DB, no network."""
from backend.app.tasks.translation import _MIN_LATIN_CONFIDENCE, _accept
def test_accept_trusts_cjk_regardless_of_confidence():
# ja/ko/zh are script-detected — accepted even at a low reported number
# (pure-kanji Japanese can come back labelled zh ~0.75, still translated).
assert _accept("ja", None) is True
assert _accept("ja", 0.10) is True
assert _accept("ko", 0.99) is True
assert _accept("zh", 0.75) is True
assert _accept("zh-CN", 0.20) is True # region suffix is still CJK
def test_accept_requires_floor_for_latin():
assert _accept("de", 0.98) is True
assert _accept("de", 0.86) is True # calibrated: ambiguous-but-correct German stays
assert _accept("de", _MIN_LATIN_CONFIDENCE) is True # exactly at floor
assert _accept("de", _MIN_LATIN_CONFIDENCE - 0.01) is False
assert _accept("fr", 0.40) is False # short mis-flag
def test_accept_is_case_insensitive():
assert _accept("JA", None) is True
assert _accept("DE", 0.40) is False
def test_accept_missing_confidence_fails_open():
# No confidence reported → don't silently drop a translation.
assert _accept("de", None) is True
assert _accept("", None) is True