From 7ddf94220db483177e177728a92dd96f7c10f91b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 9 Jul 2026 16:17:56 -0400 Subject: [PATCH 1/2] feat(translation): accept/reject gate on Interpreter's detection confidence (#1376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interpreter now returns a real per-detection confidence (source stays "auto"), so curator can reject the mis-detections it was blindly storing — e.g. a short English title mis-labelled as German and rewritten into the archive. The gate consumes ONLY Interpreter's own reported detection — curator does no language detection of its own (Scribe rule 133): a field is stored when the engine actually translated it AND either the detected language is CJK (script-detected, reliably high — ja/ko/zh trusted outright, incl. pure-kanji Japanese that lands as zh ~0.75) or the reported confidence clears a latin-script floor (_MIN_LATIN_CONFIDENCE = 0.90). A latin detection below the floor keeps the original and marks the post handled; a missing confidence fails open. The client already sent source="auto" and parsed confidence, so this is purely the gate + tests. Tests: pinned interpreter-client test now asserts source stays "auto"; new pure-unit gate tests (CJK trusted / latin floor / case-insensitive / fail-open) in the fast lane; end-to-end reject-low-latin, accept-high-latin, accept-low-cjk sweeps. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CgZP9v2otxVJymiYsnVuMy --- backend/app/tasks/translation.py | 44 +++++++++++++++++++++-- tests/test_interpreter_client.py | 3 ++ tests/test_translate_posts.py | 62 ++++++++++++++++++++++++++++++++ tests/test_translation_gate.py | 33 +++++++++++++++++ 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/test_translation_gate.py diff --git a/backend/app/tasks/translation.py b/backend/app/tasks/translation.py index e287d86..9598afd 100644 --- a/backend/app/tasks/translation.py +++ b/backend/app/tasks/translation.py @@ -6,6 +6,11 @@ viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync), same pattern as the other backfill sweeps. No-op unless translation is enabled, a base URL is set, and the service is healthy. +Curator does NO language detection of its own: each field's translation is +accepted or rejected purely on Interpreter's reported engine + detected language ++ confidence (see ``_accept``) — a short English title Interpreter mis-labels as +a European language scores below the latin-script floor and is kept as-is. + Two entry points: - ``translate_posts`` — the periodic untranslated sweep (every 8h; translated_source_lang IS NULL only), one bounded chunk per fire. The Settings @@ -45,6 +50,18 @@ _RETRANSLATE_COUNTDOWN = 5 _INTERRUPT_BACKOFF = 60 _INTERRUPT_BACKOFF_MAX = 900 +# --- Translation acceptance gate (Scribe rule 133; contract note #1347) ------- +# Curator does NO language detection of its own — it accepts or rejects a +# translation purely on Interpreter's reported detection. CJK is script-detected +# (kana/hangul/kanji) and reliably high-confidence, so ja/ko/zh are trusted +# outright (pure-kanji Japanese can even come back labelled zh ~0.75, still +# correctly translated). Latin-script detection is a real statistical +# probability, so a short English title Interpreter mis-labels as a European +# language scores low; require it to clear this floor, else keep the original. +# Tune the floor against real titles via the Settings "Test translation" box. +_CJK_LANGS = frozenset({"ja", "ko", "zh"}) +_MIN_LATIN_CONFIDENCE = 0.90 + def _interrupt_backoff(retry_after) -> int: """Seconds to wait before resuming after an Interpreter interruption: the @@ -272,18 +289,39 @@ def _summary(status: str, translated: int, scanned: int) -> str: return f"translated={translated} scanned={scanned}" +def _accept(language: str, confidence) -> bool: + """Should curator store Interpreter's translation, or keep the original? + Consumes ONLY Interpreter's own detection (curator does none, Scribe rule + 133): CJK languages are trusted regardless of the reported number; a + latin-script detection must clear ``_MIN_LATIN_CONFIDENCE``. A missing + confidence fails OPEN (accept), so a service that omits the field never causes + translations to be silently dropped.""" + if (language or "").split("-", 1)[0].lower() in _CJK_LANGS: + return True + if confidence is None: + return True + return confidence >= _MIN_LATIN_CONFIDENCE + + def _translate_field(text: str, base_url: str, target: str): """Translate ONE field independently. Returns (translated, source_lang, engine_version), or (None, None, None) when there's nothing to store — empty - text, already the target language, or a passthrough (engine "none"). Per-field - (not aggregate first-item) detection so a non-English description still gets - translated when the title is already English (mixed-language posts).""" + text, already the target language, a passthrough (engine "none"), or a + latin-script detection the acceptance gate rejects as too low-confidence (kept + as the original). Per-field (not aggregate first-item) detection so a + non-English description still gets translated when the title is already + English (mixed-language posts).""" if not text: return None, None, None res = ic.translate([text], base_url=base_url, target=target) detected = res["detected_lang"] or target if detected == target or res["engine"] == "none": return None, None, None + # Trust only Interpreter's own detection (Scribe rule 133): keep the original + # when a latin-script detection doesn't clear the confidence floor, so a short + # English title mis-labelled as e.g. German isn't rewritten into the archive. + if not _accept(detected, res.get("detected_confidence")): + return None, None, None return res["translations"][0], detected, res["engine_version"] diff --git a/tests/test_interpreter_client.py b/tests/test_interpreter_client.py index 453338f..c65f229 100644 --- a/tests/test_interpreter_client.py +++ b/tests/test_interpreter_client.py @@ -38,6 +38,9 @@ def test_translate_maps_batch_in_order(monkeypatch): assert out["engine_version"] == "ollama:x:12b" assert captured["json"]["q"] == ["ねこが可愛い", "金髪ギャル！"] assert captured["json"]["target"] == "en" + # Source MUST stay "auto": an explicit source makes Interpreter skip detection + # and report confidence 1.0, which would defeat the acceptance gate. + assert captured["json"]["source"] == "auto" def test_translate_passthrough_unchanged(monkeypatch): diff --git a/tests/test_translate_posts.py b/tests/test_translate_posts.py index 8358303..1e0c8e8 100644 --- a/tests/test_translate_posts.py +++ b/tests/test_translate_posts.py @@ -252,6 +252,68 @@ def test_translate_posts_beat_single_chunk_no_tail_chase(db_sync, monkeypatch): assert calls == [] # no self-re-enqueue +def _mock_lang(monkeypatch, *, lang, conf, ver="v1"): + # Interpreter is up and returns a translation with the given detected language + # + confidence, so the acceptance gate can be exercised end-to-end. + monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) + monkeypatch.setattr( + "backend.app.tasks.translation.ic.translate", + lambda texts, **k: { + "translations": [f"EN:{t}" for t in texts], + "detected_lang": lang, "detected_confidence": conf, + "engine": "llm", "engine_version": ver, + }, + ) + + +def test_translate_posts_rejects_low_confidence_latin(db_sync, monkeypatch): + # A short English title Interpreter mis-labels as German with LOW confidence is + # NOT stored — the original is kept, and the post is marked handled (source == + # target) so the sweep won't revisit it. + _patch(monkeypatch, db_sync) + _mock_lang(monkeypatch, lang="de", conf=0.42) + a = _artist(db_sync) + p = _post(db_sync, a.id, title="Nami Heroines - WIP Part 1", ext="p1") + _enable(db_sync) + db_sync.commit() + + translate_posts() + db_sync.refresh(p) + assert p.post_title_translated is None # mis-flag rejected, kept + assert p.translated_source_lang == "en" # marked handled (== target) + + +def test_translate_posts_accepts_high_confidence_latin(db_sync, monkeypatch): + # A confident latin-script detection (real German) clears the floor → stored. + _patch(monkeypatch, db_sync) + _mock_lang(monkeypatch, lang="de", conf=0.98) + a = _artist(db_sync) + p = _post(db_sync, a.id, title="Das Mädchen mit dem Perlenohrring", ext="p2") + _enable(db_sync) + db_sync.commit() + + translate_posts() + db_sync.refresh(p) + assert p.post_title_translated == "EN:Das Mädchen mit dem Perlenohrring" + assert p.translated_source_lang == "de" + + +def test_translate_posts_accepts_low_confidence_cjk(db_sync, monkeypatch): + # CJK is script-detected and trusted regardless of the reported number — a + # zh 0.75 (e.g. pure-kanji Japanese) is still translated, not rejected. + _patch(monkeypatch, db_sync) + _mock_lang(monkeypatch, lang="zh", conf=0.75) + a = _artist(db_sync) + p = _post(db_sync, a.id, title="猫が可愛い", ext="p3") + _enable(db_sync) + db_sync.commit() + + translate_posts() + db_sync.refresh(p) + assert p.post_title_translated == "EN:猫が可愛い" + assert p.translated_source_lang == "zh" + + def _mock_new_model(monkeypatch, *, ver="v2"): """Interpreter is up and translates with a NEW engine version.""" monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) diff --git a/tests/test_translation_gate.py b/tests/test_translation_gate.py new file mode 100644 index 0000000..c78c247 --- /dev/null +++ b/tests/test_translation_gate.py @@ -0,0 +1,33 @@ +"""Translation acceptance gate (#1376) — pure unit tests for ``_accept``, which +decides whether curator stores Interpreter's translation or keeps the original. +Curator does no detection of its own; it only thresholds Interpreter's reported +detected language + confidence (Scribe rule 133). No DB, no network.""" +from backend.app.tasks.translation import _MIN_LATIN_CONFIDENCE, _accept + + +def test_accept_trusts_cjk_regardless_of_confidence(): + # ja/ko/zh are script-detected — accepted even at a low reported number + # (pure-kanji Japanese can come back labelled zh ~0.75, still translated). + assert _accept("ja", None) is True + assert _accept("ja", 0.10) is True + assert _accept("ko", 0.99) is True + assert _accept("zh", 0.75) is True + assert _accept("zh-CN", 0.20) is True # region suffix is still CJK + + +def test_accept_requires_floor_for_latin(): + assert _accept("de", 0.98) is True + assert _accept("de", _MIN_LATIN_CONFIDENCE) is True # exactly at floor + assert _accept("de", _MIN_LATIN_CONFIDENCE - 0.01) is False + assert _accept("fr", 0.40) is False # short mis-flag + + +def test_accept_is_case_insensitive(): + assert _accept("JA", None) is True + assert _accept("DE", 0.40) is False + + +def test_accept_missing_confidence_fails_open(): + # No confidence reported → don't silently drop a translation. + assert _accept("de", None) is True + assert _accept("", None) is True From 6ab7fd5c7f96fa12a6e64dea569a10a270c428e9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 9 Jul 2026 16:40:02 -0400 Subject: [PATCH 2/2] tune(translation): set latin acceptance floor to 0.80 (#1376) Calibrated against fresh probes once Interpreter returned real langdetect confidence: genuine German detected at 1.0, a correctly-detected but ambiguous latin string at 0.86. Set _MIN_LATIN_CONFIDENCE to 0.80 (below that band) so legitimate ambiguous non-English still translates while genuinely-unsure guesses are rejected. Real langdetect also fixed the original mis-flag at the source, so this floor is a safety net, not the primary fix. Pin 0.86-accepted in the gate test to guard against bumping the floor back up. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01CgZP9v2otxVJymiYsnVuMy --- backend/app/tasks/translation.py | 11 +++++++++-- tests/test_translation_gate.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/app/tasks/translation.py b/backend/app/tasks/translation.py index 9598afd..42871c9 100644 --- a/backend/app/tasks/translation.py +++ b/backend/app/tasks/translation.py @@ -58,9 +58,16 @@ _INTERRUPT_BACKOFF_MAX = 900 # correctly translated). Latin-script detection is a real statistical # probability, so a short English title Interpreter mis-labels as a European # language scores low; require it to clear this floor, else keep the original. -# Tune the floor against real titles via the Settings "Test translation" box. +# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned +# real langdetect confidence: genuine German detected at 1.0, while a correctly- +# detected but ambiguous latin string dipped to 0.86 — so the floor sits at 0.80, +# below that band, to keep legitimate ambiguous non-English while still rejecting +# genuinely-unsure guesses. Real langdetect also fixed the original mis-flag at +# the source (short English titles now detect as English → passthrough), so this +# floor is a safety net, not the primary fix. Re-tune via the "Test translation" +# box (send fresh text — cache hits report 1.0). _CJK_LANGS = frozenset({"ja", "ko", "zh"}) -_MIN_LATIN_CONFIDENCE = 0.90 +_MIN_LATIN_CONFIDENCE = 0.80 def _interrupt_backoff(retry_after) -> int: diff --git a/tests/test_translation_gate.py b/tests/test_translation_gate.py index c78c247..2e226b7 100644 --- a/tests/test_translation_gate.py +++ b/tests/test_translation_gate.py @@ -17,6 +17,7 @@ def test_accept_trusts_cjk_regardless_of_confidence(): def test_accept_requires_floor_for_latin(): assert _accept("de", 0.98) is True + assert _accept("de", 0.86) is True # calibrated: ambiguous-but-correct German stays assert _accept("de", _MIN_LATIN_CONFIDENCE) is True # exactly at floor assert _accept("de", _MIN_LATIN_CONFIDENCE - 0.01) is False assert _accept("fr", 0.40) is False # short mis-flag