feat(translation): tunable acceptance floor (0.90) + per-post sticky override (#155) #210

Merged
bvandeusen merged 1 commits from dev into main 2026-07-10 22:47:45 -04:00
16 changed files with 579 additions and 52 deletions
@@ -0,0 +1,51 @@
"""translation strictness setting + per-post translation override (milestone 155)
ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
floor, now operator-tunable in the UI; default 0.9 — stricter than the old
hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
~0.86). Post gains ``translation_override`` — a sticky per-post choice of
auto / force / original so the operator can force a skipped translation on, or
knock a wrongly-translated one back to the original, and have it survive a
Re-translate-all.
Revision ID: 0084
Revises: 0083
Create Date: 2026-07-10
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "0084"
down_revision: Union[str, None] = "0083"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"import_settings",
sa.Column(
"translation_min_confidence", sa.Float(), nullable=False,
server_default=sa.text("0.9"),
),
)
op.add_column(
"post",
sa.Column(
"translation_override", sa.String(16), nullable=False,
server_default="auto",
),
)
op.create_check_constraint(
"ck_post_translation_override",
"post",
"translation_override IN ('auto', 'force', 'original')",
)
def downgrade() -> None:
op.drop_constraint("ck_post_translation_override", "post", type_="check")
op.drop_column("post", "translation_override")
op.drop_column("import_settings", "translation_min_confidence")
+83
View File
@@ -3,12 +3,28 @@
from quart import Blueprint, jsonify, request from quart import Blueprint, jsonify, request
from ..extensions import get_session from ..extensions import get_session
from ..models import ImportSettings, Post
from ..services import interpreter_client as ic
from ..services.post_feed_service import PostFeedService from ..services.post_feed_service import PostFeedService
from ..services.source_service import KNOWN_PLATFORMS from ..services.source_service import KNOWN_PLATFORMS
from ..utils.text import html_to_plain
from ._responses import error_response as _bad from ._responses import error_response as _bad
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts") posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
_TRANSLATION_OVERRIDES = ("auto", "force", "original")
def _queue_for_sweep(post: Post) -> None:
"""Mark a post untranslated (all translation columns NULL) so the periodic
sweep re-runs it under its new override — used when Interpreter is down and we
can't translate inline."""
post.post_title_translated = None
post.description_translated = None
post.translated_source_lang = None
post.translation_engine_version = None
post.translated_at = None
@posts_bp.route("", methods=["GET"]) @posts_bp.route("", methods=["GET"])
async def list_posts(): async def list_posts():
@@ -82,3 +98,70 @@ async def get_post(post_id: int):
if item is None: if item is None:
return _bad("not_found", status=404, detail=f"post id={post_id}") return _bad("not_found", status=404, detail=f"post id={post_id}")
return jsonify(item) return jsonify(item)
@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
async def set_translation_override(post_id: int):
"""Sticky per-post translation override (milestone 155). Body:
``{"override": "auto" | "force" | "original"}``.
'original' keeps the original (clears any stored translation now — no
Interpreter needed). 'force'/'auto' translate the post immediately if the
service is up (force bypasses the acceptance floor; auto re-runs the gate);
if it's down we save the flag and mark the post untranslated so the next sweep
applies it. The override persists, so the sweep + Re-translate-all keep
honoring it. Returns the updated translation fields + an ``applied`` status."""
body = await request.get_json(silent=True) or {}
override = body.get("override")
if override not in _TRANSLATION_OVERRIDES:
return _bad(
"invalid_override",
detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
)
# Lazy import (mirrors settings.py) so the API module doesn't pull the celery
# task graph at import time.
from ..tasks.translation import _store_translation, _translate_field
async with get_session() as session:
post = await session.get(Post, post_id)
if post is None:
return _bad("not_found", status=404, detail=f"post id={post_id}")
post.translation_override = override
cfg = await ImportSettings.load(session)
target = (cfg.translation_target_lang or "en").strip() or "en"
if override == "original":
_store_translation(post, (None, None, None), (None, None, None), target)
applied = "cleared"
else:
base_url = (cfg.interpreter_base_url or "").strip()
if cfg.translation_enabled and base_url and ic.health(base_url):
force = override == "force"
title = (post.post_title or "").strip()
desc = (html_to_plain(post.description) if post.description else "") or ""
desc = desc.strip()
mc = cfg.translation_min_confidence
try:
title_res = _translate_field(title, base_url, target, mc, force=force)
desc_res = _translate_field(desc, base_url, target, mc, force=force)
except ic.InterpreterUnavailable:
_queue_for_sweep(post)
applied = "queued"
else:
_store_translation(post, title_res, desc_res, target)
applied = "translated"
else:
# Disabled / no URL / unhealthy → let the sweep apply it later.
_queue_for_sweep(post)
applied = "queued"
await session.commit()
return jsonify({
"id": post.id,
"translation_override": post.translation_override,
"post_title_translated": post.post_title_translated,
"description_translated": post.description_translated,
"translated_source_lang": post.translated_source_lang,
"applied": applied,
})
+10
View File
@@ -47,6 +47,7 @@ _EDITABLE_FIELDS = (
"translation_enabled", "translation_enabled",
"interpreter_base_url", "interpreter_base_url",
"translation_target_lang", "translation_target_lang",
"translation_min_confidence",
) )
# Per-host external-download toggles — all plain booleans, validated uniformly. # Per-host external-download toggles — all plain booleans, validated uniformly.
@@ -87,6 +88,7 @@ async def get_import_settings():
"translation_enabled": row.translation_enabled, "translation_enabled": row.translation_enabled,
"interpreter_base_url": row.interpreter_base_url, "interpreter_base_url": row.interpreter_base_url,
"translation_target_lang": row.translation_target_lang, "translation_target_lang": row.translation_target_lang,
"translation_min_confidence": row.translation_min_confidence,
}) })
@@ -155,6 +157,14 @@ async def update_import_settings():
for key in ("interpreter_base_url", "translation_target_lang"): for key in ("interpreter_base_url", "translation_target_lang"):
if key in body and not isinstance(body[key], str): if key in body and not isinstance(body[key], str):
return jsonify({"error": f"{key} must be a string"}), 400 return jsonify({"error": f"{key} must be a string"}), 400
# Acceptance floor (milestone 155): latin-script translations below this
# Interpreter confidence are kept as the original.
if "translation_min_confidence" in body:
v = body["translation_min_confidence"]
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
return jsonify(
{"error": "translation_min_confidence must be a number in [0, 1]"}
), 400
if "series_suggest_threshold" in body: if "series_suggest_threshold" in body:
v = body["series_suggest_threshold"] v = body["series_suggest_threshold"]
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1: if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
+9
View File
@@ -106,6 +106,15 @@ class ImportSettings(Base):
translation_target_lang: Mapped[str] = mapped_column( translation_target_lang: Mapped[str] = mapped_column(
Text, nullable=False, default="en", server_default="en", Text, nullable=False, default="en", server_default="en",
) )
# The latin-script acceptance floor for the translation gate: a translation
# whose Interpreter-reported confidence is below this is kept as the original
# (operator-tunable, milestone 155). Default 0.9 — stricter than the old
# hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
# trusted regardless (script-detected). Per-post overrides handle the misses.
translation_min_confidence: Mapped[float] = mapped_column(
Float, nullable=False, default=0.9, server_default="0.9",
)
@classmethod @classmethod
async def load(cls, session) -> ImportSettings: async def load(cls, session) -> ImportSettings:
+25 -1
View File
@@ -8,7 +8,17 @@ artist-filter queries don't depend on the Source detour).
from datetime import datetime from datetime import datetime
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func from sqlalchemy import (
JSON,
CheckConstraint,
DateTime,
ForeignKey,
Integer,
String,
Text,
UniqueConstraint,
func,
)
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from .base import Base from .base import Base
@@ -23,6 +33,10 @@ class Post(Base):
# (created in alembic 0030) covers that case via # (created in alembic 0030) covers that case via
# (artist_id, external_post_id). # (artist_id, external_post_id).
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
CheckConstraint(
"translation_override IN ('auto', 'force', 'original')",
name="ck_post_translation_override",
),
) )
id: Mapped[int] = mapped_column(Integer, primary_key=True) id: Mapped[int] = mapped_column(Integer, primary_key=True)
@@ -64,6 +78,16 @@ class Post(Base):
translated_at: Mapped[datetime | None] = mapped_column( translated_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True DateTime(timezone=True), nullable=True
) )
# Sticky per-post override of the translation decision (milestone 155):
# 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
# translation even below the confidence floor (rescue a skipped legit-foreign
# title); 'original' = never translate, keep the original (kill a confidently
# mis-flagged one the floor can't catch). The sweep reads this on every run,
# and re-translate leaves 'original' posts alone, so the choice survives a
# Re-translate-all.
translation_override: Mapped[str] = mapped_column(
String(16), nullable=False, default="auto", server_default="auto",
)
downloaded_at: Mapped[datetime] = mapped_column( downloaded_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now() DateTime(timezone=True), nullable=False, server_default=func.now()
@@ -397,6 +397,8 @@ class PostFeedService:
"post_title_translated": post.post_title_translated, "post_title_translated": post.post_title_translated,
"description_translated": desc_trans_short, "description_translated": desc_trans_short,
"translated_source_lang": post.translated_source_lang, "translated_source_lang": post.translated_source_lang,
# Sticky per-post translation choice (auto/force/original, #155).
"translation_override": post.translation_override,
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
"source": ( "source": (
{"id": source.id, "platform": source.platform} {"id": source.id, "platform": source.platform}
@@ -35,6 +35,7 @@ def _post_dict(p: Post) -> dict:
"title_translated": p.post_title_translated, "title_translated": p.post_title_translated,
"description_translated": p.description_translated, "description_translated": p.description_translated,
"translated_source_lang": p.translated_source_lang, "translated_source_lang": p.translated_source_lang,
"translation_override": p.translation_override,
} }
+79 -36
View File
@@ -60,14 +60,17 @@ _INTERRUPT_BACKOFF_MAX = 900
# language scores low; require it to clear this floor, else keep the original. # language scores low; require it to clear this floor, else keep the original.
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned # Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
# real langdetect confidence: genuine German detected at 1.0, while a correctly- # real langdetect confidence: genuine German detected at 1.0, while a correctly-
# detected but ambiguous latin string dipped to 0.86 — so the floor sits at 0.80, # detected but ambiguous latin string dipped to 0.86 — so a single constant can't win.
# below that band, to keep legitimate ambiguous non-English while still rejecting # The floor is now operator-tunable (ImportSettings.translation_min_confidence,
# genuinely-unsure guesses. Real langdetect also fixed the original mis-flag at # milestone 155) with a stricter 0.90 default; per-post overrides
# the source (short English titles now detect as English → passthrough), so this # (Post.translation_override) rescue the false negatives it skips. Genuine German
# and mis-flagged short English both land ~0.86, and single-word mis-flags sit at
# a confident 1.0 no floor catches. Real langdetect fixed some cases at the
# source (some short English titles now detect as English → passthrough), so this
# floor is a safety net, not the primary fix. Re-tune via the "Test translation" # floor is a safety net, not the primary fix. Re-tune via the "Test translation"
# box (send fresh text — cache hits report 1.0). # box (send fresh text — cache hits report 1.0).
_CJK_LANGS = frozenset({"ja", "ko", "zh"}) _CJK_LANGS = frozenset({"ja", "ko", "zh"})
_MIN_LATIN_CONFIDENCE = 0.80 _DEFAULT_MIN_LATIN_CONFIDENCE = 0.90
def _interrupt_backoff(retry_after) -> int: def _interrupt_backoff(retry_after) -> int:
@@ -100,10 +103,10 @@ def translate_posts(drain: bool = False) -> str:
ready = _translation_config(session) ready = _translation_config(session)
if isinstance(ready, str): if isinstance(ready, str):
return ready return ready
base_url, target = ready base_url, target, min_confidence = ready
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN) posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
status, translated, retry_after = _translate_batch( status, translated, retry_after = _translate_batch(
session, posts, base_url, target, session, posts, base_url, target, min_confidence,
) )
# Drain mode (manual "Translate now"): chase the tail until the backlog is # Drain mode (manual "Translate now"): chase the tail until the backlog is
# zero, so one press clears the whole pile instead of one 300-chunk. # zero, so one press clears the whole pile instead of one 300-chunk.
@@ -152,7 +155,7 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
ready = _translation_config(session) ready = _translation_config(session)
if isinstance(ready, str): if isinstance(ready, str):
return ready return ready
base_url, target = ready base_url, target, min_confidence = ready
if not _reset_done: if not _reset_done:
n = _reset_translations(session, artist_ids) n = _reset_translations(session, artist_ids)
@@ -163,7 +166,7 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN) posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
status, translated, retry_after = _translate_batch( status, translated, retry_after = _translate_batch(
session, posts, base_url, target, session, posts, base_url, target, min_confidence,
) )
# run-until-done: chase the tail when the chunk finished cleanly. # run-until-done: chase the tail when the chunk finished cleanly.
@@ -201,17 +204,19 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
def _translation_config(session): def _translation_config(session):
"""Resolve (base_url, target) if translation is enabled + healthy, else a """Resolve (base_url, target, min_confidence) if translation is enabled +
short status string ("disabled" / "interpreter unavailable") for the caller healthy, else a short status string ("disabled" / "interpreter unavailable")
to return. Centralises the guard so translate/retranslate agree exactly.""" for the caller to return. Centralises the guard so translate/retranslate agree
exactly, including the operator-tunable acceptance floor."""
cfg = ImportSettings.load_sync(session) cfg = ImportSettings.load_sync(session)
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip(): if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
return "disabled" return "disabled"
base_url = cfg.interpreter_base_url.strip() base_url = cfg.interpreter_base_url.strip()
target = (cfg.translation_target_lang or "en").strip() or "en" target = (cfg.translation_target_lang or "en").strip() or "en"
min_confidence = cfg.translation_min_confidence
if not ic.health(base_url): if not ic.health(base_url):
return "interpreter unavailable" return "interpreter unavailable"
return base_url, target return base_url, target, min_confidence
def _untranslated_filter(stmt, artist_ids): def _untranslated_filter(stmt, artist_ids):
@@ -244,10 +249,13 @@ def _reset_translations(session, artist_ids) -> int:
"""Clear the stored translation columns for scoped, already-handled posts so """Clear the stored translation columns for scoped, already-handled posts so
the untranslated sweep re-runs them. Only touches rows that were translated the untranslated sweep re-runs them. Only touches rows that were translated
(translated_source_lang IS NOT NULL) — untranslated rows are already NULL. (translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
Returns the row count reset (commits it).""" Skips 'keep original' posts (translation_override = 'original') so that choice
survives a Re-translate-all (milestone 155). Returns the row count reset
(commits it)."""
stmt = ( stmt = (
update(Post) update(Post)
.where(Post.translated_source_lang.is_not(None)) .where(Post.translated_source_lang.is_not(None))
.where(Post.translation_override != "original")
.values( .values(
post_title_translated=None, post_title_translated=None,
description_translated=None, description_translated=None,
@@ -263,7 +271,7 @@ def _reset_translations(session, artist_ids) -> int:
return n return n
def _translate_batch(session, posts, base_url: str, target: str): def _translate_batch(session, posts, base_url: str, target: str, min_confidence: float):
"""Translate each post in the chunk with a per-post commit (an interrupted """Translate each post in the chunk with a per-post commit (an interrupted
run keeps progress). Returns (status, translated, retry_after) where status is run keeps progress). Returns (status, translated, retry_after) where status is
one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout"; one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout";
@@ -272,7 +280,7 @@ def _translate_batch(session, posts, base_url: str, target: str):
translated = 0 translated = 0
for post in posts: for post in posts:
try: try:
translated += _translate_one(session, post, base_url, target) translated += _translate_one(session, post, base_url, target, min_confidence)
session.commit() session.commit()
except ic.InterpreterUnavailable as e: except ic.InterpreterUnavailable as e:
session.rollback() session.rollback()
@@ -296,28 +304,36 @@ def _summary(status: str, translated: int, scanned: int) -> str:
return f"translated={translated} scanned={scanned}" return f"translated={translated} scanned={scanned}"
def _accept(language: str, confidence) -> bool: def _accept(
language: str, confidence, min_confidence: float = _DEFAULT_MIN_LATIN_CONFIDENCE,
) -> bool:
"""Should curator store Interpreter's translation, or keep the original? """Should curator store Interpreter's translation, or keep the original?
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
133): CJK languages are trusted regardless of the reported number; a 133): CJK languages are trusted regardless of the reported number; a
latin-script detection must clear ``_MIN_LATIN_CONFIDENCE``. A missing latin-script detection must clear ``min_confidence`` (the operator-tunable
confidence fails OPEN (accept), so a service that omits the field never causes ImportSettings.translation_min_confidence). A missing confidence fails OPEN
translations to be silently dropped.""" (accept), so a service that omits the field never silently drops a
translation. Per-post ``force`` overrides bypass this entirely (see
``_translate_one``)."""
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS: if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
return True return True
if confidence is None: if confidence is None:
return True return True
return confidence >= _MIN_LATIN_CONFIDENCE return confidence >= min_confidence
def _translate_field(text: str, base_url: str, target: str): def _translate_field(
text: str, base_url: str, target: str, min_confidence: float, force: bool = False,
):
"""Translate ONE field independently. Returns (translated, source_lang, """Translate ONE field independently. Returns (translated, source_lang,
engine_version), or (None, None, None) when there's nothing to store — empty engine_version), or (None, None, None) when there's nothing to store — empty
text, already the target language, a passthrough (engine "none"), or a text, already the target language, a passthrough (engine "none"), or a
latin-script detection the acceptance gate rejects as too low-confidence (kept latin-script detection the acceptance gate rejects as too low-confidence (kept
as the original). Per-field (not aggregate first-item) detection so a as the original). ``force`` (a per-post 'force' override) bypasses the
non-English description still gets translated when the title is already confidence gate so a legitimately-foreign title Interpreter is only ~0.86 sure
English (mixed-language posts).""" about is still stored; a genuine passthrough stores nothing regardless (there
is no translation to force). Per-field (not aggregate first-item) detection so
a non-English description still translates when the title is already English."""
if not text: if not text:
return None, None, None return None, None, None
res = ic.translate([text], base_url=base_url, target=target) res = ic.translate([text], base_url=base_url, target=target)
@@ -327,23 +343,30 @@ def _translate_field(text: str, base_url: str, target: str):
# Trust only Interpreter's own detection (Scribe rule 133): keep the original # Trust only Interpreter's own detection (Scribe rule 133): keep the original
# when a latin-script detection doesn't clear the confidence floor, so a short # when a latin-script detection doesn't clear the confidence floor, so a short
# English title mis-labelled as e.g. German isn't rewritten into the archive. # English title mis-labelled as e.g. German isn't rewritten into the archive.
if not _accept(detected, res.get("detected_confidence")): # A 'force' override skips the gate — the operator is overriding a rejection.
if not force and not _accept(
detected, res.get("detected_confidence"), min_confidence,
):
return None, None, None return None, None, None
return res["translations"][0], detected, res["engine_version"] return res["translations"][0], detected, res["engine_version"]
def _translate_one(session, post, base_url: str, target: str) -> int: def _store_translation(post, title_res, desc_res, target: str) -> int:
"""Translate a post's title/description in place, each field independently. """Apply per-field (translated, source_lang, engine_version) results to a post
Returns 1 if it stored at least one translation, 0 when the post is all in place. Returns 1 if a translation was stored, 0 when the post is all
passthrough/empty (still marks it handled so the sweep won't revisit it).""" passthrough/empty — in which case the translated columns are CLEARED and the
title = (post.post_title or "").strip() post is marked handled (translated_source_lang = target). Clearing (not just
desc = (html_to_plain(post.description) if post.description else "") or "" leaving) matters on re-apply: a 'keep original' override, or a re-translate
desc = desc.strip() that now rejects a previously-accepted mis-flag, must remove the stale
title_tr, title_lang, title_ev = _translate_field(title, base_url, target) translation, not keep it."""
desc_tr, desc_lang, desc_ev = _translate_field(desc, base_url, target) title_tr, title_lang, title_ev = title_res
desc_tr, desc_lang, desc_ev = desc_res
if title_tr is None and desc_tr is None: if title_tr is None and desc_tr is None:
# Nothing non-target (or nothing to translate) → handled, store nothing. post.post_title_translated = None
post.description_translated = None
post.translated_source_lang = target post.translated_source_lang = target
post.translation_engine_version = None
post.translated_at = None
return 0 return 0
post.post_title_translated = title_tr post.post_title_translated = title_tr
post.description_translated = desc_tr post.description_translated = desc_tr
@@ -353,3 +376,23 @@ def _translate_one(session, post, base_url: str, target: str) -> int:
post.translation_engine_version = title_ev or desc_ev post.translation_engine_version = title_ev or desc_ev
post.translated_at = datetime.now(UTC) post.translated_at = datetime.now(UTC)
return 1 return 1
def _translate_one(
session, post, base_url: str, target: str, min_confidence: float,
) -> int:
"""Translate a post's title/description in place, each field independently,
honoring the per-post override (milestone 155): 'original' keeps the original
(clears any stored translation, no Interpreter call); 'force' translates even
below the confidence floor; 'auto' (default) runs the acceptance gate. Returns
1 if it stored a translation, else 0 (still marks the post handled so the sweep
won't revisit it)."""
if post.translation_override == "original":
return _store_translation(post, (None, None, None), (None, None, None), target)
force = post.translation_override == "force"
title = (post.post_title or "").strip()
desc = (html_to_plain(post.description) if post.description else "") or ""
desc = desc.strip()
title_res = _translate_field(title, base_url, target, min_confidence, force=force)
desc_res = _translate_field(desc, base_url, target, min_confidence, force=force)
return _store_translation(post, title_res, desc_res, target)
@@ -70,6 +70,14 @@
@click.stop="showOriginal = !showOriginal" @click.stop="showOriginal = !showOriginal"
>{{ showOriginal ? 'Show translation' : `Show original${sourceLangLabel}` }}</button> >{{ showOriginal ? 'Show translation' : `Show original${sourceLangLabel}` }}</button>
<!-- Per-post translation override (#155): force a skipped translation on,
or keep the original for a confidently mis-flagged title. Only where
there's text to translate. -->
<PostTranslationControl
v-if="post.post_title || post.description_plain"
:post="post"
/>
<!-- Faithful (semantic) body render once expanded: backend-sanitized <!-- Faithful (semantic) body render once expanded: backend-sanitized
HTML (headings, lists, links, inline images). Collapsed and the HTML (headings, lists, links, inline images). Collapsed and the
no-detail fallback stay plain text. --> no-detail fallback stay plain text. -->
@@ -136,6 +144,7 @@ import { useModalStore } from '../../stores/modal.js'
import { usePostsStore } from '../../stores/posts.js' import { usePostsStore } from '../../stores/posts.js'
import { toPlainText } from '../../utils/htmlSanitize.js' import { toPlainText } from '../../utils/htmlSanitize.js'
import PostSeriesMenu from './PostSeriesMenu.vue' import PostSeriesMenu from './PostSeriesMenu.vue'
import PostTranslationControl from './PostTranslationControl.vue'
const props = defineProps({ const props = defineProps({
post: { type: Object, required: true }, post: { type: Object, required: true },
@@ -0,0 +1,102 @@
<template>
<!-- Sticky per-post translation override (#155). Quiet inline control under the
post title: force a skipped legit-foreign title on, or knock a wrongly
mis-translated one back to the original. The choice sticks through a
Re-translate-all. -->
<div class="fc-post-tx">
<span class="fc-post-tx__label">Translation:</span>
<v-menu :disabled="busy">
<template #activator="{ props: menuProps }">
<button
type="button" class="fc-post-tx__btn" v-bind="menuProps" :disabled="busy"
:aria-label="`Translation handling: ${currentLabel}`"
>
{{ currentLabel }}
<v-icon size="14">mdi-menu-down</v-icon>
</button>
</template>
<v-list density="compact" min-width="240" class="fc-post-tx__list">
<v-list-item
v-for="opt in OPTIONS" :key="opt.value"
:active="opt.value === current" @click="choose(opt.value)"
>
<template #prepend>
<v-icon size="small">{{ opt.icon }}</v-icon>
</template>
<v-list-item-title>{{ opt.label }}</v-list-item-title>
<v-list-item-subtitle>{{ opt.help }}</v-list-item-subtitle>
</v-list-item>
</v-list>
</v-menu>
<v-progress-circular v-if="busy" indeterminate size="13" width="2" />
</div>
</template>
<script setup>
import { computed, ref } from 'vue'
import { usePostsStore } from '../../stores/posts.js'
import { toast } from '../../utils/toast.js'
const props = defineProps({ post: { type: Object, required: true } })
const posts = usePostsStore()
const busy = ref(false)
const OPTIONS = [
{ value: 'auto', label: 'Auto', icon: 'mdi-cog-outline',
help: 'Translate only when confident enough' },
{ value: 'force', label: 'Force translate', icon: 'mdi-translate',
help: 'Always translate, even at low confidence' },
{ value: 'original', label: 'Keep original', icon: 'mdi-translate-off',
help: 'Never translate — keep the original text' },
]
const current = computed(() => props.post.translation_override || 'auto')
const currentLabel = computed(
() => (OPTIONS.find((o) => o.value === current.value) || OPTIONS[0]).label,
)
async function choose (value) {
if (value === current.value || busy.value) return
busy.value = true
try {
const res = await posts.applyTranslationOverride(props.post.id, value)
if (res.applied === 'queued') {
toast({
text: 'Saved — this post will translate on the next sweep (service offline).',
type: 'info',
})
}
} catch (e) {
toast({ text: `Couldn't update translation: ${e.message}`, type: 'error' })
} finally {
busy.value = false
}
}
</script>
<style scoped>
.fc-post-tx {
display: inline-flex; align-items: center; gap: 6px;
margin: 2px 0 6px;
}
.fc-post-tx__label {
font-size: 0.72rem;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-post-tx__btn {
display: inline-flex; align-items: center; gap: 1px;
padding: 0; background: none; border: 0; cursor: pointer;
font-size: 0.72rem; font-weight: 700;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-post-tx__btn:hover { color: rgb(var(--v-theme-accent)); }
.fc-post-tx__btn:focus-visible {
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
border-radius: 2px;
}
.fc-post-tx__btn:disabled { cursor: default; opacity: 0.6; }
.fc-post-tx__list :deep(.v-list-item-subtitle) {
font-size: 0.7rem;
}
</style>
@@ -28,6 +28,21 @@
@change="onSave" @change="onSave"
/> />
<!-- Acceptance floor (#155): latin-script translations below this Interpreter
confidence are kept as the original. Per-post overrides handle exceptions. -->
<v-text-field
v-model.number="minConfidence" label="Acceptance confidence"
type="number" min="0" max="1" step="0.01" density="compact"
hide-details style="max-width: 200px;" class="mb-1" :disabled="busy"
@change="onSaveConfidence"
/>
<div class="fc-muted text-caption mb-3">
Latin-script titles Interpreter is less than this sure about (01) are kept
as the original; CJK is always translated. Default 0.90 — raise it to reject
more mis-detections, lower it to translate more. Per-post controls in the
posts feed override this either way.
</div>
<div class="d-flex align-center flex-wrap mb-3" style="gap: 8px;"> <div class="d-flex align-center flex-wrap mb-3" style="gap: 8px;">
<v-icon size="12" :color="statusColor">mdi-circle</v-icon> <v-icon size="12" :color="statusColor">mdi-circle</v-icon>
<span class="fc-muted text-body-2">{{ statusText }}</span> <span class="fc-muted text-body-2">{{ statusText }}</span>
@@ -175,6 +190,7 @@ const store = useImportStore()
const enabled = ref(false) const enabled = ref(false)
const baseUrl = ref('') const baseUrl = ref('')
const targetLang = ref('en') const targetLang = ref('en')
const minConfidence = ref(0.9)
const busy = ref(false) const busy = ref(false)
const running = ref(false) const running = ref(false)
const retranslating = ref(false) const retranslating = ref(false)
@@ -258,6 +274,7 @@ onMounted(async () => {
enabled.value = !!s.translation_enabled enabled.value = !!s.translation_enabled
baseUrl.value = s.interpreter_base_url || '' baseUrl.value = s.interpreter_base_url || ''
targetLang.value = s.translation_target_lang || 'en' targetLang.value = s.translation_target_lang || 'en'
minConfidence.value = s.translation_min_confidence ?? 0.9
} catch { /* non-fatal */ } } catch { /* non-fatal */ }
await loadStatus() await loadStatus()
}) })
@@ -282,6 +299,14 @@ async function onSave() {
}) })
await loadStatus() await loadStatus()
} }
async function onSaveConfidence() {
// Clamp to [0, 1] so a stray value never bounces off the API 400.
let v = Number(minConfidence.value)
if (!Number.isFinite(v)) v = 0.9
v = Math.min(1, Math.max(0, v))
minConfidence.value = v
await save({ translation_min_confidence: v })
}
async function onRun() { async function onRun() {
running.value = true running.value = true
err.value = null err.value = null
+19 -1
View File
@@ -71,6 +71,24 @@ export const usePostsStore = defineStore('posts', () => {
return await api.get(`/api/posts/${id}`) return await api.get(`/api/posts/${id}`)
} }
// Set a post's sticky translation override (auto/force/original, #155) and
// patch the loaded item in place with the endpoint's result (it translates /
// clears immediately when the service is up), so the card reflects the change
// without a feed reload.
async function applyTranslationOverride(postId, override) {
const res = await api.post(`/api/posts/${postId}/translation-override`, {
body: { override },
})
const item = items.value.find((p) => p.id === postId)
if (item) {
item.translation_override = res.translation_override
item.post_title_translated = res.post_title_translated
item.description_translated = res.description_translated
item.translated_source_lang = res.translated_source_lang
}
return res
}
// Filter overlay for the around/older/newer (in-context anchored) // Filter overlay for the around/older/newer (in-context anchored)
// path. Keep this distinct from `filters.value` (the down-only feed) // path. Keep this distinct from `filters.value` (the down-only feed)
// so a normal-feed filter change doesn't leak into an active anchored // so a normal-feed filter change doesn't leak into an active anchored
@@ -168,7 +186,7 @@ export const usePostsStore = defineStore('posts', () => {
return { return {
items, cursor, loading, done, error, filters, items, cursor, loading, done, error, filters,
cursorOlder, cursorNewer, doneOlder, doneNewer, anchorId, cursorOlder, cursorNewer, doneOlder, doneNewer, anchorId,
loadInitial, loadMore, getPostFull, loadInitial, loadMore, getPostFull, applyTranslationOverride,
loadAround, loadOlder, loadNewer, loadAround, loadOlder, loadNewer,
} }
}) })
+62
View File
@@ -184,6 +184,68 @@ async def test_rejects_bad_direction(client):
assert (await resp.get_json())["error"] == "invalid_direction" assert (await resp.get_json())["error"] == "invalid_direction"
@pytest.mark.asyncio
async def test_translation_override_rejects_bad_value(client, seeded_post):
_, _, post = seeded_post
resp = await client.post(
f"/api/posts/{post.id}/translation-override", json={"override": "nope"}
)
assert resp.status_code == 400
assert (await resp.get_json())["error"] == "invalid_override"
@pytest.mark.asyncio
async def test_translation_override_404_for_unknown(client):
resp = await client.post(
"/api/posts/999999/translation-override", json={"override": "original"}
)
assert resp.status_code == 404
@pytest.mark.asyncio
async def test_translation_override_original_clears_and_keeps(client, db, seeded_post):
# 'keep original' clears a stored translation immediately (no Interpreter) and
# marks the post handled (== target). Asserts on the response, which reflects
# the endpoint's own committed session.
_, _, post = seeded_post
post.post_title_translated = "STALE"
post.description_translated = "STALE"
post.translated_source_lang = "de"
await db.commit()
resp = await client.post(
f"/api/posts/{post.id}/translation-override", json={"override": "original"}
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["translation_override"] == "original"
assert body["applied"] == "cleared"
assert body["post_title_translated"] is None
assert body["translated_source_lang"] == "en"
@pytest.mark.asyncio
async def test_translation_override_force_queues_when_disabled(client, seeded_post):
# Translation disabled (default) → can't translate inline; the override is
# saved and the post is queued (columns NULLed) for the next sweep.
_, _, post = seeded_post
resp = await client.post(
f"/api/posts/{post.id}/translation-override", json={"override": "force"}
)
assert resp.status_code == 200
body = await resp.get_json()
assert body["translation_override"] == "force"
assert body["applied"] == "queued"
assert body["translated_source_lang"] is None
@pytest.mark.asyncio
async def test_list_exposes_translation_override(client, seeded_post):
resp = await client.get("/api/posts")
assert resp.status_code == 200
body = await resp.get_json()
assert body["items"][0]["translation_override"] == "auto" # default
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_detail_returns_uncapped_thumbnails(client, db): async def test_detail_returns_uncapped_thumbnails(client, db):
"""Feed query caps thumbnails at 6 for previews; detail endpoint """Feed query caps thumbnails at 6 for previews; detail endpoint
+20
View File
@@ -53,6 +53,26 @@ async def test_translation_settings_defaults_and_patch(client):
"/api/settings/import", json={"interpreter_base_url": 123})).status_code == 400 "/api/settings/import", json={"interpreter_base_url": 123})).status_code == 400
@pytest.mark.asyncio
async def test_translation_min_confidence_default_and_patch(client):
# #155: the latin-script acceptance floor is an operator-tunable setting,
# default 0.9 (stricter than the old hardcoded 0.8).
body = await (await client.get("/api/settings/import")).get_json()
assert body["translation_min_confidence"] == 0.9
ok = await client.patch(
"/api/settings/import", json={"translation_min_confidence": 0.95}
)
assert ok.status_code == 200
assert (await ok.get_json())["translation_min_confidence"] == 0.95
# Out-of-range + wrong-type are rejected.
assert (await client.patch(
"/api/settings/import", json={"translation_min_confidence": 1.5})).status_code == 400
assert (await client.patch(
"/api/settings/import", json={"translation_min_confidence": "high"})).status_code == 400
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_translation_status_defaults(client): async def test_translation_status_defaults(client):
# Off + no URL → no network health call, healthy False (#143). # Off + no URL → no network health call, healthy False (#143).
+54
View File
@@ -314,6 +314,41 @@ def test_translate_posts_accepts_low_confidence_cjk(db_sync, monkeypatch):
assert p.translated_source_lang == "zh" assert p.translated_source_lang == "zh"
def test_translate_posts_force_override_translates_below_floor(db_sync, monkeypatch):
# override='force' stores the translation even below the confidence floor —
# rescuing a legitimately-foreign title the gate would otherwise skip.
_patch(monkeypatch, db_sync)
_mock_lang(monkeypatch, lang="de", conf=0.42) # below the 0.90 default floor
a = _artist(db_sync)
p = _post(db_sync, a.id, title="Kurz", ext="p1")
p.translation_override = "force"
_enable(db_sync)
db_sync.commit()
translate_posts()
db_sync.refresh(p)
assert p.post_title_translated == "EN:Kurz" # forced through despite 0.42
assert p.translated_source_lang == "de"
def test_translate_posts_original_override_keeps_original(db_sync, monkeypatch):
# override='original' never translates — even a confident (0.98) detection is
# kept as the original. The sweep marks it handled (== target) and stores
# nothing; Interpreter isn't consulted for the decision.
_patch(monkeypatch, db_sync)
_mock_lang(monkeypatch, lang="de", conf=0.98) # would be accepted under auto
a = _artist(db_sync)
p = _post(db_sync, a.id, title="Das Mädchen", ext="p1")
p.translation_override = "original"
_enable(db_sync)
db_sync.commit()
translate_posts()
db_sync.refresh(p)
assert p.post_title_translated is None # kept original
assert p.translated_source_lang == "en" # marked handled
def _mock_new_model(monkeypatch, *, ver="v2"): def _mock_new_model(monkeypatch, *, ver="v2"):
"""Interpreter is up and translates with a NEW engine version.""" """Interpreter is up and translates with a NEW engine version."""
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
@@ -344,6 +379,25 @@ def test_retranslate_resets_and_reruns(db_sync, monkeypatch):
assert p.translation_engine_version == "v2" assert p.translation_engine_version == "v2"
def test_retranslate_leaves_keep_original_posts(db_sync, monkeypatch):
# A 'keep original' post survives a Re-translate-all: _reset_translations skips
# override='original', so it stays handled + untranslated instead of being
# re-run (which would re-introduce the mis-flag the operator killed).
_patch(monkeypatch, db_sync)
_mock_new_model(monkeypatch)
a = _artist(db_sync)
p = _post(db_sync, a.id, title="ねこ", ext="p1")
p.translation_override = "original"
p.translated_source_lang = "en" # handled by a prior 'keep original' apply
_enable(db_sync)
db_sync.commit()
retranslate_posts(artist_ids=[a.id])
db_sync.refresh(p)
assert p.translated_source_lang == "en" # not reset
assert p.post_title_translated is None # not re-translated
def test_retranslate_artist_scope_leaves_others(db_sync, monkeypatch): def test_retranslate_artist_scope_leaves_others(db_sync, monkeypatch):
# Scoping to one artist must not touch another artist's translations. # Scoping to one artist must not touch another artist's translations.
_patch(monkeypatch, db_sync) _patch(monkeypatch, db_sync)
+28 -14
View File
@@ -1,26 +1,39 @@
"""Translation acceptance gate (#1376) — pure unit tests for ``_accept``, which """Translation acceptance gate (#1376, #155) — pure unit tests for ``_accept``,
decides whether curator stores Interpreter's translation or keeps the original. which decides whether curator stores Interpreter's translation or keeps the
Curator does no detection of its own; it only thresholds Interpreter's reported original. Curator does no detection of its own; it only thresholds Interpreter's
detected language + confidence (Scribe rule 133). No DB, no network.""" reported detected language + confidence against the operator-tunable floor
from backend.app.tasks.translation import _MIN_LATIN_CONFIDENCE, _accept (Scribe rule 133). No DB, no network."""
from backend.app.tasks.translation import _DEFAULT_MIN_LATIN_CONFIDENCE, _accept
def test_accept_trusts_cjk_regardless_of_confidence(): def test_accept_trusts_cjk_regardless_of_confidence_or_floor():
# ja/ko/zh are script-detected — accepted even at a low reported number # ja/ko/zh are script-detected — accepted even at a low reported number and
# (pure-kanji Japanese can come back labelled zh ~0.75, still translated). # even against a high floor (pure-kanji Japanese can come back labelled zh
# ~0.75, still translated).
assert _accept("ja", None) is True assert _accept("ja", None) is True
assert _accept("ja", 0.10) is True assert _accept("ja", 0.10, 0.99) is True
assert _accept("ko", 0.99) is True assert _accept("ko", 0.99) is True
assert _accept("zh", 0.75) is True assert _accept("zh", 0.75) is True
assert _accept("zh-CN", 0.20) is True # region suffix is still CJK assert _accept("zh-CN", 0.20) is True # region suffix is still CJK
def test_accept_requires_floor_for_latin(): def test_accept_default_floor_is_strict():
# Default floor is 0.90 (#155): the ~0.86 band — where genuine German AND
# mis-flagged short English collide — is now REJECTED (the whole point of the
# stricter default). Genuine high-confidence non-English (~1.0) still passes.
assert _accept("de", 0.98) is True assert _accept("de", 0.98) is True
assert _accept("de", 0.86) is True # calibrated: ambiguous-but-correct German stays assert _accept("de", 0.86) is False # the collision band, now cut
assert _accept("de", _MIN_LATIN_CONFIDENCE) is True # exactly at floor assert _accept("de", _DEFAULT_MIN_LATIN_CONFIDENCE) is True # at floor
assert _accept("de", _MIN_LATIN_CONFIDENCE - 0.01) is False assert _accept("de", _DEFAULT_MIN_LATIN_CONFIDENCE - 0.01) is False
assert _accept("fr", 0.40) is False # short mis-flag assert _accept("fr", 0.40) is False # short mis-flag
def test_accept_honors_an_explicit_floor():
# The floor is a parameter now (fed from ImportSettings). A looser floor keeps
# the 0.86 band; a stricter one rejects even a fairly confident detection.
assert _accept("de", 0.86, 0.80) is True
assert _accept("de", 0.86, 0.95) is False
assert _accept("de", 0.96, 0.95) is True
def test_accept_is_case_insensitive(): def test_accept_is_case_insensitive():
@@ -32,3 +45,4 @@ def test_accept_missing_confidence_fails_open():
# No confidence reported → don't silently drop a translation. # No confidence reported → don't silently drop a translation.
assert _accept("de", None) is True assert _accept("de", None) is True
assert _accept("", None) is True assert _accept("", None) is True
assert _accept("de", None, 0.99) is True # fail-open beats a high floor