feat(translation): tunable acceptance floor (0.90) + per-post sticky override (#155) #210
@@ -0,0 +1,51 @@
|
|||||||
|
"""translation strictness setting + per-post translation override (milestone 155)
|
||||||
|
|
||||||
|
ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
|
||||||
|
floor, now operator-tunable in the UI; default 0.9 — stricter than the old
|
||||||
|
hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
|
||||||
|
~0.86). Post gains ``translation_override`` — a sticky per-post choice of
|
||||||
|
auto / force / original so the operator can force a skipped translation on, or
|
||||||
|
knock a wrongly-translated one back to the original, and have it survive a
|
||||||
|
Re-translate-all.
|
||||||
|
|
||||||
|
Revision ID: 0084
|
||||||
|
Revises: 0083
|
||||||
|
Create Date: 2026-07-10
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0084"
|
||||||
|
down_revision: Union[str, None] = "0083"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"translation_min_confidence", sa.Float(), nullable=False,
|
||||||
|
server_default=sa.text("0.9"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"post",
|
||||||
|
sa.Column(
|
||||||
|
"translation_override", sa.String(16), nullable=False,
|
||||||
|
server_default="auto",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_check_constraint(
|
||||||
|
"ck_post_translation_override",
|
||||||
|
"post",
|
||||||
|
"translation_override IN ('auto', 'force', 'original')",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint("ck_post_translation_override", "post", type_="check")
|
||||||
|
op.drop_column("post", "translation_override")
|
||||||
|
op.drop_column("import_settings", "translation_min_confidence")
|
||||||
@@ -3,12 +3,28 @@
|
|||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
|
from ..models import ImportSettings, Post
|
||||||
|
from ..services import interpreter_client as ic
|
||||||
from ..services.post_feed_service import PostFeedService
|
from ..services.post_feed_service import PostFeedService
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
|
from ..utils.text import html_to_plain
|
||||||
from ._responses import error_response as _bad
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||||
|
|
||||||
|
_TRANSLATION_OVERRIDES = ("auto", "force", "original")
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_for_sweep(post: Post) -> None:
|
||||||
|
"""Mark a post untranslated (all translation columns NULL) so the periodic
|
||||||
|
sweep re-runs it under its new override — used when Interpreter is down and we
|
||||||
|
can't translate inline."""
|
||||||
|
post.post_title_translated = None
|
||||||
|
post.description_translated = None
|
||||||
|
post.translated_source_lang = None
|
||||||
|
post.translation_engine_version = None
|
||||||
|
post.translated_at = None
|
||||||
|
|
||||||
|
|
||||||
@posts_bp.route("", methods=["GET"])
|
@posts_bp.route("", methods=["GET"])
|
||||||
async def list_posts():
|
async def list_posts():
|
||||||
@@ -82,3 +98,70 @@ async def get_post(post_id: int):
|
|||||||
if item is None:
|
if item is None:
|
||||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||||
return jsonify(item)
|
return jsonify(item)
|
||||||
|
|
||||||
|
|
||||||
|
@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
|
||||||
|
async def set_translation_override(post_id: int):
|
||||||
|
"""Sticky per-post translation override (milestone 155). Body:
|
||||||
|
``{"override": "auto" | "force" | "original"}``.
|
||||||
|
|
||||||
|
'original' keeps the original (clears any stored translation now — no
|
||||||
|
Interpreter needed). 'force'/'auto' translate the post immediately if the
|
||||||
|
service is up (force bypasses the acceptance floor; auto re-runs the gate);
|
||||||
|
if it's down we save the flag and mark the post untranslated so the next sweep
|
||||||
|
applies it. The override persists, so the sweep + Re-translate-all keep
|
||||||
|
honoring it. Returns the updated translation fields + an ``applied`` status."""
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
override = body.get("override")
|
||||||
|
if override not in _TRANSLATION_OVERRIDES:
|
||||||
|
return _bad(
|
||||||
|
"invalid_override",
|
||||||
|
detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Lazy import (mirrors settings.py) so the API module doesn't pull the celery
|
||||||
|
# task graph at import time.
|
||||||
|
from ..tasks.translation import _store_translation, _translate_field
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
post = await session.get(Post, post_id)
|
||||||
|
if post is None:
|
||||||
|
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||||
|
post.translation_override = override
|
||||||
|
cfg = await ImportSettings.load(session)
|
||||||
|
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||||
|
|
||||||
|
if override == "original":
|
||||||
|
_store_translation(post, (None, None, None), (None, None, None), target)
|
||||||
|
applied = "cleared"
|
||||||
|
else:
|
||||||
|
base_url = (cfg.interpreter_base_url or "").strip()
|
||||||
|
if cfg.translation_enabled and base_url and ic.health(base_url):
|
||||||
|
force = override == "force"
|
||||||
|
title = (post.post_title or "").strip()
|
||||||
|
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||||
|
desc = desc.strip()
|
||||||
|
mc = cfg.translation_min_confidence
|
||||||
|
try:
|
||||||
|
title_res = _translate_field(title, base_url, target, mc, force=force)
|
||||||
|
desc_res = _translate_field(desc, base_url, target, mc, force=force)
|
||||||
|
except ic.InterpreterUnavailable:
|
||||||
|
_queue_for_sweep(post)
|
||||||
|
applied = "queued"
|
||||||
|
else:
|
||||||
|
_store_translation(post, title_res, desc_res, target)
|
||||||
|
applied = "translated"
|
||||||
|
else:
|
||||||
|
# Disabled / no URL / unhealthy → let the sweep apply it later.
|
||||||
|
_queue_for_sweep(post)
|
||||||
|
applied = "queued"
|
||||||
|
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({
|
||||||
|
"id": post.id,
|
||||||
|
"translation_override": post.translation_override,
|
||||||
|
"post_title_translated": post.post_title_translated,
|
||||||
|
"description_translated": post.description_translated,
|
||||||
|
"translated_source_lang": post.translated_source_lang,
|
||||||
|
"applied": applied,
|
||||||
|
})
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ _EDITABLE_FIELDS = (
|
|||||||
"translation_enabled",
|
"translation_enabled",
|
||||||
"interpreter_base_url",
|
"interpreter_base_url",
|
||||||
"translation_target_lang",
|
"translation_target_lang",
|
||||||
|
"translation_min_confidence",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||||
@@ -87,6 +88,7 @@ async def get_import_settings():
|
|||||||
"translation_enabled": row.translation_enabled,
|
"translation_enabled": row.translation_enabled,
|
||||||
"interpreter_base_url": row.interpreter_base_url,
|
"interpreter_base_url": row.interpreter_base_url,
|
||||||
"translation_target_lang": row.translation_target_lang,
|
"translation_target_lang": row.translation_target_lang,
|
||||||
|
"translation_min_confidence": row.translation_min_confidence,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
@@ -155,6 +157,14 @@ async def update_import_settings():
|
|||||||
for key in ("interpreter_base_url", "translation_target_lang"):
|
for key in ("interpreter_base_url", "translation_target_lang"):
|
||||||
if key in body and not isinstance(body[key], str):
|
if key in body and not isinstance(body[key], str):
|
||||||
return jsonify({"error": f"{key} must be a string"}), 400
|
return jsonify({"error": f"{key} must be a string"}), 400
|
||||||
|
# Acceptance floor (milestone 155): latin-script translations below this
|
||||||
|
# Interpreter confidence are kept as the original.
|
||||||
|
if "translation_min_confidence" in body:
|
||||||
|
v = body["translation_min_confidence"]
|
||||||
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||||
|
return jsonify(
|
||||||
|
{"error": "translation_min_confidence must be a number in [0, 1]"}
|
||||||
|
), 400
|
||||||
if "series_suggest_threshold" in body:
|
if "series_suggest_threshold" in body:
|
||||||
v = body["series_suggest_threshold"]
|
v = body["series_suggest_threshold"]
|
||||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||||
|
|||||||
@@ -106,6 +106,15 @@ class ImportSettings(Base):
|
|||||||
translation_target_lang: Mapped[str] = mapped_column(
|
translation_target_lang: Mapped[str] = mapped_column(
|
||||||
Text, nullable=False, default="en", server_default="en",
|
Text, nullable=False, default="en", server_default="en",
|
||||||
)
|
)
|
||||||
|
# The latin-script acceptance floor for the translation gate: a translation
|
||||||
|
# whose Interpreter-reported confidence is below this is kept as the original
|
||||||
|
# (operator-tunable, milestone 155). Default 0.9 — stricter than the old
|
||||||
|
# hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
|
||||||
|
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
||||||
|
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
||||||
|
translation_min_confidence: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=0.9, server_default="0.9",
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def load(cls, session) -> ImportSettings:
|
async def load(cls, session) -> ImportSettings:
|
||||||
|
|||||||
@@ -8,7 +8,17 @@ artist-filter queries don't depend on the Source detour).
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
CheckConstraint,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -23,6 +33,10 @@ class Post(Base):
|
|||||||
# (created in alembic 0030) covers that case via
|
# (created in alembic 0030) covers that case via
|
||||||
# (artist_id, external_post_id).
|
# (artist_id, external_post_id).
|
||||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
|
CheckConstraint(
|
||||||
|
"translation_override IN ('auto', 'force', 'original')",
|
||||||
|
name="ck_post_translation_override",
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
@@ -64,6 +78,16 @@ class Post(Base):
|
|||||||
translated_at: Mapped[datetime | None] = mapped_column(
|
translated_at: Mapped[datetime | None] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=True
|
DateTime(timezone=True), nullable=True
|
||||||
)
|
)
|
||||||
|
# Sticky per-post override of the translation decision (milestone 155):
|
||||||
|
# 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
|
||||||
|
# translation even below the confidence floor (rescue a skipped legit-foreign
|
||||||
|
# title); 'original' = never translate, keep the original (kill a confidently
|
||||||
|
# mis-flagged one the floor can't catch). The sweep reads this on every run,
|
||||||
|
# and re-translate leaves 'original' posts alone, so the choice survives a
|
||||||
|
# Re-translate-all.
|
||||||
|
translation_override: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="auto", server_default="auto",
|
||||||
|
)
|
||||||
|
|
||||||
downloaded_at: Mapped[datetime] = mapped_column(
|
downloaded_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -397,6 +397,8 @@ class PostFeedService:
|
|||||||
"post_title_translated": post.post_title_translated,
|
"post_title_translated": post.post_title_translated,
|
||||||
"description_translated": desc_trans_short,
|
"description_translated": desc_trans_short,
|
||||||
"translated_source_lang": post.translated_source_lang,
|
"translated_source_lang": post.translated_source_lang,
|
||||||
|
# Sticky per-post translation choice (auto/force/original, #155).
|
||||||
|
"translation_override": post.translation_override,
|
||||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
"source": (
|
"source": (
|
||||||
{"id": source.id, "platform": source.platform}
|
{"id": source.id, "platform": source.platform}
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ def _post_dict(p: Post) -> dict:
|
|||||||
"title_translated": p.post_title_translated,
|
"title_translated": p.post_title_translated,
|
||||||
"description_translated": p.description_translated,
|
"description_translated": p.description_translated,
|
||||||
"translated_source_lang": p.translated_source_lang,
|
"translated_source_lang": p.translated_source_lang,
|
||||||
|
"translation_override": p.translation_override,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -60,14 +60,17 @@ _INTERRUPT_BACKOFF_MAX = 900
|
|||||||
# language scores low; require it to clear this floor, else keep the original.
|
# language scores low; require it to clear this floor, else keep the original.
|
||||||
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
|
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
|
||||||
# real langdetect confidence: genuine German detected at 1.0, while a correctly-
|
# real langdetect confidence: genuine German detected at 1.0, while a correctly-
|
||||||
# detected but ambiguous latin string dipped to 0.86 — so the floor sits at 0.80,
|
# detected but ambiguous latin string dipped to 0.86 — so a single constant can't win.
|
||||||
# below that band, to keep legitimate ambiguous non-English while still rejecting
|
# The floor is now operator-tunable (ImportSettings.translation_min_confidence,
|
||||||
# genuinely-unsure guesses. Real langdetect also fixed the original mis-flag at
|
# milestone 155) with a stricter 0.90 default; per-post overrides
|
||||||
# the source (short English titles now detect as English → passthrough), so this
|
# (Post.translation_override) rescue the false negatives it skips. Genuine German
|
||||||
|
# and mis-flagged short English both land ~0.86, and single-word mis-flags sit at
|
||||||
|
# a confident 1.0 no floor catches. Real langdetect fixed some cases at the
|
||||||
|
# source (some short English titles now detect as English → passthrough), so this
|
||||||
# floor is a safety net, not the primary fix. Re-tune via the "Test translation"
|
# floor is a safety net, not the primary fix. Re-tune via the "Test translation"
|
||||||
# box (send fresh text — cache hits report 1.0).
|
# box (send fresh text — cache hits report 1.0).
|
||||||
_CJK_LANGS = frozenset({"ja", "ko", "zh"})
|
_CJK_LANGS = frozenset({"ja", "ko", "zh"})
|
||||||
_MIN_LATIN_CONFIDENCE = 0.80
|
_DEFAULT_MIN_LATIN_CONFIDENCE = 0.90
|
||||||
|
|
||||||
|
|
||||||
def _interrupt_backoff(retry_after) -> int:
|
def _interrupt_backoff(retry_after) -> int:
|
||||||
@@ -100,10 +103,10 @@ def translate_posts(drain: bool = False) -> str:
|
|||||||
ready = _translation_config(session)
|
ready = _translation_config(session)
|
||||||
if isinstance(ready, str):
|
if isinstance(ready, str):
|
||||||
return ready
|
return ready
|
||||||
base_url, target = ready
|
base_url, target, min_confidence = ready
|
||||||
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
||||||
status, translated, retry_after = _translate_batch(
|
status, translated, retry_after = _translate_batch(
|
||||||
session, posts, base_url, target,
|
session, posts, base_url, target, min_confidence,
|
||||||
)
|
)
|
||||||
# Drain mode (manual "Translate now"): chase the tail until the backlog is
|
# Drain mode (manual "Translate now"): chase the tail until the backlog is
|
||||||
# zero, so one press clears the whole pile instead of one 300-chunk.
|
# zero, so one press clears the whole pile instead of one 300-chunk.
|
||||||
@@ -152,7 +155,7 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
|
|||||||
ready = _translation_config(session)
|
ready = _translation_config(session)
|
||||||
if isinstance(ready, str):
|
if isinstance(ready, str):
|
||||||
return ready
|
return ready
|
||||||
base_url, target = ready
|
base_url, target, min_confidence = ready
|
||||||
|
|
||||||
if not _reset_done:
|
if not _reset_done:
|
||||||
n = _reset_translations(session, artist_ids)
|
n = _reset_translations(session, artist_ids)
|
||||||
@@ -163,7 +166,7 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
|
|||||||
|
|
||||||
posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
|
posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
|
||||||
status, translated, retry_after = _translate_batch(
|
status, translated, retry_after = _translate_batch(
|
||||||
session, posts, base_url, target,
|
session, posts, base_url, target, min_confidence,
|
||||||
)
|
)
|
||||||
|
|
||||||
# run-until-done: chase the tail when the chunk finished cleanly.
|
# run-until-done: chase the tail when the chunk finished cleanly.
|
||||||
@@ -201,17 +204,19 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _translation_config(session):
|
def _translation_config(session):
|
||||||
"""Resolve (base_url, target) if translation is enabled + healthy, else a
|
"""Resolve (base_url, target, min_confidence) if translation is enabled +
|
||||||
short status string ("disabled" / "interpreter unavailable") for the caller
|
healthy, else a short status string ("disabled" / "interpreter unavailable")
|
||||||
to return. Centralises the guard so translate/retranslate agree exactly."""
|
for the caller to return. Centralises the guard so translate/retranslate agree
|
||||||
|
exactly, including the operator-tunable acceptance floor."""
|
||||||
cfg = ImportSettings.load_sync(session)
|
cfg = ImportSettings.load_sync(session)
|
||||||
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
|
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
|
||||||
return "disabled"
|
return "disabled"
|
||||||
base_url = cfg.interpreter_base_url.strip()
|
base_url = cfg.interpreter_base_url.strip()
|
||||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||||
|
min_confidence = cfg.translation_min_confidence
|
||||||
if not ic.health(base_url):
|
if not ic.health(base_url):
|
||||||
return "interpreter unavailable"
|
return "interpreter unavailable"
|
||||||
return base_url, target
|
return base_url, target, min_confidence
|
||||||
|
|
||||||
|
|
||||||
def _untranslated_filter(stmt, artist_ids):
|
def _untranslated_filter(stmt, artist_ids):
|
||||||
@@ -244,10 +249,13 @@ def _reset_translations(session, artist_ids) -> int:
|
|||||||
"""Clear the stored translation columns for scoped, already-handled posts so
|
"""Clear the stored translation columns for scoped, already-handled posts so
|
||||||
the untranslated sweep re-runs them. Only touches rows that were translated
|
the untranslated sweep re-runs them. Only touches rows that were translated
|
||||||
(translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
|
(translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
|
||||||
Returns the row count reset (commits it)."""
|
Skips 'keep original' posts (translation_override = 'original') so that choice
|
||||||
|
survives a Re-translate-all (milestone 155). Returns the row count reset
|
||||||
|
(commits it)."""
|
||||||
stmt = (
|
stmt = (
|
||||||
update(Post)
|
update(Post)
|
||||||
.where(Post.translated_source_lang.is_not(None))
|
.where(Post.translated_source_lang.is_not(None))
|
||||||
|
.where(Post.translation_override != "original")
|
||||||
.values(
|
.values(
|
||||||
post_title_translated=None,
|
post_title_translated=None,
|
||||||
description_translated=None,
|
description_translated=None,
|
||||||
@@ -263,7 +271,7 @@ def _reset_translations(session, artist_ids) -> int:
|
|||||||
return n
|
return n
|
||||||
|
|
||||||
|
|
||||||
def _translate_batch(session, posts, base_url: str, target: str):
|
def _translate_batch(session, posts, base_url: str, target: str, min_confidence: float):
|
||||||
"""Translate each post in the chunk with a per-post commit (an interrupted
|
"""Translate each post in the chunk with a per-post commit (an interrupted
|
||||||
run keeps progress). Returns (status, translated, retry_after) where status is
|
run keeps progress). Returns (status, translated, retry_after) where status is
|
||||||
one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout";
|
one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout";
|
||||||
@@ -272,7 +280,7 @@ def _translate_batch(session, posts, base_url: str, target: str):
|
|||||||
translated = 0
|
translated = 0
|
||||||
for post in posts:
|
for post in posts:
|
||||||
try:
|
try:
|
||||||
translated += _translate_one(session, post, base_url, target)
|
translated += _translate_one(session, post, base_url, target, min_confidence)
|
||||||
session.commit()
|
session.commit()
|
||||||
except ic.InterpreterUnavailable as e:
|
except ic.InterpreterUnavailable as e:
|
||||||
session.rollback()
|
session.rollback()
|
||||||
@@ -296,28 +304,36 @@ def _summary(status: str, translated: int, scanned: int) -> str:
|
|||||||
return f"translated={translated} scanned={scanned}"
|
return f"translated={translated} scanned={scanned}"
|
||||||
|
|
||||||
|
|
||||||
def _accept(language: str, confidence) -> bool:
|
def _accept(
|
||||||
|
language: str, confidence, min_confidence: float = _DEFAULT_MIN_LATIN_CONFIDENCE,
|
||||||
|
) -> bool:
|
||||||
"""Should curator store Interpreter's translation, or keep the original?
|
"""Should curator store Interpreter's translation, or keep the original?
|
||||||
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
|
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
|
||||||
133): CJK languages are trusted regardless of the reported number; a
|
133): CJK languages are trusted regardless of the reported number; a
|
||||||
latin-script detection must clear ``_MIN_LATIN_CONFIDENCE``. A missing
|
latin-script detection must clear ``min_confidence`` (the operator-tunable
|
||||||
confidence fails OPEN (accept), so a service that omits the field never causes
|
ImportSettings.translation_min_confidence). A missing confidence fails OPEN
|
||||||
translations to be silently dropped."""
|
(accept), so a service that omits the field never silently drops a
|
||||||
|
translation. Per-post ``force`` overrides bypass this entirely (see
|
||||||
|
``_translate_one``)."""
|
||||||
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
|
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
|
||||||
return True
|
return True
|
||||||
if confidence is None:
|
if confidence is None:
|
||||||
return True
|
return True
|
||||||
return confidence >= _MIN_LATIN_CONFIDENCE
|
return confidence >= min_confidence
|
||||||
|
|
||||||
|
|
||||||
def _translate_field(text: str, base_url: str, target: str):
|
def _translate_field(
|
||||||
|
text: str, base_url: str, target: str, min_confidence: float, force: bool = False,
|
||||||
|
):
|
||||||
"""Translate ONE field independently. Returns (translated, source_lang,
|
"""Translate ONE field independently. Returns (translated, source_lang,
|
||||||
engine_version), or (None, None, None) when there's nothing to store — empty
|
engine_version), or (None, None, None) when there's nothing to store — empty
|
||||||
text, already the target language, a passthrough (engine "none"), or a
|
text, already the target language, a passthrough (engine "none"), or a
|
||||||
latin-script detection the acceptance gate rejects as too low-confidence (kept
|
latin-script detection the acceptance gate rejects as too low-confidence (kept
|
||||||
as the original). Per-field (not aggregate first-item) detection so a
|
as the original). ``force`` (a per-post 'force' override) bypasses the
|
||||||
non-English description still gets translated when the title is already
|
confidence gate so a legitimately-foreign title Interpreter is only ~0.86 sure
|
||||||
English (mixed-language posts)."""
|
about is still stored; a genuine passthrough stores nothing regardless (there
|
||||||
|
is no translation to force). Per-field (not aggregate first-item) detection so
|
||||||
|
a non-English description still translates when the title is already English."""
|
||||||
if not text:
|
if not text:
|
||||||
return None, None, None
|
return None, None, None
|
||||||
res = ic.translate([text], base_url=base_url, target=target)
|
res = ic.translate([text], base_url=base_url, target=target)
|
||||||
@@ -327,23 +343,30 @@ def _translate_field(text: str, base_url: str, target: str):
|
|||||||
# Trust only Interpreter's own detection (Scribe rule 133): keep the original
|
# Trust only Interpreter's own detection (Scribe rule 133): keep the original
|
||||||
# when a latin-script detection doesn't clear the confidence floor, so a short
|
# when a latin-script detection doesn't clear the confidence floor, so a short
|
||||||
# English title mis-labelled as e.g. German isn't rewritten into the archive.
|
# English title mis-labelled as e.g. German isn't rewritten into the archive.
|
||||||
if not _accept(detected, res.get("detected_confidence")):
|
# A 'force' override skips the gate — the operator is overriding a rejection.
|
||||||
|
if not force and not _accept(
|
||||||
|
detected, res.get("detected_confidence"), min_confidence,
|
||||||
|
):
|
||||||
return None, None, None
|
return None, None, None
|
||||||
return res["translations"][0], detected, res["engine_version"]
|
return res["translations"][0], detected, res["engine_version"]
|
||||||
|
|
||||||
|
|
||||||
def _translate_one(session, post, base_url: str, target: str) -> int:
|
def _store_translation(post, title_res, desc_res, target: str) -> int:
|
||||||
"""Translate a post's title/description in place, each field independently.
|
"""Apply per-field (translated, source_lang, engine_version) results to a post
|
||||||
Returns 1 if it stored at least one translation, 0 when the post is all
|
in place. Returns 1 if a translation was stored, 0 when the post is all
|
||||||
passthrough/empty (still marks it handled so the sweep won't revisit it)."""
|
passthrough/empty — in which case the translated columns are CLEARED and the
|
||||||
title = (post.post_title or "").strip()
|
post is marked handled (translated_source_lang = target). Clearing (not just
|
||||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
leaving) matters on re-apply: a 'keep original' override, or a re-translate
|
||||||
desc = desc.strip()
|
that now rejects a previously-accepted mis-flag, must remove the stale
|
||||||
title_tr, title_lang, title_ev = _translate_field(title, base_url, target)
|
translation, not keep it."""
|
||||||
desc_tr, desc_lang, desc_ev = _translate_field(desc, base_url, target)
|
title_tr, title_lang, title_ev = title_res
|
||||||
|
desc_tr, desc_lang, desc_ev = desc_res
|
||||||
if title_tr is None and desc_tr is None:
|
if title_tr is None and desc_tr is None:
|
||||||
# Nothing non-target (or nothing to translate) → handled, store nothing.
|
post.post_title_translated = None
|
||||||
|
post.description_translated = None
|
||||||
post.translated_source_lang = target
|
post.translated_source_lang = target
|
||||||
|
post.translation_engine_version = None
|
||||||
|
post.translated_at = None
|
||||||
return 0
|
return 0
|
||||||
post.post_title_translated = title_tr
|
post.post_title_translated = title_tr
|
||||||
post.description_translated = desc_tr
|
post.description_translated = desc_tr
|
||||||
@@ -353,3 +376,23 @@ def _translate_one(session, post, base_url: str, target: str) -> int:
|
|||||||
post.translation_engine_version = title_ev or desc_ev
|
post.translation_engine_version = title_ev or desc_ev
|
||||||
post.translated_at = datetime.now(UTC)
|
post.translated_at = datetime.now(UTC)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def _translate_one(
|
||||||
|
session, post, base_url: str, target: str, min_confidence: float,
|
||||||
|
) -> int:
|
||||||
|
"""Translate a post's title/description in place, each field independently,
|
||||||
|
honoring the per-post override (milestone 155): 'original' keeps the original
|
||||||
|
(clears any stored translation, no Interpreter call); 'force' translates even
|
||||||
|
below the confidence floor; 'auto' (default) runs the acceptance gate. Returns
|
||||||
|
1 if it stored a translation, else 0 (still marks the post handled so the sweep
|
||||||
|
won't revisit it)."""
|
||||||
|
if post.translation_override == "original":
|
||||||
|
return _store_translation(post, (None, None, None), (None, None, None), target)
|
||||||
|
force = post.translation_override == "force"
|
||||||
|
title = (post.post_title or "").strip()
|
||||||
|
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||||
|
desc = desc.strip()
|
||||||
|
title_res = _translate_field(title, base_url, target, min_confidence, force=force)
|
||||||
|
desc_res = _translate_field(desc, base_url, target, min_confidence, force=force)
|
||||||
|
return _store_translation(post, title_res, desc_res, target)
|
||||||
|
|||||||
@@ -70,6 +70,14 @@
|
|||||||
@click.stop="showOriginal = !showOriginal"
|
@click.stop="showOriginal = !showOriginal"
|
||||||
>{{ showOriginal ? 'Show translation' : `Show original${sourceLangLabel}` }}</button>
|
>{{ showOriginal ? 'Show translation' : `Show original${sourceLangLabel}` }}</button>
|
||||||
|
|
||||||
|
<!-- Per-post translation override (#155): force a skipped translation on,
|
||||||
|
or keep the original for a confidently mis-flagged title. Only where
|
||||||
|
there's text to translate. -->
|
||||||
|
<PostTranslationControl
|
||||||
|
v-if="post.post_title || post.description_plain"
|
||||||
|
:post="post"
|
||||||
|
/>
|
||||||
|
|
||||||
<!-- Faithful (semantic) body render once expanded: backend-sanitized
|
<!-- Faithful (semantic) body render once expanded: backend-sanitized
|
||||||
HTML (headings, lists, links, inline images). Collapsed and the
|
HTML (headings, lists, links, inline images). Collapsed and the
|
||||||
no-detail fallback stay plain text. -->
|
no-detail fallback stay plain text. -->
|
||||||
@@ -136,6 +144,7 @@ import { useModalStore } from '../../stores/modal.js'
|
|||||||
import { usePostsStore } from '../../stores/posts.js'
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
import { toPlainText } from '../../utils/htmlSanitize.js'
|
import { toPlainText } from '../../utils/htmlSanitize.js'
|
||||||
import PostSeriesMenu from './PostSeriesMenu.vue'
|
import PostSeriesMenu from './PostSeriesMenu.vue'
|
||||||
|
import PostTranslationControl from './PostTranslationControl.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
post: { type: Object, required: true },
|
post: { type: Object, required: true },
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
<template>
|
||||||
|
<!-- Sticky per-post translation override (#155). Quiet inline control under the
|
||||||
|
post title: force a skipped legit-foreign title on, or knock a wrongly
|
||||||
|
mis-translated one back to the original. The choice sticks through a
|
||||||
|
Re-translate-all. -->
|
||||||
|
<div class="fc-post-tx">
|
||||||
|
<span class="fc-post-tx__label">Translation:</span>
|
||||||
|
<v-menu :disabled="busy">
|
||||||
|
<template #activator="{ props: menuProps }">
|
||||||
|
<button
|
||||||
|
type="button" class="fc-post-tx__btn" v-bind="menuProps" :disabled="busy"
|
||||||
|
:aria-label="`Translation handling: ${currentLabel}`"
|
||||||
|
>
|
||||||
|
{{ currentLabel }}
|
||||||
|
<v-icon size="14">mdi-menu-down</v-icon>
|
||||||
|
</button>
|
||||||
|
</template>
|
||||||
|
<v-list density="compact" min-width="240" class="fc-post-tx__list">
|
||||||
|
<v-list-item
|
||||||
|
v-for="opt in OPTIONS" :key="opt.value"
|
||||||
|
:active="opt.value === current" @click="choose(opt.value)"
|
||||||
|
>
|
||||||
|
<template #prepend>
|
||||||
|
<v-icon size="small">{{ opt.icon }}</v-icon>
|
||||||
|
</template>
|
||||||
|
<v-list-item-title>{{ opt.label }}</v-list-item-title>
|
||||||
|
<v-list-item-subtitle>{{ opt.help }}</v-list-item-subtitle>
|
||||||
|
</v-list-item>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
<v-progress-circular v-if="busy" indeterminate size="13" width="2" />
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
|
|
||||||
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
|
|
||||||
|
const props = defineProps({ post: { type: Object, required: true } })
|
||||||
|
const posts = usePostsStore()
|
||||||
|
const busy = ref(false)
|
||||||
|
|
||||||
|
const OPTIONS = [
|
||||||
|
{ value: 'auto', label: 'Auto', icon: 'mdi-cog-outline',
|
||||||
|
help: 'Translate only when confident enough' },
|
||||||
|
{ value: 'force', label: 'Force translate', icon: 'mdi-translate',
|
||||||
|
help: 'Always translate, even at low confidence' },
|
||||||
|
{ value: 'original', label: 'Keep original', icon: 'mdi-translate-off',
|
||||||
|
help: 'Never translate — keep the original text' },
|
||||||
|
]
|
||||||
|
|
||||||
|
const current = computed(() => props.post.translation_override || 'auto')
|
||||||
|
const currentLabel = computed(
|
||||||
|
() => (OPTIONS.find((o) => o.value === current.value) || OPTIONS[0]).label,
|
||||||
|
)
|
||||||
|
|
||||||
|
async function choose (value) {
|
||||||
|
if (value === current.value || busy.value) return
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
const res = await posts.applyTranslationOverride(props.post.id, value)
|
||||||
|
if (res.applied === 'queued') {
|
||||||
|
toast({
|
||||||
|
text: 'Saved — this post will translate on the next sweep (service offline).',
|
||||||
|
type: 'info',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Couldn't update translation: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-post-tx {
|
||||||
|
display: inline-flex; align-items: center; gap: 6px;
|
||||||
|
margin: 2px 0 6px;
|
||||||
|
}
|
||||||
|
.fc-post-tx__label {
|
||||||
|
font-size: 0.72rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-post-tx__btn {
|
||||||
|
display: inline-flex; align-items: center; gap: 1px;
|
||||||
|
padding: 0; background: none; border: 0; cursor: pointer;
|
||||||
|
font-size: 0.72rem; font-weight: 700;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-post-tx__btn:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
|
.fc-post-tx__btn:focus-visible {
|
||||||
|
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
||||||
|
border-radius: 2px;
|
||||||
|
}
|
||||||
|
.fc-post-tx__btn:disabled { cursor: default; opacity: 0.6; }
|
||||||
|
.fc-post-tx__list :deep(.v-list-item-subtitle) {
|
||||||
|
font-size: 0.7rem;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -28,6 +28,21 @@
|
|||||||
@change="onSave"
|
@change="onSave"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
<!-- Acceptance floor (#155): latin-script translations below this Interpreter
|
||||||
|
confidence are kept as the original. Per-post overrides handle exceptions. -->
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="minConfidence" label="Acceptance confidence"
|
||||||
|
type="number" min="0" max="1" step="0.01" density="compact"
|
||||||
|
hide-details style="max-width: 200px;" class="mb-1" :disabled="busy"
|
||||||
|
@change="onSaveConfidence"
|
||||||
|
/>
|
||||||
|
<div class="fc-muted text-caption mb-3">
|
||||||
|
Latin-script titles Interpreter is less than this sure about (0–1) are kept
|
||||||
|
as the original; CJK is always translated. Default 0.90 — raise it to reject
|
||||||
|
more mis-detections, lower it to translate more. Per-post controls in the
|
||||||
|
posts feed override this either way.
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="d-flex align-center flex-wrap mb-3" style="gap: 8px;">
|
<div class="d-flex align-center flex-wrap mb-3" style="gap: 8px;">
|
||||||
<v-icon size="12" :color="statusColor">mdi-circle</v-icon>
|
<v-icon size="12" :color="statusColor">mdi-circle</v-icon>
|
||||||
<span class="fc-muted text-body-2">{{ statusText }}</span>
|
<span class="fc-muted text-body-2">{{ statusText }}</span>
|
||||||
@@ -175,6 +190,7 @@ const store = useImportStore()
|
|||||||
const enabled = ref(false)
|
const enabled = ref(false)
|
||||||
const baseUrl = ref('')
|
const baseUrl = ref('')
|
||||||
const targetLang = ref('en')
|
const targetLang = ref('en')
|
||||||
|
const minConfidence = ref(0.9)
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const running = ref(false)
|
const running = ref(false)
|
||||||
const retranslating = ref(false)
|
const retranslating = ref(false)
|
||||||
@@ -258,6 +274,7 @@ onMounted(async () => {
|
|||||||
enabled.value = !!s.translation_enabled
|
enabled.value = !!s.translation_enabled
|
||||||
baseUrl.value = s.interpreter_base_url || ''
|
baseUrl.value = s.interpreter_base_url || ''
|
||||||
targetLang.value = s.translation_target_lang || 'en'
|
targetLang.value = s.translation_target_lang || 'en'
|
||||||
|
minConfidence.value = s.translation_min_confidence ?? 0.9
|
||||||
} catch { /* non-fatal */ }
|
} catch { /* non-fatal */ }
|
||||||
await loadStatus()
|
await loadStatus()
|
||||||
})
|
})
|
||||||
@@ -282,6 +299,14 @@ async function onSave() {
|
|||||||
})
|
})
|
||||||
await loadStatus()
|
await loadStatus()
|
||||||
}
|
}
|
||||||
|
async function onSaveConfidence() {
|
||||||
|
// Clamp to [0, 1] so a stray value never bounces off the API 400.
|
||||||
|
let v = Number(minConfidence.value)
|
||||||
|
if (!Number.isFinite(v)) v = 0.9
|
||||||
|
v = Math.min(1, Math.max(0, v))
|
||||||
|
minConfidence.value = v
|
||||||
|
await save({ translation_min_confidence: v })
|
||||||
|
}
|
||||||
async function onRun() {
|
async function onRun() {
|
||||||
running.value = true
|
running.value = true
|
||||||
err.value = null
|
err.value = null
|
||||||
|
|||||||
@@ -71,6 +71,24 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
return await api.get(`/api/posts/${id}`)
|
return await api.get(`/api/posts/${id}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set a post's sticky translation override (auto/force/original, #155) and
|
||||||
|
// patch the loaded item in place with the endpoint's result (it translates /
|
||||||
|
// clears immediately when the service is up), so the card reflects the change
|
||||||
|
// without a feed reload.
|
||||||
|
async function applyTranslationOverride(postId, override) {
|
||||||
|
const res = await api.post(`/api/posts/${postId}/translation-override`, {
|
||||||
|
body: { override },
|
||||||
|
})
|
||||||
|
const item = items.value.find((p) => p.id === postId)
|
||||||
|
if (item) {
|
||||||
|
item.translation_override = res.translation_override
|
||||||
|
item.post_title_translated = res.post_title_translated
|
||||||
|
item.description_translated = res.description_translated
|
||||||
|
item.translated_source_lang = res.translated_source_lang
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
// Filter overlay for the around/older/newer (in-context anchored)
|
// Filter overlay for the around/older/newer (in-context anchored)
|
||||||
// path. Keep this distinct from `filters.value` (the down-only feed)
|
// path. Keep this distinct from `filters.value` (the down-only feed)
|
||||||
// so a normal-feed filter change doesn't leak into an active anchored
|
// so a normal-feed filter change doesn't leak into an active anchored
|
||||||
@@ -168,7 +186,7 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
return {
|
return {
|
||||||
items, cursor, loading, done, error, filters,
|
items, cursor, loading, done, error, filters,
|
||||||
cursorOlder, cursorNewer, doneOlder, doneNewer, anchorId,
|
cursorOlder, cursorNewer, doneOlder, doneNewer, anchorId,
|
||||||
loadInitial, loadMore, getPostFull,
|
loadInitial, loadMore, getPostFull, applyTranslationOverride,
|
||||||
loadAround, loadOlder, loadNewer,
|
loadAround, loadOlder, loadNewer,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -184,6 +184,68 @@ async def test_rejects_bad_direction(client):
|
|||||||
assert (await resp.get_json())["error"] == "invalid_direction"
|
assert (await resp.get_json())["error"] == "invalid_direction"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_override_rejects_bad_value(client, seeded_post):
|
||||||
|
_, _, post = seeded_post
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/posts/{post.id}/translation-override", json={"override": "nope"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert (await resp.get_json())["error"] == "invalid_override"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_override_404_for_unknown(client):
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/posts/999999/translation-override", json={"override": "original"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_override_original_clears_and_keeps(client, db, seeded_post):
|
||||||
|
# 'keep original' clears a stored translation immediately (no Interpreter) and
|
||||||
|
# marks the post handled (== target). Asserts on the response, which reflects
|
||||||
|
# the endpoint's own committed session.
|
||||||
|
_, _, post = seeded_post
|
||||||
|
post.post_title_translated = "STALE"
|
||||||
|
post.description_translated = "STALE"
|
||||||
|
post.translated_source_lang = "de"
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/posts/{post.id}/translation-override", json={"override": "original"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["translation_override"] == "original"
|
||||||
|
assert body["applied"] == "cleared"
|
||||||
|
assert body["post_title_translated"] is None
|
||||||
|
assert body["translated_source_lang"] == "en"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_override_force_queues_when_disabled(client, seeded_post):
|
||||||
|
# Translation disabled (default) → can't translate inline; the override is
|
||||||
|
# saved and the post is queued (columns NULLed) for the next sweep.
|
||||||
|
_, _, post = seeded_post
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/posts/{post.id}/translation-override", json={"override": "force"}
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["translation_override"] == "force"
|
||||||
|
assert body["applied"] == "queued"
|
||||||
|
assert body["translated_source_lang"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_list_exposes_translation_override(client, seeded_post):
|
||||||
|
resp = await client.get("/api/posts")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["items"][0]["translation_override"] == "auto" # default
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_detail_returns_uncapped_thumbnails(client, db):
|
async def test_detail_returns_uncapped_thumbnails(client, db):
|
||||||
"""Feed query caps thumbnails at 6 for previews; detail endpoint
|
"""Feed query caps thumbnails at 6 for previews; detail endpoint
|
||||||
|
|||||||
@@ -53,6 +53,26 @@ async def test_translation_settings_defaults_and_patch(client):
|
|||||||
"/api/settings/import", json={"interpreter_base_url": 123})).status_code == 400
|
"/api/settings/import", json={"interpreter_base_url": 123})).status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_min_confidence_default_and_patch(client):
|
||||||
|
# #155: the latin-script acceptance floor is an operator-tunable setting,
|
||||||
|
# default 0.9 (stricter than the old hardcoded 0.8).
|
||||||
|
body = await (await client.get("/api/settings/import")).get_json()
|
||||||
|
assert body["translation_min_confidence"] == 0.9
|
||||||
|
|
||||||
|
ok = await client.patch(
|
||||||
|
"/api/settings/import", json={"translation_min_confidence": 0.95}
|
||||||
|
)
|
||||||
|
assert ok.status_code == 200
|
||||||
|
assert (await ok.get_json())["translation_min_confidence"] == 0.95
|
||||||
|
|
||||||
|
# Out-of-range + wrong-type are rejected.
|
||||||
|
assert (await client.patch(
|
||||||
|
"/api/settings/import", json={"translation_min_confidence": 1.5})).status_code == 400
|
||||||
|
assert (await client.patch(
|
||||||
|
"/api/settings/import", json={"translation_min_confidence": "high"})).status_code == 400
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_translation_status_defaults(client):
|
async def test_translation_status_defaults(client):
|
||||||
# Off + no URL → no network health call, healthy False (#143).
|
# Off + no URL → no network health call, healthy False (#143).
|
||||||
|
|||||||
@@ -314,6 +314,41 @@ def test_translate_posts_accepts_low_confidence_cjk(db_sync, monkeypatch):
|
|||||||
assert p.translated_source_lang == "zh"
|
assert p.translated_source_lang == "zh"
|
||||||
|
|
||||||
|
|
||||||
|
def test_translate_posts_force_override_translates_below_floor(db_sync, monkeypatch):
|
||||||
|
# override='force' stores the translation even below the confidence floor —
|
||||||
|
# rescuing a legitimately-foreign title the gate would otherwise skip.
|
||||||
|
_patch(monkeypatch, db_sync)
|
||||||
|
_mock_lang(monkeypatch, lang="de", conf=0.42) # below the 0.90 default floor
|
||||||
|
a = _artist(db_sync)
|
||||||
|
p = _post(db_sync, a.id, title="Kurz", ext="p1")
|
||||||
|
p.translation_override = "force"
|
||||||
|
_enable(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
translate_posts()
|
||||||
|
db_sync.refresh(p)
|
||||||
|
assert p.post_title_translated == "EN:Kurz" # forced through despite 0.42
|
||||||
|
assert p.translated_source_lang == "de"
|
||||||
|
|
||||||
|
|
||||||
|
def test_translate_posts_original_override_keeps_original(db_sync, monkeypatch):
|
||||||
|
# override='original' never translates — even a confident (0.98) detection is
|
||||||
|
# kept as the original. The sweep marks it handled (== target) and stores
|
||||||
|
# nothing; Interpreter isn't consulted for the decision.
|
||||||
|
_patch(monkeypatch, db_sync)
|
||||||
|
_mock_lang(monkeypatch, lang="de", conf=0.98) # would be accepted under auto
|
||||||
|
a = _artist(db_sync)
|
||||||
|
p = _post(db_sync, a.id, title="Das Mädchen", ext="p1")
|
||||||
|
p.translation_override = "original"
|
||||||
|
_enable(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
translate_posts()
|
||||||
|
db_sync.refresh(p)
|
||||||
|
assert p.post_title_translated is None # kept original
|
||||||
|
assert p.translated_source_lang == "en" # marked handled
|
||||||
|
|
||||||
|
|
||||||
def _mock_new_model(monkeypatch, *, ver="v2"):
|
def _mock_new_model(monkeypatch, *, ver="v2"):
|
||||||
"""Interpreter is up and translates with a NEW engine version."""
|
"""Interpreter is up and translates with a NEW engine version."""
|
||||||
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
||||||
@@ -344,6 +379,25 @@ def test_retranslate_resets_and_reruns(db_sync, monkeypatch):
|
|||||||
assert p.translation_engine_version == "v2"
|
assert p.translation_engine_version == "v2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_retranslate_leaves_keep_original_posts(db_sync, monkeypatch):
|
||||||
|
# A 'keep original' post survives a Re-translate-all: _reset_translations skips
|
||||||
|
# override='original', so it stays handled + untranslated instead of being
|
||||||
|
# re-run (which would re-introduce the mis-flag the operator killed).
|
||||||
|
_patch(monkeypatch, db_sync)
|
||||||
|
_mock_new_model(monkeypatch)
|
||||||
|
a = _artist(db_sync)
|
||||||
|
p = _post(db_sync, a.id, title="ねこ", ext="p1")
|
||||||
|
p.translation_override = "original"
|
||||||
|
p.translated_source_lang = "en" # handled by a prior 'keep original' apply
|
||||||
|
_enable(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
retranslate_posts(artist_ids=[a.id])
|
||||||
|
db_sync.refresh(p)
|
||||||
|
assert p.translated_source_lang == "en" # not reset
|
||||||
|
assert p.post_title_translated is None # not re-translated
|
||||||
|
|
||||||
|
|
||||||
def test_retranslate_artist_scope_leaves_others(db_sync, monkeypatch):
|
def test_retranslate_artist_scope_leaves_others(db_sync, monkeypatch):
|
||||||
# Scoping to one artist must not touch another artist's translations.
|
# Scoping to one artist must not touch another artist's translations.
|
||||||
_patch(monkeypatch, db_sync)
|
_patch(monkeypatch, db_sync)
|
||||||
|
|||||||
@@ -1,26 +1,39 @@
|
|||||||
"""Translation acceptance gate (#1376) — pure unit tests for ``_accept``, which
|
"""Translation acceptance gate (#1376, #155) — pure unit tests for ``_accept``,
|
||||||
decides whether curator stores Interpreter's translation or keeps the original.
|
which decides whether curator stores Interpreter's translation or keeps the
|
||||||
Curator does no detection of its own; it only thresholds Interpreter's reported
|
original. Curator does no detection of its own; it only thresholds Interpreter's
|
||||||
detected language + confidence (Scribe rule 133). No DB, no network."""
|
reported detected language + confidence against the operator-tunable floor
|
||||||
from backend.app.tasks.translation import _MIN_LATIN_CONFIDENCE, _accept
|
(Scribe rule 133). No DB, no network."""
|
||||||
|
from backend.app.tasks.translation import _DEFAULT_MIN_LATIN_CONFIDENCE, _accept
|
||||||
|
|
||||||
|
|
||||||
def test_accept_trusts_cjk_regardless_of_confidence():
|
def test_accept_trusts_cjk_regardless_of_confidence_or_floor():
|
||||||
# ja/ko/zh are script-detected — accepted even at a low reported number
|
# ja/ko/zh are script-detected — accepted even at a low reported number and
|
||||||
# (pure-kanji Japanese can come back labelled zh ~0.75, still translated).
|
# even against a high floor (pure-kanji Japanese can come back labelled zh
|
||||||
|
# ~0.75, still translated).
|
||||||
assert _accept("ja", None) is True
|
assert _accept("ja", None) is True
|
||||||
assert _accept("ja", 0.10) is True
|
assert _accept("ja", 0.10, 0.99) is True
|
||||||
assert _accept("ko", 0.99) is True
|
assert _accept("ko", 0.99) is True
|
||||||
assert _accept("zh", 0.75) is True
|
assert _accept("zh", 0.75) is True
|
||||||
assert _accept("zh-CN", 0.20) is True # region suffix is still CJK
|
assert _accept("zh-CN", 0.20) is True # region suffix is still CJK
|
||||||
|
|
||||||
|
|
||||||
def test_accept_requires_floor_for_latin():
|
def test_accept_default_floor_is_strict():
|
||||||
|
# Default floor is 0.90 (#155): the ~0.86 band — where genuine German AND
|
||||||
|
# mis-flagged short English collide — is now REJECTED (the whole point of the
|
||||||
|
# stricter default). Genuine high-confidence non-English (~1.0) still passes.
|
||||||
assert _accept("de", 0.98) is True
|
assert _accept("de", 0.98) is True
|
||||||
assert _accept("de", 0.86) is True # calibrated: ambiguous-but-correct German stays
|
assert _accept("de", 0.86) is False # the collision band, now cut
|
||||||
assert _accept("de", _MIN_LATIN_CONFIDENCE) is True # exactly at floor
|
assert _accept("de", _DEFAULT_MIN_LATIN_CONFIDENCE) is True # at floor
|
||||||
assert _accept("de", _MIN_LATIN_CONFIDENCE - 0.01) is False
|
assert _accept("de", _DEFAULT_MIN_LATIN_CONFIDENCE - 0.01) is False
|
||||||
assert _accept("fr", 0.40) is False # short mis-flag
|
assert _accept("fr", 0.40) is False # short mis-flag
|
||||||
|
|
||||||
|
|
||||||
|
def test_accept_honors_an_explicit_floor():
|
||||||
|
# The floor is a parameter now (fed from ImportSettings). A looser floor keeps
|
||||||
|
# the 0.86 band; a stricter one rejects even a fairly confident detection.
|
||||||
|
assert _accept("de", 0.86, 0.80) is True
|
||||||
|
assert _accept("de", 0.86, 0.95) is False
|
||||||
|
assert _accept("de", 0.96, 0.95) is True
|
||||||
|
|
||||||
|
|
||||||
def test_accept_is_case_insensitive():
|
def test_accept_is_case_insensitive():
|
||||||
@@ -32,3 +45,4 @@ def test_accept_missing_confidence_fails_open():
|
|||||||
# No confidence reported → don't silently drop a translation.
|
# No confidence reported → don't silently drop a translation.
|
||||||
assert _accept("de", None) is True
|
assert _accept("de", None) is True
|
||||||
assert _accept("", None) is True
|
assert _accept("", None) is True
|
||||||
|
assert _accept("de", None, 0.99) is True # fail-open beats a high floor
|
||||||
|
|||||||
Reference in New Issue
Block a user