Compare commits
9 Commits
d631ed023c
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
| a017771621 | |||
| 25e555cab6 | |||
| 3c7ab44e74 | |||
| 06f98acf3e | |||
| 4371ddb7e7 | |||
| 40cc11be5b | |||
| 0b78264d62 | |||
| 9eae636047 | |||
| f8105046dc |
@@ -145,6 +145,9 @@ async def similar():
|
|||||||
filters, _sort = _parse_filters()
|
filters, _sort = _parse_filters()
|
||||||
except (KeyError, ValueError):
|
except (KeyError, ValueError):
|
||||||
return jsonify({"error": "similar_to query param required"}), 400
|
return jsonify({"error": "similar_to query param required"}), 400
|
||||||
|
# Explore passes exclude_wip=1 to also drop work-in-progress from the
|
||||||
|
# rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
|
||||||
|
exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
|
||||||
# post_id is the exclusive post-detail view — not a similarity scope.
|
# post_id is the exclusive post-detail view — not a similarity scope.
|
||||||
# include_hidden is a gallery-browse flag; similar() has its OWN presentation
|
# include_hidden is a gallery-browse flag; similar() has its OWN presentation
|
||||||
# exclusion (a similarity-quality concern, #1274), so drop it here (#141).
|
# exclusion (a similarity-quality concern, #1274), so drop it here (#141).
|
||||||
@@ -154,7 +157,8 @@ async def similar():
|
|||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
svc = GalleryService(session)
|
svc = GalleryService(session)
|
||||||
try:
|
try:
|
||||||
images = await svc.similar(image_id=similar_to, limit=limit, **scope)
|
images = await svc.similar(
|
||||||
|
image_id=similar_to, limit=limit, exclude_wip=exclude_wip, **scope)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return jsonify({"error": str(exc)}), 400
|
return jsonify({"error": str(exc)}), 400
|
||||||
if images is None:
|
if images is None:
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from ..models import (
|
|||||||
ImportTask,
|
ImportTask,
|
||||||
Post,
|
Post,
|
||||||
Tag,
|
Tag,
|
||||||
|
TaskRun,
|
||||||
)
|
)
|
||||||
from ..services import interpreter_client as ic
|
from ..services import interpreter_client as ic
|
||||||
|
|
||||||
@@ -306,6 +307,10 @@ async def translation_status():
|
|||||||
"""For the Settings card: is it on, is a URL set, is the service reachable,
|
"""For the Settings card: is it on, is a URL set, is the service reachable,
|
||||||
and how many posts still await translation. Health runs the sync client in a
|
and how many posts still await translation. Health runs the sync client in a
|
||||||
thread so the event loop isn't blocked."""
|
thread so the event loop isn't blocked."""
|
||||||
|
translation_tasks = (
|
||||||
|
"backend.app.tasks.translation.translate_posts",
|
||||||
|
"backend.app.tasks.translation.retranslate_posts",
|
||||||
|
)
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
cfg = await ImportSettings.load(session)
|
cfg = await ImportSettings.load(session)
|
||||||
untranslated = (await session.execute(
|
untranslated = (await session.execute(
|
||||||
@@ -315,6 +320,21 @@ async def translation_status():
|
|||||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
Post.post_title.is_not(None), Post.description.is_not(None),
|
||||||
))
|
))
|
||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
|
# Live progress: is a sweep running now, and what did the last one do?
|
||||||
|
# (run-until-done re-enqueues itself, so `active` stays true across a
|
||||||
|
# bulk re-translate; `last_run` surfaces a completed run's outcome.)
|
||||||
|
active = (await session.execute(
|
||||||
|
select(func.count(TaskRun.id))
|
||||||
|
.where(TaskRun.task_name.in_(translation_tasks))
|
||||||
|
.where(TaskRun.status == "running")
|
||||||
|
)).scalar_one()
|
||||||
|
last = (await session.execute(
|
||||||
|
select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
|
||||||
|
.where(TaskRun.task_name.in_(translation_tasks))
|
||||||
|
.where(TaskRun.finished_at.is_not(None))
|
||||||
|
.order_by(TaskRun.finished_at.desc())
|
||||||
|
.limit(1)
|
||||||
|
)).first()
|
||||||
base_url = (cfg.interpreter_base_url or "").strip()
|
base_url = (cfg.interpreter_base_url or "").strip()
|
||||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
||||||
return jsonify({
|
return jsonify({
|
||||||
@@ -322,6 +342,12 @@ async def translation_status():
|
|||||||
"base_url_set": bool(base_url),
|
"base_url_set": bool(base_url),
|
||||||
"healthy": healthy,
|
"healthy": healthy,
|
||||||
"untranslated_count": int(untranslated),
|
"untranslated_count": int(untranslated),
|
||||||
|
"active": int(active) > 0,
|
||||||
|
"last_run": {
|
||||||
|
"task": last[0].rsplit(".", 1)[-1],
|
||||||
|
"status": last[1],
|
||||||
|
"finished_at": last[2].isoformat() if last[2] else None,
|
||||||
|
} if last else None,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -115,6 +115,11 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
|
"cleanup-orphaned-temp-files": {
|
||||||
|
"task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
|
||||||
|
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
||||||
|
# download/import killed mid-write (graceful-shutdown fallout)
|
||||||
|
},
|
||||||
"train-heads-nightly": {
|
"train-heads-nightly": {
|
||||||
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
||||||
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
||||||
|
|||||||
@@ -48,6 +48,10 @@ class TagKind(StrEnum):
|
|||||||
# content. `wip` is real art: only the training pipelines exclude it.
|
# content. `wip` is real art: only the training pipelines exclude it.
|
||||||
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
||||||
PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot")
|
PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot")
|
||||||
|
# `wip` marks real-but-unfinished art. It's kept in the gallery's own "similar"
|
||||||
|
# results (#1274), but the Explore rabbit-hole opts to hide it (exclude_wip) so a
|
||||||
|
# browse doesn't keep surfacing work-in-progress (operator, 2026-07-08).
|
||||||
|
WIP_SYSTEM_TAG = "wip"
|
||||||
|
|
||||||
image_tag = Table(
|
image_tag = Table(
|
||||||
"image_tag",
|
"image_tag",
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ from ..models import (
|
|||||||
Tag,
|
Tag,
|
||||||
TagPositiveConfirmation,
|
TagPositiveConfirmation,
|
||||||
)
|
)
|
||||||
from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag
|
from ..models.tag import PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG, image_tag
|
||||||
from .pagination import decode_cursor, encode_cursor
|
from .pagination import decode_cursor, encode_cursor
|
||||||
from .tag_query import (
|
from .tag_query import (
|
||||||
fandom_join_alias,
|
fandom_join_alias,
|
||||||
@@ -715,6 +715,7 @@ class GalleryService:
|
|||||||
platform: str | None = None,
|
platform: str | None = None,
|
||||||
untagged: bool = False, no_artist: bool = False,
|
untagged: bool = False, no_artist: bool = False,
|
||||||
date_from: datetime | None = None, date_to: datetime | None = None,
|
date_from: datetime | None = None, date_to: datetime | None = None,
|
||||||
|
exclude_wip: bool = False,
|
||||||
) -> list[GalleryImage] | None:
|
) -> list[GalleryImage] | None:
|
||||||
"""Visual "more like this": images near `image_id`'s SigLIP embedding
|
"""Visual "more like this": images near `image_id`'s SigLIP embedding
|
||||||
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
|
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
|
||||||
@@ -751,14 +752,18 @@ class GalleryService:
|
|||||||
# Presentation images (banner / editor-screenshot system tags, #128)
|
# Presentation images (banner / editor-screenshot system tags, #128)
|
||||||
# cluster on UI chrome rather than content, so near any one of them
|
# cluster on UI chrome rather than content, so near any one of them
|
||||||
# they'd fill the grid. Excluded from CANDIDATES only — the anchor
|
# they'd fill the grid. Excluded from CANDIDATES only — the anchor
|
||||||
# itself may be a banner, and `wip` stays surfaced (real art; only
|
# itself may be a banner. `wip` stays surfaced here by default (real art;
|
||||||
# the training pipelines exclude it).
|
# only the training pipelines exclude it), but the Explore rabbit-hole
|
||||||
|
# passes exclude_wip to also drop work-in-progress (operator, 2026-07-08).
|
||||||
|
excluded_system_tags = PRESENTATION_SYSTEM_TAGS
|
||||||
|
if exclude_wip:
|
||||||
|
excluded_system_tags = (*PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG)
|
||||||
presentation = (
|
presentation = (
|
||||||
select(image_tag.c.image_record_id)
|
select(image_tag.c.image_record_id)
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||||
.where(
|
.where(
|
||||||
Tag.is_system.is_(True),
|
Tag.is_system.is_(True),
|
||||||
Tag.name.in_(PRESENTATION_SYSTEM_TAGS),
|
Tag.name.in_(excluded_system_tags),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
stmt = stmt.where(
|
stmt = stmt.where(
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ from datetime import UTC, datetime
|
|||||||
from email.utils import parsedate_to_datetime
|
from email.utils import parsedate_to_datetime
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
|
||||||
class InterpreterUnavailable(Exception):
|
class InterpreterUnavailable(Exception):
|
||||||
@@ -57,13 +59,29 @@ def _parse_retry_after(resp) -> float | None:
|
|||||||
return max(0.0, (when - datetime.now(UTC)).total_seconds())
|
return max(0.0, (when - datetime.now(UTC)).total_seconds())
|
||||||
|
|
||||||
|
|
||||||
|
# A shared session pools the keep-alive connection across the per-post sweep
|
||||||
|
# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing
|
||||||
|
# the instant a reverse proxy reloads. Status codes are deliberately NOT retried
|
||||||
|
# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable
|
||||||
|
# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After
|
||||||
|
# backoff we honour upstream.
|
||||||
|
_retry = Retry(
|
||||||
|
total=None, connect=2, read=0, redirect=0, status=0,
|
||||||
|
backoff_factor=0.3, raise_on_status=False,
|
||||||
|
)
|
||||||
|
session = requests.Session()
|
||||||
|
_adapter = HTTPAdapter(max_retries=_retry)
|
||||||
|
session.mount("http://", _adapter)
|
||||||
|
session.mount("https://", _adapter)
|
||||||
|
|
||||||
|
|
||||||
def health(base_url: str, *, timeout: float = 5.0) -> bool:
|
def health(base_url: str, *, timeout: float = 5.0) -> bool:
|
||||||
"""True iff the Interpreter LLM engine is up. Any error (unset URL, network,
|
"""True iff the Interpreter LLM engine is up. Any error (unset URL, network,
|
||||||
non-200, engine down) → False, so the sweep just no-ops rather than raising."""
|
non-200, engine down) → False, so the sweep just no-ops rather than raising."""
|
||||||
if not base_url:
|
if not base_url:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
r = requests.get(_url(base_url, "/v1/health"), timeout=timeout)
|
r = session.get(_url(base_url, "/v1/health"), timeout=timeout)
|
||||||
except requests.RequestException:
|
except requests.RequestException:
|
||||||
return False
|
return False
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
@@ -94,7 +112,7 @@ def translate(
|
|||||||
return {"translations": [], "detected_lang": None,
|
return {"translations": [], "detected_lang": None,
|
||||||
"engine": None, "engine_version": None}
|
"engine": None, "engine_version": None}
|
||||||
try:
|
try:
|
||||||
r = requests.post(
|
r = session.post(
|
||||||
_url(base_url, "/v1/translate"),
|
_url(base_url, "/v1/translate"),
|
||||||
json={"q": list(texts), "source": source,
|
json={"q": list(texts), "source": source,
|
||||||
"target": target, "engine": "auto"},
|
"target": target, "engine": "auto"},
|
||||||
|
|||||||
@@ -79,6 +79,14 @@ VERIFY_PAGE = 200
|
|||||||
FFPROBE_TIMEOUT_SECONDS = 10
|
FFPROBE_TIMEOUT_SECONDS = 10
|
||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
|
||||||
|
# then os.replace() into place (importer / external_fetch / native_ingest_common /
|
||||||
|
# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
|
||||||
|
# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
|
||||||
|
# keeps it from deleting an in-flight download's staging file mid-write.
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
||||||
|
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
||||||
|
|
||||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||||
# > the entity's longest legitimate runtime (its task's time_limit + a
|
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||||
@@ -339,6 +347,34 @@ def cleanup_old_tasks() -> int:
|
|||||||
return result.rowcount or 0
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
|
||||||
|
def cleanup_orphaned_temp_files() -> int:
|
||||||
|
"""Delete orphaned .part/.partial staging files under the images root, left by
|
||||||
|
a download/import killed mid-write. Only removes files older than
|
||||||
|
ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
|
||||||
|
pulled out from under it. Returns the count removed."""
|
||||||
|
if not IMAGES_ROOT.is_dir():
|
||||||
|
return 0
|
||||||
|
cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
|
||||||
|
removed = 0
|
||||||
|
for path in IMAGES_ROOT.rglob("*"):
|
||||||
|
if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if path.stat().st_mtime >= cutoff:
|
||||||
|
continue # too fresh — may be an active download
|
||||||
|
path.unlink()
|
||||||
|
removed += 1
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
|
||||||
|
if removed:
|
||||||
|
log.info(
|
||||||
|
"cleanup_orphaned_temp_files: removed %d orphaned staging file(s)",
|
||||||
|
removed,
|
||||||
|
)
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
def recover_stalled_task_runs() -> int:
|
def recover_stalled_task_runs() -> int:
|
||||||
"""Flip task_run rows stuck in 'running' past their queue-specific
|
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||||
|
|||||||
@@ -70,10 +70,21 @@ def translate_posts() -> str:
|
|||||||
return ready
|
return ready
|
||||||
base_url, target = ready
|
base_url, target = ready
|
||||||
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
||||||
# Steady-state daily sweep: one chunk per run. On interruption the
|
status, translated, retry_after = _translate_batch(
|
||||||
# untranslated rows stay NULL and the next daily beat (or a fresh
|
session, posts, base_url, target,
|
||||||
# "Translate now") resumes them — no self-chase needed here.
|
)
|
||||||
status, translated, _retry = _translate_batch(session, posts, base_url, target)
|
# Steady-state daily sweep: one chunk per run on success (the beat drives
|
||||||
|
# the rest — no tail-chase). But if Interpreter drained mid-chunk, don't
|
||||||
|
# make a manual "Translate now" wait a whole day — re-enqueue after its
|
||||||
|
# Retry-After hint (or the default backoff). Self-terminating: once the
|
||||||
|
# service is down the health gate returns "interpreter unavailable" early.
|
||||||
|
if status == "interrupted" and _count_untranslated(session, None):
|
||||||
|
delay = _interrupt_backoff(retry_after)
|
||||||
|
translate_posts.apply_async((), {}, countdown=delay)
|
||||||
|
log.info(
|
||||||
|
"translate_posts: interrupted (service draining) → retry in %ss",
|
||||||
|
delay,
|
||||||
|
)
|
||||||
return _summary(status, translated, len(posts))
|
return _summary(status, translated, len(posts))
|
||||||
|
|
||||||
|
|
||||||
@@ -239,29 +250,39 @@ def _summary(status: str, translated: int, scanned: int) -> str:
|
|||||||
return f"translated={translated} scanned={scanned}"
|
return f"translated={translated} scanned={scanned}"
|
||||||
|
|
||||||
|
|
||||||
|
def _translate_field(text: str, base_url: str, target: str):
|
||||||
|
"""Translate ONE field independently. Returns (translated, source_lang,
|
||||||
|
engine_version), or (None, None, None) when there's nothing to store — empty
|
||||||
|
text, already the target language, or a passthrough (engine "none"). Per-field
|
||||||
|
(not aggregate first-item) detection so a non-English description still gets
|
||||||
|
translated when the title is already English (mixed-language posts)."""
|
||||||
|
if not text:
|
||||||
|
return None, None, None
|
||||||
|
res = ic.translate([text], base_url=base_url, target=target)
|
||||||
|
detected = res["detected_lang"] or target
|
||||||
|
if detected == target or res["engine"] == "none":
|
||||||
|
return None, None, None
|
||||||
|
return res["translations"][0], detected, res["engine_version"]
|
||||||
|
|
||||||
|
|
||||||
def _translate_one(session, post, base_url: str, target: str) -> int:
|
def _translate_one(session, post, base_url: str, target: str) -> int:
|
||||||
"""Translate one post's title/description in place. Returns 1 if it stored a
|
"""Translate a post's title/description in place, each field independently.
|
||||||
translation, 0 for passthrough/empty (which still marks the post handled)."""
|
Returns 1 if it stored at least one translation, 0 when the post is all
|
||||||
|
passthrough/empty (still marks it handled so the sweep won't revisit it)."""
|
||||||
title = (post.post_title or "").strip()
|
title = (post.post_title or "").strip()
|
||||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||||
desc = desc.strip()
|
desc = desc.strip()
|
||||||
fields: list[tuple[str, str]] = []
|
title_tr, title_lang, title_ev = _translate_field(title, base_url, target)
|
||||||
if title:
|
desc_tr, desc_lang, desc_ev = _translate_field(desc, base_url, target)
|
||||||
fields.append(("title", title))
|
if title_tr is None and desc_tr is None:
|
||||||
if desc:
|
# Nothing non-target (or nothing to translate) → handled, store nothing.
|
||||||
fields.append(("desc", desc))
|
post.translated_source_lang = target
|
||||||
if not fields:
|
|
||||||
post.translated_source_lang = target # nothing to translate → handled
|
|
||||||
return 0
|
return 0
|
||||||
res = ic.translate([t for _, t in fields], base_url=base_url, target=target)
|
post.post_title_translated = title_tr
|
||||||
detected = res["detected_lang"] or target
|
post.description_translated = desc_tr
|
||||||
if detected == target or res["engine"] == "none":
|
# Source lang = whichever field was actually non-target (for a mixed post,
|
||||||
post.translated_source_lang = target # already target language → handled
|
# the translated field's language, not the already-English one).
|
||||||
return 0
|
post.translated_source_lang = title_lang or desc_lang
|
||||||
by_field = {f: res["translations"][i] for i, (f, _) in enumerate(fields)}
|
post.translation_engine_version = title_ev or desc_ev
|
||||||
post.post_title_translated = by_field.get("title")
|
|
||||||
post.description_translated = by_field.get("desc")
|
|
||||||
post.translated_source_lang = detected
|
|
||||||
post.translation_engine_version = res["engine_version"]
|
|
||||||
post.translated_at = datetime.now(UTC)
|
post.translated_at = datetime.now(UTC)
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
@@ -8,6 +8,35 @@
|
|||||||
# run `docker compose up` from this directory and switches images to
|
# run `docker compose up` from this directory and switches images to
|
||||||
# local builds + DEBUG logging.
|
# local builds + DEBUG logging.
|
||||||
|
|
||||||
|
# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
|
||||||
|
# time, START the new task before stopping the old (zero-downtime via the ingress
|
||||||
|
# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll
|
||||||
|
# back to the previous image automatically. `monitor` is sized above web's
|
||||||
|
# healthcheck start_period so a broken image that never goes healthy is caught.
|
||||||
|
# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev
|
||||||
|
# override is unaffected. Referenced by each long-lived service below.
|
||||||
|
x-deploy-policy: &deploy_policy
|
||||||
|
update_config:
|
||||||
|
order: start-first
|
||||||
|
failure_action: rollback
|
||||||
|
monitor: 90s
|
||||||
|
rollback_config:
|
||||||
|
order: start-first
|
||||||
|
restart_policy:
|
||||||
|
condition: any
|
||||||
|
delay: 10s
|
||||||
|
|
||||||
|
# Worker liveness: ping THIS container's celery node over the broker. Lenient
|
||||||
|
# (60s interval, 3 retries, 60s start_period) so a transient broker blip never
|
||||||
|
# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell;
|
||||||
|
# celery's default node name is celery@<hostname> (the container id).
|
||||||
|
x-celery-healthcheck: &celery_healthcheck
|
||||||
|
test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"]
|
||||||
|
interval: 60s
|
||||||
|
timeout: 15s
|
||||||
|
retries: 3
|
||||||
|
start_period: 60s
|
||||||
|
|
||||||
services:
|
services:
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
@@ -55,6 +84,16 @@ services:
|
|||||||
# by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
|
# by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
|
||||||
# requests + the occasional file download.
|
# requests + the occasional file download.
|
||||||
stop_grace_period: 30s
|
stop_grace_period: 30s
|
||||||
|
# Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the
|
||||||
|
# app booted + serves HTTP after `alembic upgrade head`). start_period covers
|
||||||
|
# the migration + boot so a slow start isn't mis-flagged.
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 6s
|
||||||
|
retries: 3
|
||||||
|
start_period: 40s
|
||||||
|
deploy: *deploy_policy
|
||||||
ports:
|
ports:
|
||||||
- "${PORT:-8080}:8080"
|
- "${PORT:-8080}:8080"
|
||||||
environment: &app_env
|
environment: &app_env
|
||||||
@@ -87,6 +126,8 @@ services:
|
|||||||
command: ["worker"]
|
command: ["worker"]
|
||||||
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
||||||
stop_grace_period: 90s
|
stop_grace_period: 90s
|
||||||
|
healthcheck: *celery_healthcheck
|
||||||
|
deploy: *deploy_policy
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: default,import,thumbnail,download
|
CELERY_QUEUES: default,import,thumbnail,download
|
||||||
@@ -105,6 +146,8 @@ services:
|
|||||||
command: ["scheduler"]
|
command: ["scheduler"]
|
||||||
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
||||||
stop_grace_period: 60s
|
stop_grace_period: 60s
|
||||||
|
healthcheck: *celery_healthcheck
|
||||||
|
deploy: *deploy_policy
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: maintenance,scan
|
CELERY_QUEUES: maintenance,scan
|
||||||
@@ -126,6 +169,8 @@ services:
|
|||||||
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
||||||
# that still outruns this resumes cleanly next run rather than corrupting.
|
# that still outruns this resumes cleanly next run rather than corrupting.
|
||||||
stop_grace_period: 180s
|
stop_grace_period: 180s
|
||||||
|
healthcheck: *celery_healthcheck
|
||||||
|
deploy: *deploy_policy
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: maintenance_long
|
CELERY_QUEUES: maintenance_long
|
||||||
@@ -143,6 +188,8 @@ services:
|
|||||||
command: ["ml-worker"]
|
command: ["ml-worker"]
|
||||||
# A single GPU inference pass can run tens of seconds — let it finish.
|
# A single GPU inference pass can run tens of seconds — let it finish.
|
||||||
stop_grace_period: 120s
|
stop_grace_period: 120s
|
||||||
|
healthcheck: *celery_healthcheck
|
||||||
|
deploy: *deploy_policy
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<template>
|
<template>
|
||||||
<span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
|
<span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
|
||||||
<v-chip
|
<v-chip
|
||||||
size="small" :closable="!unconfirmedAuto"
|
size="default" :closable="!unconfirmedAuto"
|
||||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||||
class="fc-tag-chip__nav"
|
class="fc-tag-chip__nav"
|
||||||
role="link"
|
role="link"
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
@click="$emit('navigate', tag)"
|
@click="$emit('navigate', tag)"
|
||||||
@click:close="$emit('remove', tag.id)"
|
@click:close="$emit('remove', tag.id)"
|
||||||
>
|
>
|
||||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
<v-icon start size="small">{{ iconFor(tag.kind) }}</v-icon>
|
||||||
<span class="fc-tag-chip__name">{{ tag.name }}</span><v-icon
|
<span class="fc-tag-chip__name">{{ tag.name }}</span><v-icon
|
||||||
v-if="tag.is_system" end size="x-small" class="fc-tag-chip__system"
|
v-if="tag.is_system" end size="x-small" class="fc-tag-chip__system"
|
||||||
title="System tag — tagged items are excluded from training other concepts"
|
title="System tag — tagged items are excluded from training other concepts"
|
||||||
@@ -27,13 +27,13 @@
|
|||||||
:title="`Yes — keep “${tag.name}” (trains the model, won't be retracted)`"
|
:title="`Yes — keep “${tag.name}” (trains the model, won't be retracted)`"
|
||||||
:aria-label="`Confirm ${tag.name}`"
|
:aria-label="`Confirm ${tag.name}`"
|
||||||
@click.stop="$emit('confirm', tag)"
|
@click.stop="$emit('confirm', tag)"
|
||||||
><v-icon size="13">mdi-check</v-icon></button>
|
><v-icon size="15">mdi-check</v-icon></button>
|
||||||
<button
|
<button
|
||||||
type="button" class="fc-tag-chip__no"
|
type="button" class="fc-tag-chip__no"
|
||||||
:title="`No — remove “${tag.name}”`"
|
:title="`No — remove “${tag.name}”`"
|
||||||
:aria-label="`Reject ${tag.name}`"
|
:aria-label="`Reject ${tag.name}`"
|
||||||
@click.stop="$emit('remove', tag.id)"
|
@click.stop="$emit('remove', tag.id)"
|
||||||
><v-icon size="13">mdi-close</v-icon></button>
|
><v-icon size="15">mdi-close</v-icon></button>
|
||||||
</span>
|
</span>
|
||||||
</v-chip>
|
</v-chip>
|
||||||
<!-- Modal-safe kebab is baked into KebabMenu (this chip lives in the
|
<!-- Modal-safe kebab is baked into KebabMenu (this chip lives in the
|
||||||
@@ -143,7 +143,19 @@ function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
|||||||
chip's hover title. */
|
chip's hover title. */
|
||||||
.fc-tag-chip { display: inline-flex; align-items: center; gap: 1px; max-width: 100%; min-width: 0; }
|
.fc-tag-chip { display: inline-flex; align-items: center; gap: 1px; max-width: 100%; min-width: 0; }
|
||||||
.fc-tag-chip__nav { max-width: 100%; min-width: 0; }
|
.fc-tag-chip__nav { max-width: 100%; min-width: 0; }
|
||||||
|
/* Tonal chips (esp. character = info) wash out against the dark rail — the fill
|
||||||
|
is intentionally faint. Give every tag chip a thin border in its OWN kind
|
||||||
|
colour (currentColor = the tonal chip's themed foreground) so the edge reads
|
||||||
|
clearly without touching the fill (operator-asked 2026-07-08). Theme-aware:
|
||||||
|
currentColor tracks the kind colour in either light or dark. */
|
||||||
|
.fc-tag-chip__nav { border: thin solid color-mix(in srgb, currentColor 55%, transparent); }
|
||||||
.fc-tag-chip__nav :deep(.v-chip__content) { min-width: 0; overflow: hidden; }
|
.fc-tag-chip__nav :deep(.v-chip__content) { min-width: 0; overflow: hidden; }
|
||||||
|
/* The bigger size=default chip widens Vuetify's negative start-margin on the
|
||||||
|
leading kind-icon, pulling it LEFT of the content box above — where that
|
||||||
|
overflow:hidden then clips the icon's left edge as a vertical slice
|
||||||
|
(operator-flagged 2026-07-08). Zero the pull so the icon sits inside the clip
|
||||||
|
box; the chip's own 12px padding keeps a comfortable left inset. */
|
||||||
|
.fc-tag-chip__nav :deep(.v-icon--start) { margin-inline-start: 0; }
|
||||||
.fc-tag-chip__name {
|
.fc-tag-chip__name {
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
min-width: 0; flex: 0 1 auto;
|
min-width: 0; flex: 0 1 auto;
|
||||||
@@ -159,23 +171,27 @@ function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
|||||||
.fc-tag-chip__kebab { opacity: 0.7; }
|
.fc-tag-chip__kebab { opacity: 0.7; }
|
||||||
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
|
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
|
||||||
.fc-tag-chip__fandom { opacity: 0.7; font-size: 0.85em; }
|
.fc-tag-chip__fandom { opacity: 0.7; font-size: 0.85em; }
|
||||||
/* Provisional auto-tag: a compact green ✓ / red ✗ pair in place of the ✕. The
|
/* Provisional auto-tag: a green ✓ / red ✗ pair in place of the ✕. The yes/no is
|
||||||
yes/no is obvious enough on its own, so there's no "auto" label (operator-asked
|
obvious enough on its own, so there's no "auto" label (operator-asked
|
||||||
2026-07-07). flex:0 0 auto keeps it visible while the name ellipsis-truncates. */
|
2026-07-07). flex:0 0 auto keeps it visible while the name ellipsis-truncates. */
|
||||||
.fc-tag-chip__verdict {
|
.fc-tag-chip__verdict {
|
||||||
flex: 0 0 auto; display: inline-flex; align-items: center; gap: 1px;
|
flex: 0 0 auto; display: inline-flex; align-items: center; gap: 3px;
|
||||||
margin-left: 3px;
|
margin-left: 5px;
|
||||||
}
|
}
|
||||||
|
/* Solid-filled (white glyph on a full success/error circle) so accept/reject
|
||||||
|
stay well-defined even on a muted tonal chip — e.g. character (info) — instead
|
||||||
|
of a faint icon that only lit up on hover. Mirrors the Suggestions rail's
|
||||||
|
verdict buttons so the affordance reads identically (operator-asked 2026-07-08). */
|
||||||
.fc-tag-chip__yes, .fc-tag-chip__no {
|
.fc-tag-chip__yes, .fc-tag-chip__no {
|
||||||
display: inline-flex; align-items: center; justify-content: center;
|
display: inline-flex; align-items: center; justify-content: center;
|
||||||
width: 18px; height: 18px; padding: 0; border: none; border-radius: 50%;
|
width: 22px; height: 22px; padding: 0; border: none; border-radius: 50%;
|
||||||
background: transparent; cursor: pointer; transition: background 0.1s;
|
color: #fff; cursor: pointer; opacity: 0.92;
|
||||||
|
transition: transform 0.1s, opacity 0.1s;
|
||||||
}
|
}
|
||||||
.fc-tag-chip__yes { color: rgb(var(--v-theme-success)); }
|
.fc-tag-chip__yes { background: rgb(var(--v-theme-success)); }
|
||||||
.fc-tag-chip__no { color: rgb(var(--v-theme-error)); }
|
.fc-tag-chip__no { background: rgb(var(--v-theme-error)); }
|
||||||
.fc-tag-chip__yes:hover { background: rgb(var(--v-theme-success), 0.16); }
|
.fc-tag-chip__yes:hover, .fc-tag-chip__no:hover { opacity: 1; transform: scale(1.1); }
|
||||||
.fc-tag-chip__no:hover { background: rgb(var(--v-theme-error), 0.16); }
|
|
||||||
.fc-tag-chip__yes:focus-visible, .fc-tag-chip__no:focus-visible {
|
.fc-tag-chip__yes:focus-visible, .fc-tag-chip__no:focus-visible {
|
||||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 2px;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -52,11 +52,25 @@
|
|||||||
prepend-icon="mdi-refresh" :loading="retranslating"
|
prepend-icon="mdi-refresh" :loading="retranslating"
|
||||||
:disabled="!enabled || !baseUrl" @click="confirmAll = true"
|
:disabled="!enabled || !baseUrl" @click="confirmAll = true"
|
||||||
>Re-translate all</v-btn>
|
>Re-translate all</v-btn>
|
||||||
<span v-if="status" class="fc-muted text-caption">
|
<span
|
||||||
|
v-if="status && status.active"
|
||||||
|
class="fc-muted text-caption d-inline-flex align-center" style="gap: 6px;"
|
||||||
|
>
|
||||||
|
<v-progress-circular indeterminate size="12" width="2" color="accent" />
|
||||||
|
Translating… {{ status.untranslated_count }} remaining
|
||||||
|
</span>
|
||||||
|
<span v-else-if="status" class="fc-muted text-caption">
|
||||||
{{ status.untranslated_count }}
|
{{ status.untranslated_count }}
|
||||||
post{{ status.untranslated_count === 1 ? '' : 's' }} awaiting translation
|
post{{ status.untranslated_count === 1 ? '' : 's' }} awaiting translation
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
<p
|
||||||
|
v-if="status && status.last_run && ['error', 'timeout'].includes(status.last_run.status)"
|
||||||
|
class="text-caption text-error mt-1 mb-0"
|
||||||
|
>
|
||||||
|
Last translation run ended with “{{ status.last_run.status }}”. It resumes on
|
||||||
|
the next sweep; check the Interpreter connection if it persists.
|
||||||
|
</p>
|
||||||
<p class="fc-muted text-caption mt-2 mb-0">
|
<p class="fc-muted text-caption mt-2 mb-0">
|
||||||
Use <strong>Re-translate all</strong> after switching the Interpreter model —
|
Use <strong>Re-translate all</strong> after switching the Interpreter model —
|
||||||
it clears every stored translation and re-runs it through the new model
|
it clears every stored translation and re-runs it through the new model
|
||||||
@@ -94,7 +108,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
import { toast } from '../../utils/toast.js'
|
import { toast } from '../../utils/toast.js'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
import { useApi } from '../../composables/useApi.js'
|
||||||
import { useImportStore } from '../../stores/import.js'
|
import { useImportStore } from '../../stores/import.js'
|
||||||
@@ -126,10 +140,16 @@ const statusText = computed(() => {
|
|||||||
return status.value.healthy ? 'Interpreter reachable' : 'Interpreter unreachable'
|
return status.value.healthy ? 'Interpreter reachable' : 'Interpreter unreachable'
|
||||||
})
|
})
|
||||||
|
|
||||||
|
let pollTimer = null
|
||||||
async function loadStatus() {
|
async function loadStatus() {
|
||||||
try { status.value = await api.get('/api/settings/translation/status') }
|
try { status.value = await api.get('/api/settings/translation/status') }
|
||||||
catch { status.value = null }
|
catch { status.value = null }
|
||||||
|
// Poll live while a sweep is running so the remaining count ticks down; stop
|
||||||
|
// when idle (a fresh trigger restarts it via its own setTimeout(loadStatus)).
|
||||||
|
clearTimeout(pollTimer)
|
||||||
|
if (status.value?.active) pollTimer = setTimeout(loadStatus, 3000)
|
||||||
}
|
}
|
||||||
|
onUnmounted(() => clearTimeout(pollTimer))
|
||||||
|
|
||||||
async function onTest() {
|
async function onTest() {
|
||||||
// Ping /v1/health for the CURRENTLY-typed URL (not the saved one), so a new
|
// Ping /v1/health for the CURRENTLY-typed URL (not the saved one), so a new
|
||||||
|
|||||||
@@ -44,7 +44,9 @@ export const useExploreStore = defineStore('explore', () => {
|
|||||||
// empty and let the view explain why (anchor.has_embedding === false).
|
// empty and let the view explain why (anchor.has_embedding === false).
|
||||||
if (detail.has_embedding) {
|
if (detail.has_embedding) {
|
||||||
const body = await api.get('/api/gallery/similar', {
|
const body = await api.get('/api/gallery/similar', {
|
||||||
params: { similar_to: numId, limit: NEIGHBOR_LIMIT },
|
// exclude_wip: keep work-in-progress out of the Explore rabbit-hole
|
||||||
|
// (the gallery's own "similar" button still shows it) — operator 2026-07-08.
|
||||||
|
params: { similar_to: numId, limit: NEIGHBOR_LIMIT, exclude_wip: 1 },
|
||||||
})
|
})
|
||||||
if (!t.isCurrent()) return
|
if (!t.isCurrent()) return
|
||||||
neighbors.value = body.images || []
|
neighbors.value = body.images || []
|
||||||
|
|||||||
@@ -61,6 +61,34 @@ async def test_translation_status_defaults(client):
|
|||||||
assert body["base_url_set"] is False
|
assert body["base_url_set"] is False
|
||||||
assert body["healthy"] is False
|
assert body["healthy"] is False
|
||||||
assert "untranslated_count" in body
|
assert "untranslated_count" in body
|
||||||
|
assert body["active"] is False # no sweep running
|
||||||
|
assert body["last_run"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_translation_status_reports_active_and_last_run(client, db):
|
||||||
|
# A running translate/retranslate TaskRun → active True; the most recent
|
||||||
|
# finished one → last_run (task basename + status).
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from backend.app.models import TaskRun
|
||||||
|
|
||||||
|
db.add(TaskRun(
|
||||||
|
celery_task_id="r-run", queue="maintenance_long",
|
||||||
|
task_name="backend.app.tasks.translation.retranslate_posts",
|
||||||
|
started_at=datetime.now(UTC), status="running",
|
||||||
|
))
|
||||||
|
db.add(TaskRun(
|
||||||
|
celery_task_id="r-done", queue="maintenance_long",
|
||||||
|
task_name="backend.app.tasks.translation.translate_posts",
|
||||||
|
started_at=datetime.now(UTC), finished_at=datetime.now(UTC),
|
||||||
|
status="success",
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
body = await (await client.get("/api/settings/translation/status")).get_json()
|
||||||
|
assert body["active"] is True
|
||||||
|
assert body["last_run"]["task"] == "translate_posts"
|
||||||
|
assert body["last_run"]["status"] == "success"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -98,6 +98,30 @@ async def test_similar_excludes_presentation_tagged_images(db):
|
|||||||
assert {i.id for i in res_from_banner} == {src.id, wipped.id, plain.id}
|
assert {i.id for i in res_from_banner} == {src.id, wipped.id, plain.id}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_similar_exclude_wip_drops_wip_neighbors(db):
|
||||||
|
"""Explore passes exclude_wip=True to also hide work-in-progress from the
|
||||||
|
rabbit-hole (banner is always hidden; wip only when asked)."""
|
||||||
|
src = await _img(db, 1, _vec(1, 0))
|
||||||
|
bannered = await _img(db, 2, _vec(1, 0.02)) # always hidden
|
||||||
|
wipped = await _img(db, 3, _vec(1, 0.3)) # hidden only with exclude_wip
|
||||||
|
plain = await _img(db, 4, _vec(1, 0.6))
|
||||||
|
banner_tag = (await db.execute(select(Tag).where(
|
||||||
|
Tag.is_system.is_(True), Tag.name == "banner"))).scalar_one()
|
||||||
|
wip_tag = (await db.execute(select(Tag).where(
|
||||||
|
Tag.is_system.is_(True), Tag.name == "wip"))).scalar_one()
|
||||||
|
await db.execute(image_tag.insert().values(
|
||||||
|
image_record_id=bannered.id, tag_id=banner_tag.id, source="manual"))
|
||||||
|
await db.execute(image_tag.insert().values(
|
||||||
|
image_record_id=wipped.id, tag_id=wip_tag.id, source="manual"))
|
||||||
|
svc = GalleryService(db)
|
||||||
|
res = await svc.similar(src.id, limit=10, exclude_wip=True)
|
||||||
|
assert [i.id for i in res] == [plain.id] # wip + banner both gone
|
||||||
|
# Default (gallery "similar" button) still keeps wip (#1274).
|
||||||
|
res_default = await svc.similar(src.id, limit=10)
|
||||||
|
assert wipped.id in {i.id for i in res_default}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_similar_composes_with_tag_filter(db):
|
async def test_similar_composes_with_tag_filter(db):
|
||||||
src = await _img(db, 1, _vec(1, 0))
|
src = await _img(db, 1, _vec(1, 0))
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ def test_translate_maps_batch_in_order(monkeypatch):
|
|||||||
"interpreter": {"engine": "llm", "engine_version": "ollama:x:12b"},
|
"interpreter": {"engine": "llm", "engine_version": "ollama:x:12b"},
|
||||||
})
|
})
|
||||||
|
|
||||||
monkeypatch.setattr(ic.requests, "post", fake_post)
|
monkeypatch.setattr(ic.session, "post", fake_post)
|
||||||
out = ic.translate(["ねこが可愛い", "金髪ギャル!"], base_url="http://i.lan")
|
out = ic.translate(["ねこが可愛い", "金髪ギャル!"], base_url="http://i.lan")
|
||||||
assert out["translations"] == ["The cat is cute", "Blonde gal!"]
|
assert out["translations"] == ["The cat is cute", "Blonde gal!"]
|
||||||
assert out["detected_lang"] == "ja"
|
assert out["detected_lang"] == "ja"
|
||||||
@@ -41,7 +41,7 @@ def test_translate_maps_batch_in_order(monkeypatch):
|
|||||||
|
|
||||||
def test_translate_passthrough_unchanged(monkeypatch):
|
def test_translate_passthrough_unchanged(monkeypatch):
|
||||||
# Already-English items come back unchanged in their slot (engine "none").
|
# Already-English items come back unchanged in their slot (engine "none").
|
||||||
monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(200, {
|
monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(200, {
|
||||||
"translatedText": ["already english"],
|
"translatedText": ["already english"],
|
||||||
"detectedLanguage": {"language": "en"},
|
"detectedLanguage": {"language": "en"},
|
||||||
"interpreter": {"engine": "none", "engine_version": None},
|
"interpreter": {"engine": "none", "engine_version": None},
|
||||||
@@ -53,7 +53,7 @@ def test_translate_passthrough_unchanged(monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_translate_503_raises_unavailable(monkeypatch):
|
def test_translate_503_raises_unavailable(monkeypatch):
|
||||||
monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(503, {}))
|
monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(503, {}))
|
||||||
with pytest.raises(ic.InterpreterUnavailable):
|
with pytest.raises(ic.InterpreterUnavailable):
|
||||||
ic.translate(["x"], base_url="http://i.lan")
|
ic.translate(["x"], base_url="http://i.lan")
|
||||||
|
|
||||||
@@ -62,7 +62,7 @@ def test_translate_connection_error_raises_unavailable(monkeypatch):
|
|||||||
def boom(*a, **k):
|
def boom(*a, **k):
|
||||||
raise ic.requests.ConnectionError("refused")
|
raise ic.requests.ConnectionError("refused")
|
||||||
|
|
||||||
monkeypatch.setattr(ic.requests, "post", boom)
|
monkeypatch.setattr(ic.session, "post", boom)
|
||||||
with pytest.raises(ic.InterpreterUnavailable):
|
with pytest.raises(ic.InterpreterUnavailable):
|
||||||
ic.translate(["x"], base_url="http://i.lan")
|
ic.translate(["x"], base_url="http://i.lan")
|
||||||
|
|
||||||
@@ -71,7 +71,7 @@ def test_translate_connection_error_raises_unavailable(monkeypatch):
|
|||||||
def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code):
|
def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code):
|
||||||
# A gracefully-draining service behind a reverse proxy returns 429/502/503/504
|
# A gracefully-draining service behind a reverse proxy returns 429/502/503/504
|
||||||
# — all mean "retry later", not an opaque error.
|
# — all mean "retry later", not an opaque error.
|
||||||
monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(code, {}))
|
monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(code, {}))
|
||||||
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
||||||
ic.translate(["x"], base_url="http://i.lan")
|
ic.translate(["x"], base_url="http://i.lan")
|
||||||
assert ei.value.retry_after is None
|
assert ei.value.retry_after is None
|
||||||
@@ -79,7 +79,7 @@ def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code):
|
|||||||
|
|
||||||
def test_translate_honours_retry_after_seconds(monkeypatch):
|
def test_translate_honours_retry_after_seconds(monkeypatch):
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
ic.requests, "post",
|
ic.session, "post",
|
||||||
lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}),
|
lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}),
|
||||||
)
|
)
|
||||||
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
with pytest.raises(ic.InterpreterUnavailable) as ei:
|
||||||
@@ -90,7 +90,7 @@ def test_translate_honours_retry_after_seconds(monkeypatch):
|
|||||||
def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch):
|
def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch):
|
||||||
# HTTP-date form; a past date → non-negative clamp to 0 (deterministic).
|
# HTTP-date form; a past date → non-negative clamp to 0 (deterministic).
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
ic.requests, "post",
|
ic.session, "post",
|
||||||
lambda *a, **k: _Resp(
|
lambda *a, **k: _Resp(
|
||||||
503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}),
|
503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}),
|
||||||
)
|
)
|
||||||
@@ -101,7 +101,7 @@ def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch):
|
|||||||
|
|
||||||
def test_translate_400_raises_bad_request(monkeypatch):
|
def test_translate_400_raises_bad_request(monkeypatch):
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
ic.requests, "post", lambda *a, **k: _Resp(400, {"error": "bad target"})
|
ic.session, "post", lambda *a, **k: _Resp(400, {"error": "bad target"})
|
||||||
)
|
)
|
||||||
with pytest.raises(ic.InterpreterBadRequest):
|
with pytest.raises(ic.InterpreterBadRequest):
|
||||||
ic.translate(["x"], base_url="http://i.lan")
|
ic.translate(["x"], base_url="http://i.lan")
|
||||||
@@ -111,20 +111,20 @@ def test_translate_empty_is_noop(monkeypatch):
|
|||||||
def boom(*a, **k):
|
def boom(*a, **k):
|
||||||
raise AssertionError("should not call the service for an empty batch")
|
raise AssertionError("should not call the service for an empty batch")
|
||||||
|
|
||||||
monkeypatch.setattr(ic.requests, "post", boom)
|
monkeypatch.setattr(ic.session, "post", boom)
|
||||||
assert ic.translate([], base_url="http://i.lan")["translations"] == []
|
assert ic.translate([], base_url="http://i.lan")["translations"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_health_true_when_llm_up(monkeypatch):
|
def test_health_true_when_llm_up(monkeypatch):
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
ic.requests, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": True}})
|
ic.session, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": True}})
|
||||||
)
|
)
|
||||||
assert ic.health("http://i.lan") is True
|
assert ic.health("http://i.lan") is True
|
||||||
|
|
||||||
|
|
||||||
def test_health_false_when_engine_down(monkeypatch):
|
def test_health_false_when_engine_down(monkeypatch):
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
ic.requests, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": False}})
|
ic.session, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": False}})
|
||||||
)
|
)
|
||||||
assert ic.health("http://i.lan") is False
|
assert ic.health("http://i.lan") is False
|
||||||
|
|
||||||
@@ -133,7 +133,7 @@ def test_health_false_on_error(monkeypatch):
|
|||||||
def boom(*a, **k):
|
def boom(*a, **k):
|
||||||
raise ic.requests.ConnectionError("refused")
|
raise ic.requests.ConnectionError("refused")
|
||||||
|
|
||||||
monkeypatch.setattr(ic.requests, "get", boom)
|
monkeypatch.setattr(ic.session, "get", boom)
|
||||||
assert ic.health("http://i.lan") is False
|
assert ic.health("http://i.lan") is False
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,28 @@ def _make_batch(session) -> int:
|
|||||||
return batch.id
|
return batch.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch):
|
||||||
|
import os
|
||||||
|
|
||||||
|
from backend.app.tasks import maintenance as m
|
||||||
|
|
||||||
|
monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path)
|
||||||
|
stale = tmp_path / "artist" / "img.jpg.part" # killed download → orphan
|
||||||
|
stale.parent.mkdir(parents=True)
|
||||||
|
stale.write_bytes(b"x")
|
||||||
|
old = datetime.now(UTC).timestamp() - 8 * 3600 # older than the 6h guard
|
||||||
|
os.utime(stale, (old, old))
|
||||||
|
fresh = tmp_path / "in_progress.jpg.partial" # active download → keep
|
||||||
|
fresh.write_bytes(b"x")
|
||||||
|
keep = tmp_path / "real.jpg" # a real image → keep
|
||||||
|
keep.write_bytes(b"x")
|
||||||
|
|
||||||
|
assert m.cleanup_orphaned_temp_files() == 1
|
||||||
|
assert not stale.exists()
|
||||||
|
assert fresh.exists()
|
||||||
|
assert keep.exists()
|
||||||
|
|
||||||
|
|
||||||
def test_recover_interrupted_only_old(db_sync, monkeypatch):
|
def test_recover_interrupted_only_old(db_sync, monkeypatch):
|
||||||
batch_id = _make_batch(db_sync)
|
batch_id = _make_batch(db_sync)
|
||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
|
|||||||
@@ -115,6 +115,35 @@ def test_translate_posts_passthrough_english_marks_handled(db_sync, monkeypatch)
|
|||||||
assert p.post_title_translated is None # ...but nothing to show
|
assert p.post_title_translated is None # ...but nothing to show
|
||||||
|
|
||||||
|
|
||||||
|
def test_translate_posts_mixed_language_translates_nonenglish_field(db_sync, monkeypatch):
|
||||||
|
# English title + Japanese description: the description is still translated
|
||||||
|
# even though the title is already English (per-field detection, not the old
|
||||||
|
# aggregate first-item bail). Source lang comes from the translated field.
|
||||||
|
_patch(monkeypatch, db_sync)
|
||||||
|
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
||||||
|
|
||||||
|
def fake_translate(texts, **k):
|
||||||
|
t = texts[0]
|
||||||
|
if t.isascii(): # already English → passthrough
|
||||||
|
return {"translations": [t], "detected_lang": "en",
|
||||||
|
"engine": "none", "engine_version": None}
|
||||||
|
return {"translations": [f"EN:{t}"], "detected_lang": "ja",
|
||||||
|
"engine": "llm", "engine_version": "v9"}
|
||||||
|
|
||||||
|
monkeypatch.setattr("backend.app.tasks.translation.ic.translate", fake_translate)
|
||||||
|
a = _artist(db_sync)
|
||||||
|
p = _post(db_sync, a.id, title="hello", desc="ねこ")
|
||||||
|
_enable(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
assert "translated=1" in translate_posts()
|
||||||
|
db_sync.refresh(p)
|
||||||
|
assert p.post_title_translated is None # English title untouched
|
||||||
|
assert p.description_translated == "EN:ねこ" # Japanese desc translated
|
||||||
|
assert p.translated_source_lang == "ja" # from the translated field
|
||||||
|
assert p.translation_engine_version == "v9"
|
||||||
|
|
||||||
|
|
||||||
def test_translate_posts_disabled_is_noop(db_sync, monkeypatch):
|
def test_translate_posts_disabled_is_noop(db_sync, monkeypatch):
|
||||||
_patch(monkeypatch, db_sync)
|
_patch(monkeypatch, db_sync)
|
||||||
a = _artist(db_sync)
|
a = _artist(db_sync)
|
||||||
@@ -137,6 +166,31 @@ def test_translate_posts_service_down_leaves_untranslated(db_sync, monkeypatch):
|
|||||||
assert p.translated_source_lang is None # will retry next run
|
assert p.translated_source_lang is None # will retry next run
|
||||||
|
|
||||||
|
|
||||||
|
def test_translate_posts_interrupt_reenqueues_after_backoff(db_sync, monkeypatch):
|
||||||
|
# A drain mid-sweep (health passed, translate 503s w/ Retry-After) re-enqueues
|
||||||
|
# the daily sweep after the backoff instead of waiting for tomorrow's beat.
|
||||||
|
_patch(monkeypatch, db_sync)
|
||||||
|
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
||||||
|
|
||||||
|
def _drain(texts, **k):
|
||||||
|
raise ic.InterpreterUnavailable("draining", retry_after=30)
|
||||||
|
|
||||||
|
monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain)
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"backend.app.tasks.translation.translate_posts.apply_async",
|
||||||
|
lambda *a, **k: calls.append((a, k)),
|
||||||
|
)
|
||||||
|
a = _artist(db_sync)
|
||||||
|
_post(db_sync, a.id, title="ねこ", ext="p1")
|
||||||
|
_enable(db_sync)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
assert "interrupted" in translate_posts()
|
||||||
|
assert len(calls) == 1
|
||||||
|
assert calls[0][1]["countdown"] == 30
|
||||||
|
|
||||||
|
|
||||||
def _mock_new_model(monkeypatch, *, ver="v2"):
|
def _mock_new_model(monkeypatch, *, ver="v2"):
|
||||||
"""Interpreter is up and translates with a NEW engine version."""
|
"""Interpreter is up and translates with a NEW engine version."""
|
||||||
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True)
|
||||||
|
|||||||
Reference in New Issue
Block a user