From c64261593d687eb6402e49592dd4bad07c1fa5e0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 21:01:00 -0400 Subject: [PATCH 1/7] =?UTF-8?q?feat(ops):=20graceful=20shutdown=20?= =?UTF-8?q?=E2=80=94=20worker=20stop-grace=20+=20Interpreter=20drain=20res?= =?UTF-8?q?ilience?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploys (docker SIGTERM→SIGKILL, default 10s) were killing Celery jobs mid-flight. Give in-flight work room to drain and make interrupted work resume cleanly instead of stalling. - docker-compose.yml: stop_grace_period per lane (web 30s / worker 90s / scheduler 60s / maintenance-long 180s / ml-worker 120s) so warm shutdown can actually drain before SIGKILL. - celery_app.py: task_reject_on_worker_lost=True — a task killed past the grace window is re-queued (safe: idempotent + chunked, recovery sweeps re-drive stragglers). - interpreter_client.py: map 429/5xx (502/503/504) → InterpreterUnavailable and parse Retry-After (delta-seconds or HTTP-date); a draining Interpreter behind a reverse proxy no longer raises an opaque HTTPError. - translation.py: thread retry_after out of _translate_batch; retranslate_posts resumes after the Retry-After hint (or 60s default, capped 900s) on an interrupt with _reset_done=True, self-terminating via the health gate. - tests: 429/5xx mapping + Retry-After parse; interrupt-resume + default backoff. No migration. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/celery_app.py | 8 +++ backend/app/services/interpreter_client.py | 54 ++++++++++++++++--- backend/app/tasks/translation.py | 60 +++++++++++++++++----- docker-compose.yml | 18 +++++++ tests/test_interpreter_client.py | 35 ++++++++++++- tests/test_translate_posts.py | 59 +++++++++++++++++++++ 6 files changed, 215 insertions(+), 19 deletions(-) diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 0e47473..fb15685 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -70,6 +70,14 @@ def make_celery() -> Celery: }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, + # Deploy graceful-shutdown safety: with acks_late, a task killed because + # it outran the container's stop-grace window (SIGKILL) is re-queued + # rather than silently lost. Safe because our long tasks are idempotent + + # chunked (translation per-post commit, downloads terminal-status, audits + # chunk) and the 5-min recovery sweeps re-drive anything left non-terminal + # — a re-run resumes cleanly and never corrupts. No redeliver-loop risk: + # heavy GPU work is tombstoned via gpu_queue, not run inline in a worker. + task_reject_on_worker_lost=True, worker_prefetch_multiplier=1, # Broker resilience (2026-06-24): a swarm overlay-network blip after a # redeploy left Redis healthy but transiently unreachable, and a worker diff --git a/backend/app/services/interpreter_client.py b/backend/app/services/interpreter_client.py index 0e02029..70008d2 100644 --- a/backend/app/services/interpreter_client.py +++ b/backend/app/services/interpreter_client.py @@ -8,12 +8,22 @@ behind a reverse proxy). Full API contract: Scribe note #1347. """ from __future__ import annotations +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime + import requests class InterpreterUnavailable(Exception): - """The translation engine is down / unreachable (HTTP 503 or a connection - error) — retry later, don't drop the item.""" + """The translation engine is down / unreachable / draining — a connection + error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while + the service restarts). Retry later, don't drop the item. ``retry_after`` holds + the server's Retry-After hint in seconds when it sent one, so the caller can + back off exactly as long as it's asked to.""" + + def __init__(self, message: str, *, retry_after: float | None = None): + super().__init__(message) + self.retry_after = retry_after class InterpreterBadRequest(Exception): @@ -24,6 +34,29 @@ def _url(base_url: str, path: str) -> str: return f"{base_url.rstrip('/')}{path}" +def _parse_retry_after(resp) -> float | None: + """Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into + non-negative seconds, or None if absent/unparseable — so a gracefully- + draining Interpreter can tell Curator exactly how long to wait before it + tries again.""" + raw = (resp.headers.get("Retry-After") or "").strip() + if not raw: + return None + try: + return max(0.0, float(int(raw))) # delta-seconds form + except ValueError: + pass + try: # HTTP-date form + when = parsedate_to_datetime(raw) + except (TypeError, ValueError): + return None + if when is None: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + return max(0.0, (when - datetime.now(timezone.utc)).total_seconds()) + + def health(base_url: str, *, timeout: float = 5.0) -> bool: """True iff the Interpreter LLM engine is up. Any error (unset URL, network, non-200, engine down) → False, so the sweep just no-ops rather than raising.""" @@ -53,8 +86,9 @@ def translate( Interpreter batch metadata is aggregate (first item only) — fine for one post's ``[title, description]`` batch since they share a language. Passthrough items (already target-language / emoji-only) come back UNCHANGED in their - slot. Raises InterpreterUnavailable on 503 / connection error (retry later), - InterpreterBadRequest on 400. (Scribe note #1347.) + slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry + later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note + #1347.) """ if not texts: return {"translations": [], "detected_lang": None, @@ -68,10 +102,18 @@ def translate( ) except requests.RequestException as e: raise InterpreterUnavailable(str(e)) from e - if r.status_code == 503: - raise InterpreterUnavailable("interpreter engine unavailable") if r.status_code == 400: raise InterpreterBadRequest((r.json() or {}).get("error", "bad request")) + # A gracefully-draining service — often behind a reverse proxy — returns 429 + # or a 5xx gateway error (502/503/504), not always a clean 503. Treat every + # one as "unavailable, retry later" and honour any Retry-After it sends, so a + # rolling Interpreter restart interrupts the sweep cleanly (instead of raising + # an opaque HTTPError) and resumes exactly when the service says it's ready. + if r.status_code == 429 or r.status_code >= 500: + raise InterpreterUnavailable( + f"interpreter unavailable (HTTP {r.status_code})", + retry_after=_parse_retry_after(r), + ) r.raise_for_status() body = r.json() or {} out = body.get("translatedText") diff --git a/backend/app/tasks/translation.py b/backend/app/tasks/translation.py index 15dd499..70f5b51 100644 --- a/backend/app/tasks/translation.py +++ b/backend/app/tasks/translation.py @@ -36,6 +36,20 @@ _MAX_POSTS_PER_RUN = 300 # Short gap between run-until-done chunks so a big re-translate finishes on its # own without monopolising the maintenance_long worker in one uninterrupted run. _RETRANSLATE_COUNTDOWN = 5 +# When Interpreter drains mid-chunk (graceful restart), resume the bulk +# re-translate after the service's Retry-After hint if it sent one, else this +# default; capped so a bogus/huge hint can't park the work longer than the daily +# beat's own safety net would. +_INTERRUPT_BACKOFF = 60 +_INTERRUPT_BACKOFF_MAX = 900 + + +def _interrupt_backoff(retry_after) -> int: + """Seconds to wait before resuming after an Interpreter interruption: the + server's Retry-After hint (capped), else the default.""" + if retry_after and retry_after > 0: + return int(min(retry_after, _INTERRUPT_BACKOFF_MAX)) + return _INTERRUPT_BACKOFF @celery.task( @@ -56,7 +70,10 @@ def translate_posts() -> str: return ready base_url, target = ready posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN) - status, translated = _translate_batch(session, posts, base_url, target) + # Steady-state daily sweep: one chunk per run. On interruption the + # untranslated rows stay NULL and the next daily beat (or a fresh + # "Translate now") resumes them — no self-chase needed here. + status, translated, _retry = _translate_batch(session, posts, base_url, target) return _summary(status, translated, len(posts)) @@ -88,11 +105,11 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str: ) posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN) - status, translated = _translate_batch(session, posts, base_url, target) + status, translated, retry_after = _translate_batch( + session, posts, base_url, target, + ) - # run-until-done: only chase the tail when the chunk finished cleanly. A - # service interruption / bad request stops here and leaves the rest NULL - # for the daily beat or a fresh trigger to resume (rule-89 recovery). + # run-until-done: chase the tail when the chunk finished cleanly. # Termination is guaranteed: every handled post leaves the NULL set, so # the remaining count strictly decreases each chunk. if status == "ok": @@ -106,6 +123,23 @@ def retranslate_posts(artist_ids=None, _reset_done=False) -> str: log.info( "retranslate_posts: %d remaining → re-enqueued", remaining, ) + # Interpreter drained mid-chunk (graceful restart / 429 / 5xx): don't + # stall the bulk re-translate until the daily beat — resume it after the + # service's Retry-After hint (or the default backoff), with + # _reset_done=True so the fresh work is never re-wiped. Self-terminating: + # if the service is still down next run, _translation_config's health gate + # returns "interpreter unavailable" early and nothing re-enqueues. + elif status == "interrupted" and _count_untranslated(session, artist_ids): + delay = _interrupt_backoff(retry_after) + retranslate_posts.apply_async( + (), + {"artist_ids": artist_ids, "_reset_done": True}, + countdown=delay, + ) + log.info( + "retranslate_posts: interrupted (service draining) → retry in %ss", + delay, + ) return _summary(status, translated, len(posts)) @@ -174,23 +208,25 @@ def _reset_translations(session, artist_ids) -> int: def _translate_batch(session, posts, base_url: str, target: str): """Translate each post in the chunk with a per-post commit (an interrupted - run keeps progress). Returns (status, translated) where status is one of - "ok" / "interrupted" (503/conn) / "stopped" (400) / "timeout".""" + run keeps progress). Returns (status, translated, retry_after) where status is + one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout"; + retry_after is the server's Retry-After hint in seconds on an interrupt, else + None.""" translated = 0 for post in posts: try: translated += _translate_one(session, post, base_url, target) session.commit() - except ic.InterpreterUnavailable: + except ic.InterpreterUnavailable as e: session.rollback() - return "interrupted", translated + return "interrupted", translated, getattr(e, "retry_after", None) except ic.InterpreterBadRequest as e: session.rollback() log.warning("translate bad request: %s", e) - return "stopped", translated + return "stopped", translated, None except SoftTimeLimitExceeded: - return "timeout", translated - return "ok", translated + return "timeout", translated, None + return "ok", translated, None def _summary(status: str, translated: int, scanned: int) -> str: diff --git a/docker-compose.yml b/docker-compose.yml index ee793f7..60eeda1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,14 @@ services: web: image: git.fabledsword.com/bvandeusen/fabledcurator:dev command: ["web"] + # Graceful shutdown: give the container time to drain in-flight work on a + # deploy (docker SIGTERMs, then SIGKILLs after this window — default is only + # 10s, far too short for real jobs). Hypercorn/Celery both warm-shut-down on + # SIGTERM; per-lane values sized to typical task length. Anything that still + # outruns the window is re-queued (task_reject_on_worker_lost) and re-driven + # by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP + # requests + the occasional file download. + stop_grace_period: 30s ports: - "${PORT:-8080}:8080" environment: &app_env @@ -77,6 +85,8 @@ services: worker: image: git.fabledsword.com/bvandeusen/fabledcurator:dev command: ["worker"] + # Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy. + stop_grace_period: 90s environment: <<: *app_env CELERY_QUEUES: default,import,thumbnail,download @@ -93,6 +103,8 @@ services: scheduler: image: git.fabledsword.com/bvandeusen/fabledcurator:dev command: ["scheduler"] + # Quick maintenance/scan lane + beat — short tasks, modest drain window. + stop_grace_period: 60s environment: <<: *app_env CELERY_QUEUES: maintenance,scan @@ -110,6 +122,10 @@ services: maintenance-long: image: git.fabledsword.com/bvandeusen/fabledcurator:dev command: ["worker"] + # Longest lane (DB backups, library audits, translation backfill) — give it + # the most room to finish a chunk gracefully. Chunked + idempotent, so a job + # that still outruns this resumes cleanly next run rather than corrupting. + stop_grace_period: 180s environment: <<: *app_env CELERY_QUEUES: maintenance_long @@ -125,6 +141,8 @@ services: ml-worker: image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev command: ["ml-worker"] + # A single GPU inference pass can run tens of seconds — let it finish. + stop_grace_period: 120s environment: <<: *app_env volumes: diff --git a/tests/test_interpreter_client.py b/tests/test_interpreter_client.py index 39b2e07..b84cdeb 100644 --- a/tests/test_interpreter_client.py +++ b/tests/test_interpreter_client.py @@ -6,9 +6,10 @@ from backend.app.services import interpreter_client as ic class _Resp: - def __init__(self, status_code=200, payload=None): + def __init__(self, status_code=200, payload=None, headers=None): self.status_code = status_code self._payload = payload or {} + self.headers = headers or {} def json(self): return self._payload @@ -66,6 +67,38 @@ def test_translate_connection_error_raises_unavailable(monkeypatch): ic.translate(["x"], base_url="http://i.lan") +@pytest.mark.parametrize("code", [429, 500, 502, 503, 504]) +def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code): + # A gracefully-draining service behind a reverse proxy returns 429/502/503/504 + # — all mean "retry later", not an opaque error. + monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(code, {})) + with pytest.raises(ic.InterpreterUnavailable) as ei: + ic.translate(["x"], base_url="http://i.lan") + assert ei.value.retry_after is None + + +def test_translate_honours_retry_after_seconds(monkeypatch): + monkeypatch.setattr( + ic.requests, "post", + lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}), + ) + with pytest.raises(ic.InterpreterUnavailable) as ei: + ic.translate(["x"], base_url="http://i.lan") + assert ei.value.retry_after == 42 + + +def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch): + # HTTP-date form; a past date → non-negative clamp to 0 (deterministic). + monkeypatch.setattr( + ic.requests, "post", + lambda *a, **k: _Resp( + 503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}), + ) + with pytest.raises(ic.InterpreterUnavailable) as ei: + ic.translate(["x"], base_url="http://i.lan") + assert ei.value.retry_after == 0.0 + + def test_translate_400_raises_bad_request(monkeypatch): monkeypatch.setattr( ic.requests, "post", lambda *a, **k: _Resp(400, {"error": "bad target"}) diff --git a/tests/test_translate_posts.py b/tests/test_translate_posts.py index 26f2c8b..4218c4b 100644 --- a/tests/test_translate_posts.py +++ b/tests/test_translate_posts.py @@ -4,6 +4,7 @@ import pytest from sqlalchemy import select from backend.app.models import Artist, ImportSettings, Post +from backend.app.services import interpreter_client as ic from backend.app.tasks.translation import retranslate_posts, translate_posts pytestmark = pytest.mark.integration @@ -239,3 +240,61 @@ def test_retranslate_unhealthy_does_not_reset(db_sync, monkeypatch): db_sync.refresh(p) assert p.translated_source_lang == "ja" assert p.post_title_translated == "OLD" + + +def test_retranslate_interrupt_resumes_after_retry_after(db_sync, monkeypatch): + # Interpreter drains mid-chunk (health passed, then translate 503s with a + # Retry-After): the bulk re-translate re-enqueues itself after the hinted + # backoff, with _reset_done=True so it never re-wipes fresh work — instead of + # stalling until the daily beat. + _patch(monkeypatch, db_sync) + monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) + + def _drain(texts, **k): + raise ic.InterpreterUnavailable("draining", retry_after=42) + + monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain) + calls = [] + monkeypatch.setattr( + "backend.app.tasks.translation.retranslate_posts.apply_async", + lambda *a, **k: calls.append((a, k)), + ) + a = _artist(db_sync) + p = _post(db_sync, a.id, title="ねこ", ext="p1") + _mark_translated(p) + _enable(db_sync) + db_sync.commit() + + assert "interrupted" in retranslate_posts(artist_ids=[a.id]) + assert len(calls) == 1 # resume re-enqueued + args, kw = calls[0] + assert args[1]["_reset_done"] is True + assert args[1]["artist_ids"] == [a.id] + assert kw["countdown"] == 42 # honoured the Retry-After + db_sync.refresh(p) + assert p.translated_source_lang is None # reset happened, not re-run + + +def test_retranslate_interrupt_default_backoff_without_hint(db_sync, monkeypatch): + # No Retry-After header → fall back to the default backoff (60s). + _patch(monkeypatch, db_sync) + monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) + + def _drain(texts, **k): + raise ic.InterpreterUnavailable("draining") + + monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain) + calls = [] + monkeypatch.setattr( + "backend.app.tasks.translation.retranslate_posts.apply_async", + lambda *a, **k: calls.append((a, k)), + ) + a = _artist(db_sync) + p = _post(db_sync, a.id, title="ねこ", ext="p1") + _mark_translated(p) + _enable(db_sync) + db_sync.commit() + + retranslate_posts(artist_ids=[a.id]) + assert len(calls) == 1 + assert calls[0][1]["countdown"] == 60 -- 2.52.0 From d631ed023ca0457a9b94807757547954e2263ae9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 21:04:17 -0400 Subject: [PATCH 2/7] style(translation): use datetime.UTC alias (ruff UP017) Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/interpreter_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/services/interpreter_client.py b/backend/app/services/interpreter_client.py index 70008d2..aeb3a4e 100644 --- a/backend/app/services/interpreter_client.py +++ b/backend/app/services/interpreter_client.py @@ -8,7 +8,7 @@ behind a reverse proxy). Full API contract: Scribe note #1347. """ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from email.utils import parsedate_to_datetime import requests @@ -53,8 +53,8 @@ def _parse_retry_after(resp) -> float | None: if when is None: return None if when.tzinfo is None: - when = when.replace(tzinfo=timezone.utc) - return max(0.0, (when - datetime.now(timezone.utc)).total_seconds()) + when = when.replace(tzinfo=UTC) + return max(0.0, (when - datetime.now(UTC)).total_seconds()) def health(base_url: str, *, timeout: float = 5.0) -> bool: -- 2.52.0 From f8105046dccee752b11e818235bddb51db6c7407 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 22:10:05 -0400 Subject: [PATCH 3/7] feat(explore): exclude WIP-tagged work from the Explore rabbit-hole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explore's neighbour grid (/api/gallery/similar → gallery_service.similar) now takes an Explore-only exclude_wip flag that drops `wip` system-tagged images from the candidates, alongside the banner/editor presentation tags. The gallery's own "similar" button is unchanged (keeps wip, #1274) — only the Explore store passes exclude_wip=1. The anchor itself may still be a WIP; only neighbours are filtered. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/gallery.py | 6 +++++- backend/app/models/tag.py | 4 ++++ backend/app/services/gallery_service.py | 13 +++++++++---- frontend/src/stores/explore.js | 4 +++- tests/test_gallery_similar.py | 24 ++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index 6bf91fe..410852e 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -145,6 +145,9 @@ async def similar(): filters, _sort = _parse_filters() except (KeyError, ValueError): return jsonify({"error": "similar_to query param required"}), 400 + # Explore passes exclude_wip=1 to also drop work-in-progress from the + # rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274). + exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True") # post_id is the exclusive post-detail view — not a similarity scope. # include_hidden is a gallery-browse flag; similar() has its OWN presentation # exclusion (a similarity-quality concern, #1274), so drop it here (#141). @@ -154,7 +157,8 @@ async def similar(): async with get_session() as session: svc = GalleryService(session) try: - images = await svc.similar(image_id=similar_to, limit=limit, **scope) + images = await svc.similar( + image_id=similar_to, limit=limit, exclude_wip=exclude_wip, **scope) except ValueError as exc: return jsonify({"error": str(exc)}), 400 if images is None: diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index 2479578..c27ebd3 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -48,6 +48,10 @@ class TagKind(StrEnum): # content. `wip` is real art: only the training pipelines exclude it. SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot") +# `wip` marks real-but-unfinished art. It's kept in the gallery's own "similar" +# results (#1274), but the Explore rabbit-hole opts to hide it (exclude_wip) so a +# browse doesn't keep surfacing work-in-progress (operator, 2026-07-08). +WIP_SYSTEM_TAG = "wip" image_tag = Table( "image_tag", diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index a05e6c7..8d11a95 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -31,7 +31,7 @@ from ..models import ( Tag, TagPositiveConfirmation, ) -from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag +from ..models.tag import PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG, image_tag from .pagination import decode_cursor, encode_cursor from .tag_query import ( fandom_join_alias, @@ -715,6 +715,7 @@ class GalleryService: platform: str | None = None, untagged: bool = False, no_artist: bool = False, date_from: datetime | None = None, date_to: datetime | None = None, + exclude_wip: bool = False, ) -> list[GalleryImage] | None: """Visual "more like this": images near `image_id`'s SigLIP embedding (pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result @@ -751,14 +752,18 @@ class GalleryService: # Presentation images (banner / editor-screenshot system tags, #128) # cluster on UI chrome rather than content, so near any one of them # they'd fill the grid. Excluded from CANDIDATES only — the anchor - # itself may be a banner, and `wip` stays surfaced (real art; only - # the training pipelines exclude it). + # itself may be a banner. `wip` stays surfaced here by default (real art; + # only the training pipelines exclude it), but the Explore rabbit-hole + # passes exclude_wip to also drop work-in-progress (operator, 2026-07-08). + excluded_system_tags = PRESENTATION_SYSTEM_TAGS + if exclude_wip: + excluded_system_tags = (*PRESENTATION_SYSTEM_TAGS, WIP_SYSTEM_TAG) presentation = ( select(image_tag.c.image_record_id) .join(Tag, Tag.id == image_tag.c.tag_id) .where( Tag.is_system.is_(True), - Tag.name.in_(PRESENTATION_SYSTEM_TAGS), + Tag.name.in_(excluded_system_tags), ) ) stmt = stmt.where( diff --git a/frontend/src/stores/explore.js b/frontend/src/stores/explore.js index 36ee046..5adb26b 100644 --- a/frontend/src/stores/explore.js +++ b/frontend/src/stores/explore.js @@ -44,7 +44,9 @@ export const useExploreStore = defineStore('explore', () => { // empty and let the view explain why (anchor.has_embedding === false). if (detail.has_embedding) { const body = await api.get('/api/gallery/similar', { - params: { similar_to: numId, limit: NEIGHBOR_LIMIT }, + // exclude_wip: keep work-in-progress out of the Explore rabbit-hole + // (the gallery's own "similar" button still shows it) — operator 2026-07-08. + params: { similar_to: numId, limit: NEIGHBOR_LIMIT, exclude_wip: 1 }, }) if (!t.isCurrent()) return neighbors.value = body.images || [] diff --git a/tests/test_gallery_similar.py b/tests/test_gallery_similar.py index 35e75d6..a54ac1e 100644 --- a/tests/test_gallery_similar.py +++ b/tests/test_gallery_similar.py @@ -98,6 +98,30 @@ async def test_similar_excludes_presentation_tagged_images(db): assert {i.id for i in res_from_banner} == {src.id, wipped.id, plain.id} +@pytest.mark.asyncio +async def test_similar_exclude_wip_drops_wip_neighbors(db): + """Explore passes exclude_wip=True to also hide work-in-progress from the + rabbit-hole (banner is always hidden; wip only when asked).""" + src = await _img(db, 1, _vec(1, 0)) + bannered = await _img(db, 2, _vec(1, 0.02)) # always hidden + wipped = await _img(db, 3, _vec(1, 0.3)) # hidden only with exclude_wip + plain = await _img(db, 4, _vec(1, 0.6)) + banner_tag = (await db.execute(select(Tag).where( + Tag.is_system.is_(True), Tag.name == "banner"))).scalar_one() + wip_tag = (await db.execute(select(Tag).where( + Tag.is_system.is_(True), Tag.name == "wip"))).scalar_one() + await db.execute(image_tag.insert().values( + image_record_id=bannered.id, tag_id=banner_tag.id, source="manual")) + await db.execute(image_tag.insert().values( + image_record_id=wipped.id, tag_id=wip_tag.id, source="manual")) + svc = GalleryService(db) + res = await svc.similar(src.id, limit=10, exclude_wip=True) + assert [i.id for i in res] == [plain.id] # wip + banner both gone + # Default (gallery "similar" button) still keeps wip (#1274). + res_default = await svc.similar(src.id, limit=10) + assert wipped.id in {i.id for i in res_default} + + @pytest.mark.asyncio async def test_similar_composes_with_tag_filter(db): src = await _img(db, 1, _vec(1, 0)) -- 2.52.0 From 9eae6360475421cad2c916b674642d6997e4aef7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 22:10:05 -0400 Subject: [PATCH 4/7] feat(translation): pooled Interpreter session + manual sweep resumes after drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - interpreter_client: shared requests.Session with a connect-only retry (connect=2, no status retries — we map 429/5xx ourselves) so a proxy reload is smoothed and the keep-alive connection is pooled across the sweep. - translate_posts: on an interrupt (drain), re-enqueue after the Retry-After hint / default backoff instead of waiting for the daily beat; self-terminates via the health gate. Steady-state one-chunk-per-run on success is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/interpreter_client.py | 22 +++++++++++++++++-- backend/app/tasks/translation.py | 19 ++++++++++++---- tests/test_interpreter_client.py | 24 ++++++++++----------- tests/test_translate_posts.py | 25 ++++++++++++++++++++++ 4 files changed, 72 insertions(+), 18 deletions(-) diff --git a/backend/app/services/interpreter_client.py b/backend/app/services/interpreter_client.py index aeb3a4e..c4db1a3 100644 --- a/backend/app/services/interpreter_client.py +++ b/backend/app/services/interpreter_client.py @@ -12,6 +12,8 @@ from datetime import UTC, datetime from email.utils import parsedate_to_datetime import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry class InterpreterUnavailable(Exception): @@ -57,13 +59,29 @@ def _parse_retry_after(resp) -> float | None: return max(0.0, (when - datetime.now(UTC)).total_seconds()) +# A shared session pools the keep-alive connection across the per-post sweep +# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing +# the instant a reverse proxy reloads. Status codes are deliberately NOT retried +# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable +# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After +# backoff we honour upstream. +_retry = Retry( + total=None, connect=2, read=0, redirect=0, status=0, + backoff_factor=0.3, raise_on_status=False, +) +session = requests.Session() +_adapter = HTTPAdapter(max_retries=_retry) +session.mount("http://", _adapter) +session.mount("https://", _adapter) + + def health(base_url: str, *, timeout: float = 5.0) -> bool: """True iff the Interpreter LLM engine is up. Any error (unset URL, network, non-200, engine down) → False, so the sweep just no-ops rather than raising.""" if not base_url: return False try: - r = requests.get(_url(base_url, "/v1/health"), timeout=timeout) + r = session.get(_url(base_url, "/v1/health"), timeout=timeout) except requests.RequestException: return False if r.status_code != 200: @@ -94,7 +112,7 @@ def translate( return {"translations": [], "detected_lang": None, "engine": None, "engine_version": None} try: - r = requests.post( + r = session.post( _url(base_url, "/v1/translate"), json={"q": list(texts), "source": source, "target": target, "engine": "auto"}, diff --git a/backend/app/tasks/translation.py b/backend/app/tasks/translation.py index 70f5b51..02cc153 100644 --- a/backend/app/tasks/translation.py +++ b/backend/app/tasks/translation.py @@ -70,10 +70,21 @@ def translate_posts() -> str: return ready base_url, target = ready posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN) - # Steady-state daily sweep: one chunk per run. On interruption the - # untranslated rows stay NULL and the next daily beat (or a fresh - # "Translate now") resumes them — no self-chase needed here. - status, translated, _retry = _translate_batch(session, posts, base_url, target) + status, translated, retry_after = _translate_batch( + session, posts, base_url, target, + ) + # Steady-state daily sweep: one chunk per run on success (the beat drives + # the rest — no tail-chase). But if Interpreter drained mid-chunk, don't + # make a manual "Translate now" wait a whole day — re-enqueue after its + # Retry-After hint (or the default backoff). Self-terminating: once the + # service is down the health gate returns "interpreter unavailable" early. + if status == "interrupted" and _count_untranslated(session, None): + delay = _interrupt_backoff(retry_after) + translate_posts.apply_async((), {}, countdown=delay) + log.info( + "translate_posts: interrupted (service draining) → retry in %ss", + delay, + ) return _summary(status, translated, len(posts)) diff --git a/tests/test_interpreter_client.py b/tests/test_interpreter_client.py index b84cdeb..0c4adbc 100644 --- a/tests/test_interpreter_client.py +++ b/tests/test_interpreter_client.py @@ -30,7 +30,7 @@ def test_translate_maps_batch_in_order(monkeypatch): "interpreter": {"engine": "llm", "engine_version": "ollama:x:12b"}, }) - monkeypatch.setattr(ic.requests, "post", fake_post) + monkeypatch.setattr(ic.session, "post", fake_post) out = ic.translate(["ねこが可愛い", "金髪ギャル！"], base_url="http://i.lan") assert out["translations"] == ["The cat is cute", "Blonde gal!"] assert out["detected_lang"] == "ja" @@ -41,7 +41,7 @@ def test_translate_maps_batch_in_order(monkeypatch): def test_translate_passthrough_unchanged(monkeypatch): # Already-English items come back unchanged in their slot (engine "none"). - monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(200, { + monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(200, { "translatedText": ["already english"], "detectedLanguage": {"language": "en"}, "interpreter": {"engine": "none", "engine_version": None}, @@ -53,7 +53,7 @@ def test_translate_passthrough_unchanged(monkeypatch): def test_translate_503_raises_unavailable(monkeypatch): - monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(503, {})) + monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(503, {})) with pytest.raises(ic.InterpreterUnavailable): ic.translate(["x"], base_url="http://i.lan") @@ -62,7 +62,7 @@ def test_translate_connection_error_raises_unavailable(monkeypatch): def boom(*a, **k): raise ic.requests.ConnectionError("refused") - monkeypatch.setattr(ic.requests, "post", boom) + monkeypatch.setattr(ic.session, "post", boom) with pytest.raises(ic.InterpreterUnavailable): ic.translate(["x"], base_url="http://i.lan") @@ -71,7 +71,7 @@ def test_translate_connection_error_raises_unavailable(monkeypatch): def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code): # A gracefully-draining service behind a reverse proxy returns 429/502/503/504 # — all mean "retry later", not an opaque error. - monkeypatch.setattr(ic.requests, "post", lambda *a, **k: _Resp(code, {})) + monkeypatch.setattr(ic.session, "post", lambda *a, **k: _Resp(code, {})) with pytest.raises(ic.InterpreterUnavailable) as ei: ic.translate(["x"], base_url="http://i.lan") assert ei.value.retry_after is None @@ -79,7 +79,7 @@ def test_translate_429_and_5xx_raise_unavailable(monkeypatch, code): def test_translate_honours_retry_after_seconds(monkeypatch): monkeypatch.setattr( - ic.requests, "post", + ic.session, "post", lambda *a, **k: _Resp(503, {}, headers={"Retry-After": "42"}), ) with pytest.raises(ic.InterpreterUnavailable) as ei: @@ -90,7 +90,7 @@ def test_translate_honours_retry_after_seconds(monkeypatch): def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch): # HTTP-date form; a past date → non-negative clamp to 0 (deterministic). monkeypatch.setattr( - ic.requests, "post", + ic.session, "post", lambda *a, **k: _Resp( 503, {}, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}), ) @@ -101,7 +101,7 @@ def test_translate_retry_after_past_http_date_clamps_to_zero(monkeypatch): def test_translate_400_raises_bad_request(monkeypatch): monkeypatch.setattr( - ic.requests, "post", lambda *a, **k: _Resp(400, {"error": "bad target"}) + ic.session, "post", lambda *a, **k: _Resp(400, {"error": "bad target"}) ) with pytest.raises(ic.InterpreterBadRequest): ic.translate(["x"], base_url="http://i.lan") @@ -111,20 +111,20 @@ def test_translate_empty_is_noop(monkeypatch): def boom(*a, **k): raise AssertionError("should not call the service for an empty batch") - monkeypatch.setattr(ic.requests, "post", boom) + monkeypatch.setattr(ic.session, "post", boom) assert ic.translate([], base_url="http://i.lan")["translations"] == [] def test_health_true_when_llm_up(monkeypatch): monkeypatch.setattr( - ic.requests, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": True}}) + ic.session, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": True}}) ) assert ic.health("http://i.lan") is True def test_health_false_when_engine_down(monkeypatch): monkeypatch.setattr( - ic.requests, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": False}}) + ic.session, "get", lambda *a, **k: _Resp(200, {"engines": {"llm": False}}) ) assert ic.health("http://i.lan") is False @@ -133,7 +133,7 @@ def test_health_false_on_error(monkeypatch): def boom(*a, **k): raise ic.requests.ConnectionError("refused") - monkeypatch.setattr(ic.requests, "get", boom) + monkeypatch.setattr(ic.session, "get", boom) assert ic.health("http://i.lan") is False diff --git a/tests/test_translate_posts.py b/tests/test_translate_posts.py index 4218c4b..2973241 100644 --- a/tests/test_translate_posts.py +++ b/tests/test_translate_posts.py @@ -137,6 +137,31 @@ def test_translate_posts_service_down_leaves_untranslated(db_sync, monkeypatch): assert p.translated_source_lang is None # will retry next run +def test_translate_posts_interrupt_reenqueues_after_backoff(db_sync, monkeypatch): + # A drain mid-sweep (health passed, translate 503s w/ Retry-After) re-enqueues + # the daily sweep after the backoff instead of waiting for tomorrow's beat. + _patch(monkeypatch, db_sync) + monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) + + def _drain(texts, **k): + raise ic.InterpreterUnavailable("draining", retry_after=30) + + monkeypatch.setattr("backend.app.tasks.translation.ic.translate", _drain) + calls = [] + monkeypatch.setattr( + "backend.app.tasks.translation.translate_posts.apply_async", + lambda *a, **k: calls.append((a, k)), + ) + a = _artist(db_sync) + _post(db_sync, a.id, title="ねこ", ext="p1") + _enable(db_sync) + db_sync.commit() + + assert "interrupted" in translate_posts() + assert len(calls) == 1 + assert calls[0][1]["countdown"] == 30 + + def _mock_new_model(monkeypatch, *, ver="v2"): """Interpreter is up and translates with a NEW engine version.""" monkeypatch.setattr("backend.app.tasks.translation.ic.health", lambda *a, **k: True) -- 2.52.0 From 0b78264d62ae931b47962479e2002602a0903c94 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 22:15:33 -0400 Subject: [PATCH 5/7] feat(maintenance): daily janitor for orphaned .part/.partial staging files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Downloads/imports stage into .part / .partial then os.replace() into place, so a kill mid-write leaves a discardable temp — never a corrupt final. cleanup_orphaned_temp_files sweeps ones left behind under the images root, only older than 6h so an in-flight download's staging file is never removed. Daily beat. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/celery_app.py | 5 +++++ backend/app/tasks/maintenance.py | 36 ++++++++++++++++++++++++++++++++ tests/test_maintenance.py | 22 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index fb15685..967ece2 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -115,6 +115,11 @@ def make_celery() -> Celery: "task": "backend.app.tasks.maintenance.cleanup_old_tasks", "schedule": 86400.0, # daily }, + "cleanup-orphaned-temp-files": { + "task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files", + "schedule": 86400.0, # daily — sweep .part/.partial left by a + # download/import killed mid-write (graceful-shutdown fallout) + }, "train-heads-nightly": { "task": "backend.app.tasks.ml.scheduled_train_heads", "schedule": 86400.0, # passive cadence; manual retrain stays available diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index f9549d9..d0a6c24 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -79,6 +79,14 @@ VERIFY_PAGE = 200 FFPROBE_TIMEOUT_SECONDS = 10 TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days +# Orphaned staging files: downloads/imports stage into .part / .partial +# then os.replace() into place (importer / external_fetch / native_ingest_common / +# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt +# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard +# keeps it from deleting an in-flight download's staging file mid-write. +IMAGES_ROOT = Path("/images") +TEMP_STAGING_SUFFIXES = (".part", ".partial") +ORPHAN_TEMP_MIN_AGE_HOURS = 6 # Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be # > the entity's longest legitimate runtime (its task's time_limit + a @@ -339,6 +347,34 @@ def cleanup_old_tasks() -> int: return result.rowcount or 0 +@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files") +def cleanup_orphaned_temp_files() -> int: + """Delete orphaned .part/.partial staging files under the images root, left by + a download/import killed mid-write. Only removes files older than + ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never + pulled out from under it. Returns the count removed.""" + if not IMAGES_ROOT.is_dir(): + return 0 + cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600 + removed = 0 + for path in IMAGES_ROOT.rglob("*"): + if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file(): + continue + try: + if path.stat().st_mtime >= cutoff: + continue # too fresh — may be an active download + path.unlink() + removed += 1 + except OSError as exc: + log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc) + if removed: + log.info( + "cleanup_orphaned_temp_files: removed %d orphaned staging file(s)", + removed, + ) + return removed + + @celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs") def recover_stalled_task_runs() -> int: """Flip task_run rows stuck in 'running' past their queue-specific diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index f6c78df..1972667 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -26,6 +26,28 @@ def _make_batch(session) -> int: return batch.id +def test_cleanup_orphaned_temp_files_removes_stale_only(tmp_path, monkeypatch): + import os + + from backend.app.tasks import maintenance as m + + monkeypatch.setattr(m, "IMAGES_ROOT", tmp_path) + stale = tmp_path / "artist" / "img.jpg.part" # killed download → orphan + stale.parent.mkdir(parents=True) + stale.write_bytes(b"x") + old = datetime.now(UTC).timestamp() - 8 * 3600 # older than the 6h guard + os.utime(stale, (old, old)) + fresh = tmp_path / "in_progress.jpg.partial" # active download → keep + fresh.write_bytes(b"x") + keep = tmp_path / "real.jpg" # a real image → keep + keep.write_bytes(b"x") + + assert m.cleanup_orphaned_temp_files() == 1 + assert not stale.exists() + assert fresh.exists() + assert keep.exists() + + def test_recover_interrupted_only_old(db_sync, monkeypatch): batch_id = _make_batch(db_sync) now = datetime.now(UTC) -- 2.52.0 From 40cc11be5b2a1cd21b66b36c225b5b92aed59964 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 22:15:33 -0400 Subject: [PATCH 6/7] feat(deploy): container healthchecks + Swarm rolling-update auto-rollback web gets a /api/health liveness check; workers a lenient celery-ping check. A shared deploy policy (update_config order=start-first, failure_action=rollback, monitor 90s; rollback_config; restart_policy) means a bad image that never goes healthy is rolled back automatically instead of taking the service down. Ignored by plain `docker compose up` (deploy: is swarm-only), so the dev override is unaffected. Assumes prod deploys from this file via docker stack deploy. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 60eeda1..1fb24d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,35 @@ # run `docker compose up` from this directory and switches images to # local builds + DEBUG logging. +# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a +# time, START the new task before stopping the old (zero-downtime via the ingress +# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll +# back to the previous image automatically. `monitor` is sized above web's +# healthcheck start_period so a broken image that never goes healthy is caught. +# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev +# override is unaffected. Referenced by each long-lived service below. +x-deploy-policy: &deploy_policy + update_config: + order: start-first + failure_action: rollback + monitor: 90s + rollback_config: + order: start-first + restart_policy: + condition: any + delay: 10s + +# Worker liveness: ping THIS container's celery node over the broker. Lenient +# (60s interval, 3 retries, 60s start_period) so a transient broker blip never +# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell; +# celery's default node name is celery@ (the container id). +x-celery-healthcheck: &celery_healthcheck + test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"] + interval: 60s + timeout: 15s + retries: 3 + start_period: 60s + services: redis: image: redis:7-alpine @@ -55,6 +84,16 @@ services: # by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP # requests + the occasional file download. stop_grace_period: 30s + # Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the + # app booted + serves HTTP after `alembic upgrade head`). start_period covers + # the migration + boot so a slow start isn't mis-flagged. + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""] + interval: 15s + timeout: 6s + retries: 3 + start_period: 40s + deploy: *deploy_policy ports: - "${PORT:-8080}:8080" environment: &app_env @@ -87,6 +126,8 @@ services: command: ["worker"] # Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy. stop_grace_period: 90s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: default,import,thumbnail,download @@ -105,6 +146,8 @@ services: command: ["scheduler"] # Quick maintenance/scan lane + beat — short tasks, modest drain window. stop_grace_period: 60s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: maintenance,scan @@ -126,6 +169,8 @@ services: # the most room to finish a chunk gracefully. Chunked + idempotent, so a job # that still outruns this resumes cleanly next run rather than corrupting. stop_grace_period: 180s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env CELERY_QUEUES: maintenance_long @@ -143,6 +188,8 @@ services: command: ["ml-worker"] # A single GPU inference pass can run tens of seconds — let it finish. stop_grace_period: 120s + healthcheck: *celery_healthcheck + deploy: *deploy_policy environment: <<: *app_env volumes: -- 2.52.0 From 4371ddb7e7c0808f028f1b8fd8e5034d928c7bfb Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 7 Jul 2026 22:20:32 -0400 Subject: [PATCH 7/7] feat(translation): live re-translate progress in the Settings card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit translation/status now reports `active` (a translate/retranslate sweep is running, from the TaskRun table) and `last_run` (the most recent finished run's task + status). The Settings card polls live while a sweep runs, showing a spinner + "Translating… N remaining" that ticks down, and flags a last run that ended in error/timeout. No migration. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/settings.py | 26 +++++++++++++++++ .../components/settings/TranslationCard.vue | 24 ++++++++++++++-- tests/test_api_settings.py | 28 +++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 6eb7294..53e4a8c 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -16,6 +16,7 @@ from ..models import ( ImportTask, Post, Tag, + TaskRun, ) from ..services import interpreter_client as ic @@ -306,6 +307,10 @@ async def translation_status(): """For the Settings card: is it on, is a URL set, is the service reachable, and how many posts still await translation. Health runs the sync client in a thread so the event loop isn't blocked.""" + translation_tasks = ( + "backend.app.tasks.translation.translate_posts", + "backend.app.tasks.translation.retranslate_posts", + ) async with get_session() as session: cfg = await ImportSettings.load(session) untranslated = (await session.execute( @@ -315,6 +320,21 @@ async def translation_status(): Post.post_title.is_not(None), Post.description.is_not(None), )) )).scalar_one() + # Live progress: is a sweep running now, and what did the last one do? + # (run-until-done re-enqueues itself, so `active` stays true across a + # bulk re-translate; `last_run` surfaces a completed run's outcome.) + active = (await session.execute( + select(func.count(TaskRun.id)) + .where(TaskRun.task_name.in_(translation_tasks)) + .where(TaskRun.status == "running") + )).scalar_one() + last = (await session.execute( + select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at) + .where(TaskRun.task_name.in_(translation_tasks)) + .where(TaskRun.finished_at.is_not(None)) + .order_by(TaskRun.finished_at.desc()) + .limit(1) + )).first() base_url = (cfg.interpreter_base_url or "").strip() healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False return jsonify({ @@ -322,6 +342,12 @@ async def translation_status(): "base_url_set": bool(base_url), "healthy": healthy, "untranslated_count": int(untranslated), + "active": int(active) > 0, + "last_run": { + "task": last[0].rsplit(".", 1)[-1], + "status": last[1], + "finished_at": last[2].isoformat() if last[2] else None, + } if last else None, }) diff --git a/frontend/src/components/settings/TranslationCard.vue b/frontend/src/components/settings/TranslationCard.vue index 754b92e..1e5db35 100644 --- a/frontend/src/components/settings/TranslationCard.vue +++ b/frontend/src/components/settings/TranslationCard.vue @@ -52,11 +52,25 @@ prepend-icon="mdi-refresh" :loading="retranslating" :disabled="!enabled || !baseUrl" @click="confirmAll = true" >Re-translate all - + + + Translating… {{ status.untranslated_count }} remaining + + {{ status.untranslated_count }} post{{ status.untranslated_count === 1 ? '' : 's' }} awaiting translation +

+ Last translation run ended with “{{ status.last_run.status }}”. It resumes on + the next sweep; check the Interpreter connection if it persists. +

Use Re-translate all after switching the Interpreter model — it clears every stored translation and re-runs it through the new model @@ -94,7 +108,7 @@