From fb05c5eef7aef26b78f9d166e79767b6ad8d83c4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 8 Jun 2026 08:41:27 -0400 Subject: [PATCH 1/5] fix(cleanup): don't flag a character's fandom (or a chaptered series) as unused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit find_unused_tags only excluded tags with image_tag or series_page references, so it flagged every fandom as 'unused' — fandoms are NEVER applied to images (a character carries its fandom via tag.fandom_id), and the FK is ondelete=SET NULL, so deleting one silently strips the fandom off all its characters (operator-flagged 2026-06-08: artist-OC fandoms showing as unused). Exclude tags referenced as a character's fandom_id, and (same class of gap) tags referenced by a series_chapter (an all-placeholder series has chapters but no pages yet). A genuinely orphaned fandom with no characters is still swept. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/cleanup_service.py | 26 ++++++++++++++++++++----- tests/test_cleanup_service.py | 25 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 05c55c8..4ed3b11 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -21,6 +21,7 @@ from sqlalchemy import func, or_, select, update from sqlalchemy.orm import Session from ..models import Artist, ImageRecord, LibraryAuditRun, Tag +from ..models.series_chapter import SeriesChapter from ..models.series_page import SeriesPage from ..models.tag import image_tag @@ -141,21 +142,36 @@ def count_tag_associations(session: Session, *, tag_id: int) -> int: def find_unused_tags( session: Session, *, limit: int | None = None, ) -> list[Tag]: - """Tags with no image_tag rows AND no series_page rows. + """Tags genuinely referenced by nothing — safe to sweep. - Sorted by name. Used by both dry-run preview and the live prune. - A tag is "unused" iff it has zero rows in image_tag AND zero rows - in series_page (so we don't accidentally prune a series tag that - happens to have no images yet). + Sorted by name. Used by both dry-run preview and the live prune. A tag is + "unused" iff it has zero references across ALL the ways a tag can be in use: + - image_tag (applied to an image) + - series_page (a series tag with ordered pages) + - series_chapter (a series tag with chapters but no pages yet, e.g. + all-placeholder) + - tag.fandom_id (a fandom referenced by a character) + + The fandom check is essential: fandom tags are NEVER applied to images — a + character carries its fandom via fandom_id — so without it every assigned + fandom looked "unused", and the FK is ondelete=SET NULL, so deleting one + would silently strip the fandom off all its characters (operator-flagged + 2026-06-08). """ used_via_image_tag = select(image_tag.c.tag_id).distinct() used_via_series = select(SeriesPage.series_tag_id).where( SeriesPage.series_tag_id.is_not(None) ).distinct() + used_via_chapter = select(SeriesChapter.series_tag_id).distinct() + used_via_fandom = select(Tag.fandom_id).where( + Tag.fandom_id.is_not(None) + ).distinct() stmt = ( select(Tag) .where(Tag.id.not_in(used_via_image_tag)) .where(Tag.id.not_in(used_via_series)) + .where(Tag.id.not_in(used_via_chapter)) + .where(Tag.id.not_in(used_via_fandom)) .order_by(Tag.name) ) if limit is not None: diff --git a/tests/test_cleanup_service.py b/tests/test_cleanup_service.py index ba6242f..25285b1 100644 --- a/tests/test_cleanup_service.py +++ b/tests/test_cleanup_service.py @@ -155,6 +155,31 @@ def test_find_unused_tags_returns_only_unreferenced(db_sync, tmp_path): assert names.index("aaa-unused") < names.index("zzz-unused") +def test_find_unused_tags_excludes_fandom_referenced_by_character(db_sync): + # A fandom is never on an image — a character carries it via fandom_id — so + # an assigned fandom must NOT be flagged unused (deleting it SET-NULLs the + # fandom off its characters). Operator-flagged 2026-06-08. + fandom = _make_tag(db_sync, name="Creux", kind=TagKind.fandom) + char = _make_tag(db_sync, name="OcChar", kind=TagKind.character) + char.fandom_id = fandom.id + orphan_fandom = _make_tag(db_sync, name="NobodysFandom", kind=TagKind.fandom) + db_sync.commit() + + names = [t.name for t in cleanup_service.find_unused_tags(db_sync)] + assert "Creux" not in names # referenced by a character → kept + assert "NobodysFandom" in names # genuinely orphaned → still swept + + +def test_find_unused_tags_excludes_series_with_only_chapters(db_sync): + # An all-placeholder series has chapters but no pages yet — not unused. + series = _make_tag(db_sync, name="EmptySeries", kind=TagKind.series) + db_sync.add(SeriesChapter(series_tag_id=series.id, chapter_number=1)) + db_sync.commit() + + names = [t.name for t in cleanup_service.find_unused_tags(db_sync)] + assert "EmptySeries" not in names + + # --- unlink_image_files --------------------------------------------- From fe0ed525956b4e98e05923cf023fad0f95d536c8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 8 Jun 2026 08:45:36 -0400 Subject: [PATCH 2/5] test: drop unused binding in find_unused_tags test (ruff F841) Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_cleanup_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cleanup_service.py b/tests/test_cleanup_service.py index 25285b1..e979ceb 100644 --- a/tests/test_cleanup_service.py +++ b/tests/test_cleanup_service.py @@ -162,7 +162,7 @@ def test_find_unused_tags_excludes_fandom_referenced_by_character(db_sync): fandom = _make_tag(db_sync, name="Creux", kind=TagKind.fandom) char = _make_tag(db_sync, name="OcChar", kind=TagKind.character) char.fandom_id = fandom.id - orphan_fandom = _make_tag(db_sync, name="NobodysFandom", kind=TagKind.fandom) + _make_tag(db_sync, name="NobodysFandom", kind=TagKind.fandom) db_sync.commit() names = [t.name for t in cleanup_service.find_unused_tags(db_sync)] From b1778ca9f2a91c73f106ee86c0e463bc08912a7b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 8 Jun 2026 08:49:37 -0400 Subject: [PATCH 3/5] obs(ml): tag_and_embed logs file + phase + timing; failures name them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task logged nothing and SoftTimeLimitExceeded stringifies to empty, so a timeout surfaced as a bare 'SoftTimeLimitExceeded()' with no clue which file or why (operator-flagged 2026-06-08). - Log start (id/path/mime/bytes/video?), per-phase timing (load_models, video probe/sample/infer, tag, embed, persist), and a success summary. - Track a + file ; on SoftTimeLimitExceeded log it and re-raise SoftTimeLimitExceeded WITH that context (keeps the 'timeout' task_run status but gives the activity a real error_message: which file, which phase, elapsed). - On other exceptions, log context then re-raise the ORIGINAL (preserves autoretry for OSError/DBAPIError/OperationalError). Now a stuck run names the culprit — most likely a slow video (frame sampling is up to 10x60s ffmpeg) or a huge image; the phase log will say which. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/tasks/ml.py | 173 ++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 50 deletions(-) diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index 8867cee..3be9f7b 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -6,8 +6,10 @@ apply_allowlist_tags sweeps which are 'maintenance' lane. Sync sessions (Celery workers are sync processes), same pattern as FC-2a tasks. """ +import logging from pathlib import Path +from celery.exceptions import SoftTimeLimitExceeded from sqlalchemy import select from sqlalchemy.exc import DBAPIError, OperationalError @@ -15,6 +17,8 @@ from ..celery_app import celery from ..models import ImageRecord, MLSettings from ._sync_engine import sync_session_factory as _sync_session_factory +log = logging.getLogger(__name__) + IMAGES_ROOT = Path("/images") VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"} @@ -50,67 +54,136 @@ def tag_and_embed(self, image_id: int) -> dict: SigLIP embeddings. On no-frames returns status='no_frames' (not an error). """ import os + import time from ..services.ml.embedder import get_embedder from ..services.ml.tagger import get_tagger - SessionLocal = _sync_session_factory() - with SessionLocal() as session: - record = session.get(ImageRecord, image_id) - if record is None: - return {"status": "missing", "image_id": image_id} - settings = session.execute( - select(MLSettings).where(MLSettings.id == 1) - ).scalar_one() + # Phase + file context, so a timeout/crash names WHICH file and WHERE it + # died instead of a bare SoftTimeLimitExceeded() (operator-flagged 2026-06-08: + # the activity told them nothing about the file or why). `ctx` is enriched + # once the record is loaded; both feed the worker log AND the re-raised + # exception message (which becomes the activity's error_message). + started = time.monotonic() + phase = "open_session" + ctx = f"image_id={image_id}" - src = Path(record.path) - if not src.is_file(): - return {"status": "file_missing", "image_id": image_id} + def _elapsed() -> float: + return time.monotonic() - started - tagger = get_tagger() - embedder = get_embedder() + try: + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + record = session.get(ImageRecord, image_id) + if record is None: + return {"status": "missing", "image_id": image_id} + settings = session.execute( + select(MLSettings).where(MLSettings.id == 1) + ).scalar_one() - if _is_video(src): - # Layer-3 isolation: ffprobe (a separate process) validates - # the container before we burn ~20 GPU ops sampling frames - # from it. A corrupt video that would crash the frame - # decoder is rejected cleanly here instead of taking down - # the ml-worker. Operator-flagged 2026-05-28. - from ..utils import safe_probe - vprobe = safe_probe.probe_video(src) - if not vprobe.ok: - return { - "status": "bad_video", "image_id": image_id, - "reason": vprobe.reason, - } - frames = _sample_video_frames( - src, int(os.environ.get("VIDEO_ML_FRAMES", "10")) + src = Path(record.path) + is_vid = _is_video(src) + ctx = ( + f"image_id={image_id} path={record.path} mime={record.mime} " + f"bytes={record.size_bytes} video={is_vid}" ) - if not frames: - return {"status": "no_frames", "image_id": image_id} - preds = _maxpool_predictions([tagger.infer(f) for f in frames]) - import numpy as np + log.info("tag_and_embed start: %s", ctx) + if not src.is_file(): + log.warning("tag_and_embed file missing on disk: %s", ctx) + return {"status": "file_missing", "image_id": image_id} - embedding = np.mean( - [embedder.infer(f) for f in frames], axis=0 - ).astype("float32") - for f in frames: - f.unlink(missing_ok=True) - else: - raw = tagger.infer(src) - preds = { - name: {"category": p.category, "confidence": p.confidence} - for name, p in raw.items() - } - embedding = embedder.infer(src) + phase = "load_models" + tagger = get_tagger() + embedder = get_embedder() - record.tagger_predictions = preds - record.tagger_model_version = settings.tagger_model_version - record.siglip_embedding = embedding.tolist() - record.siglip_model_version = settings.embedder_model_version - session.add(record) - session.commit() + if is_vid: + # Layer-3 isolation: ffprobe (a separate process) validates + # the container before we burn ~20 GPU ops sampling frames + # from it. A corrupt video that would crash the frame + # decoder is rejected cleanly here instead of taking down + # the ml-worker. Operator-flagged 2026-05-28. + phase = "video_probe" + from ..utils import safe_probe + vprobe = safe_probe.probe_video(src) + if not vprobe.ok: + log.warning( + "tag_and_embed bad video (%s): %s", vprobe.reason, ctx + ) + return { + "status": "bad_video", "image_id": image_id, + "reason": vprobe.reason, + } + phase = "video_sample_frames" + t0 = time.monotonic() + frames = _sample_video_frames( + src, int(os.environ.get("VIDEO_ML_FRAMES", "10")) + ) + log.info( + "tag_and_embed sampled %d frame(s) in %.1fs: %s", + len(frames), time.monotonic() - t0, ctx, + ) + if not frames: + return {"status": "no_frames", "image_id": image_id} + phase = "video_infer" + import numpy as np + preds = _maxpool_predictions([tagger.infer(f) for f in frames]) + embedding = np.mean( + [embedder.infer(f) for f in frames], axis=0 + ).astype("float32") + for f in frames: + f.unlink(missing_ok=True) + else: + phase = "tag" + t0 = time.monotonic() + raw = tagger.infer(src) + log.info( + "tag_and_embed tagged in %.1fs (%d tags): %s", + time.monotonic() - t0, len(raw), ctx, + ) + preds = { + name: {"category": p.category, "confidence": p.confidence} + for name, p in raw.items() + } + phase = "embed" + t0 = time.monotonic() + embedding = embedder.infer(src) + log.info( + "tag_and_embed embedded in %.1fs: %s", + time.monotonic() - t0, ctx, + ) + + phase = "persist" + record.tagger_predictions = preds + record.tagger_model_version = settings.tagger_model_version + record.siglip_embedding = embedding.tolist() + record.siglip_model_version = settings.embedder_model_version + session.add(record) + session.commit() + except SoftTimeLimitExceeded: + log.error( + "tag_and_embed TIMED OUT after %.0fs in phase=%s: %s", + _elapsed(), phase, ctx, + ) + # Re-raise as SoftTimeLimitExceeded (preserves the 'timeout' status in + # the task_run signal) but WITH context, so the activity error_message + # names the file + phase instead of being empty. + raise SoftTimeLimitExceeded( + f"timed out in phase={phase} after {_elapsed():.0f}s ({ctx})" + ) from None + except Exception: + # OSError/DBAPIError/OperationalError are autoretried — re-raise the + # ORIGINAL so the type is preserved; just make sure it's logged with + # context first. + log.exception( + "tag_and_embed FAILED in phase=%s after %.0fs: %s", + phase, _elapsed(), ctx, + ) + raise + + log.info( + "tag_and_embed ok in %.1fs (%d tags): %s", _elapsed(), len(preds), ctx + ) apply_allowlist_tags.delay(image_id=image_id) return {"status": "ok", "image_id": image_id, "tags": len(preds)} From f2fbe2ae6e9fe7282fbec1b8fb87baf361839727 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 8 Jun 2026 08:52:39 -0400 Subject: [PATCH 4/5] tweak(ml): default video frame samples 10 to 6 Operator: 10-frame max-pooled tagging on video produces a lot of noisy tags, and the sampling burns time/GPU. Drop the VIDEO_ML_FRAMES default to 6 (still env- overridable). Fewer frames = less per-frame noise into the max-pool and a smaller frame-sampling budget. Quality/perf of the whole video path is being reviewed separately. --- backend/app/tasks/ml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index 3be9f7b..7703447 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -35,8 +35,8 @@ def _is_video(path: Path) -> bool: retry_backoff_max=60, retry_jitter=True, max_retries=3, - # Sized for the video branch: sample 10 frames, run tagger + - # embedder on each (≈20 GPU ops vs 2 for an image). A loaded + # Sized for the video branch: sample 6 frames, run tagger + + # embedder on each (≈12 GPU ops vs 2 for an image). A loaded # ml-worker can take 5-10 min on a long video; bumped from # 5min/7min on 2026-05-28 after operator-flagged image 6288 (a # .mp4) hit the recovery sweep at 5 min while still legitimately @@ -50,7 +50,7 @@ def tag_and_embed(self, image_id: int) -> dict: then enqueue per-image allowlist application. Video: sample frames between 10% and 90% of duration (VIDEO_ML_FRAMES, - default 10). Max-pool tagger confidences across frames, mean-pool the + default 6). Max-pool tagger confidences across frames, mean-pool the SigLIP embeddings. On no-frames returns status='no_frames' (not an error). """ import os @@ -116,7 +116,7 @@ def tag_and_embed(self, image_id: int) -> dict: phase = "video_sample_frames" t0 = time.monotonic() frames = _sample_video_frames( - src, int(os.environ.get("VIDEO_ML_FRAMES", "10")) + src, int(os.environ.get("VIDEO_ML_FRAMES", "6")) ) log.info( "tag_and_embed sampled %d frame(s) in %.1fs: %s", From 408fcd488a1086e8de536d5737e04476dec2878c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 8 Jun 2026 08:59:55 -0400 Subject: [PATCH 5/5] refactor(ui): unify confirm-dropdown Enter behavior via useAcceptOnEnter Operator-flagged (again) on the tag-merge picker: Enter on the dropdown re-opens it instead of accepting the selection. I'd already patched this twice (fandom picker + fandom set dialog) with copy-pasted capture-phase handlers, so DRY it. New composable useAcceptOnEnter(accept): tracks the menu state and, on a capture-phase Enter, lets Vuetify pick when the menu is open but calls accept() (and blocks the re-open) when it's closed. Applied to every confirm-style picker: - TagsView merge-into picker (the reported one) - AliasPickerDialog - PostSeriesMenu add-to-existing - FandomPicker + FandomSetDialog (refactored off their bespoke handlers) One behavior, one place to change it. --- .../components/modal/AliasPickerDialog.vue | 10 +++++- .../src/components/modal/FandomPicker.vue | 21 +++-------- .../src/components/modal/FandomSetDialog.vue | 22 ++++-------- .../src/components/posts/PostSeriesMenu.vue | 9 +++++ frontend/src/composables/useAcceptOnEnter.js | 35 +++++++++++++++++++ frontend/src/views/TagsView.vue | 7 ++++ 6 files changed, 71 insertions(+), 33 deletions(-) create mode 100644 frontend/src/composables/useAcceptOnEnter.js diff --git a/frontend/src/components/modal/AliasPickerDialog.vue b/frontend/src/components/modal/AliasPickerDialog.vue index 37fae6f..0890529 100644 --- a/frontend/src/components/modal/AliasPickerDialog.vue +++ b/frontend/src/components/modal/AliasPickerDialog.vue @@ -8,6 +8,7 @@

@@ -31,9 +33,10 @@