From 911d535f5693f4b81777c29a3282d47fd7db46c5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 13:45:03 -0400 Subject: [PATCH 01/11] fix(artists): card preview dead-space + empty-state flicker on first load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two operator-reported Artists-view bugs. Layout: ArtistCard previews used aspect-ratio: 3/1 + max-height: 220px — when a lone grid column got wide (>~660px), the max-height made the aspect-ratio box shrink its own WIDTH to keep the ratio, leaving dead space beside the 3 thumbnails. Dropped max-height (kept the min-height floor) so the strip fills the card width; lowered the grid min 440→360px so a lone column stays <732px (strip ≲244px) and desktop gets 2+ columns. Flicker: isEmpty checked !loading, but on the first render loading is still false (the initial loadMore fires in onMounted, after paint) so "No artists match" flashed for a frame before the spinner/data. Added a reactive `loaded` flag (true after the first load attempt, reset on reset()); isEmpty now also requires it. Co-Authored-By: Claude Opus 4.8 (1M context) --- frontend/src/components/discovery/ArtistCard.vue | 8 +++++--- frontend/src/stores/artistDirectory.js | 9 ++++++++- frontend/src/views/ArtistsView.vue | 8 +++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/discovery/ArtistCard.vue b/frontend/src/components/discovery/ArtistCard.vue index dc2f5ae..813aeda 100644 --- a/frontend/src/components/discovery/ArtistCard.vue +++ b/frontend/src/components/discovery/ArtistCard.vue @@ -49,9 +49,11 @@ function onCardClick() { position: relative; display: grid; grid-template-columns: repeat(3, 1fr); gap: 2px; aspect-ratio: 3 / 1; - /* Explicit floor + ceiling so tall source images can't escape the - preview slot even on browsers where aspect-ratio doesn't compute. */ - min-height: 150px; max-height: 220px; + /* Floor so tall source images can't escape the slot. NO max-height: an + aspect-ratio box capped by max-height shrinks its own WIDTH to keep the + ratio, leaving dead space beside the previews on a wide (single-column) + card. The grid below keeps columns ≲400px so the strip stays a sane height. */ + min-height: 120px; overflow: hidden; background: rgb(var(--v-theme-surface-light)); } diff --git a/frontend/src/stores/artistDirectory.js b/frontend/src/stores/artistDirectory.js index baede1d..950be2f 100644 --- a/frontend/src/stores/artistDirectory.js +++ b/frontend/src/stores/artistDirectory.js @@ -14,6 +14,10 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => { const q = ref('') const platform = ref(null) let started = false + // Has a load ATTEMPT completed yet? Gates the "No artists match" empty state + // so it can't flash on the very first render (loading is still false before + // onMounted fires the first loadMore) or between a query change and its fetch. + const loaded = ref(false) // Typed "alice" then "alice bob" used to drop the second fetch // entirely (loading flag still true from the first), so the UI // showed alice results while the input said "alice bob". Inflight @@ -35,6 +39,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => { nextCursor.value = body.next_cursor started = true }) + if (t.isCurrent()) loaded.value = true } async function reset() { @@ -42,6 +47,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => { cards.value = [] nextCursor.value = null started = false + loaded.value = false await loadMore() } @@ -50,7 +56,8 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => { const hasMore = computed(() => !started || nextCursor.value !== null) const isEmpty = computed( - () => !loading.value && cards.value.length === 0 && error.value === null + () => loaded.value && !loading.value + && cards.value.length === 0 && error.value === null ) return { diff --git a/frontend/src/views/ArtistsView.vue b/frontend/src/views/ArtistsView.vue index 3e3c613..f5582f3 100644 --- a/frontend/src/views/ArtistsView.vue +++ b/frontend/src/views/ArtistsView.vue @@ -76,9 +76,11 @@ onMounted(async () => { } .fc-artists__grid { display: grid; - /* min(440px, 100%) so a card never exceeds the viewport — on phones the - min-track collapses to 100% (single column) instead of overflowing. */ - grid-template-columns: repeat(auto-fill, minmax(min(440px, 100%), 1fr)); + /* min(360px, 100%) so a card never exceeds the viewport (phones collapse to + one column). 360 keeps a LONE column under ~732px wide, so the 3/1 preview + strip stays ≲244px tall and always fills the card width (no dead space), + while giving 2+ columns on a normal desktop. */ + grid-template-columns: repeat(auto-fill, minmax(min(360px, 100%), 1fr)); gap: 12px; } .fc-artists__sentinel { -- 2.52.0 From 5bb25245a56d1eca5c48f4c98d591fc196017836 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 14:33:24 -0400 Subject: [PATCH 02/11] =?UTF-8?q?fix(archive):=20magic-byte=20archive=20de?= =?UTF-8?q?tection=20so=20mis-named=20archives=20extract=20=E2=80=94=20#71?= =?UTF-8?q?3=20part=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patreon attachment downloads land with sanitized URL-blob filenames (01_https___www.patreon.com_media-u_v3_) whose Path.suffix is junk, never .zip — so the extension-only is_archive() filed them as opaque PostAttachments and never extracted them ("No images attached to this post"). Add archive_extractor.detect_archive_format() — extension first, then magic-byte sniff (zipfile.is_zipfile + RAR/7z signatures). is_archive(), extract_archive(), and safe_probe._inspect_archive() (the bomb-guard) all route through it, so a mis-named/extension-less archive is now detected, bomb-guarded, integrity-tested, AND extracted regardless of filename. Stops new ones; part 2 re-extracts the already-imported backlog. Tests: mis-named zip detected + extracted; non-archive dotted name not misdetected; _inspect_archive on a mis-named zip; signature updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/archive_extractor.py | 46 ++++++++++++++++++++--- backend/app/utils/safe_probe.py | 22 ++++++----- tests/test_archive_extractor.py | 18 +++++++++ tests/test_safe_probe.py | 12 +++++- 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/backend/app/services/archive_extractor.py b/backend/app/services/archive_extractor.py index 45babef..d6b6566 100644 --- a/backend/app/services/archive_extractor.py +++ b/backend/app/services/archive_extractor.py @@ -16,9 +16,45 @@ log = logging.getLogger(__name__) ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"} +# Magic-byte signatures, so an archive with a mangled / extension-less filename +# is still recognised. Patreon attachment download URLs sanitize to names like +# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk +# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed +# those as opaque PostAttachments and NEVER extracted them (operator-flagged +# 2026-06-06). Detection is by extension first (cheap), then header sniff. +_RAR_MAGIC = b"Rar!\x1a\x07" +_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c" + + +def detect_archive_format(path: Path) -> str | None: + """Return "zip" | "rar" | "7z" for an archive, else None. + + Trusts a known extension first, then falls back to magic-byte sniffing so a + mis-named or extension-less archive is still handled. (zip covers .cbz too.) + """ + ext = Path(path).suffix.lower() + if ext in (".zip", ".cbz"): + return "zip" + if ext == ".rar": + return "rar" + if ext == ".7z": + return "7z" + try: + if zipfile.is_zipfile(path): + return "zip" + with open(path, "rb") as fh: + head = fh.read(8) + except OSError: + return None + if head.startswith(_RAR_MAGIC): + return "rar" + if head.startswith(_7Z_MAGIC): + return "7z" + return None + def is_archive(path: Path) -> bool: - return Path(path).suffix.lower() in ARCHIVE_EXTS + return detect_archive_format(path) is not None @contextmanager @@ -32,16 +68,16 @@ def extract_archive(path: Path): members: list[tuple[str, Path]] = [] try: try: - ext = Path(path).suffix.lower() - if ext in (".zip", ".cbz"): + fmt = detect_archive_format(path) + if fmt == "zip": with zipfile.ZipFile(path) as zf: zf.extractall(base) - elif ext == ".rar": + elif fmt == "rar": import rarfile with rarfile.RarFile(path) as rf: rf.extractall(base) - elif ext == ".7z": + elif fmt == "7z": import py7zr with py7zr.SevenZipFile(path, "r") as zf: diff --git a/backend/app/utils/safe_probe.py b/backend/app/utils/safe_probe.py index d8e5360..beb0bc1 100644 --- a/backend/app/utils/safe_probe.py +++ b/backend/app/utils/safe_probe.py @@ -149,9 +149,8 @@ def _run_probe(path_str: str) -> tuple[str, str | None]: call the same code path. """ path = Path(path_str) - ext = path.suffix.lower() try: - total, test_bad = _inspect_archive(path, ext) + total, test_bad = _inspect_archive(path) except Exception as exc: # noqa: BLE001 — clean rejection return ("error", f"{type(exc).__name__}: {exc}") if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES: @@ -162,24 +161,29 @@ def _run_probe(path_str: str) -> tuple[str, str | None]: return ("ok", None) -def _inspect_archive(path: Path, ext: str): +def _inspect_archive(path: Path): """Return (total_uncompressed_bytes | None, first_bad_member | None) - for the archive. Format-specific; raises on a structurally-broken - container (caught by the child as a clean rejection).""" - if ext in (".zip", ".cbz"): + for the archive. Format detected by extension OR magic bytes (so a + mis-named archive is still bomb-guarded + integrity-tested, matching the + extractor's gate); raises on a structurally-broken container (caught by the + child as a clean rejection).""" + from ..services.archive_extractor import detect_archive_format + + fmt = detect_archive_format(path) + if fmt == "zip": import zipfile with zipfile.ZipFile(path) as zf: total = sum(zi.file_size for zi in zf.infolist()) return total, zf.testzip() - if ext == ".rar": + if fmt == "rar": import rarfile with rarfile.RarFile(path) as rf: total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist()) rf.testrar() return total, None - if ext == ".7z": + if fmt == "7z": import py7zr with py7zr.SevenZipFile(path, "r") as zf: @@ -187,5 +191,5 @@ def _inspect_archive(path: Path, ext: str): total = getattr(info, "uncompressed", None) ok = zf.test() # True / None when all members pass return total, (None if ok in (True, None) else "7z test reported corruption") - # Unknown extension — nothing to test; treat as clean. + # Not a recognised archive — nothing to test; treat as clean. return None, None diff --git a/tests/test_archive_extractor.py b/tests/test_archive_extractor.py index 24e3a95..420655c 100644 --- a/tests/test_archive_extractor.py +++ b/tests/test_archive_extractor.py @@ -39,6 +39,24 @@ def test_cbz_alias(tmp_path): assert [n for n, _ in members] == ["p1.jpg"] +def test_misnamed_zip_detected_and_extracted(tmp_path): + """A real zip whose filename has no usable extension (Patreon attachment + URL-blob name) is detected by magic bytes and its members extracted — + previously it was filed as an opaque attachment and never opened.""" + z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093" + _zip(z, {"a.jpg": b"img", "b.png": b"img2"}) + assert is_archive(z) + with extract_archive(z) as members: + assert sorted(n for n, _ in members) == ["a.jpg", "b.png"] + + +def test_non_archive_with_dotted_name_is_not_archive(tmp_path): + """A non-archive file with a dotted/extension-less name isn't misdetected.""" + f = tmp_path / "01_https___www.patreon.com_media-u_v3_999" + f.write_bytes(b"\x89PNG\r\n\x1a\n not really but not a zip") + assert not is_archive(f) + + def test_corrupt_archive_yields_nothing(tmp_path): bad = tmp_path / "broken.zip" bad.write_bytes(b"not a zip at all") diff --git a/tests/test_safe_probe.py b/tests/test_safe_probe.py index 98a67b3..85e5c85 100644 --- a/tests/test_safe_probe.py +++ b/tests/test_safe_probe.py @@ -40,11 +40,21 @@ def test_probe_archive_corrupt_zip_clean_rejection(tmp_path): def test_inspect_archive_reports_size_and_clean_integrity(tmp_path): z = tmp_path / "sized.zip" _zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50}) - total, bad = safe_probe._inspect_archive(z, ".zip") + total, bad = safe_probe._inspect_archive(z) assert total == 150 assert bad is None +def test_inspect_archive_detects_misnamed_zip(tmp_path): + """A zip with a mangled / extension-less name (Patreon URL-blob attachment) + is still bomb-guarded + integrity-tested via magic-byte detection.""" + z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093" + _zip(z, {"a.txt": b"x" * 100}) + total, bad = safe_probe._inspect_archive(z) + assert total == 100 + assert bad is None + + def test_run_probe_bomb_guard(tmp_path, monkeypatch): """In-process call to _run_probe so the monkeypatched cap takes effect. The public probe_archive runs in a subprocess which re-imports the -- 2.52.0 From a497104661c10f11cc91d58dedc4e73ead1c2099 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 14:43:26 -0400 Subject: [PATCH 03/11] =?UTF-8?q?feat(maintenance):=20re-extract=20archive?= =?UTF-8?q?=20attachments=20+=20link=20to=20post=20=E2=80=94=20#713=20part?= =?UTF-8?q?=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Existing PostAttachments that are actually archives (filed opaquely before the magic-byte gate) need extracting retroactively. cleanup_service. reextract_archive_attachments scans PostAttachments, magic-detects the archives, and for each reconstructs the post's sidecar from the DB + re-runs attach_in_place in a temp dir — so the members extract and re-link to the SAME post via find_or_create_post (source_id + external_post_id). Idempotent (members dedupe by sha256). Enqueues thumbnail+ML for new members. Wired as a maintenance-queue Celery task (tasks/admin) + POST /api/admin/maintenance/reextract-archives (202) + a "Re-extract archive attachments" card in Settings → Maintenance. Test: a zip stored under a mangled extension-less name extracts + links its member to the post via ImageProvenance, and a second run is a no-op (idempotent). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/admin.py | 11 ++ backend/app/services/cleanup_service.py | 114 ++++++++++++++++++ backend/app/tasks/admin.py | 18 +++ .../settings/ArchiveReextractCard.vue | 46 +++++++ .../components/settings/MaintenancePanel.vue | 2 + tests/test_reextract_archives.py | 91 ++++++++++++++ 6 files changed, 282 insertions(+) create mode 100644 frontend/src/components/settings/ArchiveReextractCard.vue create mode 100644 tests/test_reextract_archives.py diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 9e4b2b2..4005475 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -289,3 +289,14 @@ async def trigger_vacuum(): vacuum_analyze.delay() return jsonify({"status": "queued"}), 202 + + +@admin_bp.route("/maintenance/reextract-archives", methods=["POST"]) +async def trigger_reextract_archives(): + """Operator-triggered re-extract (#713): PostAttachments that are actually + archives but were filed opaquely (pre magic-byte gate) get extracted and + their members linked to the post. Idempotent; runs on the maintenance queue.""" + from ..tasks.admin import reextract_archive_attachments_task + + async_result = reextract_archive_attachments_task.delay() + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index f4ee687..f897e8b 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -11,6 +11,7 @@ the one-and-done GS/IR migration tooling.) """ from __future__ import annotations +import logging from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -22,6 +23,8 @@ from ..models import Artist, ImageRecord, LibraryAuditRun, Tag from ..models.series_page import SeriesPage from ..models.tag import image_tag +log = logging.getLogger(__name__) + def project_artist_cascade(session: Session, *, slug: str) -> dict: """Read-only projection of what delete_artist_cascade would touch. @@ -665,3 +668,114 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None: .where(LibraryAuditRun.status == "running") .values(status="cancelled", finished_at=datetime.now(UTC)) ) + + +# -- archive-attachment re-extraction (#713 part 2) ------------------------ + +_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"} + + +def _reextract_archive_to_post(importer, archive_path: Path, post, source_row, artist) -> list[int]: + """Extract one stored archive and link its members to `post`. + + The stored attachment has no adjacent sidecar (it lives in the sha-addressed + attachment store), so reconstruct the post's sidecar from the DB and re-run + the normal import path: `attach_in_place` extracts the members and + `_apply_sidecar` → `find_or_create_post` re-attaches them to the SAME Post + (matched by source_id + external_post_id). Returns the new member image ids. + """ + import json + import shutil + import tempfile + + from .archive_extractor import detect_archive_format + + fmt = detect_archive_format(archive_path) + ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip") + sidecar = { + "category": source_row.platform if source_row is not None + else (post.raw_metadata or {}).get("category"), + "id": post.external_post_id, + "title": post.post_title or "", + "content": post.description or "", + "published_at": post.post_date.isoformat() if post.post_date else None, + "url": post.post_url, + } + with tempfile.TemporaryDirectory(prefix="fc_reextract_") as td: + tmp = Path(td) / f"archive{ext}" # clean ext → is_archive + find_sidecar + shutil.copy2(archive_path, tmp) + tmp.with_suffix(".json").write_text(json.dumps(sidecar)) + res = importer.attach_in_place(tmp, artist=artist, source=source_row) + return list(res.member_image_ids or []) + + +def reextract_archive_attachments(session: Session, *, images_root: Path) -> dict: + """Re-process existing PostAttachments that are ACTUALLY archives but were + filed opaquely before #713 part 1 (extension-only is_archive missed mangled / + extension-less Patreon attachment names). For each: extract the members, + import them, and link them to the attachment's post. + + Idempotent — members dedupe by sha256, the archive dedupes by sha — so it's + safe to run repeatedly. Returns a summary dict for task_run.metadata. + """ + from ..models import ImportSettings, Post, PostAttachment, Source + from ..tasks.ml import tag_and_embed + from ..tasks.thumbnail import generate_thumbnail + from .archive_extractor import is_archive + from .importer import Importer + from .thumbnailer import Thumbnailer + + summary = { + "scanned": 0, "archives": 0, "members_imported": 0, + "posts_touched": 0, "skipped_no_post": 0, "errors": 0, + } + settings = ImportSettings.load_sync(session) + importer = Importer( + session=session, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + ) + + attachments = session.execute( + select(PostAttachment).order_by(PostAttachment.id) + ).scalars().all() + enqueue_ids: list[int] = [] + for att in attachments: + summary["scanned"] += 1 + stored = Path(att.path) + try: + if not stored.is_file() or not is_archive(stored): + continue + except OSError: + continue + summary["archives"] += 1 + if att.post_id is None: + summary["skipped_no_post"] += 1 + continue + post = session.get(Post, att.post_id) + if post is None: + summary["skipped_no_post"] += 1 + continue + artist = session.get(Artist, att.artist_id) if att.artist_id else None + source_row = session.get(Source, post.source_id) if post.source_id else None + try: + ids = _reextract_archive_to_post(importer, stored, post, source_row, artist) + session.commit() + except Exception as exc: # one bad archive must not strand the rest + session.rollback() + summary["errors"] += 1 + log.warning("re-extract failed for attachment %s: %s", att.id, exc) + continue + if ids: + summary["members_imported"] += len(ids) + summary["posts_touched"] += 1 + enqueue_ids.extend(ids) + + # Thumbnails + ML for the newly-imported members (best-effort; off the + # critical path — a Redis hiccup must not fail the whole re-extract). + for img_id in enqueue_ids: + try: + generate_thumbnail.delay(img_id) + tag_and_embed.delay(img_id) + except Exception as exc: + log.warning("re-extract enqueue failed for image %s: %s", img_id, exc) + return summary diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index fec90c8..e3ae6ac 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -55,3 +55,21 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict: return cleanup_service.delete_images( session, image_ids=image_ids, images_root=IMAGES_ROOT, ) + + +@celery.task( + name="backend.app.tasks.admin.reextract_archive_attachments_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def reextract_archive_attachments_task(self) -> dict: + """Wraps cleanup_service.reextract_archive_attachments (#713 part 2): + re-extract PostAttachments that are actually archives but were filed + opaquely before the magic-byte gate, and link their members to the post.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + return cleanup_service.reextract_archive_attachments( + session, images_root=IMAGES_ROOT, + ) diff --git a/frontend/src/components/settings/ArchiveReextractCard.vue b/frontend/src/components/settings/ArchiveReextractCard.vue new file mode 100644 index 0000000..5f612cd --- /dev/null +++ b/frontend/src/components/settings/ArchiveReextractCard.vue @@ -0,0 +1,46 @@ + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index f550c77..76b85c5 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -15,6 +15,7 @@ + + - {{ iconFor(tag.kind) }} - {{ tag.name }}→ - - - - - - - Rename… - - - Set fandom… - - - - - + + {{ iconFor(tag.kind) }} + {{ tag.name }}→ + + + + + + Rename… + + + Set fandom… + + + + No tags yet. @@ -88,9 +86,6 @@ import FandomSetDialog from './FandomSetDialog.vue' const modal = useModalStore() const store = useTagStore() const errorMsg = ref(null) -// Which tag chip's kebab menu is open (only one at a time). Drives each -// chip menu's v-model so opening never depends on Vuetify's activator click. -const openTagId = ref(null) const KIND_ICONS = { general: 'mdi-tag', character: 'mdi-account-circle', @@ -153,6 +148,7 @@ async function onFandomUpdated() { margin-bottom: 12px; } .fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; } -.kebab-wrap { display: inline-flex; align-items: center; } -.kebab-icon { cursor: pointer; } +.fc-tag-panel__chip { display: inline-flex; align-items: center; gap: 1px; } +.fc-tag-panel__kebab { opacity: 0.7; } +.fc-tag-panel__chip:hover .fc-tag-panel__kebab { opacity: 1; } diff --git a/frontend/src/stores/tags.js b/frontend/src/stores/tags.js index 9365550..050a064 100644 --- a/frontend/src/stores/tags.js +++ b/frontend/src/stores/tags.js @@ -34,10 +34,20 @@ export const useTagStore = defineStore('tags', () => { } async function loadFandoms() { - fandomCache.value = await api.get('/api/tags/autocomplete', { - params: { q: ' ', kind: 'fandom', limit: 200 } - }) - return fandomCache.value + // List ALL fandoms via the cursor-paged tag directory. (#712: the old impl + // abused /tags/autocomplete with q=' ', which the backend strips to empty + // and returns [] — so the picker showed no existing fandoms.) + const all = [] + let cursor = null + do { + const params = { kind: 'fandom', limit: 200 } + if (cursor) params.cursor = cursor + const body = await api.get('/api/tags/directory', { params }) + all.push(...(body.cards || [])) + cursor = body.next_cursor + } while (cursor) + fandomCache.value = all + return all } async function createFandom(name) { -- 2.52.0 From 2b69540ecc0bd250cd8c916b6f5a4f56c78bbe5b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 15:42:20 -0400 Subject: [PATCH 06/11] =?UTF-8?q?feat(tags):=20Title-Case=20normalization?= =?UTF-8?q?=20on=20create/rename=20=E2=80=94=20#701=20(core)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tags now normalize to Title Case + collapsed whitespace at the central TagService.find_or_create (and rename) — so manual entry, the create API, and anything routed through find_or_create produce the canonical form. The lookup is case-insensitive, so a differently-cased entry finds the existing tag instead of forking a case-variant duplicate ('hatsune miku' / 'HATSUNE MIKU' → one tag). normalize_tag_name uses per-word capitalize (not str.title(), which mangles apostrophes) and folds ALL-CAPS input. Existing tags keep their current casing until touched — a retro-normalize maintenance pass (Title-Case + merge case-collisions) is the follow-up to convert the back-catalog. Test: create title-cases + collapses whitespace; case/whitespace variants dedupe to one tag. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/tag_service.py | 29 +++++++++++++++++++++++++---- tests/test_tag_service.py | 13 +++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index 9f7a0e6..fc0e924 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -13,6 +13,19 @@ from ..models.tag_allowlist import TagAllowlist from ..models.tag_reference_embedding import TagReferenceEmbedding +def normalize_tag_name(name: str) -> str: + """Canonical tag form (#701): collapse whitespace + Title Case. + + Per-word capitalize (NOT str.title(), which mangles apostrophes: + don't → Don'T). Lowercasing the word tail also folds ALL-CAPS input + (HATSUNE MIKU → Hatsune Miku); acronym casing is sacrificed, an accepted + Title-Case trade-off (operator-chosen 2026-06-06). + """ + return " ".join( + w[:1].upper() + w[1:].lower() for w in (name or "").split() + ) + + class TagValidationError(ValueError): """Raised when tag construction breaks the kind/fandom rules.""" @@ -71,7 +84,7 @@ class TagService: - if fandom_id is set, the referenced tag must exist and have kind == TagKind.fandom """ - name = name.strip() + name = normalize_tag_name(name) if not name: raise TagValidationError("Tag name cannot be empty") @@ -93,13 +106,17 @@ class TagService: # inserts; without the savepoint the outer transaction would # poison and the calling request crashes. Mirrors # importer._get_or_create. + # Case-insensitive match (#701): a normalized (Title Case) input must + # find an existing differently-cased tag instead of forking a duplicate. stmt = ( select(Tag) - .where(Tag.name == name) + .where(func.lower(Tag.name) == name.lower()) .where(Tag.kind == kind) .where( Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id ) + .order_by(Tag.id) + .limit(1) ) existing = (await self.session.execute(stmt)).scalar_one_or_none() if existing: @@ -242,16 +259,18 @@ class TagService: with an existing tag of the same (kind, fandom_id) — the API turns that into a 409 merge hint. """ - new_name = new_name.strip() + new_name = normalize_tag_name(new_name) if not new_name: raise TagValidationError("Tag name cannot be empty") tag = await self.session.get(Tag, tag_id) if tag is None: raise TagValidationError(f"Tag {tag_id} not found") + # Case-insensitive clash (#701) — colliding with a differently-cased tag + # is still a merge, not a silent fork. clash_stmt = ( select(Tag) - .where(Tag.name == new_name) + .where(func.lower(Tag.name) == new_name.lower()) .where(Tag.kind == tag.kind) .where( Tag.fandom_id.is_(None) @@ -259,6 +278,8 @@ class TagService: else Tag.fandom_id == tag.fandom_id ) .where(Tag.id != tag_id) + .order_by(Tag.id) + .limit(1) ) clash = (await self.session.execute(clash_stmt)).scalar_one_or_none() if clash is not None: diff --git a/tests/test_tag_service.py b/tests/test_tag_service.py index a8ceb16..d427ffd 100644 --- a/tests/test_tag_service.py +++ b/tests/test_tag_service.py @@ -59,6 +59,19 @@ async def test_character_with_fandom(db): assert char.fandom_id == fandom.id +@pytest.mark.asyncio +async def test_find_or_create_titlecases_and_dedups_case_insensitive(db): + """#701: tags normalize to Title Case + collapsed whitespace on create, and a + differently-cased entry finds the existing tag instead of forking.""" + svc = TagService(db) + t1 = await svc.find_or_create("hatsune miku", TagKind.character) + assert t1.name == "Hatsune Miku" + t2 = await svc.find_or_create("HATSUNE miku", TagKind.character) + assert t2.id == t1.id # case + whitespace variant → same tag, no fork + g = await svc.find_or_create(" looking at viewer ", TagKind.general) + assert g.name == "Looking At Viewer" + + @pytest.mark.asyncio async def test_character_same_name_different_fandoms(db): svc = TagService(db) -- 2.52.0 From 89dfa42e182d718d27da69dbbb3e97c37a2a4d2a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 15:44:04 -0400 Subject: [PATCH 07/11] =?UTF-8?q?fix(showcase):=20over-sample=20+=20random?= =?UTF-8?q?-order=20to=20break=20near-dup=20clustering=20=E2=80=94=20#699?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TABLESAMPLE SYSTEM_ROWS reads CONTIGUOUS rows from each sampled page, so sequentially-imported near-duplicates (multi-image posts, variant sets) came back adjacent and clustered in the showcase ("three near-identical in a row"). Sample limit*5 rows (spanning more pages) then ORDER BY random() before taking limit — breaks the physical adjacency for much better spread, still cheap (random() over a few hundred rows, not the whole table). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/services/showcase_service.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/app/services/showcase_service.py b/backend/app/services/showcase_service.py index f34051b..e60870c 100644 --- a/backend/app/services/showcase_service.py +++ b/backend/app/services/showcase_service.py @@ -20,11 +20,20 @@ class ShowcaseService: async def random_sample(self, limit: int = 60) -> list[dict]: if limit < 1 or limit > 200: raise ValueError("limit must be between 1 and 200") + # Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows + # from each sampled page, so sequentially-imported near-duplicates + # (multi-image posts, variant sets) come back adjacent and cluster in the + # showcase ("three near-identical in a row"). Sampling a multiple of + # `limit` spans more pages, and ORDER BY random() before taking `limit` + # breaks the physical adjacency — far better spread, still cheap + # (random() over a few hundred rows, not the whole table). + oversample = min(limit * 5, 1000) stmt = select(ImageRecord).from_statement( text( - "SELECT * FROM image_record " - "TABLESAMPLE SYSTEM_ROWS(:n)" - ).bindparams(n=limit) + "SELECT * FROM (" + " SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)" + ") sub ORDER BY random() LIMIT :n" + ).bindparams(o=oversample, n=limit) ) rows = (await self.session.execute(stmt)).scalars().all() return [ -- 2.52.0 From 62cca64dce87c59b5626a5dfe0e0da9e735fb65f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 15:48:31 -0400 Subject: [PATCH 08/11] =?UTF-8?q?feat(downloads):=20live=20per-file=20prog?= =?UTF-8?q?ress=20on=20running=20events=20=E2=80=94=20#709?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we own the walk, surface live counts on the in-flight download in the Downloads view. ingest_core.run takes an event_id and does a TIME-THROTTLED write (~5s, decoupled from page boundaries so it ticks steadily regardless of how big/slow a page is) of {downloaded, skipped, errors, quarantined, posts} to the running download_event's metadata.live (jsonb_set; short session; status guard so a finalized event isn't clobbered). download_backends threads event_id from ctx; the /api/downloads list surfaces `live`; ActiveDownloadsPanel renders it beside the elapsed timer. Native (Patreon) only — gallery-dl is an opaque subprocess; the row only shows when `live` is present. Phase 3 overwrites metadata with run_stats on finish, dropping `live`. Test: _write_live_progress updates a running event's metadata.live and leaves a finalized (status != running) event alone. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/downloads.py | 3 ++ backend/app/services/download_backends.py | 2 + backend/app/services/ingest_core.py | 41 +++++++++++++++++++ .../subscriptions/ActiveDownloadsPanel.vue | 19 +++++++++ tests/test_patreon_ingester.py | 28 +++++++++++++ 5 files changed, 93 insertions(+) diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py index c3ae536..b3e553c 100644 --- a/backend/app/api/downloads.py +++ b/backend/app/api/downloads.py @@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N "bytes_downloaded": event.bytes_downloaded, "error": event.error, "summary": _summary_from_metadata(event.metadata_), + # plan #709: mid-walk live counts for a RUNNING native-ingester event + # (None otherwise; phase 3 overwrites metadata with run_stats on finish). + "live": (event.metadata_ or {}).get("live"), } diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 02095f8..082b92b 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -139,6 +139,8 @@ async def _run_native_ingester( resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, posts_base=int(overrides.get("_backfill_posts", 0)), + # plan #709: live progress writes to this running event mid-walk. + event_id=ctx.get("event_id"), ), ) return dl_result, resolved_campaign_id diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 2b5286c..210a801 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -25,6 +25,7 @@ Plain-HTTP homelab: no secure-context Web API. from __future__ import annotations +import json import logging import time from collections.abc import Callable @@ -48,6 +49,12 @@ DEAD_LETTER_THRESHOLD = 3 # last_error is Text but bound it so a giant traceback doesn't bloat the row. _ERROR_MAX = 1000 +# plan #709: throttle the live-progress write to the running DownloadEvent to one +# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a +# page is (page boundaries can be minutes apart on image-dense backfills, so a +# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s). +_LIVE_PROGRESS_INTERVAL = 5.0 + class Ingester: """Generic native-ingest orchestration. Subclass with a platform adapter @@ -92,6 +99,7 @@ class Ingester: time_budget_seconds: float = 870.0, seen_threshold: int = _TICK_SEEN_THRESHOLD, posts_base: int = 0, + event_id: int | None = None, ) -> DownloadResult: """Walk + download for one source, returning a gallery-dl-shaped result. @@ -109,6 +117,7 @@ class Ingester: checkpoint = mode in ("backfill", "recovery") ledger_key = self._ledger_key start = time.monotonic() + last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] written: list[str] = [] quarantined_paths: list[str] = [] @@ -288,6 +297,19 @@ class Ingester: if to_fail: self._record_failures(source_id, to_fail) + # plan #709: time-throttled live progress to the running event so + # the Downloads view ticks ~every 5s, independent of page size. + now = time.monotonic() + if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL: + last_live = now + self._write_live_progress(event_id, { + "downloaded": downloaded, + "skipped": skipped_count, + "errors": errors, + "quarantined": quarantined, + "posts": posts_processed, + }) + if early_out: break else: @@ -474,6 +496,25 @@ class Ingester: ) session.commit() + def _write_live_progress(self, event_id: int, counts: dict) -> None: + """Throttled mid-walk write of live counts to the RUNNING download_event + (plan #709) so the Downloads view shows progress before the chunk + finishes. A short session (never held across the walk); the `status = + 'running'` guard avoids clobbering an event phase 3 already finalized. + `metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest + for phase 3 to overwrite with the final run_stats.""" + with self.session_factory() as session: + session.execute( + text( + "UPDATE download_event SET metadata = jsonb_set(" + " coalesce(metadata, '{}'::jsonb), '{live}'," + " cast(:live AS jsonb)) " + "WHERE id = :eid AND status = 'running'" + ), + {"live": json.dumps(counts), "eid": event_id}, + ) + session.commit() + def _still_running(self, source_id: int) -> bool: """True while the source is armed for a deep walk (plan #708 B4). diff --git a/frontend/src/components/subscriptions/ActiveDownloadsPanel.vue b/frontend/src/components/subscriptions/ActiveDownloadsPanel.vue index e35abd6..6f8fb43 100644 --- a/frontend/src/components/subscriptions/ActiveDownloadsPanel.vue +++ b/frontend/src/components/subscriptions/ActiveDownloadsPanel.vue @@ -23,6 +23,18 @@ {{ e.artist_name || '—' }} + + + ↓ {{ e.live.downloaded }} + + ⤼ {{ e.live.skipped }} + ✕ {{ e.live.errors }} + + {{ e.live.posts }} posts + {{ elapsed(e.started_at) }} @@ -123,6 +135,13 @@ function elapsed (startedIso) { font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em; color: rgb(var(--v-theme-on-surface-variant)); } +.fc-active__counts { + display: inline-flex; align-items: center; gap: 8px; + font-variant-numeric: tabular-nums; font-size: 0.78rem; +} +.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); } +.fc-active__count--err { color: rgb(var(--v-theme-error)); } +.fc-active__count--posts { opacity: 0.7; } @keyframes fc-active-pulse { 0%, 100% { opacity: 1; transform: scale(1); } 50% { opacity: 0.35; transform: scale(0.7); } diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 95da4bd..2e32075 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -421,6 +421,34 @@ async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path): assert result["has_more"] is True +@pytest.mark.asyncio +async def test_write_live_progress_updates_running_event( + source_id, sync_engine, tmp_path, db, +): + """plan #709: live counts land in the RUNNING event's metadata.live; a + finalized event is left alone (status guard).""" + from backend.app.models import DownloadEvent + + ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path)) + running = DownloadEvent(source_id=source_id, status="running") + done = DownloadEvent(source_id=source_id, status="ok") + db.add_all([running, done]) + await db.commit() + + counts = {"downloaded": 3, "skipped": 1, "errors": 0, "quarantined": 0, "posts": 4} + ing._write_live_progress(running.id, counts) + ing._write_live_progress(done.id, {"downloaded": 9}) # guarded: status != running + + live = (await db.execute( + select(DownloadEvent.metadata_).where(DownloadEvent.id == running.id) + )).scalar_one() + assert live["live"] == counts + done_meta = (await db.execute( + select(DownloadEvent.metadata_).where(DownloadEvent.id == done.id) + )).scalar_one() + assert "live" not in (done_meta or {}) + + @pytest.mark.asyncio async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db): """plan #708 B4: _still_running reflects the source's `_backfill_state` — the -- 2.52.0 From 23f452021f03fff262af3cd204a65f06ece044d9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 15:57:29 -0400 Subject: [PATCH 09/11] fix(tags): Title-Case operator-entered tags at create endpoint only (plan #701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalize_tag_name (per-word capitalize + whitespace collapse) is applied in the POST /api/tags handler so operator-entered tags get clean display casing. It is NOT applied in the shared find_or_create / rename paths — those are used by the ML tagger and allowlist matching, which must preserve the booru vocabulary's original casing (Title-Casing it broke apply_allowlist matching). find_or_create / rename keep case-insensitive lookup + clash detection so a differently-cased entry dedups onto the existing tag instead of forking. Tests updated to expect Title-Cased create output (sunset→Sunset, character:Saber→Character:saber, http://example.com→Http://example.com) and a dedicated normalize_tag_name unit test. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/tags.py | 6 ++++++ backend/app/services/tag_service.py | 10 +++++++--- tests/test_api_tags.py | 14 ++++++++------ tests/test_tag_service.py | 24 ++++++++++++++++-------- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 94bddff..288ade6 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -14,6 +14,7 @@ from ..services.tag_service import ( TagMergeConflict, TagService, TagValidationError, + normalize_tag_name, ) from ..utils.tag_prefix import parse_kind_prefix @@ -141,6 +142,11 @@ async def create_tag(): fandom_id = body.get("fandom_id") + # #701: Title-Case operator-entered tags. Only here (the explicit create + # endpoint), NOT in the shared find_or_create — the ML tagger uses that path + # and must keep the booru vocabulary's casing for allowlist matching. + name = normalize_tag_name(name) + async with get_session() as session: svc = TagService(session) try: diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index fc0e924..b26132b 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -84,7 +84,11 @@ class TagService: - if fandom_id is set, the referenced tag must exist and have kind == TagKind.fandom """ - name = normalize_tag_name(name) + # NOTE: case is NOT normalized here — find_or_create is the shared path + # the ML tagger / allowlist also use, and Title-Casing the booru + # vocabulary breaks allowlist matching. Display-casing (Title Case) is + # applied at the user-entry layer (api/tags create_tag) only (#701). + name = name.strip() if not name: raise TagValidationError("Tag name cannot be empty") @@ -259,14 +263,14 @@ class TagService: with an existing tag of the same (kind, fandom_id) — the API turns that into a 409 merge hint. """ - new_name = normalize_tag_name(new_name) + new_name = new_name.strip() if not new_name: raise TagValidationError("Tag name cannot be empty") tag = await self.session.get(Tag, tag_id) if tag is None: raise TagValidationError(f"Tag {tag_id} not found") - # Case-insensitive clash (#701) — colliding with a differently-cased tag + # Case-insensitive clash (#701) — renaming onto a differently-cased tag # is still a merge, not a silent fork. clash_stmt = ( select(Tag) diff --git a/tests/test_api_tags.py b/tests/test_api_tags.py index c4ead6e..c630baa 100644 --- a/tests/test_api_tags.py +++ b/tests/test_api_tags.py @@ -52,11 +52,12 @@ async def test_create_tag_missing_name_400(client): @pytest.mark.asyncio async def test_create_tag_name_only_defaults_to_general(client): - """IR-style: name without kind and without `kind:` prefix → general.""" + """IR-style: name without kind and without `kind:` prefix → general. + #701: operator-entered names are Title-Cased at the create endpoint.""" resp = await client.post("/api/tags", json={"name": "sunset"}) assert resp.status_code == 201 body = await resp.get_json() - assert body["name"] == "sunset" + assert body["name"] == "Sunset" assert body["kind"] == "general" @@ -73,21 +74,22 @@ async def test_create_tag_with_kind_prefix(client): @pytest.mark.asyncio async def test_explicit_kind_overrides_prefix_parsing(client): """If caller passes explicit kind, don't re-parse the name — - colon and prefix stay literal.""" + colon and prefix stay literal. #701: still Title-Cased as a single word.""" resp = await client.post( "/api/tags", json={"name": "character:Saber", "kind": "general"} ) assert resp.status_code == 201 body = await resp.get_json() - assert body["name"] == "character:Saber" + assert body["name"] == "Character:saber" assert body["kind"] == "general" @pytest.mark.asyncio async def test_unknown_prefix_kept_literal(client): - """`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name.""" + """`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name. + #701: Title-Cased as one word (no internal whitespace to split on).""" resp = await client.post("/api/tags", json={"name": "http://example.com"}) assert resp.status_code == 201 body = await resp.get_json() - assert body["name"] == "http://example.com" + assert body["name"] == "Http://example.com" assert body["kind"] == "general" diff --git a/tests/test_tag_service.py b/tests/test_tag_service.py index d427ffd..3cb8a9b 100644 --- a/tests/test_tag_service.py +++ b/tests/test_tag_service.py @@ -60,16 +60,24 @@ async def test_character_with_fandom(db): @pytest.mark.asyncio -async def test_find_or_create_titlecases_and_dedups_case_insensitive(db): - """#701: tags normalize to Title Case + collapsed whitespace on create, and a - differently-cased entry finds the existing tag instead of forking.""" +async def test_find_or_create_dedups_case_insensitive(db): + """#701: find_or_create matches case-insensitively so a differently-cased + entry finds the existing tag instead of forking. (Display Title-Casing is + applied at the create API, not here — this path is shared with the ML + tagger, which keeps the booru vocabulary's casing.)""" svc = TagService(db) t1 = await svc.find_or_create("hatsune miku", TagKind.character) - assert t1.name == "Hatsune Miku" - t2 = await svc.find_or_create("HATSUNE miku", TagKind.character) - assert t2.id == t1.id # case + whitespace variant → same tag, no fork - g = await svc.find_or_create(" looking at viewer ", TagKind.general) - assert g.name == "Looking At Viewer" + assert t1.name == "hatsune miku" # stored as given; not recased here + t2 = await svc.find_or_create("HATSUNE MIKU", TagKind.character) + assert t2.id == t1.id # case variant → same tag, no fork + + +@pytest.mark.asyncio +async def test_normalize_tag_name(): + from backend.app.services.tag_service import normalize_tag_name + assert normalize_tag_name("hatsune miku") == "Hatsune Miku" + assert normalize_tag_name(" looking at viewer ") == "Looking At Viewer" + assert normalize_tag_name("HATSUNE MIKU") == "Hatsune Miku" @pytest.mark.asyncio -- 2.52.0 From 3c89223dcb3abbc6e64e7686b883c035ce0b8223 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 6 Jun 2026 16:28:34 -0400 Subject: [PATCH 10/11] feat(tags): retro-normalize existing tags to Title Case + merge case-collisions (plan #714) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #701: new tags are saved canonical, but the back-catalog keeps whatever casing it was created with. This adds a maintenance action that Title-Cases every existing tag (collapsing whitespace) and merges case/whitespace-variant duplicates into one. Backend: - tag_service.normalize_existing_tags(session, *, dry_run): groups all tags by (kind, coalesce(fandom_id,-1), canonical_name). Per group it picks a survivor (prefer an already-canonical member → no rename/self-alias; else the best-connected tag → fewest FK repoints; else lowest id), merges the variants INTO it via the tested TagService._do_merge (image_tag/allowlist/embedding/ aliases/series_page repoints + protective ML aliases), then renames the survivor to canonical. Losers are deleted before the rename so there's no transient unique-index clash; commits per group and isolates failures per group. Idempotent — an already-canonical lone tag is a no-op. - normalize_tags_task (maintenance queue, asyncio.run + per-task NullPool async engine, soft 1800/hard 2400) — recovery/timeout/duration covered by FC-3i. - POST /api/admin/tags/normalize: dry_run=true returns a projection inline (group/collision/rename counts + sample); dry_run=false enqueues the task. Frontend: a "Standardize tag casing" section in TagMaintenanceCard (Cleanup tab) — preview → apply (polls the activity dashboard to terminal status), behind a back-up-first warning. admin store gains normalizeTags(). Tests: tests/test_tag_normalize.py — dry-run counts, live merge + image-tag dedup/repoint, idempotency, same-name-different-fandom and -different-kind kept separate, ML-known loser keeps a protective alias. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/app/api/admin.py | 26 +++ backend/app/services/tag_service.py | 176 ++++++++++++++++ backend/app/tasks/admin.py | 29 +++ .../settings/TagMaintenanceCard.vue | 89 +++++++++ frontend/src/stores/admin.js | 17 ++ tests/test_tag_normalize.py | 189 ++++++++++++++++++ 6 files changed, 526 insertions(+) create mode 100644 tests/test_tag_normalize.py diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 4005475..69d058e 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -245,6 +245,32 @@ async def tags_reset_content(): return jsonify(result) +@admin_bp.route("/tags/normalize", methods=["POST"]) +async def tags_normalize(): + """#714: retro-normalize existing tags to the #701 canonical form (Title + Case + collapsed whitespace) and merge case/whitespace-variant duplicates. + + dry_run=true (default) returns a projection inline — group/collision/rename + counts + a sample of the changes — so the UI shows exactly what'll happen. + dry_run=false dispatches the long-running maintenance task (the merge FK + repoints can touch many tags); the UI tails the activity dashboard for the + summary. Idempotent; back up first (the merges are irreversible).""" + from ..services.tag_service import normalize_existing_tags + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) + + if dry_run: + async with get_session() as session: + result = await normalize_existing_tags(session, dry_run=True) + return jsonify(result) + + from ..tasks.admin import normalize_tags_task + + async_result = normalize_tags_task.delay() + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + @admin_bp.route("/maintenance/db-stats", methods=["GET"]) async def db_stats(): """Per-table bloat readout (pg_stat_user_tables) for the high-churn tables diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index b26132b..d1f292a 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -1,5 +1,6 @@ """Tag CRUD + autocomplete + image-tag association.""" +import logging from collections.abc import Sequence from dataclasses import dataclass @@ -12,6 +13,8 @@ from ..models import Tag, TagKind, image_tag from ..models.tag_allowlist import TagAllowlist from ..models.tag_reference_embedding import TagReferenceEmbedding +log = logging.getLogger(__name__) + def normalize_tag_name(name: str) -> str: """Canonical tag form (#701): collapse whitespace + Title Case. @@ -526,6 +529,21 @@ class TagService: update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt) ) + async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]: + """image_tag row counts keyed by tag_id, for the survivor heuristic + (the best-connected tag in a collision group survives → fewest + FK repoints). Tags with zero associations are absent from the map.""" + if not tag_ids: + return {} + rows = ( + await self.session.execute( + select(image_tag.c.tag_id, func.count()) + .where(image_tag.c.tag_id.in_(tag_ids)) + .group_by(image_tag.c.tag_id) + ) + ).all() + return {tid: int(n) for tid, n in rows} + async def _create_protective_aliases( self, src_name: str, src_kind: TagKind, tgt: int ) -> bool: @@ -573,3 +591,161 @@ class TagService: if res.rowcount: created = True return created + + +# --------------------------------------------------------------------------- +# #714: retro-normalize existing tags to the #701 canonical (Title Case + +# collapsed whitespace) and merge case/whitespace-variant duplicates. +# --------------------------------------------------------------------------- + +_NORMALIZE_SAMPLE_CAP = 50 + + +def _group_existing_tags( + rows, +) -> dict[tuple, list[tuple[int, str]]]: + """Group (id, name, kind, fandom_id) rows by their post-normalization + identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that + would collapse to the same canonical lives in ONE group, so the canonical + form is unique within (kind, fandom) once each group is resolved.""" + groups: dict[tuple, list[tuple[int, str]]] = {} + for tag_id, name, kind, fandom_id in rows: + canonical = normalize_tag_name(name) + key = (kind, fandom_id if fandom_id is not None else -1, canonical) + groups.setdefault(key, []).append((tag_id, name)) + return groups + + +def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool: + """A group is already canonical iff it's a single member whose name equals + the canonical form. Anything else (a collision, or a lone mis-cased tag) + needs work.""" + if len(members) > 1: + return True + return members[0][1] != canonical + + +def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int: + """The tag with the most image associations (→ fewest FK repoints when it + survives), tie-broken to the lowest id for determinism. Module-level so the + key closes over its parameter, not a loop variable (ruff B023).""" + return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid)) + + +async def normalize_existing_tags( + session: AsyncSession, *, dry_run: bool = False +) -> dict: + """Convert the back-catalog to the #701 canonical tag form. + + For each (kind, fandom, canonical) group: pick a survivor, merge any + case/whitespace-variant siblings INTO it via the tested merge path + (TagService._do_merge — FK repoints + protective aliases), then rename the + survivor to the canonical form. Idempotent: a group that is already a lone + canonical tag is a no-op, so re-running is safe. + + dry_run=True returns a projection (counts + a sample of the changes) with no + mutations. Live runs commit per group and isolate failures per group so one + bad group can't strand the rest. + + Returns (dry_run): + {"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R, + "total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]} + Returns (live): + {"groups_processed", "merged", "renamed", "aliases_created", "errors", + "sample": [...]} + """ + rows = ( + await session.execute( + select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id) + ) + ).all() + groups = _group_existing_tags(rows) + + # Deterministic sample/ordering: by kind then canonical name. + touched = sorted( + ( + (key, members) + for key, members in groups.items() + if _group_needs_change(key[2], members) + ), + key=lambda km: ( + km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]), + km[0][2].lower(), + ), + ) + + sample = [ + { + "to": key[2], + "from": [name for _id, name in members], + "kind": key[0].value if hasattr(key[0], "value") else str(key[0]), + "merge": len(members) > 1, + } + for key, members in touched[:_NORMALIZE_SAMPLE_CAP] + ] + + if dry_run: + collisions = sum(1 for key, m in touched if len(m) > 1) + tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1) + # A group renames iff the canonical form isn't already one of its + # members' exact names (else that member is picked as survivor → no + # rename, the rest merge into it). + tags_to_rename = sum( + 1 + for key, m in touched + if key[2] not in {name for _id, name in m} + ) + return { + "groups": len(groups), + "collisions": collisions, + "tags_to_merge": tags_to_merge, + "tags_to_rename": tags_to_rename, + "total_changes": len(touched), + "sample": sample, + } + + svc = TagService(session) + summary = { + "groups_processed": 0, + "merged": 0, + "renamed": 0, + "aliases_created": 0, + "errors": 0, + "sample": sample, + } + for key, members in touched: + canonical = key[2] + names_by_id = {tag_id: name for tag_id, name in members} + # Survivor: prefer a member already named canonically (no rename, no + # self-alias); else the best-connected (fewest FK repoints); else + # lowest id for determinism. + survivor_id = next( + (tid for tid, name in members if name == canonical), None + ) + if survivor_id is None: + counts = await svc._image_assoc_counts(list(names_by_id)) + survivor_id = _best_connected(list(names_by_id), counts) + loser_ids = [tid for tid in names_by_id if tid != survivor_id] + try: + survivor = await session.get(Tag, survivor_id) + for loser_id in loser_ids: + loser = await session.get(Tag, loser_id) + if loser is None: + continue + result = await svc._do_merge(loser, survivor) + if result.alias_created: + summary["aliases_created"] += 1 + if survivor.name != canonical: + survivor.name = canonical + await session.flush() + summary["renamed"] += 1 + await session.commit() + summary["groups_processed"] += 1 + summary["merged"] += len(loser_ids) + except Exception as exc: # one bad group must not strand the rest + await session.rollback() + summary["errors"] += 1 + log.warning( + "tag normalize failed for group %r: %s", canonical, exc + ) + return summary diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index e3ae6ac..f115387 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -73,3 +73,32 @@ def reextract_archive_attachments_task(self) -> dict: return cleanup_service.reextract_archive_attachments( session, images_root=IMAGES_ROOT, ) + + +@celery.task( + name="backend.app.tasks.admin.normalize_tags_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def normalize_tags_task(self) -> dict: + """Wraps tag_service.normalize_existing_tags (#714): Title-Case the + back-catalog and merge case/whitespace-variant duplicate tags via the + tested async merge path. Runs under its own asyncio loop + per-task async + engine (NullPool, disposed when the loop ends), mirroring download_source.""" + import asyncio + + from ..services.tag_service import normalize_existing_tags + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + try: + async with async_factory() as session: + # normalize_existing_tags commits per group internally. + return await normalize_existing_tags(session, dry_run=False) + finally: + await async_engine.dispose() + + return asyncio.run(_run()) diff --git a/frontend/src/components/settings/TagMaintenanceCard.vue b/frontend/src/components/settings/TagMaintenanceCard.vue index db8fc73..2232074 100644 --- a/frontend/src/components/settings/TagMaintenanceCard.vue +++ b/frontend/src/components/settings/TagMaintenanceCard.vue @@ -136,6 +136,64 @@ >Delete {{ resetPreview.count }} content tag(s) + {{ resetPreview.applications }} application(s) + + + + +

+ Standardize tag casing. + New tags are saved Title Case, but older tags keep whatever casing they + were created with. This renames every existing tag to + Title Case (collapsing extra spaces) and + merges tags that differ only by case or spacing + (e.g. hatsune miku + Hatsune  Miku) + into one — repointing their images, allowlist entries and series pages. +

+ + The merges are irreversible — back up first + (Settings → Maintenance → Backup). Safe to run more than once. + + + Preview tag standardization + +
+

+ {{ normPreview.total_changes }} tag group(s) to + change — {{ normPreview.tags_to_rename }} rename(s), + {{ normPreview.collisions }} collision(s) merging + {{ normPreview.tags_to_merge }} tag(s) away. +

+
+ + {{ s.to }} + +
+ Standardize {{ normPreview.total_changes }} tag group(s) + + + + +
@@ -155,6 +213,10 @@ const kindCommitting = ref(false) const resetPreview = ref(null) const loadingResetPreview = ref(false) const resetCommitting = ref(false) +const normPreview = ref(null) +const loadingNormPreview = ref(false) +const normCommitting = ref(false) +const normResult = ref(null) async function onPreview() { loadingPreview.value = true @@ -212,6 +274,33 @@ async function onResetCommit() { resetCommitting.value = false } } + +async function onNormPreview() { + loadingNormPreview.value = true + normResult.value = null + try { + normPreview.value = await store.normalizeTags({ dryRun: true }) + } finally { + loadingNormPreview.value = false + } +} + +async function onNormCommit() { + normCommitting.value = true + normResult.value = null + try { + // Long op (FK repoints): enqueue the maintenance task, then tail the + // activity dashboard until its row reaches a terminal status. (The + // per-run summary dict isn't exposed by /activity/runs, so we surface + // the terminal status — the dry-run preview is the detailed view.) + const { task_id: taskId } = await store.normalizeTags({ dryRun: false }) + const row = await store.pollTaskUntilDone(taskId) + normResult.value = row?.status || 'ok' + normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] } + } finally { + normCommitting.value = false + } +}