diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 9e4b2b2..69d058e 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -245,6 +245,32 @@ async def tags_reset_content(): return jsonify(result) +@admin_bp.route("/tags/normalize", methods=["POST"]) +async def tags_normalize(): + """#714: retro-normalize existing tags to the #701 canonical form (Title + Case + collapsed whitespace) and merge case/whitespace-variant duplicates. + + dry_run=true (default) returns a projection inline — group/collision/rename + counts + a sample of the changes — so the UI shows exactly what'll happen. + dry_run=false dispatches the long-running maintenance task (the merge FK + repoints can touch many tags); the UI tails the activity dashboard for the + summary. Idempotent; back up first (the merges are irreversible).""" + from ..services.tag_service import normalize_existing_tags + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) + + if dry_run: + async with get_session() as session: + result = await normalize_existing_tags(session, dry_run=True) + return jsonify(result) + + from ..tasks.admin import normalize_tags_task + + async_result = normalize_tags_task.delay() + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 + + @admin_bp.route("/maintenance/db-stats", methods=["GET"]) async def db_stats(): """Per-table bloat readout (pg_stat_user_tables) for the high-churn tables @@ -289,3 +315,14 @@ async def trigger_vacuum(): vacuum_analyze.delay() return jsonify({"status": "queued"}), 202 + + +@admin_bp.route("/maintenance/reextract-archives", methods=["POST"]) +async def trigger_reextract_archives(): + """Operator-triggered re-extract (#713): PostAttachments that are actually + archives but were filed opaquely (pre magic-byte gate) get extracted and + their members linked to the post. Idempotent; runs on the maintenance queue.""" + from ..tasks.admin import reextract_archive_attachments_task + + async_result = reextract_archive_attachments_task.delay() + return jsonify({"task_id": async_result.id, "status": "queued"}), 202 diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py index c3ae536..b3e553c 100644 --- a/backend/app/api/downloads.py +++ b/backend/app/api/downloads.py @@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N "bytes_downloaded": event.bytes_downloaded, "error": event.error, "summary": _summary_from_metadata(event.metadata_), + # plan #709: mid-walk live counts for a RUNNING native-ingester event + # (None otherwise; phase 3 overwrites metadata with run_stats on finish). + "live": (event.metadata_ or {}).get("live"), } diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 94bddff..288ade6 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -14,6 +14,7 @@ from ..services.tag_service import ( TagMergeConflict, TagService, TagValidationError, + normalize_tag_name, ) from ..utils.tag_prefix import parse_kind_prefix @@ -141,6 +142,11 @@ async def create_tag(): fandom_id = body.get("fandom_id") + # #701: Title-Case operator-entered tags. Only here (the explicit create + # endpoint), NOT in the shared find_or_create — the ML tagger uses that path + # and must keep the booru vocabulary's casing for allowlist matching. + name = normalize_tag_name(name) + async with get_session() as session: svc = TagService(session) try: diff --git a/backend/app/services/archive_extractor.py b/backend/app/services/archive_extractor.py index 45babef..d6b6566 100644 --- a/backend/app/services/archive_extractor.py +++ b/backend/app/services/archive_extractor.py @@ -16,9 +16,45 @@ log = logging.getLogger(__name__) ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"} +# Magic-byte signatures, so an archive with a mangled / extension-less filename +# is still recognised. Patreon attachment download URLs sanitize to names like +# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk +# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed +# those as opaque PostAttachments and NEVER extracted them (operator-flagged +# 2026-06-06). Detection is by extension first (cheap), then header sniff. +_RAR_MAGIC = b"Rar!\x1a\x07" +_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c" + + +def detect_archive_format(path: Path) -> str | None: + """Return "zip" | "rar" | "7z" for an archive, else None. + + Trusts a known extension first, then falls back to magic-byte sniffing so a + mis-named or extension-less archive is still handled. (zip covers .cbz too.) + """ + ext = Path(path).suffix.lower() + if ext in (".zip", ".cbz"): + return "zip" + if ext == ".rar": + return "rar" + if ext == ".7z": + return "7z" + try: + if zipfile.is_zipfile(path): + return "zip" + with open(path, "rb") as fh: + head = fh.read(8) + except OSError: + return None + if head.startswith(_RAR_MAGIC): + return "rar" + if head.startswith(_7Z_MAGIC): + return "7z" + return None + def is_archive(path: Path) -> bool: - return Path(path).suffix.lower() in ARCHIVE_EXTS + return detect_archive_format(path) is not None @contextmanager @@ -32,16 +68,16 @@ def extract_archive(path: Path): members: list[tuple[str, Path]] = [] try: try: - ext = Path(path).suffix.lower() - if ext in (".zip", ".cbz"): + fmt = detect_archive_format(path) + if fmt == "zip": with zipfile.ZipFile(path) as zf: zf.extractall(base) - elif ext == ".rar": + elif fmt == "rar": import rarfile with rarfile.RarFile(path) as rf: rf.extractall(base) - elif ext == ".7z": + elif fmt == "7z": import py7zr with py7zr.SevenZipFile(path, "r") as zf: diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index f4ee687..605c215 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -11,6 +11,7 @@ the one-and-done GS/IR migration tooling.) """ from __future__ import annotations +import logging from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -22,6 +23,8 @@ from ..models import Artist, ImageRecord, LibraryAuditRun, Tag from ..models.series_page import SeriesPage from ..models.tag import image_tag +log = logging.getLogger(__name__) + def project_artist_cascade(session: Session, *, slug: str) -> dict: """Read-only projection of what delete_artist_cascade would touch. @@ -665,3 +668,139 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None: .where(LibraryAuditRun.status == "running") .values(status="cancelled", finished_at=datetime.now(UTC)) ) + + +# -- archive-attachment re-extraction (#713 part 2) ------------------------ + +_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"} + + +def _reextract_archive_to_post( + importer, archive_path: Path, post, source_row, artist, images_root: Path, +) -> list[int]: + """Extract one stored archive and link its members to `post`. + + The stored attachment has no adjacent sidecar (it lives in the sha-addressed + attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's + library dir (`images_root////`) — the importer + re-derives the artist from the path AND copies members relative to it, so the + members land in the real library and resolve to the right artist — then re-run + `attach_in_place`: the archive extracts and `find_or_create_post` re-attaches + the members to the SAME Post (source_id + external_post_id). Removes only the + staged archive + sidecar afterward; the imported member files stay. Returns + the new member image ids. + """ + import json + import shutil + + from .archive_extractor import detect_archive_format + + fmt = detect_archive_format(archive_path) + ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip") + platform = source_row.platform if source_row is not None else "imported" + sidecar = { + "category": source_row.platform if source_row is not None + else (post.raw_metadata or {}).get("category"), + "id": post.external_post_id, + "title": post.post_title or "", + "content": post.description or "", + "published_at": post.post_date.isoformat() if post.post_date else None, + "url": post.post_url, + } + work = images_root / artist.slug / platform / str(post.external_post_id) + work.mkdir(parents=True, exist_ok=True) + staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar + sidecar_path = staged.with_suffix(".json") + try: + shutil.copy2(archive_path, staged) + sidecar_path.write_text(json.dumps(sidecar)) + res = importer.attach_in_place(staged, artist=artist, source=source_row) + return list(res.member_image_ids or []) + finally: + # Drop only the staged archive + sidecar; the extracted member files + # were copied into the library alongside them and must stay. + staged.unlink(missing_ok=True) + sidecar_path.unlink(missing_ok=True) + + +def reextract_archive_attachments(session: Session, *, images_root: Path) -> dict: + """Re-process existing PostAttachments that are ACTUALLY archives but were + filed opaquely before #713 part 1 (extension-only is_archive missed mangled / + extension-less Patreon attachment names). For each: extract the members, + import them, and link them to the attachment's post. + + Idempotent — members dedupe by sha256, the archive dedupes by sha — so it's + safe to run repeatedly. Returns a summary dict for task_run.metadata. + """ + from ..models import ImportSettings, Post, PostAttachment, Source + from ..tasks.ml import tag_and_embed + from ..tasks.thumbnail import generate_thumbnail + from .archive_extractor import is_archive + from .importer import Importer + from .thumbnailer import Thumbnailer + + summary = { + "scanned": 0, "archives": 0, "members_imported": 0, + "posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0, + "errors": 0, + } + settings = ImportSettings.load_sync(session) + importer = Importer( + session=session, images_root=images_root, import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + ) + + attachments = session.execute( + select(PostAttachment).order_by(PostAttachment.id) + ).scalars().all() + enqueue_ids: list[int] = [] + for att in attachments: + summary["scanned"] += 1 + stored = Path(att.path) + try: + if not stored.is_file() or not is_archive(stored): + continue + except OSError: + continue + summary["archives"] += 1 + if att.post_id is None: + summary["skipped_no_post"] += 1 + continue + post = session.get(Post, att.post_id) + if post is None: + summary["skipped_no_post"] += 1 + continue + artist = session.get(Artist, att.artist_id) if att.artist_id else None + if artist is None and post.artist_id: + artist = session.get(Artist, post.artist_id) + if artist is None or not artist.slug: + # The importer re-derives the artist from the staged path, so we need + # a real artist+slug to anchor under. (Shouldn't happen for + # subscription posts; skip rather than orphan the members.) + summary["skipped_no_artist"] += 1 + continue + source_row = session.get(Source, post.source_id) if post.source_id else None + try: + ids = _reextract_archive_to_post( + importer, stored, post, source_row, artist, images_root, + ) + session.commit() + except Exception as exc: # one bad archive must not strand the rest + session.rollback() + summary["errors"] += 1 + log.warning("re-extract failed for attachment %s: %s", att.id, exc) + continue + if ids: + summary["members_imported"] += len(ids) + summary["posts_touched"] += 1 + enqueue_ids.extend(ids) + + # Thumbnails + ML for the newly-imported members (best-effort; off the + # critical path — a Redis hiccup must not fail the whole re-extract). + for img_id in enqueue_ids: + try: + generate_thumbnail.delay(img_id) + tag_and_embed.delay(img_id) + except Exception as exc: + log.warning("re-extract enqueue failed for image %s: %s", img_id, exc) + return summary diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 02095f8..082b92b 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -139,6 +139,8 @@ async def _run_native_ingester( resume_cursor=source_config.resume_cursor, time_budget_seconds=source_config.timeout, posts_base=int(overrides.get("_backfill_posts", 0)), + # plan #709: live progress writes to this running event mid-walk. + event_id=ctx.get("event_id"), ), ) return dl_result, resolved_campaign_id diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 2b5286c..210a801 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -25,6 +25,7 @@ Plain-HTTP homelab: no secure-context Web API. from __future__ import annotations +import json import logging import time from collections.abc import Callable @@ -48,6 +49,12 @@ DEAD_LETTER_THRESHOLD = 3 # last_error is Text but bound it so a giant traceback doesn't bloat the row. _ERROR_MAX = 1000 +# plan #709: throttle the live-progress write to the running DownloadEvent to one +# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a +# page is (page boundaries can be minutes apart on image-dense backfills, so a +# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s). +_LIVE_PROGRESS_INTERVAL = 5.0 + class Ingester: """Generic native-ingest orchestration. Subclass with a platform adapter @@ -92,6 +99,7 @@ class Ingester: time_budget_seconds: float = 870.0, seen_threshold: int = _TICK_SEEN_THRESHOLD, posts_base: int = 0, + event_id: int | None = None, ) -> DownloadResult: """Walk + download for one source, returning a gallery-dl-shaped result. @@ -109,6 +117,7 @@ class Ingester: checkpoint = mode in ("backfill", "recovery") ledger_key = self._ledger_key start = time.monotonic() + last_live = start # plan #709: last live-progress write timestamp log_lines: list[str] = [] written: list[str] = [] quarantined_paths: list[str] = [] @@ -288,6 +297,19 @@ class Ingester: if to_fail: self._record_failures(source_id, to_fail) + # plan #709: time-throttled live progress to the running event so + # the Downloads view ticks ~every 5s, independent of page size. + now = time.monotonic() + if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL: + last_live = now + self._write_live_progress(event_id, { + "downloaded": downloaded, + "skipped": skipped_count, + "errors": errors, + "quarantined": quarantined, + "posts": posts_processed, + }) + if early_out: break else: @@ -474,6 +496,25 @@ class Ingester: ) session.commit() + def _write_live_progress(self, event_id: int, counts: dict) -> None: + """Throttled mid-walk write of live counts to the RUNNING download_event + (plan #709) so the Downloads view shows progress before the chunk + finishes. A short session (never held across the walk); the `status = + 'running'` guard avoids clobbering an event phase 3 already finalized. + `metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest + for phase 3 to overwrite with the final run_stats.""" + with self.session_factory() as session: + session.execute( + text( + "UPDATE download_event SET metadata = jsonb_set(" + " coalesce(metadata, '{}'::jsonb), '{live}'," + " cast(:live AS jsonb)) " + "WHERE id = :eid AND status = 'running'" + ), + {"live": json.dumps(counts), "eid": event_id}, + ) + session.commit() + def _still_running(self, source_id: int) -> bool: """True while the source is armed for a deep walk (plan #708 B4). diff --git a/backend/app/services/showcase_service.py b/backend/app/services/showcase_service.py index f34051b..e60870c 100644 --- a/backend/app/services/showcase_service.py +++ b/backend/app/services/showcase_service.py @@ -20,11 +20,20 @@ class ShowcaseService: async def random_sample(self, limit: int = 60) -> list[dict]: if limit < 1 or limit > 200: raise ValueError("limit must be between 1 and 200") + # Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows + # from each sampled page, so sequentially-imported near-duplicates + # (multi-image posts, variant sets) come back adjacent and cluster in the + # showcase ("three near-identical in a row"). Sampling a multiple of + # `limit` spans more pages, and ORDER BY random() before taking `limit` + # breaks the physical adjacency — far better spread, still cheap + # (random() over a few hundred rows, not the whole table). + oversample = min(limit * 5, 1000) stmt = select(ImageRecord).from_statement( text( - "SELECT * FROM image_record " - "TABLESAMPLE SYSTEM_ROWS(:n)" - ).bindparams(n=limit) + "SELECT * FROM (" + " SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)" + ") sub ORDER BY random() LIMIT :n" + ).bindparams(o=oversample, n=limit) ) rows = (await self.session.execute(stmt)).scalars().all() return [ diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py index 9f7a0e6..60ffddd 100644 --- a/backend/app/services/tag_service.py +++ b/backend/app/services/tag_service.py @@ -1,5 +1,6 @@ """Tag CRUD + autocomplete + image-tag association.""" +import logging from collections.abc import Sequence from dataclasses import dataclass @@ -12,6 +13,21 @@ from ..models import Tag, TagKind, image_tag from ..models.tag_allowlist import TagAllowlist from ..models.tag_reference_embedding import TagReferenceEmbedding +log = logging.getLogger(__name__) + + +def normalize_tag_name(name: str) -> str: + """Canonical tag form (#701): collapse whitespace + Title Case. + + Per-word capitalize (NOT str.title(), which mangles apostrophes: + don't → Don'T). Lowercasing the word tail also folds ALL-CAPS input + (HATSUNE MIKU → Hatsune Miku); acronym casing is sacrificed, an accepted + Title-Case trade-off (operator-chosen 2026-06-06). + """ + return " ".join( + w[:1].upper() + w[1:].lower() for w in (name or "").split() + ) + class TagValidationError(ValueError): """Raised when tag construction breaks the kind/fandom rules.""" @@ -71,6 +87,10 @@ class TagService: - if fandom_id is set, the referenced tag must exist and have kind == TagKind.fandom """ + # NOTE: case is NOT normalized here — find_or_create is the shared path + # the ML tagger / allowlist also use, and Title-Casing the booru + # vocabulary breaks allowlist matching. Display-casing (Title Case) is + # applied at the user-entry layer (api/tags create_tag) only (#701). name = name.strip() if not name: raise TagValidationError("Tag name cannot be empty") @@ -93,13 +113,17 @@ class TagService: # inserts; without the savepoint the outer transaction would # poison and the calling request crashes. Mirrors # importer._get_or_create. + # Case-insensitive match (#701): a normalized (Title Case) input must + # find an existing differently-cased tag instead of forking a duplicate. stmt = ( select(Tag) - .where(Tag.name == name) + .where(func.lower(Tag.name) == name.lower()) .where(Tag.kind == kind) .where( Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id ) + .order_by(Tag.id) + .limit(1) ) existing = (await self.session.execute(stmt)).scalar_one_or_none() if existing: @@ -249,9 +273,11 @@ class TagService: if tag is None: raise TagValidationError(f"Tag {tag_id} not found") + # Case-insensitive clash (#701) — renaming onto a differently-cased tag + # is still a merge, not a silent fork. clash_stmt = ( select(Tag) - .where(Tag.name == new_name) + .where(func.lower(Tag.name) == new_name.lower()) .where(Tag.kind == tag.kind) .where( Tag.fandom_id.is_(None) @@ -259,6 +285,8 @@ class TagService: else Tag.fandom_id == tag.fandom_id ) .where(Tag.id != tag_id) + .order_by(Tag.id) + .limit(1) ) clash = (await self.session.execute(clash_stmt)).scalar_one_or_none() if clash is not None: @@ -501,6 +529,21 @@ class TagService: update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt) ) + async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]: + """image_tag row counts keyed by tag_id, for the survivor heuristic + (the best-connected tag in a collision group survives → fewest + FK repoints). Tags with zero associations are absent from the map.""" + if not tag_ids: + return {} + rows = ( + await self.session.execute( + select(image_tag.c.tag_id, func.count()) + .where(image_tag.c.tag_id.in_(tag_ids)) + .group_by(image_tag.c.tag_id) + ) + ).all() + return {tid: int(n) for tid, n in rows} + async def _create_protective_aliases( self, src_name: str, src_kind: TagKind, tgt: int ) -> bool: @@ -548,3 +591,161 @@ class TagService: if res.rowcount: created = True return created + + +# --------------------------------------------------------------------------- +# #714: retro-normalize existing tags to the #701 canonical (Title Case + +# collapsed whitespace) and merge case/whitespace-variant duplicates. +# --------------------------------------------------------------------------- + +_NORMALIZE_SAMPLE_CAP = 50 + + +def _group_existing_tags( + rows, +) -> dict[tuple, list[tuple[int, str]]]: + """Group (id, name, kind, fandom_id) rows by their post-normalization + identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that + would collapse to the same canonical lives in ONE group, so the canonical + form is unique within (kind, fandom) once each group is resolved.""" + groups: dict[tuple, list[tuple[int, str]]] = {} + for tag_id, name, kind, fandom_id in rows: + canonical = normalize_tag_name(name) + key = (kind, fandom_id if fandom_id is not None else -1, canonical) + groups.setdefault(key, []).append((tag_id, name)) + return groups + + +def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool: + """A group is already canonical iff it's a single member whose name equals + the canonical form. Anything else (a collision, or a lone mis-cased tag) + needs work.""" + if len(members) > 1: + return True + return members[0][1] != canonical + + +def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int: + """The tag with the most image associations (→ fewest FK repoints when it + survives), tie-broken to the lowest id for determinism. Module-level so the + key closes over its parameter, not a loop variable (ruff B023).""" + return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid)) + + +async def normalize_existing_tags( + session: AsyncSession, *, dry_run: bool = False +) -> dict: + """Convert the back-catalog to the #701 canonical tag form. + + For each (kind, fandom, canonical) group: pick a survivor, merge any + case/whitespace-variant siblings INTO it via the tested merge path + (TagService._do_merge — FK repoints + protective aliases), then rename the + survivor to the canonical form. Idempotent: a group that is already a lone + canonical tag is a no-op, so re-running is safe. + + dry_run=True returns a projection (counts + a sample of the changes) with no + mutations. Live runs commit per group and isolate failures per group so one + bad group can't strand the rest. + + Returns (dry_run): + {"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R, + "total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]} + Returns (live): + {"groups_processed", "merged", "renamed", "aliases_created", "errors", + "sample": [...]} + """ + rows = ( + await session.execute( + select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id) + ) + ).all() + groups = _group_existing_tags(rows) + + # Deterministic sample/ordering: by kind then canonical name. + touched = sorted( + ( + (key, members) + for key, members in groups.items() + if _group_needs_change(key[2], members) + ), + key=lambda km: ( + km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]), + km[0][2].lower(), + ), + ) + + sample = [ + { + "to": key[2], + "from": [name for _id, name in members], + "kind": key[0].value if hasattr(key[0], "value") else str(key[0]), + "merge": len(members) > 1, + } + for key, members in touched[:_NORMALIZE_SAMPLE_CAP] + ] + + if dry_run: + collisions = sum(1 for key, m in touched if len(m) > 1) + tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1) + # A group renames iff the canonical form isn't already one of its + # members' exact names (else that member is picked as survivor → no + # rename, the rest merge into it). + tags_to_rename = sum( + 1 + for key, m in touched + if key[2] not in {name for _id, name in m} + ) + return { + "groups": len(groups), + "collisions": collisions, + "tags_to_merge": tags_to_merge, + "tags_to_rename": tags_to_rename, + "total_changes": len(touched), + "sample": sample, + } + + svc = TagService(session) + summary = { + "groups_processed": 0, + "merged": 0, + "renamed": 0, + "aliases_created": 0, + "errors": 0, + "sample": sample, + } + for key, members in touched: + canonical = key[2] + names_by_id = dict(members) + # Survivor: prefer a member already named canonically (no rename, no + # self-alias); else the best-connected (fewest FK repoints); else + # lowest id for determinism. + survivor_id = next( + (tid for tid, name in members if name == canonical), None + ) + if survivor_id is None: + counts = await svc._image_assoc_counts(list(names_by_id)) + survivor_id = _best_connected(list(names_by_id), counts) + loser_ids = [tid for tid in names_by_id if tid != survivor_id] + try: + survivor = await session.get(Tag, survivor_id) + for loser_id in loser_ids: + loser = await session.get(Tag, loser_id) + if loser is None: + continue + result = await svc._do_merge(loser, survivor) + if result.alias_created: + summary["aliases_created"] += 1 + if survivor.name != canonical: + survivor.name = canonical + await session.flush() + summary["renamed"] += 1 + await session.commit() + summary["groups_processed"] += 1 + summary["merged"] += len(loser_ids) + except Exception as exc: # one bad group must not strand the rest + await session.rollback() + summary["errors"] += 1 + log.warning( + "tag normalize failed for group %r: %s", canonical, exc + ) + return summary diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index fec90c8..f115387 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -55,3 +55,50 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict: return cleanup_service.delete_images( session, image_ids=image_ids, images_root=IMAGES_ROOT, ) + + +@celery.task( + name="backend.app.tasks.admin.reextract_archive_attachments_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def reextract_archive_attachments_task(self) -> dict: + """Wraps cleanup_service.reextract_archive_attachments (#713 part 2): + re-extract PostAttachments that are actually archives but were filed + opaquely before the magic-byte gate, and link their members to the post.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + return cleanup_service.reextract_archive_attachments( + session, images_root=IMAGES_ROOT, + ) + + +@celery.task( + name="backend.app.tasks.admin.normalize_tags_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def normalize_tags_task(self) -> dict: + """Wraps tag_service.normalize_existing_tags (#714): Title-Case the + back-catalog and merge case/whitespace-variant duplicate tags via the + tested async merge path. Runs under its own asyncio loop + per-task async + engine (NullPool, disposed when the loop ends), mirroring download_source.""" + import asyncio + + from ..services.tag_service import normalize_existing_tags + from ._async_session import async_session_factory + + async def _run() -> dict: + async_factory, async_engine = async_session_factory() + try: + async with async_factory() as session: + # normalize_existing_tags commits per group internally. + return await normalize_existing_tags(session, dry_run=False) + finally: + await async_engine.dispose() + + return asyncio.run(_run()) diff --git a/backend/app/utils/safe_probe.py b/backend/app/utils/safe_probe.py index d8e5360..beb0bc1 100644 --- a/backend/app/utils/safe_probe.py +++ b/backend/app/utils/safe_probe.py @@ -149,9 +149,8 @@ def _run_probe(path_str: str) -> tuple[str, str | None]: call the same code path. """ path = Path(path_str) - ext = path.suffix.lower() try: - total, test_bad = _inspect_archive(path, ext) + total, test_bad = _inspect_archive(path) except Exception as exc: # noqa: BLE001 — clean rejection return ("error", f"{type(exc).__name__}: {exc}") if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES: @@ -162,24 +161,29 @@ def _run_probe(path_str: str) -> tuple[str, str | None]: return ("ok", None) -def _inspect_archive(path: Path, ext: str): +def _inspect_archive(path: Path): """Return (total_uncompressed_bytes | None, first_bad_member | None) - for the archive. Format-specific; raises on a structurally-broken - container (caught by the child as a clean rejection).""" - if ext in (".zip", ".cbz"): + for the archive. Format detected by extension OR magic bytes (so a + mis-named archive is still bomb-guarded + integrity-tested, matching the + extractor's gate); raises on a structurally-broken container (caught by the + child as a clean rejection).""" + from ..services.archive_extractor import detect_archive_format + + fmt = detect_archive_format(path) + if fmt == "zip": import zipfile with zipfile.ZipFile(path) as zf: total = sum(zi.file_size for zi in zf.infolist()) return total, zf.testzip() - if ext == ".rar": + if fmt == "rar": import rarfile with rarfile.RarFile(path) as rf: total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist()) rf.testrar() return total, None - if ext == ".7z": + if fmt == "7z": import py7zr with py7zr.SevenZipFile(path, "r") as zf: @@ -187,5 +191,5 @@ def _inspect_archive(path: Path, ext: str): total = getattr(info, "uncompressed", None) ok = zf.test() # True / None when all members pass return total, (None if ok in (True, None) else "7z test reported corruption") - # Unknown extension — nothing to test; treat as clean. + # Not a recognised archive — nothing to test; treat as clean. return None, None diff --git a/frontend/src/components/discovery/ArtistCard.vue b/frontend/src/components/discovery/ArtistCard.vue index dc2f5ae..813aeda 100644 --- a/frontend/src/components/discovery/ArtistCard.vue +++ b/frontend/src/components/discovery/ArtistCard.vue @@ -49,9 +49,11 @@ function onCardClick() { position: relative; display: grid; grid-template-columns: repeat(3, 1fr); gap: 2px; aspect-ratio: 3 / 1; - /* Explicit floor + ceiling so tall source images can't escape the - preview slot even on browsers where aspect-ratio doesn't compute. */ - min-height: 150px; max-height: 220px; + /* Floor so tall source images can't escape the slot. NO max-height: an + aspect-ratio box capped by max-height shrinks its own WIDTH to keep the + ratio, leaving dead space beside the previews on a wide (single-column) + card. The grid below keeps columns ≲400px so the strip stays a sane height. */ + min-height: 120px; overflow: hidden; background: rgb(var(--v-theme-surface-light)); } diff --git a/frontend/src/components/modal/FandomPicker.vue b/frontend/src/components/modal/FandomPicker.vue index e9ed4a8..d412b6c 100644 --- a/frontend/src/components/modal/FandomPicker.vue +++ b/frontend/src/components/modal/FandomPicker.vue @@ -38,7 +38,8 @@ const store = useTagStore() const selectedId = ref(null) const newName = ref('') -onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() }) +// Always refresh on open so the list reflects fandoms created elsewhere (#712). +onMounted(() => store.loadFandoms()) async function onCreate() { const f = await store.createFandom(newName.value.trim()) diff --git a/frontend/src/components/modal/FandomSetDialog.vue b/frontend/src/components/modal/FandomSetDialog.vue index 4706d59..07f38da 100644 --- a/frontend/src/components/modal/FandomSetDialog.vue +++ b/frontend/src/components/modal/FandomSetDialog.vue @@ -84,7 +84,8 @@ const busy = ref(false) const error = ref(null) const collision = ref(null) -onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() }) +// Always refresh on open so the list reflects fandoms created elsewhere (#712). +onMounted(() => store.loadFandoms()) async function onCreate() { const name = newName.value.trim() diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue index 5bf9c79..53e17be 100644 --- a/frontend/src/components/modal/ImageViewer.vue +++ b/frontend/src/components/modal/ImageViewer.vue @@ -106,7 +106,11 @@ function onKeyDown(ev) { // overlay's own Esc handling fire instead of closing the whole // modal mid-interaction. Vuetify marks open overlays with // `.v-overlay--active`. - if (document.querySelector('.v-overlay--active')) return + // EXCLUDE tooltips (`.v-tooltip`): they're also `.v-overlay--active` while + // shown, so one lingering after a hover/click (e.g. just after accepting a + // suggested tag) wrongly suppressed the close (#700). Only real interactive + // overlays (menus/dialogs) should keep ESC from closing the modal. + if (document.querySelector('.v-overlay--active:not(.v-tooltip)')) return ev.preventDefault() emit('close') } else if (ev.key === 'ArrowLeft') { diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue index 319b0b9..4123c18 100644 --- a/frontend/src/components/modal/TagPanel.vue +++ b/frontend/src/components/modal/TagPanel.vue @@ -2,45 +2,43 @@