Merge pull request 'Tag-maintenance sweep + bug-fix batch: #699 #700 #701 #709 #711 #712 #713 #714' (#73) from dev into main
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 3s
Build images / build-ml (push) Successful in 8s
Build images / build-web (push) Successful in 9s
CI / frontend-build (push) Successful in 18s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m2s
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 3s
Build images / build-ml (push) Successful in 8s
Build images / build-web (push) Successful in 9s
CI / frontend-build (push) Successful in 18s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m2s
This commit was merged in pull request #73.
This commit is contained in:
@@ -245,6 +245,32 @@ async def tags_reset_content():
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/normalize", methods=["POST"])
|
||||
async def tags_normalize():
|
||||
"""#714: retro-normalize existing tags to the #701 canonical form (Title
|
||||
Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||
|
||||
dry_run=true (default) returns a projection inline — group/collision/rename
|
||||
counts + a sample of the changes — so the UI shows exactly what'll happen.
|
||||
dry_run=false dispatches the long-running maintenance task (the merge FK
|
||||
repoints can touch many tags); the UI tails the activity dashboard for the
|
||||
summary. Idempotent; back up first (the merges are irreversible)."""
|
||||
from ..services.tag_service import normalize_existing_tags
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True))
|
||||
|
||||
if dry_run:
|
||||
async with get_session() as session:
|
||||
result = await normalize_existing_tags(session, dry_run=True)
|
||||
return jsonify(result)
|
||||
|
||||
from ..tasks.admin import normalize_tags_task
|
||||
|
||||
async_result = normalize_tags_task.delay()
|
||||
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
|
||||
async def db_stats():
|
||||
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
|
||||
@@ -289,3 +315,14 @@ async def trigger_vacuum():
|
||||
|
||||
vacuum_analyze.delay()
|
||||
return jsonify({"status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
|
||||
async def trigger_reextract_archives():
|
||||
"""Operator-triggered re-extract (#713): PostAttachments that are actually
|
||||
archives but were filed opaquely (pre magic-byte gate) get extracted and
|
||||
their members linked to the post. Idempotent; runs on the maintenance queue."""
|
||||
from ..tasks.admin import reextract_archive_attachments_task
|
||||
|
||||
async_result = reextract_archive_attachments_task.delay()
|
||||
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||
|
||||
@@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
|
||||
"bytes_downloaded": event.bytes_downloaded,
|
||||
"error": event.error,
|
||||
"summary": _summary_from_metadata(event.metadata_),
|
||||
# plan #709: mid-walk live counts for a RUNNING native-ingester event
|
||||
# (None otherwise; phase 3 overwrites metadata with run_stats on finish).
|
||||
"live": (event.metadata_ or {}).get("live"),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from ..services.tag_service import (
|
||||
TagMergeConflict,
|
||||
TagService,
|
||||
TagValidationError,
|
||||
normalize_tag_name,
|
||||
)
|
||||
from ..utils.tag_prefix import parse_kind_prefix
|
||||
|
||||
@@ -141,6 +142,11 @@ async def create_tag():
|
||||
|
||||
fandom_id = body.get("fandom_id")
|
||||
|
||||
# #701: Title-Case operator-entered tags. Only here (the explicit create
|
||||
# endpoint), NOT in the shared find_or_create — the ML tagger uses that path
|
||||
# and must keep the booru vocabulary's casing for allowlist matching.
|
||||
name = normalize_tag_name(name)
|
||||
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
|
||||
@@ -16,9 +16,45 @@ log = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
|
||||
|
||||
# Magic-byte signatures, so an archive with a mangled / extension-less filename
|
||||
# is still recognised. Patreon attachment download URLs sanitize to names like
|
||||
# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk
|
||||
# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed
|
||||
# those as opaque PostAttachments and NEVER extracted them (operator-flagged
|
||||
# 2026-06-06). Detection is by extension first (cheap), then header sniff.
|
||||
_RAR_MAGIC = b"Rar!\x1a\x07"
|
||||
_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c"
|
||||
|
||||
|
||||
def detect_archive_format(path: Path) -> str | None:
|
||||
"""Return "zip" | "rar" | "7z" for an archive, else None.
|
||||
|
||||
Trusts a known extension first, then falls back to magic-byte sniffing so a
|
||||
mis-named or extension-less archive is still handled. (zip covers .cbz too.)
|
||||
"""
|
||||
ext = Path(path).suffix.lower()
|
||||
if ext in (".zip", ".cbz"):
|
||||
return "zip"
|
||||
if ext == ".rar":
|
||||
return "rar"
|
||||
if ext == ".7z":
|
||||
return "7z"
|
||||
try:
|
||||
if zipfile.is_zipfile(path):
|
||||
return "zip"
|
||||
with open(path, "rb") as fh:
|
||||
head = fh.read(8)
|
||||
except OSError:
|
||||
return None
|
||||
if head.startswith(_RAR_MAGIC):
|
||||
return "rar"
|
||||
if head.startswith(_7Z_MAGIC):
|
||||
return "7z"
|
||||
return None
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
return Path(path).suffix.lower() in ARCHIVE_EXTS
|
||||
return detect_archive_format(path) is not None
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -32,16 +68,16 @@ def extract_archive(path: Path):
|
||||
members: list[tuple[str, Path]] = []
|
||||
try:
|
||||
try:
|
||||
ext = Path(path).suffix.lower()
|
||||
if ext in (".zip", ".cbz"):
|
||||
fmt = detect_archive_format(path)
|
||||
if fmt == "zip":
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
zf.extractall(base)
|
||||
elif ext == ".rar":
|
||||
elif fmt == "rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
rf.extractall(base)
|
||||
elif ext == ".7z":
|
||||
elif fmt == "7z":
|
||||
import py7zr
|
||||
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
|
||||
@@ -11,6 +11,7 @@ the one-and-done GS/IR migration tooling.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -22,6 +23,8 @@ from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||
from ..models.series_page import SeriesPage
|
||||
from ..models.tag import image_tag
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def project_artist_cascade(session: Session, *, slug: str) -> dict:
|
||||
"""Read-only projection of what delete_artist_cascade would touch.
|
||||
@@ -665,3 +668,139 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||
.where(LibraryAuditRun.status == "running")
|
||||
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||
)
|
||||
|
||||
|
||||
# -- archive-attachment re-extraction (#713 part 2) ------------------------
|
||||
|
||||
_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
|
||||
|
||||
|
||||
def _reextract_archive_to_post(
|
||||
importer, archive_path: Path, post, source_row, artist, images_root: Path,
|
||||
) -> list[int]:
|
||||
"""Extract one stored archive and link its members to `post`.
|
||||
|
||||
The stored attachment has no adjacent sidecar (it lives in the sha-addressed
|
||||
attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
|
||||
library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
|
||||
re-derives the artist from the path AND copies members relative to it, so the
|
||||
members land in the real library and resolve to the right artist — then re-run
|
||||
`attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
|
||||
the members to the SAME Post (source_id + external_post_id). Removes only the
|
||||
staged archive + sidecar afterward; the imported member files stay. Returns
|
||||
the new member image ids.
|
||||
"""
|
||||
import json
|
||||
import shutil
|
||||
|
||||
from .archive_extractor import detect_archive_format
|
||||
|
||||
fmt = detect_archive_format(archive_path)
|
||||
ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
|
||||
platform = source_row.platform if source_row is not None else "imported"
|
||||
sidecar = {
|
||||
"category": source_row.platform if source_row is not None
|
||||
else (post.raw_metadata or {}).get("category"),
|
||||
"id": post.external_post_id,
|
||||
"title": post.post_title or "",
|
||||
"content": post.description or "",
|
||||
"published_at": post.post_date.isoformat() if post.post_date else None,
|
||||
"url": post.post_url,
|
||||
}
|
||||
work = images_root / artist.slug / platform / str(post.external_post_id)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar
|
||||
sidecar_path = staged.with_suffix(".json")
|
||||
try:
|
||||
shutil.copy2(archive_path, staged)
|
||||
sidecar_path.write_text(json.dumps(sidecar))
|
||||
res = importer.attach_in_place(staged, artist=artist, source=source_row)
|
||||
return list(res.member_image_ids or [])
|
||||
finally:
|
||||
# Drop only the staged archive + sidecar; the extracted member files
|
||||
# were copied into the library alongside them and must stay.
|
||||
staged.unlink(missing_ok=True)
|
||||
sidecar_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def reextract_archive_attachments(session: Session, *, images_root: Path) -> dict:
|
||||
"""Re-process existing PostAttachments that are ACTUALLY archives but were
|
||||
filed opaquely before #713 part 1 (extension-only is_archive missed mangled /
|
||||
extension-less Patreon attachment names). For each: extract the members,
|
||||
import them, and link them to the attachment's post.
|
||||
|
||||
Idempotent — members dedupe by sha256, the archive dedupes by sha — so it's
|
||||
safe to run repeatedly. Returns a summary dict for task_run.metadata.
|
||||
"""
|
||||
from ..models import ImportSettings, Post, PostAttachment, Source
|
||||
from ..tasks.ml import tag_and_embed
|
||||
from ..tasks.thumbnail import generate_thumbnail
|
||||
from .archive_extractor import is_archive
|
||||
from .importer import Importer
|
||||
from .thumbnailer import Thumbnailer
|
||||
|
||||
summary = {
|
||||
"scanned": 0, "archives": 0, "members_imported": 0,
|
||||
"posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
|
||||
"errors": 0,
|
||||
}
|
||||
settings = ImportSettings.load_sync(session)
|
||||
importer = Importer(
|
||||
session=session, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
|
||||
)
|
||||
|
||||
attachments = session.execute(
|
||||
select(PostAttachment).order_by(PostAttachment.id)
|
||||
).scalars().all()
|
||||
enqueue_ids: list[int] = []
|
||||
for att in attachments:
|
||||
summary["scanned"] += 1
|
||||
stored = Path(att.path)
|
||||
try:
|
||||
if not stored.is_file() or not is_archive(stored):
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
summary["archives"] += 1
|
||||
if att.post_id is None:
|
||||
summary["skipped_no_post"] += 1
|
||||
continue
|
||||
post = session.get(Post, att.post_id)
|
||||
if post is None:
|
||||
summary["skipped_no_post"] += 1
|
||||
continue
|
||||
artist = session.get(Artist, att.artist_id) if att.artist_id else None
|
||||
if artist is None and post.artist_id:
|
||||
artist = session.get(Artist, post.artist_id)
|
||||
if artist is None or not artist.slug:
|
||||
# The importer re-derives the artist from the staged path, so we need
|
||||
# a real artist+slug to anchor under. (Shouldn't happen for
|
||||
# subscription posts; skip rather than orphan the members.)
|
||||
summary["skipped_no_artist"] += 1
|
||||
continue
|
||||
source_row = session.get(Source, post.source_id) if post.source_id else None
|
||||
try:
|
||||
ids = _reextract_archive_to_post(
|
||||
importer, stored, post, source_row, artist, images_root,
|
||||
)
|
||||
session.commit()
|
||||
except Exception as exc: # one bad archive must not strand the rest
|
||||
session.rollback()
|
||||
summary["errors"] += 1
|
||||
log.warning("re-extract failed for attachment %s: %s", att.id, exc)
|
||||
continue
|
||||
if ids:
|
||||
summary["members_imported"] += len(ids)
|
||||
summary["posts_touched"] += 1
|
||||
enqueue_ids.extend(ids)
|
||||
|
||||
# Thumbnails + ML for the newly-imported members (best-effort; off the
|
||||
# critical path — a Redis hiccup must not fail the whole re-extract).
|
||||
for img_id in enqueue_ids:
|
||||
try:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
except Exception as exc:
|
||||
log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
|
||||
return summary
|
||||
|
||||
@@ -139,6 +139,8 @@ async def _run_native_ingester(
|
||||
resume_cursor=source_config.resume_cursor,
|
||||
time_budget_seconds=source_config.timeout,
|
||||
posts_base=int(overrides.get("_backfill_posts", 0)),
|
||||
# plan #709: live progress writes to this running event mid-walk.
|
||||
event_id=ctx.get("event_id"),
|
||||
),
|
||||
)
|
||||
return dl_result, resolved_campaign_id
|
||||
|
||||
@@ -25,6 +25,7 @@ Plain-HTTP homelab: no secure-context Web API.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
@@ -48,6 +49,12 @@ DEAD_LETTER_THRESHOLD = 3
|
||||
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
|
||||
_ERROR_MAX = 1000
|
||||
|
||||
# plan #709: throttle the live-progress write to the running DownloadEvent to one
|
||||
# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a
|
||||
# page is (page boundaries can be minutes apart on image-dense backfills, so a
|
||||
# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s).
|
||||
_LIVE_PROGRESS_INTERVAL = 5.0
|
||||
|
||||
|
||||
class Ingester:
|
||||
"""Generic native-ingest orchestration. Subclass with a platform adapter
|
||||
@@ -92,6 +99,7 @@ class Ingester:
|
||||
time_budget_seconds: float = 870.0,
|
||||
seen_threshold: int = _TICK_SEEN_THRESHOLD,
|
||||
posts_base: int = 0,
|
||||
event_id: int | None = None,
|
||||
) -> DownloadResult:
|
||||
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
||||
|
||||
@@ -109,6 +117,7 @@ class Ingester:
|
||||
checkpoint = mode in ("backfill", "recovery")
|
||||
ledger_key = self._ledger_key
|
||||
start = time.monotonic()
|
||||
last_live = start # plan #709: last live-progress write timestamp
|
||||
log_lines: list[str] = []
|
||||
written: list[str] = []
|
||||
quarantined_paths: list[str] = []
|
||||
@@ -288,6 +297,19 @@ class Ingester:
|
||||
if to_fail:
|
||||
self._record_failures(source_id, to_fail)
|
||||
|
||||
# plan #709: time-throttled live progress to the running event so
|
||||
# the Downloads view ticks ~every 5s, independent of page size.
|
||||
now = time.monotonic()
|
||||
if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL:
|
||||
last_live = now
|
||||
self._write_live_progress(event_id, {
|
||||
"downloaded": downloaded,
|
||||
"skipped": skipped_count,
|
||||
"errors": errors,
|
||||
"quarantined": quarantined,
|
||||
"posts": posts_processed,
|
||||
})
|
||||
|
||||
if early_out:
|
||||
break
|
||||
else:
|
||||
@@ -474,6 +496,25 @@ class Ingester:
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _write_live_progress(self, event_id: int, counts: dict) -> None:
|
||||
"""Throttled mid-walk write of live counts to the RUNNING download_event
|
||||
(plan #709) so the Downloads view shows progress before the chunk
|
||||
finishes. A short session (never held across the walk); the `status =
|
||||
'running'` guard avoids clobbering an event phase 3 already finalized.
|
||||
`metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest
|
||||
for phase 3 to overwrite with the final run_stats."""
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
text(
|
||||
"UPDATE download_event SET metadata = jsonb_set("
|
||||
" coalesce(metadata, '{}'::jsonb), '{live}',"
|
||||
" cast(:live AS jsonb)) "
|
||||
"WHERE id = :eid AND status = 'running'"
|
||||
),
|
||||
{"live": json.dumps(counts), "eid": event_id},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _still_running(self, source_id: int) -> bool:
|
||||
"""True while the source is armed for a deep walk (plan #708 B4).
|
||||
|
||||
|
||||
@@ -20,11 +20,20 @@ class ShowcaseService:
|
||||
async def random_sample(self, limit: int = 60) -> list[dict]:
|
||||
if limit < 1 or limit > 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
# Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows
|
||||
# from each sampled page, so sequentially-imported near-duplicates
|
||||
# (multi-image posts, variant sets) come back adjacent and cluster in the
|
||||
# showcase ("three near-identical in a row"). Sampling a multiple of
|
||||
# `limit` spans more pages, and ORDER BY random() before taking `limit`
|
||||
# breaks the physical adjacency — far better spread, still cheap
|
||||
# (random() over a few hundred rows, not the whole table).
|
||||
oversample = min(limit * 5, 1000)
|
||||
stmt = select(ImageRecord).from_statement(
|
||||
text(
|
||||
"SELECT * FROM image_record "
|
||||
"TABLESAMPLE SYSTEM_ROWS(:n)"
|
||||
).bindparams(n=limit)
|
||||
"SELECT * FROM ("
|
||||
" SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)"
|
||||
") sub ORDER BY random() LIMIT :n"
|
||||
).bindparams(o=oversample, n=limit)
|
||||
)
|
||||
rows = (await self.session.execute(stmt)).scalars().all()
|
||||
return [
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Tag CRUD + autocomplete + image-tag association."""
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
@@ -12,6 +13,21 @@ from ..models import Tag, TagKind, image_tag
|
||||
from ..models.tag_allowlist import TagAllowlist
|
||||
from ..models.tag_reference_embedding import TagReferenceEmbedding
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def normalize_tag_name(name: str) -> str:
|
||||
"""Canonical tag form (#701): collapse whitespace + Title Case.
|
||||
|
||||
Per-word capitalize (NOT str.title(), which mangles apostrophes:
|
||||
don't → Don'T). Lowercasing the word tail also folds ALL-CAPS input
|
||||
(HATSUNE MIKU → Hatsune Miku); acronym casing is sacrificed, an accepted
|
||||
Title-Case trade-off (operator-chosen 2026-06-06).
|
||||
"""
|
||||
return " ".join(
|
||||
w[:1].upper() + w[1:].lower() for w in (name or "").split()
|
||||
)
|
||||
|
||||
|
||||
class TagValidationError(ValueError):
|
||||
"""Raised when tag construction breaks the kind/fandom rules."""
|
||||
@@ -71,6 +87,10 @@ class TagService:
|
||||
- if fandom_id is set, the referenced tag must exist and have
|
||||
kind == TagKind.fandom
|
||||
"""
|
||||
# NOTE: case is NOT normalized here — find_or_create is the shared path
|
||||
# the ML tagger / allowlist also use, and Title-Casing the booru
|
||||
# vocabulary breaks allowlist matching. Display-casing (Title Case) is
|
||||
# applied at the user-entry layer (api/tags create_tag) only (#701).
|
||||
name = name.strip()
|
||||
if not name:
|
||||
raise TagValidationError("Tag name cannot be empty")
|
||||
@@ -93,13 +113,17 @@ class TagService:
|
||||
# inserts; without the savepoint the outer transaction would
|
||||
# poison and the calling request crashes. Mirrors
|
||||
# importer._get_or_create.
|
||||
# Case-insensitive match (#701): a normalized (Title Case) input must
|
||||
# find an existing differently-cased tag instead of forking a duplicate.
|
||||
stmt = (
|
||||
select(Tag)
|
||||
.where(Tag.name == name)
|
||||
.where(func.lower(Tag.name) == name.lower())
|
||||
.where(Tag.kind == kind)
|
||||
.where(
|
||||
Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id
|
||||
)
|
||||
.order_by(Tag.id)
|
||||
.limit(1)
|
||||
)
|
||||
existing = (await self.session.execute(stmt)).scalar_one_or_none()
|
||||
if existing:
|
||||
@@ -249,9 +273,11 @@ class TagService:
|
||||
if tag is None:
|
||||
raise TagValidationError(f"Tag {tag_id} not found")
|
||||
|
||||
# Case-insensitive clash (#701) — renaming onto a differently-cased tag
|
||||
# is still a merge, not a silent fork.
|
||||
clash_stmt = (
|
||||
select(Tag)
|
||||
.where(Tag.name == new_name)
|
||||
.where(func.lower(Tag.name) == new_name.lower())
|
||||
.where(Tag.kind == tag.kind)
|
||||
.where(
|
||||
Tag.fandom_id.is_(None)
|
||||
@@ -259,6 +285,8 @@ class TagService:
|
||||
else Tag.fandom_id == tag.fandom_id
|
||||
)
|
||||
.where(Tag.id != tag_id)
|
||||
.order_by(Tag.id)
|
||||
.limit(1)
|
||||
)
|
||||
clash = (await self.session.execute(clash_stmt)).scalar_one_or_none()
|
||||
if clash is not None:
|
||||
@@ -501,6 +529,21 @@ class TagService:
|
||||
update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
|
||||
)
|
||||
|
||||
async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]:
|
||||
"""image_tag row counts keyed by tag_id, for the survivor heuristic
|
||||
(the best-connected tag in a collision group survives → fewest
|
||||
FK repoints). Tags with zero associations are absent from the map."""
|
||||
if not tag_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await self.session.execute(
|
||||
select(image_tag.c.tag_id, func.count())
|
||||
.where(image_tag.c.tag_id.in_(tag_ids))
|
||||
.group_by(image_tag.c.tag_id)
|
||||
)
|
||||
).all()
|
||||
return {tid: int(n) for tid, n in rows}
|
||||
|
||||
async def _create_protective_aliases(
|
||||
self, src_name: str, src_kind: TagKind, tgt: int
|
||||
) -> bool:
|
||||
@@ -548,3 +591,161 @@ class TagService:
|
||||
if res.rowcount:
|
||||
created = True
|
||||
return created
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #714: retro-normalize existing tags to the #701 canonical (Title Case +
|
||||
# collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_NORMALIZE_SAMPLE_CAP = 50
|
||||
|
||||
|
||||
def _group_existing_tags(
|
||||
rows,
|
||||
) -> dict[tuple, list[tuple[int, str]]]:
|
||||
"""Group (id, name, kind, fandom_id) rows by their post-normalization
|
||||
identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that
|
||||
would collapse to the same canonical lives in ONE group, so the canonical
|
||||
form is unique within (kind, fandom) once each group is resolved."""
|
||||
groups: dict[tuple, list[tuple[int, str]]] = {}
|
||||
for tag_id, name, kind, fandom_id in rows:
|
||||
canonical = normalize_tag_name(name)
|
||||
key = (kind, fandom_id if fandom_id is not None else -1, canonical)
|
||||
groups.setdefault(key, []).append((tag_id, name))
|
||||
return groups
|
||||
|
||||
|
||||
def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool:
|
||||
"""A group is already canonical iff it's a single member whose name equals
|
||||
the canonical form. Anything else (a collision, or a lone mis-cased tag)
|
||||
needs work."""
|
||||
if len(members) > 1:
|
||||
return True
|
||||
return members[0][1] != canonical
|
||||
|
||||
|
||||
def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
|
||||
"""The tag with the most image associations (→ fewest FK repoints when it
|
||||
survives), tie-broken to the lowest id for determinism. Module-level so the
|
||||
key closes over its parameter, not a loop variable (ruff B023)."""
|
||||
return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid))
|
||||
|
||||
|
||||
async def normalize_existing_tags(
|
||||
session: AsyncSession, *, dry_run: bool = False
|
||||
) -> dict:
|
||||
"""Convert the back-catalog to the #701 canonical tag form.
|
||||
|
||||
For each (kind, fandom, canonical) group: pick a survivor, merge any
|
||||
case/whitespace-variant siblings INTO it via the tested merge path
|
||||
(TagService._do_merge — FK repoints + protective aliases), then rename the
|
||||
survivor to the canonical form. Idempotent: a group that is already a lone
|
||||
canonical tag is a no-op, so re-running is safe.
|
||||
|
||||
dry_run=True returns a projection (counts + a sample of the changes) with no
|
||||
mutations. Live runs commit per group and isolate failures per group so one
|
||||
bad group can't strand the rest.
|
||||
|
||||
Returns (dry_run):
|
||||
{"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R,
|
||||
"total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
|
||||
Returns (live):
|
||||
{"groups_processed", "merged", "renamed", "aliases_created", "errors",
|
||||
"sample": [...]}
|
||||
"""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
|
||||
)
|
||||
).all()
|
||||
groups = _group_existing_tags(rows)
|
||||
|
||||
# Deterministic sample/ordering: by kind then canonical name.
|
||||
touched = sorted(
|
||||
(
|
||||
(key, members)
|
||||
for key, members in groups.items()
|
||||
if _group_needs_change(key[2], members)
|
||||
),
|
||||
key=lambda km: (
|
||||
km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]),
|
||||
km[0][2].lower(),
|
||||
),
|
||||
)
|
||||
|
||||
sample = [
|
||||
{
|
||||
"to": key[2],
|
||||
"from": [name for _id, name in members],
|
||||
"kind": key[0].value if hasattr(key[0], "value") else str(key[0]),
|
||||
"merge": len(members) > 1,
|
||||
}
|
||||
for key, members in touched[:_NORMALIZE_SAMPLE_CAP]
|
||||
]
|
||||
|
||||
if dry_run:
|
||||
collisions = sum(1 for key, m in touched if len(m) > 1)
|
||||
tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1)
|
||||
# A group renames iff the canonical form isn't already one of its
|
||||
# members' exact names (else that member is picked as survivor → no
|
||||
# rename, the rest merge into it).
|
||||
tags_to_rename = sum(
|
||||
1
|
||||
for key, m in touched
|
||||
if key[2] not in {name for _id, name in m}
|
||||
)
|
||||
return {
|
||||
"groups": len(groups),
|
||||
"collisions": collisions,
|
||||
"tags_to_merge": tags_to_merge,
|
||||
"tags_to_rename": tags_to_rename,
|
||||
"total_changes": len(touched),
|
||||
"sample": sample,
|
||||
}
|
||||
|
||||
svc = TagService(session)
|
||||
summary = {
|
||||
"groups_processed": 0,
|
||||
"merged": 0,
|
||||
"renamed": 0,
|
||||
"aliases_created": 0,
|
||||
"errors": 0,
|
||||
"sample": sample,
|
||||
}
|
||||
for key, members in touched:
|
||||
canonical = key[2]
|
||||
names_by_id = dict(members)
|
||||
# Survivor: prefer a member already named canonically (no rename, no
|
||||
# self-alias); else the best-connected (fewest FK repoints); else
|
||||
# lowest id for determinism.
|
||||
survivor_id = next(
|
||||
(tid for tid, name in members if name == canonical), None
|
||||
)
|
||||
if survivor_id is None:
|
||||
counts = await svc._image_assoc_counts(list(names_by_id))
|
||||
survivor_id = _best_connected(list(names_by_id), counts)
|
||||
loser_ids = [tid for tid in names_by_id if tid != survivor_id]
|
||||
try:
|
||||
survivor = await session.get(Tag, survivor_id)
|
||||
for loser_id in loser_ids:
|
||||
loser = await session.get(Tag, loser_id)
|
||||
if loser is None:
|
||||
continue
|
||||
result = await svc._do_merge(loser, survivor)
|
||||
if result.alias_created:
|
||||
summary["aliases_created"] += 1
|
||||
if survivor.name != canonical:
|
||||
survivor.name = canonical
|
||||
await session.flush()
|
||||
summary["renamed"] += 1
|
||||
await session.commit()
|
||||
summary["groups_processed"] += 1
|
||||
summary["merged"] += len(loser_ids)
|
||||
except Exception as exc: # one bad group must not strand the rest
|
||||
await session.rollback()
|
||||
summary["errors"] += 1
|
||||
log.warning(
|
||||
"tag normalize failed for group %r: %s", canonical, exc
|
||||
)
|
||||
return summary
|
||||
|
||||
@@ -55,3 +55,50 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
|
||||
return cleanup_service.delete_images(
|
||||
session, image_ids=image_ids, images_root=IMAGES_ROOT,
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.reextract_archive_attachments_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def reextract_archive_attachments_task(self) -> dict:
|
||||
"""Wraps cleanup_service.reextract_archive_attachments (#713 part 2):
|
||||
re-extract PostAttachments that are actually archives but were filed
|
||||
opaquely before the magic-byte gate, and link their members to the post."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
return cleanup_service.reextract_archive_attachments(
|
||||
session, images_root=IMAGES_ROOT,
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.admin.normalize_tags_task",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||
)
|
||||
def normalize_tags_task(self) -> dict:
|
||||
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
|
||||
back-catalog and merge case/whitespace-variant duplicate tags via the
|
||||
tested async merge path. Runs under its own asyncio loop + per-task async
|
||||
engine (NullPool, disposed when the loop ends), mirroring download_source."""
|
||||
import asyncio
|
||||
|
||||
from ..services.tag_service import normalize_existing_tags
|
||||
from ._async_session import async_session_factory
|
||||
|
||||
async def _run() -> dict:
|
||||
async_factory, async_engine = async_session_factory()
|
||||
try:
|
||||
async with async_factory() as session:
|
||||
# normalize_existing_tags commits per group internally.
|
||||
return await normalize_existing_tags(session, dry_run=False)
|
||||
finally:
|
||||
await async_engine.dispose()
|
||||
|
||||
return asyncio.run(_run())
|
||||
|
||||
@@ -149,9 +149,8 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
|
||||
call the same code path.
|
||||
"""
|
||||
path = Path(path_str)
|
||||
ext = path.suffix.lower()
|
||||
try:
|
||||
total, test_bad = _inspect_archive(path, ext)
|
||||
total, test_bad = _inspect_archive(path)
|
||||
except Exception as exc: # noqa: BLE001 — clean rejection
|
||||
return ("error", f"{type(exc).__name__}: {exc}")
|
||||
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||
@@ -162,24 +161,29 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
|
||||
return ("ok", None)
|
||||
|
||||
|
||||
def _inspect_archive(path: Path, ext: str):
|
||||
def _inspect_archive(path: Path):
|
||||
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||
for the archive. Format-specific; raises on a structurally-broken
|
||||
container (caught by the child as a clean rejection)."""
|
||||
if ext in (".zip", ".cbz"):
|
||||
for the archive. Format detected by extension OR magic bytes (so a
|
||||
mis-named archive is still bomb-guarded + integrity-tested, matching the
|
||||
extractor's gate); raises on a structurally-broken container (caught by the
|
||||
child as a clean rejection)."""
|
||||
from ..services.archive_extractor import detect_archive_format
|
||||
|
||||
fmt = detect_archive_format(path)
|
||||
if fmt == "zip":
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
total = sum(zi.file_size for zi in zf.infolist())
|
||||
return total, zf.testzip()
|
||||
if ext == ".rar":
|
||||
if fmt == "rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||
rf.testrar()
|
||||
return total, None
|
||||
if ext == ".7z":
|
||||
if fmt == "7z":
|
||||
import py7zr
|
||||
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
@@ -187,5 +191,5 @@ def _inspect_archive(path: Path, ext: str):
|
||||
total = getattr(info, "uncompressed", None)
|
||||
ok = zf.test() # True / None when all members pass
|
||||
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||
# Unknown extension — nothing to test; treat as clean.
|
||||
# Not a recognised archive — nothing to test; treat as clean.
|
||||
return None, None
|
||||
|
||||
@@ -49,9 +49,11 @@ function onCardClick() {
|
||||
position: relative;
|
||||
display: grid; grid-template-columns: repeat(3, 1fr);
|
||||
gap: 2px; aspect-ratio: 3 / 1;
|
||||
/* Explicit floor + ceiling so tall source images can't escape the
|
||||
preview slot even on browsers where aspect-ratio doesn't compute. */
|
||||
min-height: 150px; max-height: 220px;
|
||||
/* Floor so tall source images can't escape the slot. NO max-height: an
|
||||
aspect-ratio box capped by max-height shrinks its own WIDTH to keep the
|
||||
ratio, leaving dead space beside the previews on a wide (single-column)
|
||||
card. The grid below keeps columns ≲400px so the strip stays a sane height. */
|
||||
min-height: 120px;
|
||||
overflow: hidden;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
|
||||
@@ -38,7 +38,8 @@ const store = useTagStore()
|
||||
const selectedId = ref(null)
|
||||
const newName = ref('')
|
||||
|
||||
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
|
||||
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
|
||||
onMounted(() => store.loadFandoms())
|
||||
|
||||
async function onCreate() {
|
||||
const f = await store.createFandom(newName.value.trim())
|
||||
|
||||
@@ -84,7 +84,8 @@ const busy = ref(false)
|
||||
const error = ref(null)
|
||||
const collision = ref(null)
|
||||
|
||||
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
|
||||
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
|
||||
onMounted(() => store.loadFandoms())
|
||||
|
||||
async function onCreate() {
|
||||
const name = newName.value.trim()
|
||||
|
||||
@@ -106,7 +106,11 @@ function onKeyDown(ev) {
|
||||
// overlay's own Esc handling fire instead of closing the whole
|
||||
// modal mid-interaction. Vuetify marks open overlays with
|
||||
// `.v-overlay--active`.
|
||||
if (document.querySelector('.v-overlay--active')) return
|
||||
// EXCLUDE tooltips (`.v-tooltip`): they're also `.v-overlay--active` while
|
||||
// shown, so one lingering after a hover/click (e.g. just after accepting a
|
||||
// suggested tag) wrongly suppressed the close (#700). Only real interactive
|
||||
// overlays (menus/dialogs) should keep ESC from closing the modal.
|
||||
if (document.querySelector('.v-overlay--active:not(.v-tooltip)')) return
|
||||
ev.preventDefault()
|
||||
emit('close')
|
||||
} else if (ev.key === 'ArrowLeft') {
|
||||
|
||||
@@ -2,45 +2,43 @@
|
||||
<aside class="fc-tag-panel" aria-label="Tags for this image">
|
||||
<h3 class="fc-tag-panel__title">Tags</h3>
|
||||
<div class="fc-tag-panel__chips">
|
||||
<v-chip
|
||||
<!-- #711: the kebab + its menu used to be NESTED inside the v-chip, which
|
||||
swallowed/mis-routed the click and mis-anchored the (teleported) menu
|
||||
so it never opened. Render the kebab as a SIBLING of the chip and use
|
||||
the standard v-menu activator slot — Vuetify wires the click and
|
||||
stacks the overlay above the modal natively. -->
|
||||
<span
|
||||
v-for="tag in modal.current?.tags || []"
|
||||
:key="tag.id" size="small" closable
|
||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||
@click:close="onRemove(tag.id)"
|
||||
:key="tag.id" class="fc-tag-panel__chip"
|
||||
>
|
||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||
<!-- Operator-flagged 2026-06-04: the `#activator` + `v-bind` menu
|
||||
never opened inside this teleported modal. Drive it explicitly
|
||||
instead (same mechanism as the dialogs below, which work): the
|
||||
icon toggles `openTagId` with @click.stop (shielding the chip's
|
||||
close button), and `activator="parent"` + `:open-on-click=false`
|
||||
anchors the menu for positioning only. One tag's menu open at a
|
||||
time, so a single id is enough. -->
|
||||
<span class="kebab-wrap">
|
||||
<v-icon
|
||||
size="x-small" class="ml-1 kebab-icon"
|
||||
icon="mdi-dots-vertical"
|
||||
@click.stop="openTagId = openTagId === tag.id ? null : tag.id"
|
||||
/>
|
||||
<v-menu
|
||||
:model-value="openTagId === tag.id"
|
||||
activator="parent" :open-on-click="false"
|
||||
@update:model-value="v => { if (!v) openTagId = null }"
|
||||
>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="openRename(tag)">
|
||||
<v-list-item-title>Rename…</v-list-item-title>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
|
||||
>
|
||||
<v-list-item-title>Set fandom…</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</span>
|
||||
</v-chip>
|
||||
<v-chip
|
||||
size="small" closable
|
||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||
@click:close="onRemove(tag.id)"
|
||||
>
|
||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||
</v-chip>
|
||||
<v-menu location="bottom end">
|
||||
<template #activator="{ props }">
|
||||
<v-btn
|
||||
v-bind="props" icon="mdi-dots-vertical" size="x-small"
|
||||
variant="text" density="comfortable"
|
||||
class="fc-tag-panel__kebab" @click.stop
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="openRename(tag)">
|
||||
<v-list-item-title>Rename…</v-list-item-title>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
|
||||
>
|
||||
<v-list-item-title>Set fandom…</v-list-item-title>
|
||||
</v-list-item>
|
||||
</v-list>
|
||||
</v-menu>
|
||||
</span>
|
||||
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||
</div>
|
||||
|
||||
@@ -88,9 +86,6 @@ import FandomSetDialog from './FandomSetDialog.vue'
|
||||
const modal = useModalStore()
|
||||
const store = useTagStore()
|
||||
const errorMsg = ref(null)
|
||||
// Which tag chip's kebab menu is open (only one at a time). Drives each
|
||||
// chip menu's v-model so opening never depends on Vuetify's activator click.
|
||||
const openTagId = ref(null)
|
||||
|
||||
const KIND_ICONS = {
|
||||
general: 'mdi-tag', character: 'mdi-account-circle',
|
||||
@@ -153,6 +148,7 @@ async function onFandomUpdated() {
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.kebab-wrap { display: inline-flex; align-items: center; }
|
||||
.kebab-icon { cursor: pointer; }
|
||||
.fc-tag-panel__chip { display: inline-flex; align-items: center; gap: 1px; }
|
||||
.fc-tag-panel__kebab { opacity: 0.7; }
|
||||
.fc-tag-panel__chip:hover .fc-tag-panel__kebab { opacity: 1; }
|
||||
</style>
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
<template>
|
||||
<!-- #713: re-extract PostAttachments that are really archives but were filed
|
||||
opaquely before the magic-byte gate, and attach their images to the post. -->
|
||||
<v-card>
|
||||
<v-card-title>Re-extract archive attachments</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="text-body-2 mb-3">
|
||||
Some posts attached an archive (zip) whose images weren't extracted
|
||||
because the downloaded file had no usable extension. This scans existing
|
||||
attachments, extracts any that are really archives, and attaches their
|
||||
images to the post. Idempotent — safe to run more than once.
|
||||
</p>
|
||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
||||
</v-btn>
|
||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
|
||||
const api = useApi()
|
||||
const busy = ref(false)
|
||||
const queued = ref(false)
|
||||
|
||||
async function run () {
|
||||
busy.value = true
|
||||
queued.value = false
|
||||
try {
|
||||
await api.post('/api/admin/maintenance/reextract-archives')
|
||||
queued.value = true
|
||||
toast({ text: 'Archive re-extract queued', type: 'success' })
|
||||
} catch (e) {
|
||||
toast({ text: e?.body?.detail || e?.message || 'Failed to queue', type: 'error' })
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
@@ -15,6 +15,7 @@
|
||||
<AllowlistTable class="mt-4" />
|
||||
<AliasTable class="mt-4" />
|
||||
<DbMaintenanceCard class="mt-6" />
|
||||
<ArchiveReextractCard class="mt-6" />
|
||||
<BackupCard class="mt-6" />
|
||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||
operates on the existing library which fits the Cleanup-tab
|
||||
@@ -33,6 +34,7 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||
import AllowlistTable from './AllowlistTable.vue'
|
||||
import AliasTable from './AliasTable.vue'
|
||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||
import BackupCard from './BackupCard.vue'
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
|
||||
|
||||
@@ -136,6 +136,64 @@
|
||||
>Delete {{ resetPreview.count }} content tag(s) +
|
||||
{{ resetPreview.applications }} application(s)</v-btn>
|
||||
</div>
|
||||
|
||||
<v-divider class="my-5" />
|
||||
|
||||
<!-- #714: standardize existing tags to the canonical Title-Case form
|
||||
and merge case/whitespace-variant duplicates. -->
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>Standardize tag casing.</strong>
|
||||
New tags are saved Title Case, but older tags keep whatever casing they
|
||||
were created with. This renames every existing tag to
|
||||
<code>Title Case</code> (collapsing extra spaces) and
|
||||
<strong>merges</strong> tags that differ only by case or spacing
|
||||
(e.g. <code>hatsune miku</code> + <code>Hatsune Miku</code>)
|
||||
into one — repointing their images, allowlist entries and series pages.
|
||||
</p>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
The merges are irreversible — back up first
|
||||
(Settings → Maintenance → Backup). Safe to run more than once.
|
||||
</v-alert>
|
||||
|
||||
<v-btn
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-magnify"
|
||||
:loading="loadingNormPreview"
|
||||
class="mb-3"
|
||||
@click="onNormPreview"
|
||||
>Preview tag standardization</v-btn>
|
||||
|
||||
<div v-if="normPreview">
|
||||
<p class="text-body-2 mb-2">
|
||||
<strong>{{ normPreview.total_changes }}</strong> tag group(s) to
|
||||
change — <strong>{{ normPreview.tags_to_rename }}</strong> rename(s),
|
||||
<strong>{{ normPreview.collisions }}</strong> collision(s) merging
|
||||
<strong>{{ normPreview.tags_to_merge }}</strong> tag(s) away.
|
||||
</p>
|
||||
<div v-if="normPreview.sample?.length" class="fc-name-grid mb-3">
|
||||
<span
|
||||
v-for="s in normPreview.sample" :key="s.to + s.kind"
|
||||
class="fc-name"
|
||||
>
|
||||
<template v-if="s.merge">{{ s.from.join(' + ') }} → </template>{{ s.to }}
|
||||
</span>
|
||||
</div>
|
||||
<v-btn
|
||||
color="error" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-format-letter-case"
|
||||
:disabled="!normPreview.total_changes"
|
||||
:loading="normCommitting"
|
||||
@click="onNormCommit"
|
||||
>Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
|
||||
<span
|
||||
v-if="normResult"
|
||||
class="ml-3 text-caption"
|
||||
:class="normResult === 'ok' ? 'text-success' : 'text-warning'"
|
||||
>
|
||||
<template v-if="normResult === 'ok'">Standardization complete ✓</template>
|
||||
<template v-else>Finished with status: {{ normResult }}</template>
|
||||
</span>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
@@ -155,6 +213,10 @@ const kindCommitting = ref(false)
|
||||
const resetPreview = ref(null)
|
||||
const loadingResetPreview = ref(false)
|
||||
const resetCommitting = ref(false)
|
||||
const normPreview = ref(null)
|
||||
const loadingNormPreview = ref(false)
|
||||
const normCommitting = ref(false)
|
||||
const normResult = ref(null)
|
||||
|
||||
async function onPreview() {
|
||||
loadingPreview.value = true
|
||||
@@ -212,6 +274,33 @@ async function onResetCommit() {
|
||||
resetCommitting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onNormPreview() {
|
||||
loadingNormPreview.value = true
|
||||
normResult.value = null
|
||||
try {
|
||||
normPreview.value = await store.normalizeTags({ dryRun: true })
|
||||
} finally {
|
||||
loadingNormPreview.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function onNormCommit() {
|
||||
normCommitting.value = true
|
||||
normResult.value = null
|
||||
try {
|
||||
// Long op (FK repoints): enqueue the maintenance task, then tail the
|
||||
// activity dashboard until its row reaches a terminal status. (The
|
||||
// per-run summary dict isn't exposed by /activity/runs, so we surface
|
||||
// the terminal status — the dry-run preview is the detailed view.)
|
||||
const { task_id: taskId } = await store.normalizeTags({ dryRun: false })
|
||||
const row = await store.pollTaskUntilDone(taskId)
|
||||
normResult.value = row?.status || 'ok'
|
||||
normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
|
||||
} finally {
|
||||
normCommitting.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -23,6 +23,18 @@
|
||||
<span class="fc-active__dot" />
|
||||
<PlatformChip :platform="e.platform" size="x-small" />
|
||||
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
|
||||
<!-- #709: live per-file progress for native (Patreon) downloads. -->
|
||||
<span v-if="e.live" class="fc-active__counts">
|
||||
<span class="fc-active__count" title="downloaded">↓ {{ e.live.downloaded }}</span>
|
||||
<span v-if="e.live.skipped" class="fc-active__count" title="skipped">
|
||||
⤼ {{ e.live.skipped }}</span>
|
||||
<span
|
||||
v-if="e.live.errors" class="fc-active__count fc-active__count--err"
|
||||
title="errors"
|
||||
>✕ {{ e.live.errors }}</span>
|
||||
<span class="fc-active__count fc-active__count--posts" title="posts scanned">
|
||||
{{ e.live.posts }} posts</span>
|
||||
</span>
|
||||
<v-spacer />
|
||||
<span class="fc-active__timer" :title="`started ${e.started_at}`">
|
||||
{{ elapsed(e.started_at) }}
|
||||
@@ -123,6 +135,13 @@ function elapsed (startedIso) {
|
||||
font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
}
|
||||
.fc-active__counts {
|
||||
display: inline-flex; align-items: center; gap: 8px;
|
||||
font-variant-numeric: tabular-nums; font-size: 0.78rem;
|
||||
}
|
||||
.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
.fc-active__count--err { color: rgb(var(--v-theme-error)); }
|
||||
.fc-active__count--posts { opacity: 0.7; }
|
||||
@keyframes fc-active-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.35; transform: scale(0.7); }
|
||||
|
||||
@@ -142,6 +142,22 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
}
|
||||
}
|
||||
|
||||
// #714: Title-Case the back-catalog + merge case/whitespace-variant tags.
|
||||
// dry-run returns a projection inline; live returns {task_id} (long op —
|
||||
// FK repoints) the caller polls via pollTaskUntilDone.
|
||||
async function normalizeTags({ dryRun = true } = {}) {
|
||||
lastError.value = null
|
||||
try {
|
||||
return await api.post(
|
||||
'/api/admin/tags/normalize',
|
||||
{ body: { dry_run: dryRun } },
|
||||
)
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||
|
||||
/**
|
||||
@@ -178,6 +194,7 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
pruneUnusedTags,
|
||||
purgeLegacyTags,
|
||||
resetContentTagging,
|
||||
normalizeTags,
|
||||
pollTaskUntilDone,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -14,6 +14,10 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
||||
const q = ref('')
|
||||
const platform = ref(null)
|
||||
let started = false
|
||||
// Has a load ATTEMPT completed yet? Gates the "No artists match" empty state
|
||||
// so it can't flash on the very first render (loading is still false before
|
||||
// onMounted fires the first loadMore) or between a query change and its fetch.
|
||||
const loaded = ref(false)
|
||||
// Typed "alice" then "alice bob" used to drop the second fetch
|
||||
// entirely (loading flag still true from the first), so the UI
|
||||
// showed alice results while the input said "alice bob". Inflight
|
||||
@@ -35,6 +39,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
||||
nextCursor.value = body.next_cursor
|
||||
started = true
|
||||
})
|
||||
if (t.isCurrent()) loaded.value = true
|
||||
}
|
||||
|
||||
async function reset() {
|
||||
@@ -42,6 +47,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
||||
cards.value = []
|
||||
nextCursor.value = null
|
||||
started = false
|
||||
loaded.value = false
|
||||
await loadMore()
|
||||
}
|
||||
|
||||
@@ -50,7 +56,8 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
||||
|
||||
const hasMore = computed(() => !started || nextCursor.value !== null)
|
||||
const isEmpty = computed(
|
||||
() => !loading.value && cards.value.length === 0 && error.value === null
|
||||
() => loaded.value && !loading.value
|
||||
&& cards.value.length === 0 && error.value === null
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -34,10 +34,20 @@ export const useTagStore = defineStore('tags', () => {
|
||||
}
|
||||
|
||||
async function loadFandoms() {
|
||||
fandomCache.value = await api.get('/api/tags/autocomplete', {
|
||||
params: { q: ' ', kind: 'fandom', limit: 200 }
|
||||
})
|
||||
return fandomCache.value
|
||||
// List ALL fandoms via the cursor-paged tag directory. (#712: the old impl
|
||||
// abused /tags/autocomplete with q=' ', which the backend strips to empty
|
||||
// and returns [] — so the picker showed no existing fandoms.)
|
||||
const all = []
|
||||
let cursor = null
|
||||
do {
|
||||
const params = { kind: 'fandom', limit: 200 }
|
||||
if (cursor) params.cursor = cursor
|
||||
const body = await api.get('/api/tags/directory', { params })
|
||||
all.push(...(body.cards || []))
|
||||
cursor = body.next_cursor
|
||||
} while (cursor)
|
||||
fandomCache.value = all
|
||||
return all
|
||||
}
|
||||
|
||||
async function createFandom(name) {
|
||||
|
||||
@@ -76,9 +76,11 @@ onMounted(async () => {
|
||||
}
|
||||
.fc-artists__grid {
|
||||
display: grid;
|
||||
/* min(440px, 100%) so a card never exceeds the viewport — on phones the
|
||||
min-track collapses to 100% (single column) instead of overflowing. */
|
||||
grid-template-columns: repeat(auto-fill, minmax(min(440px, 100%), 1fr));
|
||||
/* min(360px, 100%) so a card never exceeds the viewport (phones collapse to
|
||||
one column). 360 keeps a LONE column under ~732px wide, so the 3/1 preview
|
||||
strip stays ≲244px tall and always fills the card width (no dead space),
|
||||
while giving 2+ columns on a normal desktop. */
|
||||
grid-template-columns: repeat(auto-fill, minmax(min(360px, 100%), 1fr));
|
||||
gap: 12px;
|
||||
}
|
||||
.fc-artists__sentinel {
|
||||
|
||||
@@ -52,11 +52,12 @@ async def test_create_tag_missing_name_400(client):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_tag_name_only_defaults_to_general(client):
|
||||
"""IR-style: name without kind and without `kind:` prefix → general."""
|
||||
"""IR-style: name without kind and without `kind:` prefix → general.
|
||||
#701: operator-entered names are Title-Cased at the create endpoint."""
|
||||
resp = await client.post("/api/tags", json={"name": "sunset"})
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["name"] == "sunset"
|
||||
assert body["name"] == "Sunset"
|
||||
assert body["kind"] == "general"
|
||||
|
||||
|
||||
@@ -73,21 +74,22 @@ async def test_create_tag_with_kind_prefix(client):
|
||||
@pytest.mark.asyncio
|
||||
async def test_explicit_kind_overrides_prefix_parsing(client):
|
||||
"""If caller passes explicit kind, don't re-parse the name —
|
||||
colon and prefix stay literal."""
|
||||
colon and prefix stay literal. #701: still Title-Cased as a single word."""
|
||||
resp = await client.post(
|
||||
"/api/tags", json={"name": "character:Saber", "kind": "general"}
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["name"] == "character:Saber"
|
||||
assert body["name"] == "Character:saber"
|
||||
assert body["kind"] == "general"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_prefix_kept_literal(client):
|
||||
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name."""
|
||||
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name.
|
||||
#701: Title-Cased as one word (no internal whitespace to split on)."""
|
||||
resp = await client.post("/api/tags", json={"name": "http://example.com"})
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["name"] == "http://example.com"
|
||||
assert body["name"] == "Http://example.com"
|
||||
assert body["kind"] == "general"
|
||||
|
||||
@@ -39,6 +39,24 @@ def test_cbz_alias(tmp_path):
|
||||
assert [n for n, _ in members] == ["p1.jpg"]
|
||||
|
||||
|
||||
def test_misnamed_zip_detected_and_extracted(tmp_path):
|
||||
"""A real zip whose filename has no usable extension (Patreon attachment
|
||||
URL-blob name) is detected by magic bytes and its members extracted —
|
||||
previously it was filed as an opaque attachment and never opened."""
|
||||
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
|
||||
_zip(z, {"a.jpg": b"img", "b.png": b"img2"})
|
||||
assert is_archive(z)
|
||||
with extract_archive(z) as members:
|
||||
assert sorted(n for n, _ in members) == ["a.jpg", "b.png"]
|
||||
|
||||
|
||||
def test_non_archive_with_dotted_name_is_not_archive(tmp_path):
|
||||
"""A non-archive file with a dotted/extension-less name isn't misdetected."""
|
||||
f = tmp_path / "01_https___www.patreon.com_media-u_v3_999"
|
||||
f.write_bytes(b"\x89PNG\r\n\x1a\n not really but not a zip")
|
||||
assert not is_archive(f)
|
||||
|
||||
|
||||
def test_corrupt_archive_yields_nothing(tmp_path):
|
||||
bad = tmp_path / "broken.zip"
|
||||
bad.write_bytes(b"not a zip at all")
|
||||
|
||||
@@ -421,6 +421,34 @@ async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path):
|
||||
assert result["has_more"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_live_progress_updates_running_event(
|
||||
source_id, sync_engine, tmp_path, db,
|
||||
):
|
||||
"""plan #709: live counts land in the RUNNING event's metadata.live; a
|
||||
finalized event is left alone (status guard)."""
|
||||
from backend.app.models import DownloadEvent
|
||||
|
||||
ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path))
|
||||
running = DownloadEvent(source_id=source_id, status="running")
|
||||
done = DownloadEvent(source_id=source_id, status="ok")
|
||||
db.add_all([running, done])
|
||||
await db.commit()
|
||||
|
||||
counts = {"downloaded": 3, "skipped": 1, "errors": 0, "quarantined": 0, "posts": 4}
|
||||
ing._write_live_progress(running.id, counts)
|
||||
ing._write_live_progress(done.id, {"downloaded": 9}) # guarded: status != running
|
||||
|
||||
live = (await db.execute(
|
||||
select(DownloadEvent.metadata_).where(DownloadEvent.id == running.id)
|
||||
)).scalar_one()
|
||||
assert live["live"] == counts
|
||||
done_meta = (await db.execute(
|
||||
select(DownloadEvent.metadata_).where(DownloadEvent.id == done.id)
|
||||
)).scalar_one()
|
||||
assert "live" not in (done_meta or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
|
||||
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""#713 part 2: re-extract archive PostAttachments that were filed opaquely
|
||||
(magic-byte gate missed them) and link their members to the post."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services import cleanup_service
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _jpeg(color, size=256):
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (size, size), color).save(buf, "JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch):
|
||||
from backend.app.tasks import ml as ml_mod
|
||||
from backend.app.tasks import thumbnail as thumb_mod
|
||||
|
||||
# No broker in this path — the post-import enqueue is best-effort anyway.
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
images_root.mkdir()
|
||||
|
||||
artist = Artist(name="Bob", slug="bob")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
source = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/bob", enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add(source)
|
||||
db_sync.flush()
|
||||
post = Post(
|
||||
source_id=source.id, artist_id=artist.id, external_post_id="59102952",
|
||||
post_url="https://www.patreon.com/posts/59102952",
|
||||
)
|
||||
db_sync.add(post)
|
||||
db_sync.flush()
|
||||
|
||||
# A real zip stored under a mangled / extension-less name (the failure case).
|
||||
store_dir = images_root / "attachments" / "abc"
|
||||
store_dir.mkdir(parents=True)
|
||||
arc = store_dir / "01_https___www.patreon.com_media-u_v3_59102952"
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("inside.jpg", _jpeg("red"))
|
||||
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
|
||||
db_sync.add(PostAttachment(
|
||||
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
|
||||
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
|
||||
))
|
||||
db_sync.commit()
|
||||
|
||||
summary = cleanup_service.reextract_archive_attachments(
|
||||
db_sync, images_root=images_root,
|
||||
)
|
||||
assert summary["archives"] == 1
|
||||
assert summary["members_imported"] == 1
|
||||
assert summary["posts_touched"] == 1
|
||||
|
||||
images = db_sync.execute(select(ImageRecord)).scalars().all()
|
||||
assert len(images) == 1
|
||||
prov = db_sync.execute(
|
||||
select(ImageProvenance).where(ImageProvenance.post_id == post.id)
|
||||
).scalars().all()
|
||||
assert len(prov) == 1
|
||||
assert prov[0].image_record_id == images[0].id
|
||||
|
||||
# Idempotent — a second run imports nothing new (member dedups by sha256).
|
||||
again = cleanup_service.reextract_archive_attachments(
|
||||
db_sync, images_root=images_root,
|
||||
)
|
||||
assert again["members_imported"] == 0
|
||||
assert db_sync.execute(select(ImageRecord)).scalars().all() == images
|
||||
@@ -40,11 +40,21 @@ def test_probe_archive_corrupt_zip_clean_rejection(tmp_path):
|
||||
def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
|
||||
z = tmp_path / "sized.zip"
|
||||
_zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50})
|
||||
total, bad = safe_probe._inspect_archive(z, ".zip")
|
||||
total, bad = safe_probe._inspect_archive(z)
|
||||
assert total == 150
|
||||
assert bad is None
|
||||
|
||||
|
||||
def test_inspect_archive_detects_misnamed_zip(tmp_path):
|
||||
"""A zip with a mangled / extension-less name (Patreon URL-blob attachment)
|
||||
is still bomb-guarded + integrity-tested via magic-byte detection."""
|
||||
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
|
||||
_zip(z, {"a.txt": b"x" * 100})
|
||||
total, bad = safe_probe._inspect_archive(z)
|
||||
assert total == 100
|
||||
assert bad is None
|
||||
|
||||
|
||||
def test_run_probe_bomb_guard(tmp_path, monkeypatch):
|
||||
"""In-process call to _run_probe so the monkeypatched cap takes effect.
|
||||
The public probe_archive runs in a subprocess which re-imports the
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
"""#714: retro-normalize existing tags to the #701 canonical (Title Case +
|
||||
collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||
|
||||
normalize_existing_tags commits per group, so case-variant duplicates are
|
||||
seeded with DIRECT inserts (find_or_create would case-insensitively dedup
|
||||
them, which is exactly the pre-#701 state we're cleaning up), and assertions
|
||||
re-query via fresh selects rather than touching post-commit ORM entities.
|
||||
"""
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Tag, TagKind, image_tag
|
||||
from backend.app.models.tag_alias import TagAlias
|
||||
from backend.app.services.tag_service import (
|
||||
TagService,
|
||||
normalize_existing_tags,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
_IMG_SEQ = 0
|
||||
|
||||
|
||||
async def _img(db):
|
||||
"""Minimal valid ImageRecord; returns its id."""
|
||||
global _IMG_SEQ
|
||||
_IMG_SEQ += 1
|
||||
from backend.app.models import ImageRecord
|
||||
|
||||
rec = ImageRecord(
|
||||
path=f"/tmp/fc_norm_{_IMG_SEQ}.png",
|
||||
sha256=f"{_IMG_SEQ:064d}",
|
||||
size_bytes=1,
|
||||
mime="image/png",
|
||||
origin="uploaded",
|
||||
)
|
||||
db.add(rec)
|
||||
await db.flush()
|
||||
return rec.id
|
||||
|
||||
|
||||
async def _raw_tag(db, name, kind, fandom_id=None):
|
||||
"""Insert a Tag row verbatim — bypasses find_or_create's case-insensitive
|
||||
dedup so we can plant the case-variant duplicates #714 exists to fix."""
|
||||
t = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||
db.add(t)
|
||||
await db.flush()
|
||||
return t
|
||||
|
||||
|
||||
async def _link(db, image_id, tag_id, source="manual"):
|
||||
await db.execute(
|
||||
image_tag.insert().values(
|
||||
image_record_id=image_id, tag_id=tag_id, source=source
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _count_named(db, name, kind):
|
||||
return await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Tag)
|
||||
.where(Tag.name == name)
|
||||
.where(Tag.kind == kind)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_projection_counts(db):
|
||||
# Collision group with an already-canonical member (→ no rename, 1 merge).
|
||||
await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||
await _raw_tag(db, "Hatsune Miku", TagKind.character)
|
||||
# Lone mis-cased + extra whitespace (→ rename, no merge).
|
||||
await _raw_tag(db, "looking at viewer", TagKind.general)
|
||||
# Already canonical (→ no change).
|
||||
await _raw_tag(db, "Already Canonical", TagKind.general)
|
||||
|
||||
proj = await normalize_existing_tags(db, dry_run=True)
|
||||
|
||||
assert proj["collisions"] == 1
|
||||
assert proj["tags_to_merge"] == 1
|
||||
assert proj["tags_to_rename"] == 1 # only the whitespace group renames
|
||||
assert proj["total_changes"] == 2
|
||||
# Nothing mutated by a dry run.
|
||||
assert await _count_named(db, "hatsune miku", TagKind.character) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_merges_case_variants_and_repoints_images(db):
|
||||
a = await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||
b = await _raw_tag(db, "HATSUNE MIKU", TagKind.character)
|
||||
i1, i2, i3 = await _img(db), await _img(db), await _img(db)
|
||||
await _link(db, i1, a.id)
|
||||
await _link(db, i2, a.id)
|
||||
await _link(db, i2, b.id) # shared → must dedup on the survivor
|
||||
await _link(db, i3, b.id)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["errors"] == 0
|
||||
assert summary["merged"] == 1
|
||||
assert summary["renamed"] == 1
|
||||
assert summary["groups_processed"] == 1
|
||||
|
||||
# Exactly one canonical character tag survives; the variant is gone.
|
||||
assert await _count_named(db, "Hatsune Miku", TagKind.character) == 1
|
||||
total_chars = await db.scalar(
|
||||
select(func.count()).select_from(Tag).where(Tag.kind == TagKind.character)
|
||||
)
|
||||
assert total_chars == 1
|
||||
survivor_id = await db.scalar(
|
||||
select(Tag.id)
|
||||
.where(Tag.name == "Hatsune Miku")
|
||||
.where(Tag.kind == TagKind.character)
|
||||
)
|
||||
# Union of {i1,i2} and {i2,i3} with the shared i2 deduped == 3.
|
||||
assoc = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.where(image_tag.c.tag_id == survivor_id)
|
||||
)
|
||||
assert assoc == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_second_run_is_noop(db):
|
||||
await _raw_tag(db, "kafka", TagKind.character)
|
||||
await _raw_tag(db, "KAFKA", TagKind.character)
|
||||
|
||||
first = await normalize_existing_tags(db, dry_run=False)
|
||||
assert first["groups_processed"] == 1
|
||||
|
||||
proj = await normalize_existing_tags(db, dry_run=True)
|
||||
assert proj["total_changes"] == 0
|
||||
again = await normalize_existing_tags(db, dry_run=False)
|
||||
assert again["groups_processed"] == 0
|
||||
assert again["merged"] == 0
|
||||
assert again["renamed"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_name_different_fandom_not_merged(db):
|
||||
f1 = await _raw_tag(db, "Bleach", TagKind.fandom)
|
||||
f2 = await _raw_tag(db, "Naruto", TagKind.fandom)
|
||||
await _raw_tag(db, "alice", TagKind.character, fandom_id=f1.id)
|
||||
await _raw_tag(db, "Alice", TagKind.character, fandom_id=f2.id)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 0
|
||||
assert summary["renamed"] == 1 # only the f1 "alice" recases
|
||||
# Both characters survive — same canonical name, distinct fandoms.
|
||||
assert await _count_named(db, "Alice", TagKind.character) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_name_different_kind_not_merged(db):
|
||||
await _raw_tag(db, "forge", TagKind.artist)
|
||||
await _raw_tag(db, "Forge", TagKind.general)
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 0
|
||||
assert await _count_named(db, "Forge", TagKind.artist) == 1
|
||||
assert await _count_named(db, "Forge", TagKind.general) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ml_known_loser_keeps_protective_alias(db):
|
||||
# Survivor is the already-canonical (manual, ML-unknown) tag; the ML-sourced
|
||||
# variant merges away and must leave an alias so the tagger re-resolves it.
|
||||
canon = await _raw_tag(db, "Nemu", TagKind.general)
|
||||
canon_id = canon.id # capture before the service commits
|
||||
ml = await _raw_tag(db, "nemu", TagKind.general)
|
||||
img = await _img(db)
|
||||
await _link(db, img, ml.id, source="ml_auto")
|
||||
|
||||
summary = await normalize_existing_tags(db, dry_run=False)
|
||||
|
||||
assert summary["merged"] == 1
|
||||
assert summary["aliases_created"] == 1
|
||||
assert await _count_named(db, "Nemu", TagKind.general) == 1
|
||||
alias_rows = await db.scalar(
|
||||
select(func.count())
|
||||
.select_from(TagAlias)
|
||||
.where(TagAlias.alias_string == "nemu")
|
||||
.where(TagAlias.canonical_tag_id == canon_id)
|
||||
)
|
||||
assert alias_rows == 1
|
||||
@@ -59,6 +59,27 @@ async def test_character_with_fandom(db):
|
||||
assert char.fandom_id == fandom.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_find_or_create_dedups_case_insensitive(db):
|
||||
"""#701: find_or_create matches case-insensitively so a differently-cased
|
||||
entry finds the existing tag instead of forking. (Display Title-Casing is
|
||||
applied at the create API, not here — this path is shared with the ML
|
||||
tagger, which keeps the booru vocabulary's casing.)"""
|
||||
svc = TagService(db)
|
||||
t1 = await svc.find_or_create("hatsune miku", TagKind.character)
|
||||
assert t1.name == "hatsune miku" # stored as given; not recased here
|
||||
t2 = await svc.find_or_create("HATSUNE MIKU", TagKind.character)
|
||||
assert t2.id == t1.id # case variant → same tag, no fork
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalize_tag_name():
|
||||
from backend.app.services.tag_service import normalize_tag_name
|
||||
assert normalize_tag_name("hatsune miku") == "Hatsune Miku"
|
||||
assert normalize_tag_name(" looking at viewer ") == "Looking At Viewer"
|
||||
assert normalize_tag_name("HATSUNE MIKU") == "Hatsune Miku"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_character_same_name_different_fandoms(db):
|
||||
svc = TagService(db)
|
||||
|
||||
Reference in New Issue
Block a user