@@ -245,6 +245,32 @@ async def tags_reset_content():
|
|||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/tags/normalize", methods=["POST"])
|
||||||
|
async def tags_normalize():
|
||||||
|
"""#714: retro-normalize existing tags to the #701 canonical form (Title
|
||||||
|
Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||||
|
|
||||||
|
dry_run=true (default) returns a projection inline — group/collision/rename
|
||||||
|
counts + a sample of the changes — so the UI shows exactly what'll happen.
|
||||||
|
dry_run=false dispatches the long-running maintenance task (the merge FK
|
||||||
|
repoints can touch many tags); the UI tails the activity dashboard for the
|
||||||
|
summary. Idempotent; back up first (the merges are irreversible)."""
|
||||||
|
from ..services.tag_service import normalize_existing_tags
|
||||||
|
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
dry_run = bool(body.get("dry_run", True))
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await normalize_existing_tags(session, dry_run=True)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
from ..tasks.admin import normalize_tags_task
|
||||||
|
|
||||||
|
async_result = normalize_tags_task.delay()
|
||||||
|
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||||
|
|
||||||
|
|
||||||
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
|
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
|
||||||
async def db_stats():
|
async def db_stats():
|
||||||
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
|
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
|
||||||
@@ -289,3 +315,14 @@ async def trigger_vacuum():
|
|||||||
|
|
||||||
vacuum_analyze.delay()
|
vacuum_analyze.delay()
|
||||||
return jsonify({"status": "queued"}), 202
|
return jsonify({"status": "queued"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
|
||||||
|
async def trigger_reextract_archives():
|
||||||
|
"""Operator-triggered re-extract (#713): PostAttachments that are actually
|
||||||
|
archives but were filed opaquely (pre magic-byte gate) get extracted and
|
||||||
|
their members linked to the post. Idempotent; runs on the maintenance queue."""
|
||||||
|
from ..tasks.admin import reextract_archive_attachments_task
|
||||||
|
|
||||||
|
async_result = reextract_archive_attachments_task.delay()
|
||||||
|
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||||
|
|||||||
@@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
|
|||||||
"bytes_downloaded": event.bytes_downloaded,
|
"bytes_downloaded": event.bytes_downloaded,
|
||||||
"error": event.error,
|
"error": event.error,
|
||||||
"summary": _summary_from_metadata(event.metadata_),
|
"summary": _summary_from_metadata(event.metadata_),
|
||||||
|
# plan #709: mid-walk live counts for a RUNNING native-ingester event
|
||||||
|
# (None otherwise; phase 3 overwrites metadata with run_stats on finish).
|
||||||
|
"live": (event.metadata_ or {}).get("live"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from ..services.tag_service import (
|
|||||||
TagMergeConflict,
|
TagMergeConflict,
|
||||||
TagService,
|
TagService,
|
||||||
TagValidationError,
|
TagValidationError,
|
||||||
|
normalize_tag_name,
|
||||||
)
|
)
|
||||||
from ..utils.tag_prefix import parse_kind_prefix
|
from ..utils.tag_prefix import parse_kind_prefix
|
||||||
|
|
||||||
@@ -141,6 +142,11 @@ async def create_tag():
|
|||||||
|
|
||||||
fandom_id = body.get("fandom_id")
|
fandom_id = body.get("fandom_id")
|
||||||
|
|
||||||
|
# #701: Title-Case operator-entered tags. Only here (the explicit create
|
||||||
|
# endpoint), NOT in the shared find_or_create — the ML tagger uses that path
|
||||||
|
# and must keep the booru vocabulary's casing for allowlist matching.
|
||||||
|
name = normalize_tag_name(name)
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
svc = TagService(session)
|
svc = TagService(session)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -16,9 +16,45 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
|
ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
|
||||||
|
|
||||||
|
# Magic-byte signatures, so an archive with a mangled / extension-less filename
|
||||||
|
# is still recognised. Patreon attachment download URLs sanitize to names like
|
||||||
|
# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk
|
||||||
|
# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed
|
||||||
|
# those as opaque PostAttachments and NEVER extracted them (operator-flagged
|
||||||
|
# 2026-06-06). Detection is by extension first (cheap), then header sniff.
|
||||||
|
_RAR_MAGIC = b"Rar!\x1a\x07"
|
||||||
|
_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c"
|
||||||
|
|
||||||
|
|
||||||
|
def detect_archive_format(path: Path) -> str | None:
|
||||||
|
"""Return "zip" | "rar" | "7z" for an archive, else None.
|
||||||
|
|
||||||
|
Trusts a known extension first, then falls back to magic-byte sniffing so a
|
||||||
|
mis-named or extension-less archive is still handled. (zip covers .cbz too.)
|
||||||
|
"""
|
||||||
|
ext = Path(path).suffix.lower()
|
||||||
|
if ext in (".zip", ".cbz"):
|
||||||
|
return "zip"
|
||||||
|
if ext == ".rar":
|
||||||
|
return "rar"
|
||||||
|
if ext == ".7z":
|
||||||
|
return "7z"
|
||||||
|
try:
|
||||||
|
if zipfile.is_zipfile(path):
|
||||||
|
return "zip"
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
head = fh.read(8)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
if head.startswith(_RAR_MAGIC):
|
||||||
|
return "rar"
|
||||||
|
if head.startswith(_7Z_MAGIC):
|
||||||
|
return "7z"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def is_archive(path: Path) -> bool:
|
def is_archive(path: Path) -> bool:
|
||||||
return Path(path).suffix.lower() in ARCHIVE_EXTS
|
return detect_archive_format(path) is not None
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
@@ -32,16 +68,16 @@ def extract_archive(path: Path):
|
|||||||
members: list[tuple[str, Path]] = []
|
members: list[tuple[str, Path]] = []
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
ext = Path(path).suffix.lower()
|
fmt = detect_archive_format(path)
|
||||||
if ext in (".zip", ".cbz"):
|
if fmt == "zip":
|
||||||
with zipfile.ZipFile(path) as zf:
|
with zipfile.ZipFile(path) as zf:
|
||||||
zf.extractall(base)
|
zf.extractall(base)
|
||||||
elif ext == ".rar":
|
elif fmt == "rar":
|
||||||
import rarfile
|
import rarfile
|
||||||
|
|
||||||
with rarfile.RarFile(path) as rf:
|
with rarfile.RarFile(path) as rf:
|
||||||
rf.extractall(base)
|
rf.extractall(base)
|
||||||
elif ext == ".7z":
|
elif fmt == "7z":
|
||||||
import py7zr
|
import py7zr
|
||||||
|
|
||||||
with py7zr.SevenZipFile(path, "r") as zf:
|
with py7zr.SevenZipFile(path, "r") as zf:
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ the one-and-done GS/IR migration tooling.)
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -22,6 +23,8 @@ from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
|||||||
from ..models.series_page import SeriesPage
|
from ..models.series_page import SeriesPage
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def project_artist_cascade(session: Session, *, slug: str) -> dict:
|
def project_artist_cascade(session: Session, *, slug: str) -> dict:
|
||||||
"""Read-only projection of what delete_artist_cascade would touch.
|
"""Read-only projection of what delete_artist_cascade would touch.
|
||||||
@@ -665,3 +668,139 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
|||||||
.where(LibraryAuditRun.status == "running")
|
.where(LibraryAuditRun.status == "running")
|
||||||
.values(status="cancelled", finished_at=datetime.now(UTC))
|
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# -- archive-attachment re-extraction (#713 part 2) ------------------------
|
||||||
|
|
||||||
|
_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
|
||||||
|
|
||||||
|
|
||||||
|
def _reextract_archive_to_post(
|
||||||
|
importer, archive_path: Path, post, source_row, artist, images_root: Path,
|
||||||
|
) -> list[int]:
|
||||||
|
"""Extract one stored archive and link its members to `post`.
|
||||||
|
|
||||||
|
The stored attachment has no adjacent sidecar (it lives in the sha-addressed
|
||||||
|
attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
|
||||||
|
library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
|
||||||
|
re-derives the artist from the path AND copies members relative to it, so the
|
||||||
|
members land in the real library and resolve to the right artist — then re-run
|
||||||
|
`attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
|
||||||
|
the members to the SAME Post (source_id + external_post_id). Removes only the
|
||||||
|
staged archive + sidecar afterward; the imported member files stay. Returns
|
||||||
|
the new member image ids.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from .archive_extractor import detect_archive_format
|
||||||
|
|
||||||
|
fmt = detect_archive_format(archive_path)
|
||||||
|
ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
|
||||||
|
platform = source_row.platform if source_row is not None else "imported"
|
||||||
|
sidecar = {
|
||||||
|
"category": source_row.platform if source_row is not None
|
||||||
|
else (post.raw_metadata or {}).get("category"),
|
||||||
|
"id": post.external_post_id,
|
||||||
|
"title": post.post_title or "",
|
||||||
|
"content": post.description or "",
|
||||||
|
"published_at": post.post_date.isoformat() if post.post_date else None,
|
||||||
|
"url": post.post_url,
|
||||||
|
}
|
||||||
|
work = images_root / artist.slug / platform / str(post.external_post_id)
|
||||||
|
work.mkdir(parents=True, exist_ok=True)
|
||||||
|
staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar
|
||||||
|
sidecar_path = staged.with_suffix(".json")
|
||||||
|
try:
|
||||||
|
shutil.copy2(archive_path, staged)
|
||||||
|
sidecar_path.write_text(json.dumps(sidecar))
|
||||||
|
res = importer.attach_in_place(staged, artist=artist, source=source_row)
|
||||||
|
return list(res.member_image_ids or [])
|
||||||
|
finally:
|
||||||
|
# Drop only the staged archive + sidecar; the extracted member files
|
||||||
|
# were copied into the library alongside them and must stay.
|
||||||
|
staged.unlink(missing_ok=True)
|
||||||
|
sidecar_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def reextract_archive_attachments(session: Session, *, images_root: Path) -> dict:
|
||||||
|
"""Re-process existing PostAttachments that are ACTUALLY archives but were
|
||||||
|
filed opaquely before #713 part 1 (extension-only is_archive missed mangled /
|
||||||
|
extension-less Patreon attachment names). For each: extract the members,
|
||||||
|
import them, and link them to the attachment's post.
|
||||||
|
|
||||||
|
Idempotent — members dedupe by sha256, the archive dedupes by sha — so it's
|
||||||
|
safe to run repeatedly. Returns a summary dict for task_run.metadata.
|
||||||
|
"""
|
||||||
|
from ..models import ImportSettings, Post, PostAttachment, Source
|
||||||
|
from ..tasks.ml import tag_and_embed
|
||||||
|
from ..tasks.thumbnail import generate_thumbnail
|
||||||
|
from .archive_extractor import is_archive
|
||||||
|
from .importer import Importer
|
||||||
|
from .thumbnailer import Thumbnailer
|
||||||
|
|
||||||
|
summary = {
|
||||||
|
"scanned": 0, "archives": 0, "members_imported": 0,
|
||||||
|
"posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
|
||||||
|
"errors": 0,
|
||||||
|
}
|
||||||
|
settings = ImportSettings.load_sync(session)
|
||||||
|
importer = Importer(
|
||||||
|
session=session, images_root=images_root, import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
|
||||||
|
)
|
||||||
|
|
||||||
|
attachments = session.execute(
|
||||||
|
select(PostAttachment).order_by(PostAttachment.id)
|
||||||
|
).scalars().all()
|
||||||
|
enqueue_ids: list[int] = []
|
||||||
|
for att in attachments:
|
||||||
|
summary["scanned"] += 1
|
||||||
|
stored = Path(att.path)
|
||||||
|
try:
|
||||||
|
if not stored.is_file() or not is_archive(stored):
|
||||||
|
continue
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
summary["archives"] += 1
|
||||||
|
if att.post_id is None:
|
||||||
|
summary["skipped_no_post"] += 1
|
||||||
|
continue
|
||||||
|
post = session.get(Post, att.post_id)
|
||||||
|
if post is None:
|
||||||
|
summary["skipped_no_post"] += 1
|
||||||
|
continue
|
||||||
|
artist = session.get(Artist, att.artist_id) if att.artist_id else None
|
||||||
|
if artist is None and post.artist_id:
|
||||||
|
artist = session.get(Artist, post.artist_id)
|
||||||
|
if artist is None or not artist.slug:
|
||||||
|
# The importer re-derives the artist from the staged path, so we need
|
||||||
|
# a real artist+slug to anchor under. (Shouldn't happen for
|
||||||
|
# subscription posts; skip rather than orphan the members.)
|
||||||
|
summary["skipped_no_artist"] += 1
|
||||||
|
continue
|
||||||
|
source_row = session.get(Source, post.source_id) if post.source_id else None
|
||||||
|
try:
|
||||||
|
ids = _reextract_archive_to_post(
|
||||||
|
importer, stored, post, source_row, artist, images_root,
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except Exception as exc: # one bad archive must not strand the rest
|
||||||
|
session.rollback()
|
||||||
|
summary["errors"] += 1
|
||||||
|
log.warning("re-extract failed for attachment %s: %s", att.id, exc)
|
||||||
|
continue
|
||||||
|
if ids:
|
||||||
|
summary["members_imported"] += len(ids)
|
||||||
|
summary["posts_touched"] += 1
|
||||||
|
enqueue_ids.extend(ids)
|
||||||
|
|
||||||
|
# Thumbnails + ML for the newly-imported members (best-effort; off the
|
||||||
|
# critical path — a Redis hiccup must not fail the whole re-extract).
|
||||||
|
for img_id in enqueue_ids:
|
||||||
|
try:
|
||||||
|
generate_thumbnail.delay(img_id)
|
||||||
|
tag_and_embed.delay(img_id)
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
|
||||||
|
return summary
|
||||||
|
|||||||
@@ -139,6 +139,8 @@ async def _run_native_ingester(
|
|||||||
resume_cursor=source_config.resume_cursor,
|
resume_cursor=source_config.resume_cursor,
|
||||||
time_budget_seconds=source_config.timeout,
|
time_budget_seconds=source_config.timeout,
|
||||||
posts_base=int(overrides.get("_backfill_posts", 0)),
|
posts_base=int(overrides.get("_backfill_posts", 0)),
|
||||||
|
# plan #709: live progress writes to this running event mid-walk.
|
||||||
|
event_id=ctx.get("event_id"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return dl_result, resolved_campaign_id
|
return dl_result, resolved_campaign_id
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ Plain-HTTP homelab: no secure-context Web API.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
@@ -48,6 +49,12 @@ DEAD_LETTER_THRESHOLD = 3
|
|||||||
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
|
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
|
||||||
_ERROR_MAX = 1000
|
_ERROR_MAX = 1000
|
||||||
|
|
||||||
|
# plan #709: throttle the live-progress write to the running DownloadEvent to one
|
||||||
|
# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a
|
||||||
|
# page is (page boundaries can be minutes apart on image-dense backfills, so a
|
||||||
|
# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s).
|
||||||
|
_LIVE_PROGRESS_INTERVAL = 5.0
|
||||||
|
|
||||||
|
|
||||||
class Ingester:
|
class Ingester:
|
||||||
"""Generic native-ingest orchestration. Subclass with a platform adapter
|
"""Generic native-ingest orchestration. Subclass with a platform adapter
|
||||||
@@ -92,6 +99,7 @@ class Ingester:
|
|||||||
time_budget_seconds: float = 870.0,
|
time_budget_seconds: float = 870.0,
|
||||||
seen_threshold: int = _TICK_SEEN_THRESHOLD,
|
seen_threshold: int = _TICK_SEEN_THRESHOLD,
|
||||||
posts_base: int = 0,
|
posts_base: int = 0,
|
||||||
|
event_id: int | None = None,
|
||||||
) -> DownloadResult:
|
) -> DownloadResult:
|
||||||
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
||||||
|
|
||||||
@@ -109,6 +117,7 @@ class Ingester:
|
|||||||
checkpoint = mode in ("backfill", "recovery")
|
checkpoint = mode in ("backfill", "recovery")
|
||||||
ledger_key = self._ledger_key
|
ledger_key = self._ledger_key
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
|
last_live = start # plan #709: last live-progress write timestamp
|
||||||
log_lines: list[str] = []
|
log_lines: list[str] = []
|
||||||
written: list[str] = []
|
written: list[str] = []
|
||||||
quarantined_paths: list[str] = []
|
quarantined_paths: list[str] = []
|
||||||
@@ -288,6 +297,19 @@ class Ingester:
|
|||||||
if to_fail:
|
if to_fail:
|
||||||
self._record_failures(source_id, to_fail)
|
self._record_failures(source_id, to_fail)
|
||||||
|
|
||||||
|
# plan #709: time-throttled live progress to the running event so
|
||||||
|
# the Downloads view ticks ~every 5s, independent of page size.
|
||||||
|
now = time.monotonic()
|
||||||
|
if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL:
|
||||||
|
last_live = now
|
||||||
|
self._write_live_progress(event_id, {
|
||||||
|
"downloaded": downloaded,
|
||||||
|
"skipped": skipped_count,
|
||||||
|
"errors": errors,
|
||||||
|
"quarantined": quarantined,
|
||||||
|
"posts": posts_processed,
|
||||||
|
})
|
||||||
|
|
||||||
if early_out:
|
if early_out:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -474,6 +496,25 @@ class Ingester:
|
|||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
def _write_live_progress(self, event_id: int, counts: dict) -> None:
|
||||||
|
"""Throttled mid-walk write of live counts to the RUNNING download_event
|
||||||
|
(plan #709) so the Downloads view shows progress before the chunk
|
||||||
|
finishes. A short session (never held across the walk); the `status =
|
||||||
|
'running'` guard avoids clobbering an event phase 3 already finalized.
|
||||||
|
`metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest
|
||||||
|
for phase 3 to overwrite with the final run_stats."""
|
||||||
|
with self.session_factory() as session:
|
||||||
|
session.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE download_event SET metadata = jsonb_set("
|
||||||
|
" coalesce(metadata, '{}'::jsonb), '{live}',"
|
||||||
|
" cast(:live AS jsonb)) "
|
||||||
|
"WHERE id = :eid AND status = 'running'"
|
||||||
|
),
|
||||||
|
{"live": json.dumps(counts), "eid": event_id},
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
def _still_running(self, source_id: int) -> bool:
|
def _still_running(self, source_id: int) -> bool:
|
||||||
"""True while the source is armed for a deep walk (plan #708 B4).
|
"""True while the source is armed for a deep walk (plan #708 B4).
|
||||||
|
|
||||||
|
|||||||
@@ -20,11 +20,20 @@ class ShowcaseService:
|
|||||||
async def random_sample(self, limit: int = 60) -> list[dict]:
|
async def random_sample(self, limit: int = 60) -> list[dict]:
|
||||||
if limit < 1 or limit > 200:
|
if limit < 1 or limit > 200:
|
||||||
raise ValueError("limit must be between 1 and 200")
|
raise ValueError("limit must be between 1 and 200")
|
||||||
|
# Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows
|
||||||
|
# from each sampled page, so sequentially-imported near-duplicates
|
||||||
|
# (multi-image posts, variant sets) come back adjacent and cluster in the
|
||||||
|
# showcase ("three near-identical in a row"). Sampling a multiple of
|
||||||
|
# `limit` spans more pages, and ORDER BY random() before taking `limit`
|
||||||
|
# breaks the physical adjacency — far better spread, still cheap
|
||||||
|
# (random() over a few hundred rows, not the whole table).
|
||||||
|
oversample = min(limit * 5, 1000)
|
||||||
stmt = select(ImageRecord).from_statement(
|
stmt = select(ImageRecord).from_statement(
|
||||||
text(
|
text(
|
||||||
"SELECT * FROM image_record "
|
"SELECT * FROM ("
|
||||||
"TABLESAMPLE SYSTEM_ROWS(:n)"
|
" SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)"
|
||||||
).bindparams(n=limit)
|
") sub ORDER BY random() LIMIT :n"
|
||||||
|
).bindparams(o=oversample, n=limit)
|
||||||
)
|
)
|
||||||
rows = (await self.session.execute(stmt)).scalars().all()
|
rows = (await self.session.execute(stmt)).scalars().all()
|
||||||
return [
|
return [
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
"""Tag CRUD + autocomplete + image-tag association."""
|
"""Tag CRUD + autocomplete + image-tag association."""
|
||||||
|
|
||||||
|
import logging
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@@ -12,6 +13,21 @@ from ..models import Tag, TagKind, image_tag
|
|||||||
from ..models.tag_allowlist import TagAllowlist
|
from ..models.tag_allowlist import TagAllowlist
|
||||||
from ..models.tag_reference_embedding import TagReferenceEmbedding
|
from ..models.tag_reference_embedding import TagReferenceEmbedding
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_tag_name(name: str) -> str:
|
||||||
|
"""Canonical tag form (#701): collapse whitespace + Title Case.
|
||||||
|
|
||||||
|
Per-word capitalize (NOT str.title(), which mangles apostrophes:
|
||||||
|
don't → Don'T). Lowercasing the word tail also folds ALL-CAPS input
|
||||||
|
(HATSUNE MIKU → Hatsune Miku); acronym casing is sacrificed, an accepted
|
||||||
|
Title-Case trade-off (operator-chosen 2026-06-06).
|
||||||
|
"""
|
||||||
|
return " ".join(
|
||||||
|
w[:1].upper() + w[1:].lower() for w in (name or "").split()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TagValidationError(ValueError):
|
class TagValidationError(ValueError):
|
||||||
"""Raised when tag construction breaks the kind/fandom rules."""
|
"""Raised when tag construction breaks the kind/fandom rules."""
|
||||||
@@ -71,6 +87,10 @@ class TagService:
|
|||||||
- if fandom_id is set, the referenced tag must exist and have
|
- if fandom_id is set, the referenced tag must exist and have
|
||||||
kind == TagKind.fandom
|
kind == TagKind.fandom
|
||||||
"""
|
"""
|
||||||
|
# NOTE: case is NOT normalized here — find_or_create is the shared path
|
||||||
|
# the ML tagger / allowlist also use, and Title-Casing the booru
|
||||||
|
# vocabulary breaks allowlist matching. Display-casing (Title Case) is
|
||||||
|
# applied at the user-entry layer (api/tags create_tag) only (#701).
|
||||||
name = name.strip()
|
name = name.strip()
|
||||||
if not name:
|
if not name:
|
||||||
raise TagValidationError("Tag name cannot be empty")
|
raise TagValidationError("Tag name cannot be empty")
|
||||||
@@ -93,13 +113,17 @@ class TagService:
|
|||||||
# inserts; without the savepoint the outer transaction would
|
# inserts; without the savepoint the outer transaction would
|
||||||
# poison and the calling request crashes. Mirrors
|
# poison and the calling request crashes. Mirrors
|
||||||
# importer._get_or_create.
|
# importer._get_or_create.
|
||||||
|
# Case-insensitive match (#701): a normalized (Title Case) input must
|
||||||
|
# find an existing differently-cased tag instead of forking a duplicate.
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Tag)
|
select(Tag)
|
||||||
.where(Tag.name == name)
|
.where(func.lower(Tag.name) == name.lower())
|
||||||
.where(Tag.kind == kind)
|
.where(Tag.kind == kind)
|
||||||
.where(
|
.where(
|
||||||
Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id
|
Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id
|
||||||
)
|
)
|
||||||
|
.order_by(Tag.id)
|
||||||
|
.limit(1)
|
||||||
)
|
)
|
||||||
existing = (await self.session.execute(stmt)).scalar_one_or_none()
|
existing = (await self.session.execute(stmt)).scalar_one_or_none()
|
||||||
if existing:
|
if existing:
|
||||||
@@ -249,9 +273,11 @@ class TagService:
|
|||||||
if tag is None:
|
if tag is None:
|
||||||
raise TagValidationError(f"Tag {tag_id} not found")
|
raise TagValidationError(f"Tag {tag_id} not found")
|
||||||
|
|
||||||
|
# Case-insensitive clash (#701) — renaming onto a differently-cased tag
|
||||||
|
# is still a merge, not a silent fork.
|
||||||
clash_stmt = (
|
clash_stmt = (
|
||||||
select(Tag)
|
select(Tag)
|
||||||
.where(Tag.name == new_name)
|
.where(func.lower(Tag.name) == new_name.lower())
|
||||||
.where(Tag.kind == tag.kind)
|
.where(Tag.kind == tag.kind)
|
||||||
.where(
|
.where(
|
||||||
Tag.fandom_id.is_(None)
|
Tag.fandom_id.is_(None)
|
||||||
@@ -259,6 +285,8 @@ class TagService:
|
|||||||
else Tag.fandom_id == tag.fandom_id
|
else Tag.fandom_id == tag.fandom_id
|
||||||
)
|
)
|
||||||
.where(Tag.id != tag_id)
|
.where(Tag.id != tag_id)
|
||||||
|
.order_by(Tag.id)
|
||||||
|
.limit(1)
|
||||||
)
|
)
|
||||||
clash = (await self.session.execute(clash_stmt)).scalar_one_or_none()
|
clash = (await self.session.execute(clash_stmt)).scalar_one_or_none()
|
||||||
if clash is not None:
|
if clash is not None:
|
||||||
@@ -501,6 +529,21 @@ class TagService:
|
|||||||
update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
|
update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]:
|
||||||
|
"""image_tag row counts keyed by tag_id, for the survivor heuristic
|
||||||
|
(the best-connected tag in a collision group survives → fewest
|
||||||
|
FK repoints). Tags with zero associations are absent from the map."""
|
||||||
|
if not tag_ids:
|
||||||
|
return {}
|
||||||
|
rows = (
|
||||||
|
await self.session.execute(
|
||||||
|
select(image_tag.c.tag_id, func.count())
|
||||||
|
.where(image_tag.c.tag_id.in_(tag_ids))
|
||||||
|
.group_by(image_tag.c.tag_id)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {tid: int(n) for tid, n in rows}
|
||||||
|
|
||||||
async def _create_protective_aliases(
|
async def _create_protective_aliases(
|
||||||
self, src_name: str, src_kind: TagKind, tgt: int
|
self, src_name: str, src_kind: TagKind, tgt: int
|
||||||
) -> bool:
|
) -> bool:
|
||||||
@@ -548,3 +591,161 @@ class TagService:
|
|||||||
if res.rowcount:
|
if res.rowcount:
|
||||||
created = True
|
created = True
|
||||||
return created
|
return created
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #714: retro-normalize existing tags to the #701 canonical (Title Case +
|
||||||
|
# collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_NORMALIZE_SAMPLE_CAP = 50
|
||||||
|
|
||||||
|
|
||||||
|
def _group_existing_tags(
|
||||||
|
rows,
|
||||||
|
) -> dict[tuple, list[tuple[int, str]]]:
|
||||||
|
"""Group (id, name, kind, fandom_id) rows by their post-normalization
|
||||||
|
identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that
|
||||||
|
would collapse to the same canonical lives in ONE group, so the canonical
|
||||||
|
form is unique within (kind, fandom) once each group is resolved."""
|
||||||
|
groups: dict[tuple, list[tuple[int, str]]] = {}
|
||||||
|
for tag_id, name, kind, fandom_id in rows:
|
||||||
|
canonical = normalize_tag_name(name)
|
||||||
|
key = (kind, fandom_id if fandom_id is not None else -1, canonical)
|
||||||
|
groups.setdefault(key, []).append((tag_id, name))
|
||||||
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool:
|
||||||
|
"""A group is already canonical iff it's a single member whose name equals
|
||||||
|
the canonical form. Anything else (a collision, or a lone mis-cased tag)
|
||||||
|
needs work."""
|
||||||
|
if len(members) > 1:
|
||||||
|
return True
|
||||||
|
return members[0][1] != canonical
|
||||||
|
|
||||||
|
|
||||||
|
def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
|
||||||
|
"""The tag with the most image associations (→ fewest FK repoints when it
|
||||||
|
survives), tie-broken to the lowest id for determinism. Module-level so the
|
||||||
|
key closes over its parameter, not a loop variable (ruff B023)."""
|
||||||
|
return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid))
|
||||||
|
|
||||||
|
|
||||||
|
async def normalize_existing_tags(
|
||||||
|
session: AsyncSession, *, dry_run: bool = False
|
||||||
|
) -> dict:
|
||||||
|
"""Convert the back-catalog to the #701 canonical tag form.
|
||||||
|
|
||||||
|
For each (kind, fandom, canonical) group: pick a survivor, merge any
|
||||||
|
case/whitespace-variant siblings INTO it via the tested merge path
|
||||||
|
(TagService._do_merge — FK repoints + protective aliases), then rename the
|
||||||
|
survivor to the canonical form. Idempotent: a group that is already a lone
|
||||||
|
canonical tag is a no-op, so re-running is safe.
|
||||||
|
|
||||||
|
dry_run=True returns a projection (counts + a sample of the changes) with no
|
||||||
|
mutations. Live runs commit per group and isolate failures per group so one
|
||||||
|
bad group can't strand the rest.
|
||||||
|
|
||||||
|
Returns (dry_run):
|
||||||
|
{"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R,
|
||||||
|
"total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
|
||||||
|
Returns (live):
|
||||||
|
{"groups_processed", "merged", "renamed", "aliases_created", "errors",
|
||||||
|
"sample": [...]}
|
||||||
|
"""
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
groups = _group_existing_tags(rows)
|
||||||
|
|
||||||
|
# Deterministic sample/ordering: by kind then canonical name.
|
||||||
|
touched = sorted(
|
||||||
|
(
|
||||||
|
(key, members)
|
||||||
|
for key, members in groups.items()
|
||||||
|
if _group_needs_change(key[2], members)
|
||||||
|
),
|
||||||
|
key=lambda km: (
|
||||||
|
km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]),
|
||||||
|
km[0][2].lower(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
sample = [
|
||||||
|
{
|
||||||
|
"to": key[2],
|
||||||
|
"from": [name for _id, name in members],
|
||||||
|
"kind": key[0].value if hasattr(key[0], "value") else str(key[0]),
|
||||||
|
"merge": len(members) > 1,
|
||||||
|
}
|
||||||
|
for key, members in touched[:_NORMALIZE_SAMPLE_CAP]
|
||||||
|
]
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
collisions = sum(1 for key, m in touched if len(m) > 1)
|
||||||
|
tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1)
|
||||||
|
# A group renames iff the canonical form isn't already one of its
|
||||||
|
# members' exact names (else that member is picked as survivor → no
|
||||||
|
# rename, the rest merge into it).
|
||||||
|
tags_to_rename = sum(
|
||||||
|
1
|
||||||
|
for key, m in touched
|
||||||
|
if key[2] not in {name for _id, name in m}
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"groups": len(groups),
|
||||||
|
"collisions": collisions,
|
||||||
|
"tags_to_merge": tags_to_merge,
|
||||||
|
"tags_to_rename": tags_to_rename,
|
||||||
|
"total_changes": len(touched),
|
||||||
|
"sample": sample,
|
||||||
|
}
|
||||||
|
|
||||||
|
svc = TagService(session)
|
||||||
|
summary = {
|
||||||
|
"groups_processed": 0,
|
||||||
|
"merged": 0,
|
||||||
|
"renamed": 0,
|
||||||
|
"aliases_created": 0,
|
||||||
|
"errors": 0,
|
||||||
|
"sample": sample,
|
||||||
|
}
|
||||||
|
for key, members in touched:
|
||||||
|
canonical = key[2]
|
||||||
|
names_by_id = dict(members)
|
||||||
|
# Survivor: prefer a member already named canonically (no rename, no
|
||||||
|
# self-alias); else the best-connected (fewest FK repoints); else
|
||||||
|
# lowest id for determinism.
|
||||||
|
survivor_id = next(
|
||||||
|
(tid for tid, name in members if name == canonical), None
|
||||||
|
)
|
||||||
|
if survivor_id is None:
|
||||||
|
counts = await svc._image_assoc_counts(list(names_by_id))
|
||||||
|
survivor_id = _best_connected(list(names_by_id), counts)
|
||||||
|
loser_ids = [tid for tid in names_by_id if tid != survivor_id]
|
||||||
|
try:
|
||||||
|
survivor = await session.get(Tag, survivor_id)
|
||||||
|
for loser_id in loser_ids:
|
||||||
|
loser = await session.get(Tag, loser_id)
|
||||||
|
if loser is None:
|
||||||
|
continue
|
||||||
|
result = await svc._do_merge(loser, survivor)
|
||||||
|
if result.alias_created:
|
||||||
|
summary["aliases_created"] += 1
|
||||||
|
if survivor.name != canonical:
|
||||||
|
survivor.name = canonical
|
||||||
|
await session.flush()
|
||||||
|
summary["renamed"] += 1
|
||||||
|
await session.commit()
|
||||||
|
summary["groups_processed"] += 1
|
||||||
|
summary["merged"] += len(loser_ids)
|
||||||
|
except Exception as exc: # one bad group must not strand the rest
|
||||||
|
await session.rollback()
|
||||||
|
summary["errors"] += 1
|
||||||
|
log.warning(
|
||||||
|
"tag normalize failed for group %r: %s", canonical, exc
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|||||||
@@ -55,3 +55,50 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
|
|||||||
return cleanup_service.delete_images(
|
return cleanup_service.delete_images(
|
||||||
session, image_ids=image_ids, images_root=IMAGES_ROOT,
|
session, image_ids=image_ids, images_root=IMAGES_ROOT,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.reextract_archive_attachments_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||||
|
)
|
||||||
|
def reextract_archive_attachments_task(self) -> dict:
|
||||||
|
"""Wraps cleanup_service.reextract_archive_attachments (#713 part 2):
|
||||||
|
re-extract PostAttachments that are actually archives but were filed
|
||||||
|
opaquely before the magic-byte gate, and link their members to the post."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
return cleanup_service.reextract_archive_attachments(
|
||||||
|
session, images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.admin.normalize_tags_task",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=15, retry_backoff_max=180, max_retries=1,
|
||||||
|
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
|
||||||
|
)
|
||||||
|
def normalize_tags_task(self) -> dict:
|
||||||
|
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
|
||||||
|
back-catalog and merge case/whitespace-variant duplicate tags via the
|
||||||
|
tested async merge path. Runs under its own asyncio loop + per-task async
|
||||||
|
engine (NullPool, disposed when the loop ends), mirroring download_source."""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from ..services.tag_service import normalize_existing_tags
|
||||||
|
from ._async_session import async_session_factory
|
||||||
|
|
||||||
|
async def _run() -> dict:
|
||||||
|
async_factory, async_engine = async_session_factory()
|
||||||
|
try:
|
||||||
|
async with async_factory() as session:
|
||||||
|
# normalize_existing_tags commits per group internally.
|
||||||
|
return await normalize_existing_tags(session, dry_run=False)
|
||||||
|
finally:
|
||||||
|
await async_engine.dispose()
|
||||||
|
|
||||||
|
return asyncio.run(_run())
|
||||||
|
|||||||
@@ -149,9 +149,8 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
|
|||||||
call the same code path.
|
call the same code path.
|
||||||
"""
|
"""
|
||||||
path = Path(path_str)
|
path = Path(path_str)
|
||||||
ext = path.suffix.lower()
|
|
||||||
try:
|
try:
|
||||||
total, test_bad = _inspect_archive(path, ext)
|
total, test_bad = _inspect_archive(path)
|
||||||
except Exception as exc: # noqa: BLE001 — clean rejection
|
except Exception as exc: # noqa: BLE001 — clean rejection
|
||||||
return ("error", f"{type(exc).__name__}: {exc}")
|
return ("error", f"{type(exc).__name__}: {exc}")
|
||||||
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
|
||||||
@@ -162,24 +161,29 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
|
|||||||
return ("ok", None)
|
return ("ok", None)
|
||||||
|
|
||||||
|
|
||||||
def _inspect_archive(path: Path, ext: str):
|
def _inspect_archive(path: Path):
|
||||||
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
|
||||||
for the archive. Format-specific; raises on a structurally-broken
|
for the archive. Format detected by extension OR magic bytes (so a
|
||||||
container (caught by the child as a clean rejection)."""
|
mis-named archive is still bomb-guarded + integrity-tested, matching the
|
||||||
if ext in (".zip", ".cbz"):
|
extractor's gate); raises on a structurally-broken container (caught by the
|
||||||
|
child as a clean rejection)."""
|
||||||
|
from ..services.archive_extractor import detect_archive_format
|
||||||
|
|
||||||
|
fmt = detect_archive_format(path)
|
||||||
|
if fmt == "zip":
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
with zipfile.ZipFile(path) as zf:
|
with zipfile.ZipFile(path) as zf:
|
||||||
total = sum(zi.file_size for zi in zf.infolist())
|
total = sum(zi.file_size for zi in zf.infolist())
|
||||||
return total, zf.testzip()
|
return total, zf.testzip()
|
||||||
if ext == ".rar":
|
if fmt == "rar":
|
||||||
import rarfile
|
import rarfile
|
||||||
|
|
||||||
with rarfile.RarFile(path) as rf:
|
with rarfile.RarFile(path) as rf:
|
||||||
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
|
||||||
rf.testrar()
|
rf.testrar()
|
||||||
return total, None
|
return total, None
|
||||||
if ext == ".7z":
|
if fmt == "7z":
|
||||||
import py7zr
|
import py7zr
|
||||||
|
|
||||||
with py7zr.SevenZipFile(path, "r") as zf:
|
with py7zr.SevenZipFile(path, "r") as zf:
|
||||||
@@ -187,5 +191,5 @@ def _inspect_archive(path: Path, ext: str):
|
|||||||
total = getattr(info, "uncompressed", None)
|
total = getattr(info, "uncompressed", None)
|
||||||
ok = zf.test() # True / None when all members pass
|
ok = zf.test() # True / None when all members pass
|
||||||
return total, (None if ok in (True, None) else "7z test reported corruption")
|
return total, (None if ok in (True, None) else "7z test reported corruption")
|
||||||
# Unknown extension — nothing to test; treat as clean.
|
# Not a recognised archive — nothing to test; treat as clean.
|
||||||
return None, None
|
return None, None
|
||||||
|
|||||||
@@ -49,9 +49,11 @@ function onCardClick() {
|
|||||||
position: relative;
|
position: relative;
|
||||||
display: grid; grid-template-columns: repeat(3, 1fr);
|
display: grid; grid-template-columns: repeat(3, 1fr);
|
||||||
gap: 2px; aspect-ratio: 3 / 1;
|
gap: 2px; aspect-ratio: 3 / 1;
|
||||||
/* Explicit floor + ceiling so tall source images can't escape the
|
/* Floor so tall source images can't escape the slot. NO max-height: an
|
||||||
preview slot even on browsers where aspect-ratio doesn't compute. */
|
aspect-ratio box capped by max-height shrinks its own WIDTH to keep the
|
||||||
min-height: 150px; max-height: 220px;
|
ratio, leaving dead space beside the previews on a wide (single-column)
|
||||||
|
card. The grid below keeps columns ≲400px so the strip stays a sane height. */
|
||||||
|
min-height: 120px;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
background: rgb(var(--v-theme-surface-light));
|
background: rgb(var(--v-theme-surface-light));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,7 +38,8 @@ const store = useTagStore()
|
|||||||
const selectedId = ref(null)
|
const selectedId = ref(null)
|
||||||
const newName = ref('')
|
const newName = ref('')
|
||||||
|
|
||||||
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
|
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
|
||||||
|
onMounted(() => store.loadFandoms())
|
||||||
|
|
||||||
async function onCreate() {
|
async function onCreate() {
|
||||||
const f = await store.createFandom(newName.value.trim())
|
const f = await store.createFandom(newName.value.trim())
|
||||||
|
|||||||
@@ -84,7 +84,8 @@ const busy = ref(false)
|
|||||||
const error = ref(null)
|
const error = ref(null)
|
||||||
const collision = ref(null)
|
const collision = ref(null)
|
||||||
|
|
||||||
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
|
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
|
||||||
|
onMounted(() => store.loadFandoms())
|
||||||
|
|
||||||
async function onCreate() {
|
async function onCreate() {
|
||||||
const name = newName.value.trim()
|
const name = newName.value.trim()
|
||||||
|
|||||||
@@ -106,7 +106,11 @@ function onKeyDown(ev) {
|
|||||||
// overlay's own Esc handling fire instead of closing the whole
|
// overlay's own Esc handling fire instead of closing the whole
|
||||||
// modal mid-interaction. Vuetify marks open overlays with
|
// modal mid-interaction. Vuetify marks open overlays with
|
||||||
// `.v-overlay--active`.
|
// `.v-overlay--active`.
|
||||||
if (document.querySelector('.v-overlay--active')) return
|
// EXCLUDE tooltips (`.v-tooltip`): they're also `.v-overlay--active` while
|
||||||
|
// shown, so one lingering after a hover/click (e.g. just after accepting a
|
||||||
|
// suggested tag) wrongly suppressed the close (#700). Only real interactive
|
||||||
|
// overlays (menus/dialogs) should keep ESC from closing the modal.
|
||||||
|
if (document.querySelector('.v-overlay--active:not(.v-tooltip)')) return
|
||||||
ev.preventDefault()
|
ev.preventDefault()
|
||||||
emit('close')
|
emit('close')
|
||||||
} else if (ev.key === 'ArrowLeft') {
|
} else if (ev.key === 'ArrowLeft') {
|
||||||
|
|||||||
@@ -2,45 +2,43 @@
|
|||||||
<aside class="fc-tag-panel" aria-label="Tags for this image">
|
<aside class="fc-tag-panel" aria-label="Tags for this image">
|
||||||
<h3 class="fc-tag-panel__title">Tags</h3>
|
<h3 class="fc-tag-panel__title">Tags</h3>
|
||||||
<div class="fc-tag-panel__chips">
|
<div class="fc-tag-panel__chips">
|
||||||
<v-chip
|
<!-- #711: the kebab + its menu used to be NESTED inside the v-chip, which
|
||||||
|
swallowed/mis-routed the click and mis-anchored the (teleported) menu
|
||||||
|
so it never opened. Render the kebab as a SIBLING of the chip and use
|
||||||
|
the standard v-menu activator slot — Vuetify wires the click and
|
||||||
|
stacks the overlay above the modal natively. -->
|
||||||
|
<span
|
||||||
v-for="tag in modal.current?.tags || []"
|
v-for="tag in modal.current?.tags || []"
|
||||||
:key="tag.id" size="small" closable
|
:key="tag.id" class="fc-tag-panel__chip"
|
||||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
|
||||||
@click:close="onRemove(tag.id)"
|
|
||||||
>
|
>
|
||||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
<v-chip
|
||||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
size="small" closable
|
||||||
<!-- Operator-flagged 2026-06-04: the `#activator` + `v-bind` menu
|
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||||
never opened inside this teleported modal. Drive it explicitly
|
@click:close="onRemove(tag.id)"
|
||||||
instead (same mechanism as the dialogs below, which work): the
|
>
|
||||||
icon toggles `openTagId` with @click.stop (shielding the chip's
|
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||||
close button), and `activator="parent"` + `:open-on-click=false`
|
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||||
anchors the menu for positioning only. One tag's menu open at a
|
</v-chip>
|
||||||
time, so a single id is enough. -->
|
<v-menu location="bottom end">
|
||||||
<span class="kebab-wrap">
|
<template #activator="{ props }">
|
||||||
<v-icon
|
<v-btn
|
||||||
size="x-small" class="ml-1 kebab-icon"
|
v-bind="props" icon="mdi-dots-vertical" size="x-small"
|
||||||
icon="mdi-dots-vertical"
|
variant="text" density="comfortable"
|
||||||
@click.stop="openTagId = openTagId === tag.id ? null : tag.id"
|
class="fc-tag-panel__kebab" @click.stop
|
||||||
/>
|
/>
|
||||||
<v-menu
|
</template>
|
||||||
:model-value="openTagId === tag.id"
|
<v-list density="compact">
|
||||||
activator="parent" :open-on-click="false"
|
<v-list-item @click="openRename(tag)">
|
||||||
@update:model-value="v => { if (!v) openTagId = null }"
|
<v-list-item-title>Rename…</v-list-item-title>
|
||||||
>
|
</v-list-item>
|
||||||
<v-list density="compact">
|
<v-list-item
|
||||||
<v-list-item @click="openRename(tag)">
|
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
|
||||||
<v-list-item-title>Rename…</v-list-item-title>
|
>
|
||||||
</v-list-item>
|
<v-list-item-title>Set fandom…</v-list-item-title>
|
||||||
<v-list-item
|
</v-list-item>
|
||||||
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
|
</v-list>
|
||||||
>
|
</v-menu>
|
||||||
<v-list-item-title>Set fandom…</v-list-item-title>
|
</span>
|
||||||
</v-list-item>
|
|
||||||
</v-list>
|
|
||||||
</v-menu>
|
|
||||||
</span>
|
|
||||||
</v-chip>
|
|
||||||
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -88,9 +86,6 @@ import FandomSetDialog from './FandomSetDialog.vue'
|
|||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const store = useTagStore()
|
const store = useTagStore()
|
||||||
const errorMsg = ref(null)
|
const errorMsg = ref(null)
|
||||||
// Which tag chip's kebab menu is open (only one at a time). Drives each
|
|
||||||
// chip menu's v-model so opening never depends on Vuetify's activator click.
|
|
||||||
const openTagId = ref(null)
|
|
||||||
|
|
||||||
const KIND_ICONS = {
|
const KIND_ICONS = {
|
||||||
general: 'mdi-tag', character: 'mdi-account-circle',
|
general: 'mdi-tag', character: 'mdi-account-circle',
|
||||||
@@ -153,6 +148,7 @@ async function onFandomUpdated() {
|
|||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
}
|
}
|
||||||
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||||
.kebab-wrap { display: inline-flex; align-items: center; }
|
.fc-tag-panel__chip { display: inline-flex; align-items: center; gap: 1px; }
|
||||||
.kebab-icon { cursor: pointer; }
|
.fc-tag-panel__kebab { opacity: 0.7; }
|
||||||
|
.fc-tag-panel__chip:hover .fc-tag-panel__kebab { opacity: 1; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
<template>
|
||||||
|
<!-- #713: re-extract PostAttachments that are really archives but were filed
|
||||||
|
opaquely before the magic-byte gate, and attach their images to the post. -->
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Re-extract archive attachments</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="text-body-2 mb-3">
|
||||||
|
Some posts attached an archive (zip) whose images weren't extracted
|
||||||
|
because the downloaded file had no usable extension. This scans existing
|
||||||
|
attachments, extracts any that are really archives, and attaches their
|
||||||
|
images to the post. Idempotent — safe to run more than once.
|
||||||
|
</p>
|
||||||
|
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||||
|
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
||||||
|
</v-btn>
|
||||||
|
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||||
|
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||||
|
</v-card-text>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../../composables/useApi.js'
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
|
|
||||||
|
const api = useApi()
|
||||||
|
const busy = ref(false)
|
||||||
|
const queued = ref(false)
|
||||||
|
|
||||||
|
async function run () {
|
||||||
|
busy.value = true
|
||||||
|
queued.value = false
|
||||||
|
try {
|
||||||
|
await api.post('/api/admin/maintenance/reextract-archives')
|
||||||
|
queued.value = true
|
||||||
|
toast({ text: 'Archive re-extract queued', type: 'success' })
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: e?.body?.detail || e?.message || 'Failed to queue', type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
@@ -15,6 +15,7 @@
|
|||||||
<AllowlistTable class="mt-4" />
|
<AllowlistTable class="mt-4" />
|
||||||
<AliasTable class="mt-4" />
|
<AliasTable class="mt-4" />
|
||||||
<DbMaintenanceCard class="mt-6" />
|
<DbMaintenanceCard class="mt-6" />
|
||||||
|
<ArchiveReextractCard class="mt-6" />
|
||||||
<BackupCard class="mt-6" />
|
<BackupCard class="mt-6" />
|
||||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||||
operates on the existing library which fits the Cleanup-tab
|
operates on the existing library which fits the Cleanup-tab
|
||||||
@@ -33,6 +34,7 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
|
|||||||
import AllowlistTable from './AllowlistTable.vue'
|
import AllowlistTable from './AllowlistTable.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||||
|
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||||
import BackupCard from './BackupCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
|
||||||
|
|||||||
@@ -136,6 +136,64 @@
|
|||||||
>Delete {{ resetPreview.count }} content tag(s) +
|
>Delete {{ resetPreview.count }} content tag(s) +
|
||||||
{{ resetPreview.applications }} application(s)</v-btn>
|
{{ resetPreview.applications }} application(s)</v-btn>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<v-divider class="my-5" />
|
||||||
|
|
||||||
|
<!-- #714: standardize existing tags to the canonical Title-Case form
|
||||||
|
and merge case/whitespace-variant duplicates. -->
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
<strong>Standardize tag casing.</strong>
|
||||||
|
New tags are saved Title Case, but older tags keep whatever casing they
|
||||||
|
were created with. This renames every existing tag to
|
||||||
|
<code>Title Case</code> (collapsing extra spaces) and
|
||||||
|
<strong>merges</strong> tags that differ only by case or spacing
|
||||||
|
(e.g. <code>hatsune miku</code> + <code>Hatsune Miku</code>)
|
||||||
|
into one — repointing their images, allowlist entries and series pages.
|
||||||
|
</p>
|
||||||
|
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||||
|
The merges are irreversible — back up first
|
||||||
|
(Settings → Maintenance → Backup). Safe to run more than once.
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify"
|
||||||
|
:loading="loadingNormPreview"
|
||||||
|
class="mb-3"
|
||||||
|
@click="onNormPreview"
|
||||||
|
>Preview tag standardization</v-btn>
|
||||||
|
|
||||||
|
<div v-if="normPreview">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
<strong>{{ normPreview.total_changes }}</strong> tag group(s) to
|
||||||
|
change — <strong>{{ normPreview.tags_to_rename }}</strong> rename(s),
|
||||||
|
<strong>{{ normPreview.collisions }}</strong> collision(s) merging
|
||||||
|
<strong>{{ normPreview.tags_to_merge }}</strong> tag(s) away.
|
||||||
|
</p>
|
||||||
|
<div v-if="normPreview.sample?.length" class="fc-name-grid mb-3">
|
||||||
|
<span
|
||||||
|
v-for="s in normPreview.sample" :key="s.to + s.kind"
|
||||||
|
class="fc-name"
|
||||||
|
>
|
||||||
|
<template v-if="s.merge">{{ s.from.join(' + ') }} → </template>{{ s.to }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<v-btn
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-format-letter-case"
|
||||||
|
:disabled="!normPreview.total_changes"
|
||||||
|
:loading="normCommitting"
|
||||||
|
@click="onNormCommit"
|
||||||
|
>Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
|
||||||
|
<span
|
||||||
|
v-if="normResult"
|
||||||
|
class="ml-3 text-caption"
|
||||||
|
:class="normResult === 'ok' ? 'text-success' : 'text-warning'"
|
||||||
|
>
|
||||||
|
<template v-if="normResult === 'ok'">Standardization complete ✓</template>
|
||||||
|
<template v-else>Finished with status: {{ normResult }}</template>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
@@ -155,6 +213,10 @@ const kindCommitting = ref(false)
|
|||||||
const resetPreview = ref(null)
|
const resetPreview = ref(null)
|
||||||
const loadingResetPreview = ref(false)
|
const loadingResetPreview = ref(false)
|
||||||
const resetCommitting = ref(false)
|
const resetCommitting = ref(false)
|
||||||
|
const normPreview = ref(null)
|
||||||
|
const loadingNormPreview = ref(false)
|
||||||
|
const normCommitting = ref(false)
|
||||||
|
const normResult = ref(null)
|
||||||
|
|
||||||
async function onPreview() {
|
async function onPreview() {
|
||||||
loadingPreview.value = true
|
loadingPreview.value = true
|
||||||
@@ -212,6 +274,33 @@ async function onResetCommit() {
|
|||||||
resetCommitting.value = false
|
resetCommitting.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onNormPreview() {
|
||||||
|
loadingNormPreview.value = true
|
||||||
|
normResult.value = null
|
||||||
|
try {
|
||||||
|
normPreview.value = await store.normalizeTags({ dryRun: true })
|
||||||
|
} finally {
|
||||||
|
loadingNormPreview.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onNormCommit() {
|
||||||
|
normCommitting.value = true
|
||||||
|
normResult.value = null
|
||||||
|
try {
|
||||||
|
// Long op (FK repoints): enqueue the maintenance task, then tail the
|
||||||
|
// activity dashboard until its row reaches a terminal status. (The
|
||||||
|
// per-run summary dict isn't exposed by /activity/runs, so we surface
|
||||||
|
// the terminal status — the dry-run preview is the detailed view.)
|
||||||
|
const { task_id: taskId } = await store.normalizeTags({ dryRun: false })
|
||||||
|
const row = await store.pollTaskUntilDone(taskId)
|
||||||
|
normResult.value = row?.status || 'ok'
|
||||||
|
normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
|
||||||
|
} finally {
|
||||||
|
normCommitting.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -23,6 +23,18 @@
|
|||||||
<span class="fc-active__dot" />
|
<span class="fc-active__dot" />
|
||||||
<PlatformChip :platform="e.platform" size="x-small" />
|
<PlatformChip :platform="e.platform" size="x-small" />
|
||||||
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
|
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
|
||||||
|
<!-- #709: live per-file progress for native (Patreon) downloads. -->
|
||||||
|
<span v-if="e.live" class="fc-active__counts">
|
||||||
|
<span class="fc-active__count" title="downloaded">↓ {{ e.live.downloaded }}</span>
|
||||||
|
<span v-if="e.live.skipped" class="fc-active__count" title="skipped">
|
||||||
|
⤼ {{ e.live.skipped }}</span>
|
||||||
|
<span
|
||||||
|
v-if="e.live.errors" class="fc-active__count fc-active__count--err"
|
||||||
|
title="errors"
|
||||||
|
>✕ {{ e.live.errors }}</span>
|
||||||
|
<span class="fc-active__count fc-active__count--posts" title="posts scanned">
|
||||||
|
{{ e.live.posts }} posts</span>
|
||||||
|
</span>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
<span class="fc-active__timer" :title="`started ${e.started_at}`">
|
<span class="fc-active__timer" :title="`started ${e.started_at}`">
|
||||||
{{ elapsed(e.started_at) }}
|
{{ elapsed(e.started_at) }}
|
||||||
@@ -123,6 +135,13 @@ function elapsed (startedIso) {
|
|||||||
font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em;
|
font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
}
|
}
|
||||||
|
.fc-active__counts {
|
||||||
|
display: inline-flex; align-items: center; gap: 8px;
|
||||||
|
font-variant-numeric: tabular-nums; font-size: 0.78rem;
|
||||||
|
}
|
||||||
|
.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-active__count--err { color: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-active__count--posts { opacity: 0.7; }
|
||||||
@keyframes fc-active-pulse {
|
@keyframes fc-active-pulse {
|
||||||
0%, 100% { opacity: 1; transform: scale(1); }
|
0%, 100% { opacity: 1; transform: scale(1); }
|
||||||
50% { opacity: 0.35; transform: scale(0.7); }
|
50% { opacity: 0.35; transform: scale(0.7); }
|
||||||
|
|||||||
@@ -142,6 +142,22 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #714: Title-Case the back-catalog + merge case/whitespace-variant tags.
|
||||||
|
// dry-run returns a projection inline; live returns {task_id} (long op —
|
||||||
|
// FK repoints) the caller polls via pollTaskUntilDone.
|
||||||
|
async function normalizeTags({ dryRun = true } = {}) {
|
||||||
|
lastError.value = null
|
||||||
|
try {
|
||||||
|
return await api.post(
|
||||||
|
'/api/admin/tags/normalize',
|
||||||
|
{ body: { dry_run: dryRun } },
|
||||||
|
)
|
||||||
|
} catch (e) {
|
||||||
|
lastError.value = e.message
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -178,6 +194,7 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
pruneUnusedTags,
|
pruneUnusedTags,
|
||||||
purgeLegacyTags,
|
purgeLegacyTags,
|
||||||
resetContentTagging,
|
resetContentTagging,
|
||||||
|
normalizeTags,
|
||||||
pollTaskUntilDone,
|
pollTaskUntilDone,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
const q = ref('')
|
const q = ref('')
|
||||||
const platform = ref(null)
|
const platform = ref(null)
|
||||||
let started = false
|
let started = false
|
||||||
|
// Has a load ATTEMPT completed yet? Gates the "No artists match" empty state
|
||||||
|
// so it can't flash on the very first render (loading is still false before
|
||||||
|
// onMounted fires the first loadMore) or between a query change and its fetch.
|
||||||
|
const loaded = ref(false)
|
||||||
// Typed "alice" then "alice bob" used to drop the second fetch
|
// Typed "alice" then "alice bob" used to drop the second fetch
|
||||||
// entirely (loading flag still true from the first), so the UI
|
// entirely (loading flag still true from the first), so the UI
|
||||||
// showed alice results while the input said "alice bob". Inflight
|
// showed alice results while the input said "alice bob". Inflight
|
||||||
@@ -35,6 +39,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
nextCursor.value = body.next_cursor
|
nextCursor.value = body.next_cursor
|
||||||
started = true
|
started = true
|
||||||
})
|
})
|
||||||
|
if (t.isCurrent()) loaded.value = true
|
||||||
}
|
}
|
||||||
|
|
||||||
async function reset() {
|
async function reset() {
|
||||||
@@ -42,6 +47,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
cards.value = []
|
cards.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
started = false
|
started = false
|
||||||
|
loaded.value = false
|
||||||
await loadMore()
|
await loadMore()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,7 +56,8 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
|
|
||||||
const hasMore = computed(() => !started || nextCursor.value !== null)
|
const hasMore = computed(() => !started || nextCursor.value !== null)
|
||||||
const isEmpty = computed(
|
const isEmpty = computed(
|
||||||
() => !loading.value && cards.value.length === 0 && error.value === null
|
() => loaded.value && !loading.value
|
||||||
|
&& cards.value.length === 0 && error.value === null
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -34,10 +34,20 @@ export const useTagStore = defineStore('tags', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function loadFandoms() {
|
async function loadFandoms() {
|
||||||
fandomCache.value = await api.get('/api/tags/autocomplete', {
|
// List ALL fandoms via the cursor-paged tag directory. (#712: the old impl
|
||||||
params: { q: ' ', kind: 'fandom', limit: 200 }
|
// abused /tags/autocomplete with q=' ', which the backend strips to empty
|
||||||
})
|
// and returns [] — so the picker showed no existing fandoms.)
|
||||||
return fandomCache.value
|
const all = []
|
||||||
|
let cursor = null
|
||||||
|
do {
|
||||||
|
const params = { kind: 'fandom', limit: 200 }
|
||||||
|
if (cursor) params.cursor = cursor
|
||||||
|
const body = await api.get('/api/tags/directory', { params })
|
||||||
|
all.push(...(body.cards || []))
|
||||||
|
cursor = body.next_cursor
|
||||||
|
} while (cursor)
|
||||||
|
fandomCache.value = all
|
||||||
|
return all
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createFandom(name) {
|
async function createFandom(name) {
|
||||||
|
|||||||
@@ -76,9 +76,11 @@ onMounted(async () => {
|
|||||||
}
|
}
|
||||||
.fc-artists__grid {
|
.fc-artists__grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
/* min(440px, 100%) so a card never exceeds the viewport — on phones the
|
/* min(360px, 100%) so a card never exceeds the viewport (phones collapse to
|
||||||
min-track collapses to 100% (single column) instead of overflowing. */
|
one column). 360 keeps a LONE column under ~732px wide, so the 3/1 preview
|
||||||
grid-template-columns: repeat(auto-fill, minmax(min(440px, 100%), 1fr));
|
strip stays ≲244px tall and always fills the card width (no dead space),
|
||||||
|
while giving 2+ columns on a normal desktop. */
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(min(360px, 100%), 1fr));
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
}
|
}
|
||||||
.fc-artists__sentinel {
|
.fc-artists__sentinel {
|
||||||
|
|||||||
@@ -52,11 +52,12 @@ async def test_create_tag_missing_name_400(client):
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_create_tag_name_only_defaults_to_general(client):
|
async def test_create_tag_name_only_defaults_to_general(client):
|
||||||
"""IR-style: name without kind and without `kind:` prefix → general."""
|
"""IR-style: name without kind and without `kind:` prefix → general.
|
||||||
|
#701: operator-entered names are Title-Cased at the create endpoint."""
|
||||||
resp = await client.post("/api/tags", json={"name": "sunset"})
|
resp = await client.post("/api/tags", json={"name": "sunset"})
|
||||||
assert resp.status_code == 201
|
assert resp.status_code == 201
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["name"] == "sunset"
|
assert body["name"] == "Sunset"
|
||||||
assert body["kind"] == "general"
|
assert body["kind"] == "general"
|
||||||
|
|
||||||
|
|
||||||
@@ -73,21 +74,22 @@ async def test_create_tag_with_kind_prefix(client):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_explicit_kind_overrides_prefix_parsing(client):
|
async def test_explicit_kind_overrides_prefix_parsing(client):
|
||||||
"""If caller passes explicit kind, don't re-parse the name —
|
"""If caller passes explicit kind, don't re-parse the name —
|
||||||
colon and prefix stay literal."""
|
colon and prefix stay literal. #701: still Title-Cased as a single word."""
|
||||||
resp = await client.post(
|
resp = await client.post(
|
||||||
"/api/tags", json={"name": "character:Saber", "kind": "general"}
|
"/api/tags", json={"name": "character:Saber", "kind": "general"}
|
||||||
)
|
)
|
||||||
assert resp.status_code == 201
|
assert resp.status_code == 201
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["name"] == "character:Saber"
|
assert body["name"] == "Character:saber"
|
||||||
assert body["kind"] == "general"
|
assert body["kind"] == "general"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_unknown_prefix_kept_literal(client):
|
async def test_unknown_prefix_kept_literal(client):
|
||||||
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name."""
|
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name.
|
||||||
|
#701: Title-Cased as one word (no internal whitespace to split on)."""
|
||||||
resp = await client.post("/api/tags", json={"name": "http://example.com"})
|
resp = await client.post("/api/tags", json={"name": "http://example.com"})
|
||||||
assert resp.status_code == 201
|
assert resp.status_code == 201
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["name"] == "http://example.com"
|
assert body["name"] == "Http://example.com"
|
||||||
assert body["kind"] == "general"
|
assert body["kind"] == "general"
|
||||||
|
|||||||
@@ -39,6 +39,24 @@ def test_cbz_alias(tmp_path):
|
|||||||
assert [n for n, _ in members] == ["p1.jpg"]
|
assert [n for n, _ in members] == ["p1.jpg"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_misnamed_zip_detected_and_extracted(tmp_path):
|
||||||
|
"""A real zip whose filename has no usable extension (Patreon attachment
|
||||||
|
URL-blob name) is detected by magic bytes and its members extracted —
|
||||||
|
previously it was filed as an opaque attachment and never opened."""
|
||||||
|
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
|
||||||
|
_zip(z, {"a.jpg": b"img", "b.png": b"img2"})
|
||||||
|
assert is_archive(z)
|
||||||
|
with extract_archive(z) as members:
|
||||||
|
assert sorted(n for n, _ in members) == ["a.jpg", "b.png"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_archive_with_dotted_name_is_not_archive(tmp_path):
|
||||||
|
"""A non-archive file with a dotted/extension-less name isn't misdetected."""
|
||||||
|
f = tmp_path / "01_https___www.patreon.com_media-u_v3_999"
|
||||||
|
f.write_bytes(b"\x89PNG\r\n\x1a\n not really but not a zip")
|
||||||
|
assert not is_archive(f)
|
||||||
|
|
||||||
|
|
||||||
def test_corrupt_archive_yields_nothing(tmp_path):
|
def test_corrupt_archive_yields_nothing(tmp_path):
|
||||||
bad = tmp_path / "broken.zip"
|
bad = tmp_path / "broken.zip"
|
||||||
bad.write_bytes(b"not a zip at all")
|
bad.write_bytes(b"not a zip at all")
|
||||||
|
|||||||
@@ -421,6 +421,34 @@ async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path):
|
|||||||
assert result["has_more"] is True
|
assert result["has_more"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_write_live_progress_updates_running_event(
|
||||||
|
source_id, sync_engine, tmp_path, db,
|
||||||
|
):
|
||||||
|
"""plan #709: live counts land in the RUNNING event's metadata.live; a
|
||||||
|
finalized event is left alone (status guard)."""
|
||||||
|
from backend.app.models import DownloadEvent
|
||||||
|
|
||||||
|
ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path))
|
||||||
|
running = DownloadEvent(source_id=source_id, status="running")
|
||||||
|
done = DownloadEvent(source_id=source_id, status="ok")
|
||||||
|
db.add_all([running, done])
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
counts = {"downloaded": 3, "skipped": 1, "errors": 0, "quarantined": 0, "posts": 4}
|
||||||
|
ing._write_live_progress(running.id, counts)
|
||||||
|
ing._write_live_progress(done.id, {"downloaded": 9}) # guarded: status != running
|
||||||
|
|
||||||
|
live = (await db.execute(
|
||||||
|
select(DownloadEvent.metadata_).where(DownloadEvent.id == running.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert live["live"] == counts
|
||||||
|
done_meta = (await db.execute(
|
||||||
|
select(DownloadEvent.metadata_).where(DownloadEvent.id == done.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert "live" not in (done_meta or {})
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
|
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
|
||||||
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
|
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
"""#713 part 2: re-extract archive PostAttachments that were filed opaquely
|
||||||
|
(magic-byte gate missed them) and link their members to the post."""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from backend.app.models import (
|
||||||
|
Artist,
|
||||||
|
ImageProvenance,
|
||||||
|
ImageRecord,
|
||||||
|
Post,
|
||||||
|
PostAttachment,
|
||||||
|
Source,
|
||||||
|
)
|
||||||
|
from backend.app.services import cleanup_service
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _jpeg(color, size=256):
|
||||||
|
buf = io.BytesIO()
|
||||||
|
Image.new("RGB", (size, size), color).save(buf, "JPEG")
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch):
|
||||||
|
from backend.app.tasks import ml as ml_mod
|
||||||
|
from backend.app.tasks import thumbnail as thumb_mod
|
||||||
|
|
||||||
|
# No broker in this path — the post-import enqueue is best-effort anyway.
|
||||||
|
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||||
|
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
||||||
|
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
images_root.mkdir()
|
||||||
|
|
||||||
|
artist = Artist(name="Bob", slug="bob")
|
||||||
|
db_sync.add(artist)
|
||||||
|
db_sync.flush()
|
||||||
|
source = Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/bob", enabled=True, config_overrides={},
|
||||||
|
)
|
||||||
|
db_sync.add(source)
|
||||||
|
db_sync.flush()
|
||||||
|
post = Post(
|
||||||
|
source_id=source.id, artist_id=artist.id, external_post_id="59102952",
|
||||||
|
post_url="https://www.patreon.com/posts/59102952",
|
||||||
|
)
|
||||||
|
db_sync.add(post)
|
||||||
|
db_sync.flush()
|
||||||
|
|
||||||
|
# A real zip stored under a mangled / extension-less name (the failure case).
|
||||||
|
store_dir = images_root / "attachments" / "abc"
|
||||||
|
store_dir.mkdir(parents=True)
|
||||||
|
arc = store_dir / "01_https___www.patreon.com_media-u_v3_59102952"
|
||||||
|
with zipfile.ZipFile(arc, "w") as zf:
|
||||||
|
zf.writestr("inside.jpg", _jpeg("red"))
|
||||||
|
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
|
||||||
|
db_sync.add(PostAttachment(
|
||||||
|
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
|
||||||
|
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
|
||||||
|
))
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
summary = cleanup_service.reextract_archive_attachments(
|
||||||
|
db_sync, images_root=images_root,
|
||||||
|
)
|
||||||
|
assert summary["archives"] == 1
|
||||||
|
assert summary["members_imported"] == 1
|
||||||
|
assert summary["posts_touched"] == 1
|
||||||
|
|
||||||
|
images = db_sync.execute(select(ImageRecord)).scalars().all()
|
||||||
|
assert len(images) == 1
|
||||||
|
prov = db_sync.execute(
|
||||||
|
select(ImageProvenance).where(ImageProvenance.post_id == post.id)
|
||||||
|
).scalars().all()
|
||||||
|
assert len(prov) == 1
|
||||||
|
assert prov[0].image_record_id == images[0].id
|
||||||
|
|
||||||
|
# Idempotent — a second run imports nothing new (member dedups by sha256).
|
||||||
|
again = cleanup_service.reextract_archive_attachments(
|
||||||
|
db_sync, images_root=images_root,
|
||||||
|
)
|
||||||
|
assert again["members_imported"] == 0
|
||||||
|
assert db_sync.execute(select(ImageRecord)).scalars().all() == images
|
||||||
@@ -40,11 +40,21 @@ def test_probe_archive_corrupt_zip_clean_rejection(tmp_path):
|
|||||||
def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
|
def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
|
||||||
z = tmp_path / "sized.zip"
|
z = tmp_path / "sized.zip"
|
||||||
_zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50})
|
_zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50})
|
||||||
total, bad = safe_probe._inspect_archive(z, ".zip")
|
total, bad = safe_probe._inspect_archive(z)
|
||||||
assert total == 150
|
assert total == 150
|
||||||
assert bad is None
|
assert bad is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_inspect_archive_detects_misnamed_zip(tmp_path):
|
||||||
|
"""A zip with a mangled / extension-less name (Patreon URL-blob attachment)
|
||||||
|
is still bomb-guarded + integrity-tested via magic-byte detection."""
|
||||||
|
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
|
||||||
|
_zip(z, {"a.txt": b"x" * 100})
|
||||||
|
total, bad = safe_probe._inspect_archive(z)
|
||||||
|
assert total == 100
|
||||||
|
assert bad is None
|
||||||
|
|
||||||
|
|
||||||
def test_run_probe_bomb_guard(tmp_path, monkeypatch):
|
def test_run_probe_bomb_guard(tmp_path, monkeypatch):
|
||||||
"""In-process call to _run_probe so the monkeypatched cap takes effect.
|
"""In-process call to _run_probe so the monkeypatched cap takes effect.
|
||||||
The public probe_archive runs in a subprocess which re-imports the
|
The public probe_archive runs in a subprocess which re-imports the
|
||||||
|
|||||||
@@ -0,0 +1,189 @@
|
|||||||
|
"""#714: retro-normalize existing tags to the #701 canonical (Title Case +
|
||||||
|
collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||||
|
|
||||||
|
normalize_existing_tags commits per group, so case-variant duplicates are
|
||||||
|
seeded with DIRECT inserts (find_or_create would case-insensitively dedup
|
||||||
|
them, which is exactly the pre-#701 state we're cleaning up), and assertions
|
||||||
|
re-query via fresh selects rather than touching post-commit ORM entities.
|
||||||
|
"""
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.models import Tag, TagKind, image_tag
|
||||||
|
from backend.app.models.tag_alias import TagAlias
|
||||||
|
from backend.app.services.tag_service import (
|
||||||
|
TagService,
|
||||||
|
normalize_existing_tags,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
_IMG_SEQ = 0
|
||||||
|
|
||||||
|
|
||||||
|
async def _img(db):
|
||||||
|
"""Minimal valid ImageRecord; returns its id."""
|
||||||
|
global _IMG_SEQ
|
||||||
|
_IMG_SEQ += 1
|
||||||
|
from backend.app.models import ImageRecord
|
||||||
|
|
||||||
|
rec = ImageRecord(
|
||||||
|
path=f"/tmp/fc_norm_{_IMG_SEQ}.png",
|
||||||
|
sha256=f"{_IMG_SEQ:064d}",
|
||||||
|
size_bytes=1,
|
||||||
|
mime="image/png",
|
||||||
|
origin="uploaded",
|
||||||
|
)
|
||||||
|
db.add(rec)
|
||||||
|
await db.flush()
|
||||||
|
return rec.id
|
||||||
|
|
||||||
|
|
||||||
|
async def _raw_tag(db, name, kind, fandom_id=None):
|
||||||
|
"""Insert a Tag row verbatim — bypasses find_or_create's case-insensitive
|
||||||
|
dedup so we can plant the case-variant duplicates #714 exists to fix."""
|
||||||
|
t = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||||
|
db.add(t)
|
||||||
|
await db.flush()
|
||||||
|
return t
|
||||||
|
|
||||||
|
|
||||||
|
async def _link(db, image_id, tag_id, source="manual"):
|
||||||
|
await db.execute(
|
||||||
|
image_tag.insert().values(
|
||||||
|
image_record_id=image_id, tag_id=tag_id, source=source
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _count_named(db, name, kind):
|
||||||
|
return await db.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Tag)
|
||||||
|
.where(Tag.name == name)
|
||||||
|
.where(Tag.kind == kind)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_dry_run_projection_counts(db):
|
||||||
|
# Collision group with an already-canonical member (→ no rename, 1 merge).
|
||||||
|
await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||||
|
await _raw_tag(db, "Hatsune Miku", TagKind.character)
|
||||||
|
# Lone mis-cased + extra whitespace (→ rename, no merge).
|
||||||
|
await _raw_tag(db, "looking at viewer", TagKind.general)
|
||||||
|
# Already canonical (→ no change).
|
||||||
|
await _raw_tag(db, "Already Canonical", TagKind.general)
|
||||||
|
|
||||||
|
proj = await normalize_existing_tags(db, dry_run=True)
|
||||||
|
|
||||||
|
assert proj["collisions"] == 1
|
||||||
|
assert proj["tags_to_merge"] == 1
|
||||||
|
assert proj["tags_to_rename"] == 1 # only the whitespace group renames
|
||||||
|
assert proj["total_changes"] == 2
|
||||||
|
# Nothing mutated by a dry run.
|
||||||
|
assert await _count_named(db, "hatsune miku", TagKind.character) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_live_merges_case_variants_and_repoints_images(db):
|
||||||
|
a = await _raw_tag(db, "hatsune miku", TagKind.character)
|
||||||
|
b = await _raw_tag(db, "HATSUNE MIKU", TagKind.character)
|
||||||
|
i1, i2, i3 = await _img(db), await _img(db), await _img(db)
|
||||||
|
await _link(db, i1, a.id)
|
||||||
|
await _link(db, i2, a.id)
|
||||||
|
await _link(db, i2, b.id) # shared → must dedup on the survivor
|
||||||
|
await _link(db, i3, b.id)
|
||||||
|
|
||||||
|
summary = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
|
||||||
|
assert summary["errors"] == 0
|
||||||
|
assert summary["merged"] == 1
|
||||||
|
assert summary["renamed"] == 1
|
||||||
|
assert summary["groups_processed"] == 1
|
||||||
|
|
||||||
|
# Exactly one canonical character tag survives; the variant is gone.
|
||||||
|
assert await _count_named(db, "Hatsune Miku", TagKind.character) == 1
|
||||||
|
total_chars = await db.scalar(
|
||||||
|
select(func.count()).select_from(Tag).where(Tag.kind == TagKind.character)
|
||||||
|
)
|
||||||
|
assert total_chars == 1
|
||||||
|
survivor_id = await db.scalar(
|
||||||
|
select(Tag.id)
|
||||||
|
.where(Tag.name == "Hatsune Miku")
|
||||||
|
.where(Tag.kind == TagKind.character)
|
||||||
|
)
|
||||||
|
# Union of {i1,i2} and {i2,i3} with the shared i2 deduped == 3.
|
||||||
|
assoc = await db.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(image_tag)
|
||||||
|
.where(image_tag.c.tag_id == survivor_id)
|
||||||
|
)
|
||||||
|
assert assoc == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_idempotent_second_run_is_noop(db):
|
||||||
|
await _raw_tag(db, "kafka", TagKind.character)
|
||||||
|
await _raw_tag(db, "KAFKA", TagKind.character)
|
||||||
|
|
||||||
|
first = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
assert first["groups_processed"] == 1
|
||||||
|
|
||||||
|
proj = await normalize_existing_tags(db, dry_run=True)
|
||||||
|
assert proj["total_changes"] == 0
|
||||||
|
again = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
assert again["groups_processed"] == 0
|
||||||
|
assert again["merged"] == 0
|
||||||
|
assert again["renamed"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_same_name_different_fandom_not_merged(db):
|
||||||
|
f1 = await _raw_tag(db, "Bleach", TagKind.fandom)
|
||||||
|
f2 = await _raw_tag(db, "Naruto", TagKind.fandom)
|
||||||
|
await _raw_tag(db, "alice", TagKind.character, fandom_id=f1.id)
|
||||||
|
await _raw_tag(db, "Alice", TagKind.character, fandom_id=f2.id)
|
||||||
|
|
||||||
|
summary = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
|
||||||
|
assert summary["merged"] == 0
|
||||||
|
assert summary["renamed"] == 1 # only the f1 "alice" recases
|
||||||
|
# Both characters survive — same canonical name, distinct fandoms.
|
||||||
|
assert await _count_named(db, "Alice", TagKind.character) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_same_name_different_kind_not_merged(db):
|
||||||
|
await _raw_tag(db, "forge", TagKind.artist)
|
||||||
|
await _raw_tag(db, "Forge", TagKind.general)
|
||||||
|
|
||||||
|
summary = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
|
||||||
|
assert summary["merged"] == 0
|
||||||
|
assert await _count_named(db, "Forge", TagKind.artist) == 1
|
||||||
|
assert await _count_named(db, "Forge", TagKind.general) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ml_known_loser_keeps_protective_alias(db):
|
||||||
|
# Survivor is the already-canonical (manual, ML-unknown) tag; the ML-sourced
|
||||||
|
# variant merges away and must leave an alias so the tagger re-resolves it.
|
||||||
|
canon = await _raw_tag(db, "Nemu", TagKind.general)
|
||||||
|
canon_id = canon.id # capture before the service commits
|
||||||
|
ml = await _raw_tag(db, "nemu", TagKind.general)
|
||||||
|
img = await _img(db)
|
||||||
|
await _link(db, img, ml.id, source="ml_auto")
|
||||||
|
|
||||||
|
summary = await normalize_existing_tags(db, dry_run=False)
|
||||||
|
|
||||||
|
assert summary["merged"] == 1
|
||||||
|
assert summary["aliases_created"] == 1
|
||||||
|
assert await _count_named(db, "Nemu", TagKind.general) == 1
|
||||||
|
alias_rows = await db.scalar(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(TagAlias)
|
||||||
|
.where(TagAlias.alias_string == "nemu")
|
||||||
|
.where(TagAlias.canonical_tag_id == canon_id)
|
||||||
|
)
|
||||||
|
assert alias_rows == 1
|
||||||
@@ -59,6 +59,27 @@ async def test_character_with_fandom(db):
|
|||||||
assert char.fandom_id == fandom.id
|
assert char.fandom_id == fandom.id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_find_or_create_dedups_case_insensitive(db):
|
||||||
|
"""#701: find_or_create matches case-insensitively so a differently-cased
|
||||||
|
entry finds the existing tag instead of forking. (Display Title-Casing is
|
||||||
|
applied at the create API, not here — this path is shared with the ML
|
||||||
|
tagger, which keeps the booru vocabulary's casing.)"""
|
||||||
|
svc = TagService(db)
|
||||||
|
t1 = await svc.find_or_create("hatsune miku", TagKind.character)
|
||||||
|
assert t1.name == "hatsune miku" # stored as given; not recased here
|
||||||
|
t2 = await svc.find_or_create("HATSUNE MIKU", TagKind.character)
|
||||||
|
assert t2.id == t1.id # case variant → same tag, no fork
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_normalize_tag_name():
|
||||||
|
from backend.app.services.tag_service import normalize_tag_name
|
||||||
|
assert normalize_tag_name("hatsune miku") == "Hatsune Miku"
|
||||||
|
assert normalize_tag_name(" looking at viewer ") == "Looking At Viewer"
|
||||||
|
assert normalize_tag_name("HATSUNE MIKU") == "Hatsune Miku"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_character_same_name_different_fandoms(db):
|
async def test_character_same_name_different_fandoms(db):
|
||||||
svc = TagService(db)
|
svc = TagService(db)
|
||||||
|
|||||||
Reference in New Issue
Block a user