Merge pull request 'Tag-maintenance sweep + bug-fix batch: #699 #700 #701 #709 #711 #712 #713 #714' (#73) from dev into main
Build images / sign-extension (push) Successful in 2s
CI / lint (push) Successful in 3s
Build images / build-ml (push) Successful in 8s
Build images / build-web (push) Successful in 9s
CI / frontend-build (push) Successful in 18s
CI / backend-lint-and-test (push) Successful in 27s
CI / integration (push) Successful in 3m2s

This commit was merged in pull request #73.
This commit is contained in:
2026-06-06 16:49:00 -04:00
31 changed files with 1162 additions and 82 deletions
+37
View File
@@ -245,6 +245,32 @@ async def tags_reset_content():
return jsonify(result)
@admin_bp.route("/tags/normalize", methods=["POST"])
async def tags_normalize():
"""#714: retro-normalize existing tags to the #701 canonical form (Title
Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
dry_run=true (default) returns a projection inline — group/collision/rename
counts + a sample of the changes — so the UI shows exactly what'll happen.
dry_run=false dispatches the long-running maintenance task (the merge FK
repoints can touch many tags); the UI tails the activity dashboard for the
summary. Idempotent; back up first (the merges are irreversible)."""
from ..services.tag_service import normalize_existing_tags
body = await request.get_json(silent=True) or {}
dry_run = bool(body.get("dry_run", True))
if dry_run:
async with get_session() as session:
result = await normalize_existing_tags(session, dry_run=True)
return jsonify(result)
from ..tasks.admin import normalize_tags_task
async_result = normalize_tags_task.delay()
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
async def db_stats():
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
@@ -289,3 +315,14 @@ async def trigger_vacuum():
vacuum_analyze.delay()
return jsonify({"status": "queued"}), 202
@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
async def trigger_reextract_archives():
"""Operator-triggered re-extract (#713): PostAttachments that are actually
archives but were filed opaquely (pre magic-byte gate) get extracted and
their members linked to the post. Idempotent; runs on the maintenance queue."""
from ..tasks.admin import reextract_archive_attachments_task
async_result = reextract_archive_attachments_task.delay()
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
+3
View File
@@ -44,6 +44,9 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
"bytes_downloaded": event.bytes_downloaded,
"error": event.error,
"summary": _summary_from_metadata(event.metadata_),
# plan #709: mid-walk live counts for a RUNNING native-ingester event
# (None otherwise; phase 3 overwrites metadata with run_stats on finish).
"live": (event.metadata_ or {}).get("live"),
}
+6
View File
@@ -14,6 +14,7 @@ from ..services.tag_service import (
TagMergeConflict,
TagService,
TagValidationError,
normalize_tag_name,
)
from ..utils.tag_prefix import parse_kind_prefix
@@ -141,6 +142,11 @@ async def create_tag():
fandom_id = body.get("fandom_id")
# #701: Title-Case operator-entered tags. Only here (the explicit create
# endpoint), NOT in the shared find_or_create — the ML tagger uses that path
# and must keep the booru vocabulary's casing for allowlist matching.
name = normalize_tag_name(name)
async with get_session() as session:
svc = TagService(session)
try:
+41 -5
View File
@@ -16,9 +16,45 @@ log = logging.getLogger(__name__)
ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
# Magic-byte signatures, so an archive with a mangled / extension-less filename
# is still recognised. Patreon attachment download URLs sanitize to names like
# `01_https___www.patreon.com_media-u_v3_131083093`, whose `Path.suffix` is junk
# (`.com_media-u_v3_131083093`), never `.zip` — an extension-only gate filed
# those as opaque PostAttachments and NEVER extracted them (operator-flagged
# 2026-06-06). Detection is by extension first (cheap), then header sniff.
_RAR_MAGIC = b"Rar!\x1a\x07"
_7Z_MAGIC = b"7z\xbc\xaf\x27\x1c"
def detect_archive_format(path: Path) -> str | None:
"""Return "zip" | "rar" | "7z" for an archive, else None.
Trusts a known extension first, then falls back to magic-byte sniffing so a
mis-named or extension-less archive is still handled. (zip covers .cbz too.)
"""
ext = Path(path).suffix.lower()
if ext in (".zip", ".cbz"):
return "zip"
if ext == ".rar":
return "rar"
if ext == ".7z":
return "7z"
try:
if zipfile.is_zipfile(path):
return "zip"
with open(path, "rb") as fh:
head = fh.read(8)
except OSError:
return None
if head.startswith(_RAR_MAGIC):
return "rar"
if head.startswith(_7Z_MAGIC):
return "7z"
return None
def is_archive(path: Path) -> bool:
return Path(path).suffix.lower() in ARCHIVE_EXTS
return detect_archive_format(path) is not None
@contextmanager
@@ -32,16 +68,16 @@ def extract_archive(path: Path):
members: list[tuple[str, Path]] = []
try:
try:
ext = Path(path).suffix.lower()
if ext in (".zip", ".cbz"):
fmt = detect_archive_format(path)
if fmt == "zip":
with zipfile.ZipFile(path) as zf:
zf.extractall(base)
elif ext == ".rar":
elif fmt == "rar":
import rarfile
with rarfile.RarFile(path) as rf:
rf.extractall(base)
elif ext == ".7z":
elif fmt == "7z":
import py7zr
with py7zr.SevenZipFile(path, "r") as zf:
+139
View File
@@ -11,6 +11,7 @@ the one-and-done GS/IR migration tooling.)
"""
from __future__ import annotations
import logging
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
@@ -22,6 +23,8 @@ from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
from ..models.series_page import SeriesPage
from ..models.tag import image_tag
log = logging.getLogger(__name__)
def project_artist_cascade(session: Session, *, slug: str) -> dict:
"""Read-only projection of what delete_artist_cascade would touch.
@@ -665,3 +668,139 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
.where(LibraryAuditRun.status == "running")
.values(status="cancelled", finished_at=datetime.now(UTC))
)
# -- archive-attachment re-extraction (#713 part 2) ------------------------
_ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
def _reextract_archive_to_post(
importer, archive_path: Path, post, source_row, artist, images_root: Path,
) -> list[int]:
"""Extract one stored archive and link its members to `post`.
The stored attachment has no adjacent sidecar (it lives in the sha-addressed
attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
re-derives the artist from the path AND copies members relative to it, so the
members land in the real library and resolve to the right artist — then re-run
`attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
the members to the SAME Post (source_id + external_post_id). Removes only the
staged archive + sidecar afterward; the imported member files stay. Returns
the new member image ids.
"""
import json
import shutil
from .archive_extractor import detect_archive_format
fmt = detect_archive_format(archive_path)
ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
platform = source_row.platform if source_row is not None else "imported"
sidecar = {
"category": source_row.platform if source_row is not None
else (post.raw_metadata or {}).get("category"),
"id": post.external_post_id,
"title": post.post_title or "",
"content": post.description or "",
"published_at": post.post_date.isoformat() if post.post_date else None,
"url": post.post_url,
}
work = images_root / artist.slug / platform / str(post.external_post_id)
work.mkdir(parents=True, exist_ok=True)
staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar
sidecar_path = staged.with_suffix(".json")
try:
shutil.copy2(archive_path, staged)
sidecar_path.write_text(json.dumps(sidecar))
res = importer.attach_in_place(staged, artist=artist, source=source_row)
return list(res.member_image_ids or [])
finally:
# Drop only the staged archive + sidecar; the extracted member files
# were copied into the library alongside them and must stay.
staged.unlink(missing_ok=True)
sidecar_path.unlink(missing_ok=True)
def reextract_archive_attachments(session: Session, *, images_root: Path) -> dict:
"""Re-process existing PostAttachments that are ACTUALLY archives but were
filed opaquely before #713 part 1 (extension-only is_archive missed mangled /
extension-less Patreon attachment names). For each: extract the members,
import them, and link them to the attachment's post.
Idempotent — members dedupe by sha256, the archive dedupes by sha — so it's
safe to run repeatedly. Returns a summary dict for task_run.metadata.
"""
from ..models import ImportSettings, Post, PostAttachment, Source
from ..tasks.ml import tag_and_embed
from ..tasks.thumbnail import generate_thumbnail
from .archive_extractor import is_archive
from .importer import Importer
from .thumbnailer import Thumbnailer
summary = {
"scanned": 0, "archives": 0, "members_imported": 0,
"posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
"errors": 0,
}
settings = ImportSettings.load_sync(session)
importer = Importer(
session=session, images_root=images_root, import_root=images_root,
thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
)
attachments = session.execute(
select(PostAttachment).order_by(PostAttachment.id)
).scalars().all()
enqueue_ids: list[int] = []
for att in attachments:
summary["scanned"] += 1
stored = Path(att.path)
try:
if not stored.is_file() or not is_archive(stored):
continue
except OSError:
continue
summary["archives"] += 1
if att.post_id is None:
summary["skipped_no_post"] += 1
continue
post = session.get(Post, att.post_id)
if post is None:
summary["skipped_no_post"] += 1
continue
artist = session.get(Artist, att.artist_id) if att.artist_id else None
if artist is None and post.artist_id:
artist = session.get(Artist, post.artist_id)
if artist is None or not artist.slug:
# The importer re-derives the artist from the staged path, so we need
# a real artist+slug to anchor under. (Shouldn't happen for
# subscription posts; skip rather than orphan the members.)
summary["skipped_no_artist"] += 1
continue
source_row = session.get(Source, post.source_id) if post.source_id else None
try:
ids = _reextract_archive_to_post(
importer, stored, post, source_row, artist, images_root,
)
session.commit()
except Exception as exc: # one bad archive must not strand the rest
session.rollback()
summary["errors"] += 1
log.warning("re-extract failed for attachment %s: %s", att.id, exc)
continue
if ids:
summary["members_imported"] += len(ids)
summary["posts_touched"] += 1
enqueue_ids.extend(ids)
# Thumbnails + ML for the newly-imported members (best-effort; off the
# critical path — a Redis hiccup must not fail the whole re-extract).
for img_id in enqueue_ids:
try:
generate_thumbnail.delay(img_id)
tag_and_embed.delay(img_id)
except Exception as exc:
log.warning("re-extract enqueue failed for image %s: %s", img_id, exc)
return summary
@@ -139,6 +139,8 @@ async def _run_native_ingester(
resume_cursor=source_config.resume_cursor,
time_budget_seconds=source_config.timeout,
posts_base=int(overrides.get("_backfill_posts", 0)),
# plan #709: live progress writes to this running event mid-walk.
event_id=ctx.get("event_id"),
),
)
return dl_result, resolved_campaign_id
+41
View File
@@ -25,6 +25,7 @@ Plain-HTTP homelab: no secure-context Web API.
from __future__ import annotations
import json
import logging
import time
from collections.abc import Callable
@@ -48,6 +49,12 @@ DEAD_LETTER_THRESHOLD = 3
# last_error is Text but bound it so a giant traceback doesn't bloat the row.
_ERROR_MAX = 1000
# plan #709: throttle the live-progress write to the running DownloadEvent to one
# every ~5s — a steady cadence for the Downloads view regardless of how big/slow a
# page is (page boundaries can be minutes apart on image-dense backfills, so a
# page-tied update would lurch). Trivial churn (~one single-row UPDATE / 5s).
_LIVE_PROGRESS_INTERVAL = 5.0
class Ingester:
"""Generic native-ingest orchestration. Subclass with a platform adapter
@@ -92,6 +99,7 @@ class Ingester:
time_budget_seconds: float = 870.0,
seen_threshold: int = _TICK_SEEN_THRESHOLD,
posts_base: int = 0,
event_id: int | None = None,
) -> DownloadResult:
"""Walk + download for one source, returning a gallery-dl-shaped result.
@@ -109,6 +117,7 @@ class Ingester:
checkpoint = mode in ("backfill", "recovery")
ledger_key = self._ledger_key
start = time.monotonic()
last_live = start # plan #709: last live-progress write timestamp
log_lines: list[str] = []
written: list[str] = []
quarantined_paths: list[str] = []
@@ -288,6 +297,19 @@ class Ingester:
if to_fail:
self._record_failures(source_id, to_fail)
# plan #709: time-throttled live progress to the running event so
# the Downloads view ticks ~every 5s, independent of page size.
now = time.monotonic()
if event_id is not None and (now - last_live) >= _LIVE_PROGRESS_INTERVAL:
last_live = now
self._write_live_progress(event_id, {
"downloaded": downloaded,
"skipped": skipped_count,
"errors": errors,
"quarantined": quarantined,
"posts": posts_processed,
})
if early_out:
break
else:
@@ -474,6 +496,25 @@ class Ingester:
)
session.commit()
def _write_live_progress(self, event_id: int, counts: dict) -> None:
"""Throttled mid-walk write of live counts to the RUNNING download_event
(plan #709) so the Downloads view shows progress before the chunk
finishes. A short session (never held across the walk); the `status =
'running'` guard avoids clobbering an event phase 3 already finalized.
`metadata` is JSONB — jsonb_set sets just the `live` key, leaving the rest
for phase 3 to overwrite with the final run_stats."""
with self.session_factory() as session:
session.execute(
text(
"UPDATE download_event SET metadata = jsonb_set("
" coalesce(metadata, '{}'::jsonb), '{live}',"
" cast(:live AS jsonb)) "
"WHERE id = :eid AND status = 'running'"
),
{"live": json.dumps(counts), "eid": event_id},
)
session.commit()
def _still_running(self, source_id: int) -> bool:
"""True while the source is armed for a deep walk (plan #708 B4).
+12 -3
View File
@@ -20,11 +20,20 @@ class ShowcaseService:
async def random_sample(self, limit: int = 60) -> list[dict]:
if limit < 1 or limit > 200:
raise ValueError("limit must be between 1 and 200")
# Over-sample then random-order (#699): SYSTEM_ROWS reads CONTIGUOUS rows
# from each sampled page, so sequentially-imported near-duplicates
# (multi-image posts, variant sets) come back adjacent and cluster in the
# showcase ("three near-identical in a row"). Sampling a multiple of
# `limit` spans more pages, and ORDER BY random() before taking `limit`
# breaks the physical adjacency — far better spread, still cheap
# (random() over a few hundred rows, not the whole table).
oversample = min(limit * 5, 1000)
stmt = select(ImageRecord).from_statement(
text(
"SELECT * FROM image_record "
"TABLESAMPLE SYSTEM_ROWS(:n)"
).bindparams(n=limit)
"SELECT * FROM ("
" SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(:o)"
") sub ORDER BY random() LIMIT :n"
).bindparams(o=oversample, n=limit)
)
rows = (await self.session.execute(stmt)).scalars().all()
return [
+203 -2
View File
@@ -1,5 +1,6 @@
"""Tag CRUD + autocomplete + image-tag association."""
import logging
from collections.abc import Sequence
from dataclasses import dataclass
@@ -12,6 +13,21 @@ from ..models import Tag, TagKind, image_tag
from ..models.tag_allowlist import TagAllowlist
from ..models.tag_reference_embedding import TagReferenceEmbedding
log = logging.getLogger(__name__)
def normalize_tag_name(name: str) -> str:
"""Canonical tag form (#701): collapse whitespace + Title Case.
Per-word capitalize (NOT str.title(), which mangles apostrophes:
don't → Don'T). Lowercasing the word tail also folds ALL-CAPS input
(HATSUNE MIKU → Hatsune Miku); acronym casing is sacrificed, an accepted
Title-Case trade-off (operator-chosen 2026-06-06).
"""
return " ".join(
w[:1].upper() + w[1:].lower() for w in (name or "").split()
)
class TagValidationError(ValueError):
"""Raised when tag construction breaks the kind/fandom rules."""
@@ -71,6 +87,10 @@ class TagService:
- if fandom_id is set, the referenced tag must exist and have
kind == TagKind.fandom
"""
# NOTE: case is NOT normalized here — find_or_create is the shared path
# the ML tagger / allowlist also use, and Title-Casing the booru
# vocabulary breaks allowlist matching. Display-casing (Title Case) is
# applied at the user-entry layer (api/tags create_tag) only (#701).
name = name.strip()
if not name:
raise TagValidationError("Tag name cannot be empty")
@@ -93,13 +113,17 @@ class TagService:
# inserts; without the savepoint the outer transaction would
# poison and the calling request crashes. Mirrors
# importer._get_or_create.
# Case-insensitive match (#701): a normalized (Title Case) input must
# find an existing differently-cased tag instead of forking a duplicate.
stmt = (
select(Tag)
.where(Tag.name == name)
.where(func.lower(Tag.name) == name.lower())
.where(Tag.kind == kind)
.where(
Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id
)
.order_by(Tag.id)
.limit(1)
)
existing = (await self.session.execute(stmt)).scalar_one_or_none()
if existing:
@@ -249,9 +273,11 @@ class TagService:
if tag is None:
raise TagValidationError(f"Tag {tag_id} not found")
# Case-insensitive clash (#701) — renaming onto a differently-cased tag
# is still a merge, not a silent fork.
clash_stmt = (
select(Tag)
.where(Tag.name == new_name)
.where(func.lower(Tag.name) == new_name.lower())
.where(Tag.kind == tag.kind)
.where(
Tag.fandom_id.is_(None)
@@ -259,6 +285,8 @@ class TagService:
else Tag.fandom_id == tag.fandom_id
)
.where(Tag.id != tag_id)
.order_by(Tag.id)
.limit(1)
)
clash = (await self.session.execute(clash_stmt)).scalar_one_or_none()
if clash is not None:
@@ -501,6 +529,21 @@ class TagService:
update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt)
)
async def _image_assoc_counts(self, tag_ids: list[int]) -> dict[int, int]:
"""image_tag row counts keyed by tag_id, for the survivor heuristic
(the best-connected tag in a collision group survives → fewest
FK repoints). Tags with zero associations are absent from the map."""
if not tag_ids:
return {}
rows = (
await self.session.execute(
select(image_tag.c.tag_id, func.count())
.where(image_tag.c.tag_id.in_(tag_ids))
.group_by(image_tag.c.tag_id)
)
).all()
return {tid: int(n) for tid, n in rows}
async def _create_protective_aliases(
self, src_name: str, src_kind: TagKind, tgt: int
) -> bool:
@@ -548,3 +591,161 @@ class TagService:
if res.rowcount:
created = True
return created
# ---------------------------------------------------------------------------
# #714: retro-normalize existing tags to the #701 canonical (Title Case +
# collapsed whitespace) and merge case/whitespace-variant duplicates.
# ---------------------------------------------------------------------------
_NORMALIZE_SAMPLE_CAP = 50
def _group_existing_tags(
rows,
) -> dict[tuple, list[tuple[int, str]]]:
"""Group (id, name, kind, fandom_id) rows by their post-normalization
identity: (kind, COALESCE(fandom_id, -1), canonical_name). Every tag that
would collapse to the same canonical lives in ONE group, so the canonical
form is unique within (kind, fandom) once each group is resolved."""
groups: dict[tuple, list[tuple[int, str]]] = {}
for tag_id, name, kind, fandom_id in rows:
canonical = normalize_tag_name(name)
key = (kind, fandom_id if fandom_id is not None else -1, canonical)
groups.setdefault(key, []).append((tag_id, name))
return groups
def _group_needs_change(canonical: str, members: list[tuple[int, str]]) -> bool:
"""A group is already canonical iff it's a single member whose name equals
the canonical form. Anything else (a collision, or a lone mis-cased tag)
needs work."""
if len(members) > 1:
return True
return members[0][1] != canonical
def _best_connected(tag_ids: list[int], counts: dict[int, int]) -> int:
"""The tag with the most image associations (→ fewest FK repoints when it
survives), tie-broken to the lowest id for determinism. Module-level so the
key closes over its parameter, not a loop variable (ruff B023)."""
return max(tag_ids, key=lambda tid: (counts.get(tid, 0), -tid))
async def normalize_existing_tags(
session: AsyncSession, *, dry_run: bool = False
) -> dict:
"""Convert the back-catalog to the #701 canonical tag form.
For each (kind, fandom, canonical) group: pick a survivor, merge any
case/whitespace-variant siblings INTO it via the tested merge path
(TagService._do_merge — FK repoints + protective aliases), then rename the
survivor to the canonical form. Idempotent: a group that is already a lone
canonical tag is a no-op, so re-running is safe.
dry_run=True returns a projection (counts + a sample of the changes) with no
mutations. Live runs commit per group and isolate failures per group so one
bad group can't strand the rest.
Returns (dry_run):
{"groups": N, "collisions": M, "tags_to_merge": K, "tags_to_rename": R,
"total_changes": T, "sample": [{"to", "from": [...], "kind", "merge"}]}
Returns (live):
{"groups_processed", "merged", "renamed", "aliases_created", "errors",
"sample": [...]}
"""
rows = (
await session.execute(
select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
)
).all()
groups = _group_existing_tags(rows)
# Deterministic sample/ordering: by kind then canonical name.
touched = sorted(
(
(key, members)
for key, members in groups.items()
if _group_needs_change(key[2], members)
),
key=lambda km: (
km[0][0].value if hasattr(km[0][0], "value") else str(km[0][0]),
km[0][2].lower(),
),
)
sample = [
{
"to": key[2],
"from": [name for _id, name in members],
"kind": key[0].value if hasattr(key[0], "value") else str(key[0]),
"merge": len(members) > 1,
}
for key, members in touched[:_NORMALIZE_SAMPLE_CAP]
]
if dry_run:
collisions = sum(1 for key, m in touched if len(m) > 1)
tags_to_merge = sum(len(m) - 1 for key, m in touched if len(m) > 1)
# A group renames iff the canonical form isn't already one of its
# members' exact names (else that member is picked as survivor → no
# rename, the rest merge into it).
tags_to_rename = sum(
1
for key, m in touched
if key[2] not in {name for _id, name in m}
)
return {
"groups": len(groups),
"collisions": collisions,
"tags_to_merge": tags_to_merge,
"tags_to_rename": tags_to_rename,
"total_changes": len(touched),
"sample": sample,
}
svc = TagService(session)
summary = {
"groups_processed": 0,
"merged": 0,
"renamed": 0,
"aliases_created": 0,
"errors": 0,
"sample": sample,
}
for key, members in touched:
canonical = key[2]
names_by_id = dict(members)
# Survivor: prefer a member already named canonically (no rename, no
# self-alias); else the best-connected (fewest FK repoints); else
# lowest id for determinism.
survivor_id = next(
(tid for tid, name in members if name == canonical), None
)
if survivor_id is None:
counts = await svc._image_assoc_counts(list(names_by_id))
survivor_id = _best_connected(list(names_by_id), counts)
loser_ids = [tid for tid in names_by_id if tid != survivor_id]
try:
survivor = await session.get(Tag, survivor_id)
for loser_id in loser_ids:
loser = await session.get(Tag, loser_id)
if loser is None:
continue
result = await svc._do_merge(loser, survivor)
if result.alias_created:
summary["aliases_created"] += 1
if survivor.name != canonical:
survivor.name = canonical
await session.flush()
summary["renamed"] += 1
await session.commit()
summary["groups_processed"] += 1
summary["merged"] += len(loser_ids)
except Exception as exc: # one bad group must not strand the rest
await session.rollback()
summary["errors"] += 1
log.warning(
"tag normalize failed for group %r: %s", canonical, exc
)
return summary
+47
View File
@@ -55,3 +55,50 @@ def bulk_delete_images_task(self, *, image_ids: list[int]) -> dict:
return cleanup_service.delete_images(
session, image_ids=image_ids, images_root=IMAGES_ROOT,
)
@celery.task(
name="backend.app.tasks.admin.reextract_archive_attachments_task",
bind=True,
autoretry_for=(OperationalError, DBAPIError),
retry_backoff=15, retry_backoff_max=180, max_retries=1,
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
)
def reextract_archive_attachments_task(self) -> dict:
"""Wraps cleanup_service.reextract_archive_attachments (#713 part 2):
re-extract PostAttachments that are actually archives but were filed
opaquely before the magic-byte gate, and link their members to the post."""
SessionLocal = _sync_session_factory()
with SessionLocal() as session:
return cleanup_service.reextract_archive_attachments(
session, images_root=IMAGES_ROOT,
)
@celery.task(
name="backend.app.tasks.admin.normalize_tags_task",
bind=True,
autoretry_for=(OperationalError, DBAPIError),
retry_backoff=15, retry_backoff_max=180, max_retries=1,
soft_time_limit=1800, time_limit=2400, # 30 min / 40 min
)
def normalize_tags_task(self) -> dict:
"""Wraps tag_service.normalize_existing_tags (#714): Title-Case the
back-catalog and merge case/whitespace-variant duplicate tags via the
tested async merge path. Runs under its own asyncio loop + per-task async
engine (NullPool, disposed when the loop ends), mirroring download_source."""
import asyncio
from ..services.tag_service import normalize_existing_tags
from ._async_session import async_session_factory
async def _run() -> dict:
async_factory, async_engine = async_session_factory()
try:
async with async_factory() as session:
# normalize_existing_tags commits per group internally.
return await normalize_existing_tags(session, dry_run=False)
finally:
await async_engine.dispose()
return asyncio.run(_run())
+13 -9
View File
@@ -149,9 +149,8 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
call the same code path.
"""
path = Path(path_str)
ext = path.suffix.lower()
try:
total, test_bad = _inspect_archive(path, ext)
total, test_bad = _inspect_archive(path)
except Exception as exc: # noqa: BLE001 — clean rejection
return ("error", f"{type(exc).__name__}: {exc}")
if total is not None and total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
@@ -162,24 +161,29 @@ def _run_probe(path_str: str) -> tuple[str, str | None]:
return ("ok", None)
def _inspect_archive(path: Path, ext: str):
def _inspect_archive(path: Path):
"""Return (total_uncompressed_bytes | None, first_bad_member | None)
for the archive. Format-specific; raises on a structurally-broken
container (caught by the child as a clean rejection)."""
if ext in (".zip", ".cbz"):
for the archive. Format detected by extension OR magic bytes (so a
mis-named archive is still bomb-guarded + integrity-tested, matching the
extractor's gate); raises on a structurally-broken container (caught by the
child as a clean rejection)."""
from ..services.archive_extractor import detect_archive_format
fmt = detect_archive_format(path)
if fmt == "zip":
import zipfile
with zipfile.ZipFile(path) as zf:
total = sum(zi.file_size for zi in zf.infolist())
return total, zf.testzip()
if ext == ".rar":
if fmt == "rar":
import rarfile
with rarfile.RarFile(path) as rf:
total = sum(getattr(ri, "file_size", 0) for ri in rf.infolist())
rf.testrar()
return total, None
if ext == ".7z":
if fmt == "7z":
import py7zr
with py7zr.SevenZipFile(path, "r") as zf:
@@ -187,5 +191,5 @@ def _inspect_archive(path: Path, ext: str):
total = getattr(info, "uncompressed", None)
ok = zf.test() # True / None when all members pass
return total, (None if ok in (True, None) else "7z test reported corruption")
# Unknown extension — nothing to test; treat as clean.
# Not a recognised archive — nothing to test; treat as clean.
return None, None
@@ -49,9 +49,11 @@ function onCardClick() {
position: relative;
display: grid; grid-template-columns: repeat(3, 1fr);
gap: 2px; aspect-ratio: 3 / 1;
/* Explicit floor + ceiling so tall source images can't escape the
preview slot even on browsers where aspect-ratio doesn't compute. */
min-height: 150px; max-height: 220px;
/* Floor so tall source images can't escape the slot. NO max-height: an
aspect-ratio box capped by max-height shrinks its own WIDTH to keep the
ratio, leaving dead space beside the previews on a wide (single-column)
card. The grid below keeps columns ≲400px so the strip stays a sane height. */
min-height: 120px;
overflow: hidden;
background: rgb(var(--v-theme-surface-light));
}
@@ -38,7 +38,8 @@ const store = useTagStore()
const selectedId = ref(null)
const newName = ref('')
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
onMounted(() => store.loadFandoms())
async function onCreate() {
const f = await store.createFandom(newName.value.trim())
@@ -84,7 +84,8 @@ const busy = ref(false)
const error = ref(null)
const collision = ref(null)
onMounted(() => { if (store.fandomCache.length === 0) store.loadFandoms() })
// Always refresh on open so the list reflects fandoms created elsewhere (#712).
onMounted(() => store.loadFandoms())
async function onCreate() {
const name = newName.value.trim()
@@ -106,7 +106,11 @@ function onKeyDown(ev) {
// overlay's own Esc handling fire instead of closing the whole
// modal mid-interaction. Vuetify marks open overlays with
// `.v-overlay--active`.
if (document.querySelector('.v-overlay--active')) return
// EXCLUDE tooltips (`.v-tooltip`): they're also `.v-overlay--active` while
// shown, so one lingering after a hover/click (e.g. just after accepting a
// suggested tag) wrongly suppressed the close (#700). Only real interactive
// overlays (menus/dialogs) should keep ESC from closing the modal.
if (document.querySelector('.v-overlay--active:not(.v-tooltip)')) return
ev.preventDefault()
emit('close')
} else if (ev.key === 'ArrowLeft') {
+38 -42
View File
@@ -2,45 +2,43 @@
<aside class="fc-tag-panel" aria-label="Tags for this image">
<h3 class="fc-tag-panel__title">Tags</h3>
<div class="fc-tag-panel__chips">
<v-chip
<!-- #711: the kebab + its menu used to be NESTED inside the v-chip, which
swallowed/mis-routed the click and mis-anchored the (teleported) menu
so it never opened. Render the kebab as a SIBLING of the chip and use
the standard v-menu activator slot Vuetify wires the click and
stacks the overlay above the modal natively. -->
<span
v-for="tag in modal.current?.tags || []"
:key="tag.id" size="small" closable
:color="store.colorFor(tag.kind)" variant="tonal"
@click:close="onRemove(tag.id)"
:key="tag.id" class="fc-tag-panel__chip"
>
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
{{ tag.name }}<span v-if="tag.fandom_id"></span>
<!-- Operator-flagged 2026-06-04: the `#activator` + `v-bind` menu
never opened inside this teleported modal. Drive it explicitly
instead (same mechanism as the dialogs below, which work): the
icon toggles `openTagId` with @click.stop (shielding the chip's
close button), and `activator="parent"` + `:open-on-click=false`
anchors the menu for positioning only. One tag's menu open at a
time, so a single id is enough. -->
<span class="kebab-wrap">
<v-icon
size="x-small" class="ml-1 kebab-icon"
icon="mdi-dots-vertical"
@click.stop="openTagId = openTagId === tag.id ? null : tag.id"
/>
<v-menu
:model-value="openTagId === tag.id"
activator="parent" :open-on-click="false"
@update:model-value="v => { if (!v) openTagId = null }"
>
<v-list density="compact">
<v-list-item @click="openRename(tag)">
<v-list-item-title>Rename…</v-list-item-title>
</v-list-item>
<v-list-item
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
>
<v-list-item-title>Set fandom…</v-list-item-title>
</v-list-item>
</v-list>
</v-menu>
</span>
</v-chip>
<v-chip
size="small" closable
:color="store.colorFor(tag.kind)" variant="tonal"
@click:close="onRemove(tag.id)"
>
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
{{ tag.name }}<span v-if="tag.fandom_id"></span>
</v-chip>
<v-menu location="bottom end">
<template #activator="{ props }">
<v-btn
v-bind="props" icon="mdi-dots-vertical" size="x-small"
variant="text" density="comfortable"
class="fc-tag-panel__kebab" @click.stop
/>
</template>
<v-list density="compact">
<v-list-item @click="openRename(tag)">
<v-list-item-title>Rename</v-list-item-title>
</v-list-item>
<v-list-item
v-if="tag.kind === 'character'" @click="openSetFandom(tag)"
>
<v-list-item-title>Set fandom</v-list-item-title>
</v-list-item>
</v-list>
</v-menu>
</span>
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
</div>
@@ -88,9 +86,6 @@ import FandomSetDialog from './FandomSetDialog.vue'
const modal = useModalStore()
const store = useTagStore()
const errorMsg = ref(null)
// Which tag chip's kebab menu is open (only one at a time). Drives each
// chip menu's v-model so opening never depends on Vuetify's activator click.
const openTagId = ref(null)
const KIND_ICONS = {
general: 'mdi-tag', character: 'mdi-account-circle',
@@ -153,6 +148,7 @@ async function onFandomUpdated() {
margin-bottom: 12px;
}
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
.kebab-wrap { display: inline-flex; align-items: center; }
.kebab-icon { cursor: pointer; }
.fc-tag-panel__chip { display: inline-flex; align-items: center; gap: 1px; }
.fc-tag-panel__kebab { opacity: 0.7; }
.fc-tag-panel__chip:hover .fc-tag-panel__kebab { opacity: 1; }
</style>
@@ -0,0 +1,46 @@
<template>
<!-- #713: re-extract PostAttachments that are really archives but were filed
opaquely before the magic-byte gate, and attach their images to the post. -->
<v-card>
<v-card-title>Re-extract archive attachments</v-card-title>
<v-card-text>
<p class="text-body-2 mb-3">
Some posts attached an archive (zip) whose images weren't extracted
because the downloaded file had no usable extension. This scans existing
attachments, extracts any that are really archives, and attaches their
images to the post. Idempotent — safe to run more than once.
</p>
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
</v-btn>
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
</v-card-text>
</v-card>
</template>
<script setup>
import { ref } from 'vue'
import { useApi } from '../../composables/useApi.js'
import { toast } from '../../utils/toast.js'
import QueueStatusBar from './QueueStatusBar.vue'
const api = useApi()
const busy = ref(false)
const queued = ref(false)
async function run () {
busy.value = true
queued.value = false
try {
await api.post('/api/admin/maintenance/reextract-archives')
queued.value = true
toast({ text: 'Archive re-extract queued', type: 'success' })
} catch (e) {
toast({ text: e?.body?.detail || e?.message || 'Failed to queue', type: 'error' })
} finally {
busy.value = false
}
}
</script>
@@ -15,6 +15,7 @@
<AllowlistTable class="mt-4" />
<AliasTable class="mt-4" />
<DbMaintenanceCard class="mt-6" />
<ArchiveReextractCard class="mt-6" />
<BackupCard class="mt-6" />
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) it
operates on the existing library which fits the Cleanup-tab
@@ -33,6 +34,7 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
import AllowlistTable from './AllowlistTable.vue'
import AliasTable from './AliasTable.vue'
import DbMaintenanceCard from './DbMaintenanceCard.vue'
import ArchiveReextractCard from './ArchiveReextractCard.vue'
import BackupCard from './BackupCard.vue'
import { useSystemActivityStore } from '../../stores/systemActivity.js'
@@ -136,6 +136,64 @@
>Delete {{ resetPreview.count }} content tag(s) +
{{ resetPreview.applications }} application(s)</v-btn>
</div>
<v-divider class="my-5" />
<!-- #714: standardize existing tags to the canonical Title-Case form
and merge case/whitespace-variant duplicates. -->
<p class="text-body-2 mb-2">
<strong>Standardize tag casing.</strong>
New tags are saved Title Case, but older tags keep whatever casing they
were created with. This renames every existing tag to
<code>Title Case</code> (collapsing extra spaces) and
<strong>merges</strong> tags that differ only by case or spacing
(e.g. <code>hatsune miku</code> + <code>Hatsune&nbsp;&nbsp;Miku</code>)
into one — repointing their images, allowlist entries and series pages.
</p>
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
The merges are irreversible — back up first
(Settings → Maintenance → Backup). Safe to run more than once.
</v-alert>
<v-btn
color="accent" variant="flat" rounded="pill"
prepend-icon="mdi-magnify"
:loading="loadingNormPreview"
class="mb-3"
@click="onNormPreview"
>Preview tag standardization</v-btn>
<div v-if="normPreview">
<p class="text-body-2 mb-2">
<strong>{{ normPreview.total_changes }}</strong> tag group(s) to
change — <strong>{{ normPreview.tags_to_rename }}</strong> rename(s),
<strong>{{ normPreview.collisions }}</strong> collision(s) merging
<strong>{{ normPreview.tags_to_merge }}</strong> tag(s) away.
</p>
<div v-if="normPreview.sample?.length" class="fc-name-grid mb-3">
<span
v-for="s in normPreview.sample" :key="s.to + s.kind"
class="fc-name"
>
<template v-if="s.merge">{{ s.from.join(' + ') }} → </template>{{ s.to }}
</span>
</div>
<v-btn
color="error" variant="flat" rounded="pill"
prepend-icon="mdi-format-letter-case"
:disabled="!normPreview.total_changes"
:loading="normCommitting"
@click="onNormCommit"
>Standardize {{ normPreview.total_changes }} tag group(s)</v-btn>
<span
v-if="normResult"
class="ml-3 text-caption"
:class="normResult === 'ok' ? 'text-success' : 'text-warning'"
>
<template v-if="normResult === 'ok'">Standardization complete ✓</template>
<template v-else>Finished with status: {{ normResult }}</template>
</span>
</div>
</v-card-text>
</v-card>
</template>
@@ -155,6 +213,10 @@ const kindCommitting = ref(false)
const resetPreview = ref(null)
const loadingResetPreview = ref(false)
const resetCommitting = ref(false)
const normPreview = ref(null)
const loadingNormPreview = ref(false)
const normCommitting = ref(false)
const normResult = ref(null)
async function onPreview() {
loadingPreview.value = true
@@ -212,6 +274,33 @@ async function onResetCommit() {
resetCommitting.value = false
}
}
async function onNormPreview() {
loadingNormPreview.value = true
normResult.value = null
try {
normPreview.value = await store.normalizeTags({ dryRun: true })
} finally {
loadingNormPreview.value = false
}
}
async function onNormCommit() {
normCommitting.value = true
normResult.value = null
try {
// Long op (FK repoints): enqueue the maintenance task, then tail the
// activity dashboard until its row reaches a terminal status. (The
// per-run summary dict isn't exposed by /activity/runs, so we surface
// the terminal status — the dry-run preview is the detailed view.)
const { task_id: taskId } = await store.normalizeTags({ dryRun: false })
const row = await store.pollTaskUntilDone(taskId)
normResult.value = row?.status || 'ok'
normPreview.value = { total_changes: 0, tags_to_rename: 0, collisions: 0, tags_to_merge: 0, sample: [] }
} finally {
normCommitting.value = false
}
}
</script>
<style scoped>
@@ -23,6 +23,18 @@
<span class="fc-active__dot" />
<PlatformChip :platform="e.platform" size="x-small" />
<span class="fc-active__artist">{{ e.artist_name || '—' }}</span>
<!-- #709: live per-file progress for native (Patreon) downloads. -->
<span v-if="e.live" class="fc-active__counts">
<span class="fc-active__count" title="downloaded"> {{ e.live.downloaded }}</span>
<span v-if="e.live.skipped" class="fc-active__count" title="skipped">
{{ e.live.skipped }}</span>
<span
v-if="e.live.errors" class="fc-active__count fc-active__count--err"
title="errors"
> {{ e.live.errors }}</span>
<span class="fc-active__count fc-active__count--posts" title="posts scanned">
{{ e.live.posts }} posts</span>
</span>
<v-spacer />
<span class="fc-active__timer" :title="`started ${e.started_at}`">
{{ elapsed(e.started_at) }}
@@ -123,6 +135,13 @@ function elapsed (startedIso) {
font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.05em;
color: rgb(var(--v-theme-on-surface-variant));
}
.fc-active__counts {
display: inline-flex; align-items: center; gap: 8px;
font-variant-numeric: tabular-nums; font-size: 0.78rem;
}
.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); }
.fc-active__count--err { color: rgb(var(--v-theme-error)); }
.fc-active__count--posts { opacity: 0.7; }
@keyframes fc-active-pulse {
0%, 100% { opacity: 1; transform: scale(1); }
50% { opacity: 0.35; transform: scale(0.7); }
+17
View File
@@ -142,6 +142,22 @@ export const useAdminStore = defineStore('admin', () => {
}
}
// #714: Title-Case the back-catalog + merge case/whitespace-variant tags.
// dry-run returns a projection inline; live returns {task_id} (long op —
// FK repoints) the caller polls via pollTaskUntilDone.
async function normalizeTags({ dryRun = true } = {}) {
lastError.value = null
try {
return await api.post(
'/api/admin/tags/normalize',
{ body: { dry_run: dryRun } },
)
} catch (e) {
lastError.value = e.message
throw e
}
}
// --- Task progress polling (taps FC-3i activity dashboard) --------
/**
@@ -178,6 +194,7 @@ export const useAdminStore = defineStore('admin', () => {
pruneUnusedTags,
purgeLegacyTags,
resetContentTagging,
normalizeTags,
pollTaskUntilDone,
}
})
+8 -1
View File
@@ -14,6 +14,10 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
const q = ref('')
const platform = ref(null)
let started = false
// Has a load ATTEMPT completed yet? Gates the "No artists match" empty state
// so it can't flash on the very first render (loading is still false before
// onMounted fires the first loadMore) or between a query change and its fetch.
const loaded = ref(false)
// Typed "alice" then "alice bob" used to drop the second fetch
// entirely (loading flag still true from the first), so the UI
// showed alice results while the input said "alice bob". Inflight
@@ -35,6 +39,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
nextCursor.value = body.next_cursor
started = true
})
if (t.isCurrent()) loaded.value = true
}
async function reset() {
@@ -42,6 +47,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
cards.value = []
nextCursor.value = null
started = false
loaded.value = false
await loadMore()
}
@@ -50,7 +56,8 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
const hasMore = computed(() => !started || nextCursor.value !== null)
const isEmpty = computed(
() => !loading.value && cards.value.length === 0 && error.value === null
() => loaded.value && !loading.value
&& cards.value.length === 0 && error.value === null
)
return {
+14 -4
View File
@@ -34,10 +34,20 @@ export const useTagStore = defineStore('tags', () => {
}
async function loadFandoms() {
fandomCache.value = await api.get('/api/tags/autocomplete', {
params: { q: ' ', kind: 'fandom', limit: 200 }
})
return fandomCache.value
// List ALL fandoms via the cursor-paged tag directory. (#712: the old impl
// abused /tags/autocomplete with q=' ', which the backend strips to empty
// and returns [] — so the picker showed no existing fandoms.)
const all = []
let cursor = null
do {
const params = { kind: 'fandom', limit: 200 }
if (cursor) params.cursor = cursor
const body = await api.get('/api/tags/directory', { params })
all.push(...(body.cards || []))
cursor = body.next_cursor
} while (cursor)
fandomCache.value = all
return all
}
async function createFandom(name) {
+5 -3
View File
@@ -76,9 +76,11 @@ onMounted(async () => {
}
.fc-artists__grid {
display: grid;
/* min(440px, 100%) so a card never exceeds the viewport — on phones the
min-track collapses to 100% (single column) instead of overflowing. */
grid-template-columns: repeat(auto-fill, minmax(min(440px, 100%), 1fr));
/* min(360px, 100%) so a card never exceeds the viewport (phones collapse to
one column). 360 keeps a LONE column under ~732px wide, so the 3/1 preview
strip stays ≲244px tall and always fills the card width (no dead space),
while giving 2+ columns on a normal desktop. */
grid-template-columns: repeat(auto-fill, minmax(min(360px, 100%), 1fr));
gap: 12px;
}
.fc-artists__sentinel {
+8 -6
View File
@@ -52,11 +52,12 @@ async def test_create_tag_missing_name_400(client):
@pytest.mark.asyncio
async def test_create_tag_name_only_defaults_to_general(client):
"""IR-style: name without kind and without `kind:` prefix → general."""
"""IR-style: name without kind and without `kind:` prefix → general.
#701: operator-entered names are Title-Cased at the create endpoint."""
resp = await client.post("/api/tags", json={"name": "sunset"})
assert resp.status_code == 201
body = await resp.get_json()
assert body["name"] == "sunset"
assert body["name"] == "Sunset"
assert body["kind"] == "general"
@@ -73,21 +74,22 @@ async def test_create_tag_with_kind_prefix(client):
@pytest.mark.asyncio
async def test_explicit_kind_overrides_prefix_parsing(client):
"""If caller passes explicit kind, don't re-parse the name —
colon and prefix stay literal."""
colon and prefix stay literal. #701: still Title-Cased as a single word."""
resp = await client.post(
"/api/tags", json={"name": "character:Saber", "kind": "general"}
)
assert resp.status_code == 201
body = await resp.get_json()
assert body["name"] == "character:Saber"
assert body["name"] == "Character:saber"
assert body["kind"] == "general"
@pytest.mark.asyncio
async def test_unknown_prefix_kept_literal(client):
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name."""
"""`http:example` — `http` not in KNOWN_KINDS → kind=general, literal name.
#701: Title-Cased as one word (no internal whitespace to split on)."""
resp = await client.post("/api/tags", json={"name": "http://example.com"})
assert resp.status_code == 201
body = await resp.get_json()
assert body["name"] == "http://example.com"
assert body["name"] == "Http://example.com"
assert body["kind"] == "general"
+18
View File
@@ -39,6 +39,24 @@ def test_cbz_alias(tmp_path):
assert [n for n, _ in members] == ["p1.jpg"]
def test_misnamed_zip_detected_and_extracted(tmp_path):
"""A real zip whose filename has no usable extension (Patreon attachment
URL-blob name) is detected by magic bytes and its members extracted —
previously it was filed as an opaque attachment and never opened."""
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
_zip(z, {"a.jpg": b"img", "b.png": b"img2"})
assert is_archive(z)
with extract_archive(z) as members:
assert sorted(n for n, _ in members) == ["a.jpg", "b.png"]
def test_non_archive_with_dotted_name_is_not_archive(tmp_path):
"""A non-archive file with a dotted/extension-less name isn't misdetected."""
f = tmp_path / "01_https___www.patreon.com_media-u_v3_999"
f.write_bytes(b"\x89PNG\r\n\x1a\n not really but not a zip")
assert not is_archive(f)
def test_corrupt_archive_yields_nothing(tmp_path):
bad = tmp_path / "broken.zip"
bad.write_bytes(b"not a zip at all")
+28
View File
@@ -421,6 +421,34 @@ async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path):
assert result["has_more"] is True
@pytest.mark.asyncio
async def test_write_live_progress_updates_running_event(
source_id, sync_engine, tmp_path, db,
):
"""plan #709: live counts land in the RUNNING event's metadata.live; a
finalized event is left alone (status guard)."""
from backend.app.models import DownloadEvent
ing = _ingester(sync_engine, tmp_path, _FakeClient([]), _FakeDownloader(tmp_path))
running = DownloadEvent(source_id=source_id, status="running")
done = DownloadEvent(source_id=source_id, status="ok")
db.add_all([running, done])
await db.commit()
counts = {"downloaded": 3, "skipped": 1, "errors": 0, "quarantined": 0, "posts": 4}
ing._write_live_progress(running.id, counts)
ing._write_live_progress(done.id, {"downloaded": 9}) # guarded: status != running
live = (await db.execute(
select(DownloadEvent.metadata_).where(DownloadEvent.id == running.id)
)).scalar_one()
assert live["live"] == counts
done_meta = (await db.execute(
select(DownloadEvent.metadata_).where(DownloadEvent.id == done.id)
)).scalar_one()
assert "live" not in (done_meta or {})
@pytest.mark.asyncio
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
+91
View File
@@ -0,0 +1,91 @@
"""#713 part 2: re-extract archive PostAttachments that were filed opaquely
(magic-byte gate missed them) and link their members to the post."""
import hashlib
import io
import zipfile
import pytest
from PIL import Image
from sqlalchemy import select
from backend.app.models import (
Artist,
ImageProvenance,
ImageRecord,
Post,
PostAttachment,
Source,
)
from backend.app.services import cleanup_service
pytestmark = pytest.mark.integration
def _jpeg(color, size=256):
buf = io.BytesIO()
Image.new("RGB", (size, size), color).save(buf, "JPEG")
return buf.getvalue()
def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch):
from backend.app.tasks import ml as ml_mod
from backend.app.tasks import thumbnail as thumb_mod
# No broker in this path — the post-import enqueue is best-effort anyway.
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
images_root = tmp_path / "images"
images_root.mkdir()
artist = Artist(name="Bob", slug="bob")
db_sync.add(artist)
db_sync.flush()
source = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/bob", enabled=True, config_overrides={},
)
db_sync.add(source)
db_sync.flush()
post = Post(
source_id=source.id, artist_id=artist.id, external_post_id="59102952",
post_url="https://www.patreon.com/posts/59102952",
)
db_sync.add(post)
db_sync.flush()
# A real zip stored under a mangled / extension-less name (the failure case).
store_dir = images_root / "attachments" / "abc"
store_dir.mkdir(parents=True)
arc = store_dir / "01_https___www.patreon.com_media-u_v3_59102952"
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("inside.jpg", _jpeg("red"))
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
db_sync.add(PostAttachment(
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
))
db_sync.commit()
summary = cleanup_service.reextract_archive_attachments(
db_sync, images_root=images_root,
)
assert summary["archives"] == 1
assert summary["members_imported"] == 1
assert summary["posts_touched"] == 1
images = db_sync.execute(select(ImageRecord)).scalars().all()
assert len(images) == 1
prov = db_sync.execute(
select(ImageProvenance).where(ImageProvenance.post_id == post.id)
).scalars().all()
assert len(prov) == 1
assert prov[0].image_record_id == images[0].id
# Idempotent — a second run imports nothing new (member dedups by sha256).
again = cleanup_service.reextract_archive_attachments(
db_sync, images_root=images_root,
)
assert again["members_imported"] == 0
assert db_sync.execute(select(ImageRecord)).scalars().all() == images
+11 -1
View File
@@ -40,11 +40,21 @@ def test_probe_archive_corrupt_zip_clean_rejection(tmp_path):
def test_inspect_archive_reports_size_and_clean_integrity(tmp_path):
z = tmp_path / "sized.zip"
_zip(z, {"a.txt": b"x" * 100, "b.txt": b"y" * 50})
total, bad = safe_probe._inspect_archive(z, ".zip")
total, bad = safe_probe._inspect_archive(z)
assert total == 150
assert bad is None
def test_inspect_archive_detects_misnamed_zip(tmp_path):
"""A zip with a mangled / extension-less name (Patreon URL-blob attachment)
is still bomb-guarded + integrity-tested via magic-byte detection."""
z = tmp_path / "01_https___www.patreon.com_media-u_v3_131083093"
_zip(z, {"a.txt": b"x" * 100})
total, bad = safe_probe._inspect_archive(z)
assert total == 100
assert bad is None
def test_run_probe_bomb_guard(tmp_path, monkeypatch):
"""In-process call to _run_probe so the monkeypatched cap takes effect.
The public probe_archive runs in a subprocess which re-imports the
+189
View File
@@ -0,0 +1,189 @@
"""#714: retro-normalize existing tags to the #701 canonical (Title Case +
collapsed whitespace) and merge case/whitespace-variant duplicates.
normalize_existing_tags commits per group, so case-variant duplicates are
seeded with DIRECT inserts (find_or_create would case-insensitively dedup
them, which is exactly the pre-#701 state we're cleaning up), and assertions
re-query via fresh selects rather than touching post-commit ORM entities.
"""
import pytest
from sqlalchemy import func, select
from backend.app.models import Tag, TagKind, image_tag
from backend.app.models.tag_alias import TagAlias
from backend.app.services.tag_service import (
TagService,
normalize_existing_tags,
)
pytestmark = pytest.mark.integration
_IMG_SEQ = 0
async def _img(db):
"""Minimal valid ImageRecord; returns its id."""
global _IMG_SEQ
_IMG_SEQ += 1
from backend.app.models import ImageRecord
rec = ImageRecord(
path=f"/tmp/fc_norm_{_IMG_SEQ}.png",
sha256=f"{_IMG_SEQ:064d}",
size_bytes=1,
mime="image/png",
origin="uploaded",
)
db.add(rec)
await db.flush()
return rec.id
async def _raw_tag(db, name, kind, fandom_id=None):
"""Insert a Tag row verbatim — bypasses find_or_create's case-insensitive
dedup so we can plant the case-variant duplicates #714 exists to fix."""
t = Tag(name=name, kind=kind, fandom_id=fandom_id)
db.add(t)
await db.flush()
return t
async def _link(db, image_id, tag_id, source="manual"):
await db.execute(
image_tag.insert().values(
image_record_id=image_id, tag_id=tag_id, source=source
)
)
async def _count_named(db, name, kind):
return await db.scalar(
select(func.count())
.select_from(Tag)
.where(Tag.name == name)
.where(Tag.kind == kind)
)
@pytest.mark.asyncio
async def test_dry_run_projection_counts(db):
# Collision group with an already-canonical member (→ no rename, 1 merge).
await _raw_tag(db, "hatsune miku", TagKind.character)
await _raw_tag(db, "Hatsune Miku", TagKind.character)
# Lone mis-cased + extra whitespace (→ rename, no merge).
await _raw_tag(db, "looking at viewer", TagKind.general)
# Already canonical (→ no change).
await _raw_tag(db, "Already Canonical", TagKind.general)
proj = await normalize_existing_tags(db, dry_run=True)
assert proj["collisions"] == 1
assert proj["tags_to_merge"] == 1
assert proj["tags_to_rename"] == 1 # only the whitespace group renames
assert proj["total_changes"] == 2
# Nothing mutated by a dry run.
assert await _count_named(db, "hatsune miku", TagKind.character) == 1
@pytest.mark.asyncio
async def test_live_merges_case_variants_and_repoints_images(db):
a = await _raw_tag(db, "hatsune miku", TagKind.character)
b = await _raw_tag(db, "HATSUNE MIKU", TagKind.character)
i1, i2, i3 = await _img(db), await _img(db), await _img(db)
await _link(db, i1, a.id)
await _link(db, i2, a.id)
await _link(db, i2, b.id) # shared → must dedup on the survivor
await _link(db, i3, b.id)
summary = await normalize_existing_tags(db, dry_run=False)
assert summary["errors"] == 0
assert summary["merged"] == 1
assert summary["renamed"] == 1
assert summary["groups_processed"] == 1
# Exactly one canonical character tag survives; the variant is gone.
assert await _count_named(db, "Hatsune Miku", TagKind.character) == 1
total_chars = await db.scalar(
select(func.count()).select_from(Tag).where(Tag.kind == TagKind.character)
)
assert total_chars == 1
survivor_id = await db.scalar(
select(Tag.id)
.where(Tag.name == "Hatsune Miku")
.where(Tag.kind == TagKind.character)
)
# Union of {i1,i2} and {i2,i3} with the shared i2 deduped == 3.
assoc = await db.scalar(
select(func.count())
.select_from(image_tag)
.where(image_tag.c.tag_id == survivor_id)
)
assert assoc == 3
@pytest.mark.asyncio
async def test_idempotent_second_run_is_noop(db):
await _raw_tag(db, "kafka", TagKind.character)
await _raw_tag(db, "KAFKA", TagKind.character)
first = await normalize_existing_tags(db, dry_run=False)
assert first["groups_processed"] == 1
proj = await normalize_existing_tags(db, dry_run=True)
assert proj["total_changes"] == 0
again = await normalize_existing_tags(db, dry_run=False)
assert again["groups_processed"] == 0
assert again["merged"] == 0
assert again["renamed"] == 0
@pytest.mark.asyncio
async def test_same_name_different_fandom_not_merged(db):
f1 = await _raw_tag(db, "Bleach", TagKind.fandom)
f2 = await _raw_tag(db, "Naruto", TagKind.fandom)
await _raw_tag(db, "alice", TagKind.character, fandom_id=f1.id)
await _raw_tag(db, "Alice", TagKind.character, fandom_id=f2.id)
summary = await normalize_existing_tags(db, dry_run=False)
assert summary["merged"] == 0
assert summary["renamed"] == 1 # only the f1 "alice" recases
# Both characters survive — same canonical name, distinct fandoms.
assert await _count_named(db, "Alice", TagKind.character) == 2
@pytest.mark.asyncio
async def test_same_name_different_kind_not_merged(db):
await _raw_tag(db, "forge", TagKind.artist)
await _raw_tag(db, "Forge", TagKind.general)
summary = await normalize_existing_tags(db, dry_run=False)
assert summary["merged"] == 0
assert await _count_named(db, "Forge", TagKind.artist) == 1
assert await _count_named(db, "Forge", TagKind.general) == 1
@pytest.mark.asyncio
async def test_ml_known_loser_keeps_protective_alias(db):
# Survivor is the already-canonical (manual, ML-unknown) tag; the ML-sourced
# variant merges away and must leave an alias so the tagger re-resolves it.
canon = await _raw_tag(db, "Nemu", TagKind.general)
canon_id = canon.id # capture before the service commits
ml = await _raw_tag(db, "nemu", TagKind.general)
img = await _img(db)
await _link(db, img, ml.id, source="ml_auto")
summary = await normalize_existing_tags(db, dry_run=False)
assert summary["merged"] == 1
assert summary["aliases_created"] == 1
assert await _count_named(db, "Nemu", TagKind.general) == 1
alias_rows = await db.scalar(
select(func.count())
.select_from(TagAlias)
.where(TagAlias.alias_string == "nemu")
.where(TagAlias.canonical_tag_id == canon_id)
)
assert alias_rows == 1
+21
View File
@@ -59,6 +59,27 @@ async def test_character_with_fandom(db):
assert char.fandom_id == fandom.id
@pytest.mark.asyncio
async def test_find_or_create_dedups_case_insensitive(db):
"""#701: find_or_create matches case-insensitively so a differently-cased
entry finds the existing tag instead of forking. (Display Title-Casing is
applied at the create API, not here — this path is shared with the ML
tagger, which keeps the booru vocabulary's casing.)"""
svc = TagService(db)
t1 = await svc.find_or_create("hatsune miku", TagKind.character)
assert t1.name == "hatsune miku" # stored as given; not recased here
t2 = await svc.find_or_create("HATSUNE MIKU", TagKind.character)
assert t2.id == t1.id # case variant → same tag, no fork
@pytest.mark.asyncio
async def test_normalize_tag_name():
from backend.app.services.tag_service import normalize_tag_name
assert normalize_tag_name("hatsune miku") == "Hatsune Miku"
assert normalize_tag_name(" looking at viewer ") == "Looking At Viewer"
assert normalize_tag_name("HATSUNE MIKU") == "Hatsune Miku"
@pytest.mark.asyncio
async def test_character_same_name_different_fandoms(db):
svc = TagService(db)