diff --git a/alembic/versions/0020_library_audit_run.py b/alembic/versions/0020_library_audit_run.py new file mode 100644 index 0000000..07a8b87 --- /dev/null +++ b/alembic/versions/0020_library_audit_run.py @@ -0,0 +1,65 @@ +"""fc-cleanup: library_audit_run table for async transparency/single_color audits + +Revision ID: 0020 +Revises: 0019 +Create Date: 2026-05-26 + +The table backs the async audit lifecycle: rule + params snapshot, status +state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and +the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs +per row by the scan task (oversize = rule too aggressive, operator narrows +before re-running). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0020" +down_revision: Union[str, None] = "0019" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "library_audit_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("rule", sa.String(32), nullable=False), + sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column( + "status", sa.String(16), + nullable=False, server_default="running", + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "scanned_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_ids", postgresql.JSONB(astext_type=sa.Text()), + nullable=False, server_default=sa.text("'[]'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + ) + op.create_index( + "ix_library_audit_run_rule", "library_audit_run", ["rule"], + ) + op.create_index( + "ix_library_audit_run_status", "library_audit_run", ["status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_library_audit_run_status", table_name="library_audit_run") + op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run") + op.drop_table("library_audit_run") diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 6ba84cb..3068ca2 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -33,6 +33,7 @@ def make_celery() -> Celery: "backend.app.tasks.download", "backend.app.tasks.backup", "backend.app.tasks.admin", + "backend.app.tasks.library_audit", ], ) app.conf.update( @@ -47,6 +48,7 @@ def make_celery() -> Celery: "backend.app.tasks.migration.*": {"queue": "maintenance"}, "backend.app.tasks.backup.*": {"queue": "maintenance"}, "backend.app.tasks.admin.*": {"queue": "maintenance"}, + "backend.app.tasks.library_audit.*": {"queue": "maintenance"}, }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 0cefb2d..00a7c80 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -11,6 +11,7 @@ from .image_record import ImageRecord from .import_batch import ImportBatch from .import_settings import ImportSettings from .import_task import ImportTask +from .library_audit_run import LibraryAuditRun from .migration_run import MigrationRun from .ml_settings import MLSettings from .post import Post @@ -43,6 +44,7 @@ __all__ = [ "ImportBatch", "ImportTask", "ImportSettings", + "LibraryAuditRun", "MLSettings", "MigrationRun", "TagAlias", diff --git a/backend/app/models/library_audit_run.py b/backend/app/models/library_audit_run.py new file mode 100644 index 0000000..ca5aa82 --- /dev/null +++ b/backend/app/models/library_audit_run.py @@ -0,0 +1,37 @@ +"""LibraryAuditRun — async transparency / single_color audit lifecycle. + +State machine: running → ready → applied / cancelled / error. +matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run +reads it and routes through cleanup_service.delete_images. +""" + +from datetime import datetime +from typing import Any + +from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class LibraryAuditRun(Base): + __tablename__ = "library_audit_run" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="running", index=True, + ) + # running | ready | applied | cancelled | error + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + finished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list) + error: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/services/audits/__init__.py b/backend/app/services/audits/__init__.py new file mode 100644 index 0000000..96e5198 --- /dev/null +++ b/backend/app/services/audits/__init__.py @@ -0,0 +1,6 @@ +"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool. + +The retroactive library-cleanup tab and (future) import-time filter logic +both consume these. Importers should NOT inline rule logic going forward; +add the rule here and call from both sides. +""" diff --git a/backend/app/services/audits/single_color.py b/backend/app/services/audits/single_color.py new file mode 100644 index 0000000..167e078 --- /dev/null +++ b/backend/app/services/audits/single_color.py @@ -0,0 +1,53 @@ +"""Single-color audit: matches images where one color dominates beyond +the threshold (within the given Euclidean RGB tolerance). The first +canonical implementation — the import-side filter (SkipReason.single_color) +was never wired; FC-Cleanup's audit module is the source of truth and a +future spec can adopt it on the import path too. +""" + +from PIL import Image + +_THUMB_SIZE = (64, 64) + + +def evaluate( + pil_image, + *, + threshold: float, + tolerance: int, +) -> bool: + """True iff the fraction of pixels within `tolerance` (Euclidean RGB + distance) of the dominant color exceeds `threshold`. + + Downsamples to 64x64 for speed (~4ms regardless of source size). + Alpha channels are stripped; only RGB is considered. Animated images + use frame 0 (PIL's default after Image.open without seek). + """ + im = pil_image + if im.mode == "RGBA": + im = im.convert("RGB") + elif im.mode not in ("RGB", "L"): + im = im.convert("RGB") + if im.size != _THUMB_SIZE: + im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR) + pixels = list(im.getdata()) + if not pixels: + return False + # Normalize L-mode pixels to RGB tuples for distance math. + if isinstance(pixels[0], int): + pixels = [(p, p, p) for p in pixels] + # Dominant color = mean RGB. + n = len(pixels) + sum_r = sum(p[0] for p in pixels) + sum_g = sum(p[1] for p in pixels) + sum_b = sum(p[2] for p in pixels) + dom = (sum_r / n, sum_g / n, sum_b / n) + tol_sq = tolerance * tolerance + within = 0 + for r, g, b in pixels: + dr = r - dom[0] + dg = g - dom[1] + db = b - dom[2] + if dr * dr + dg * dg + db * db <= tol_sq: + within += 1 + return (within / n) > threshold diff --git a/backend/app/services/audits/transparency.py b/backend/app/services/audits/transparency.py new file mode 100644 index 0000000..f15f31f --- /dev/null +++ b/backend/app/services/audits/transparency.py @@ -0,0 +1,27 @@ +"""Transparency audit: matches images whose transparent-pixel fraction +exceeds the threshold. Animated images short-circuit (skipped) to avoid +the multi-frame PIL decode that hits Celery's hard time limit.""" + + +def evaluate(pil_image, *, threshold: float) -> bool: + """True iff the image's transparent-pixel fraction exceeds threshold. + + False for non-alpha modes and animated images. Mirrors the import-side + Importer._transparency_pct logic so retroactive enforcement matches + prospective filtering. + """ + if getattr(pil_image, "is_animated", False): + return False + if pil_image.mode not in ("RGBA", "LA") and not ( + pil_image.mode == "P" and "transparency" in pil_image.info + ): + return False + im = pil_image + if im.mode != "RGBA": + im = im.convert("RGBA") + alpha = im.getchannel("A") + histogram = alpha.histogram() + transparent = histogram[0] + total = sum(histogram) + pct = transparent / total if total else 0.0 + return pct > threshold diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 3f749ad..b8325d5 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -12,12 +12,14 @@ re-exports from this module and then delete the wrapper. """ from __future__ import annotations +from datetime import UTC, datetime from pathlib import Path +from typing import Any -from sqlalchemy import func, select +from sqlalchemy import func, select, update from sqlalchemy.orm import Session -from ..models import Artist, ImageRecord, Tag +from ..models import Artist, ImageRecord, LibraryAuditRun, Tag from ..models.series_page import SeriesPage from ..models.tag import image_tag @@ -365,3 +367,142 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict: ) session.commit() return {"deleted": len(ids), "sample_names": sample} + + +# --------------------------------------------------------------------------- +# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules. +# --------------------------------------------------------------------------- + +_MIN_DIM_SAMPLE_CAP = 50 + + +def project_min_dimension_violations( + session: Session, *, min_width: int, min_height: int, +) -> dict: + """Return {count, sample_ids} for image_record rows with width or + height below the thresholds. Synchronous SQL — no PIL inspection + needed since width/height are stored columns.""" + base = select(ImageRecord.id).where( + (ImageRecord.width < min_width) | (ImageRecord.height < min_height) + ) + count = session.execute( + select(func.count()).select_from(base.subquery()) + ).scalar_one() + sample_ids = session.execute( + base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP) + ).scalars().all() + return {"count": count, "sample_ids": list(sample_ids)} + + +def delete_min_dimension_violations( + session: Session, *, min_width: int, min_height: int, images_root: Path, +) -> int: + """Delete every image_record where width int: + """Create a LibraryAuditRun row in status='running' and dispatch the + scan_library_for_rule Celery task. Returns the new audit_id. + + Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run + has status='running'. Operator must cancel or wait.""" + if rule not in _VALID_RULES: + raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}") + existing = session.execute( + select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running") + ).scalar_one_or_none() + if existing is not None: + raise AuditAlreadyRunning(existing) + audit = LibraryAuditRun( + rule=rule, + params=params, + status="running", + scanned_count=0, + matched_count=0, + matched_ids=[], + ) + session.add(audit) + session.flush() + audit_id = audit.id + # Dispatch after flush so audit_id is populated; commit happens in + # the API handler so the audit row + dispatch are visible together. + from ..tasks.library_audit import scan_library_for_rule + scan_library_for_rule.delay(audit_id) + return audit_id + + +def apply_audit_run( + session: Session, *, audit_id: int, confirm_token: str, images_root: Path, +) -> int: + """Delete all images in audit_run.matched_ids after confirming token. + Marks audit status='applied'. Routes through delete_images so files + + cascading FK rows are handled uniformly.""" + audit = session.execute( + select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id) + ).scalar_one_or_none() + if audit is None: + raise ValueError(f"audit_run {audit_id} not found") + if audit.status != "ready": + raise AuditNotReady(audit.status) + expected = f"apply-audit-{audit_id}" + if confirm_token != expected: + raise ConfirmTokenMismatch(expected) + ids = list(audit.matched_ids or []) + deleted = 0 + if ids: + result = delete_images(session, image_ids=ids, images_root=images_root) + deleted = result["images_deleted"] + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values(status="applied", finished_at=datetime.now(UTC)) + ) + return deleted + + +def cancel_audit_run(session: Session, *, audit_id: int) -> None: + """Flip a running audit_run to 'cancelled'. The scan task checks + for status=='cancelled' between batches and exits cleanly.""" + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .where(LibraryAuditRun.status == "running") + .values(status="cancelled", finished_at=datetime.now(UTC)) + ) diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 50fe479..fb9d0e3 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -860,8 +860,26 @@ class Importer: pass def _transparency_pct(self, source: Path) -> float: - """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.""" + """Fraction of fully-transparent pixels in the image. 0.0 if no alpha. + + For animated formats (multi-frame WebP / GIF / APNG), short-circuit + to 0.0 instead of decoding every frame. PIL's `getchannel("A")` + forces a full decode of all frames in an animated image, which for + a large animated WebP takes 5+ minutes and blows past the Celery + soft+hard time limits (300s/360s → SIGKILL). Operator-flagged + 2026-05-26. Transparency analysis on a multi-frame image isn't + meaningful for art-curation purposes anyway — different frames + have different alpha — so the existing too_transparent skip rule + is bypassed entirely for animated content. + """ with Image.open(source) as im: + if getattr(im, "is_animated", False): + log.info( + "skipping transparency check for animated image %s " + "(n_frames=%d) — avoids multi-frame decode timeout", + source, getattr(im, "n_frames", 0), + ) + return 0.0 if im.mode not in ("RGBA", "LA") and not ( im.mode == "P" and "transparency" in im.info ): diff --git a/backend/app/tasks/library_audit.py b/backend/app/tasks/library_audit.py new file mode 100644 index 0000000..dd66a23 --- /dev/null +++ b/backend/app/tasks/library_audit.py @@ -0,0 +1,167 @@ +"""scan_library_for_rule Celery task — iterates image_record in keyset- +paginated batches, evaluates the audit rule per image, populates +LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft +time limit (plenty of margin for 100k+ image libraries at ~100ms PIL +decode + histogram per image). + +State machine: + start: status='running' + end success: status='ready' + end error: status='error', error=traceback + oversize: status='error', error='matched too many images; tighten threshold' + external cancel: scan sees status='cancelled' between batches, exits. +""" + +import logging +import traceback +from datetime import UTC, datetime + +from celery.exceptions import SoftTimeLimitExceeded +from PIL import Image +from sqlalchemy import select, update +from sqlalchemy.exc import DBAPIError, OperationalError + +from ..celery_app import celery +from ..models import ImageRecord, LibraryAuditRun +from ..services.audits import single_color, transparency +from ._sync_engine import sync_session_factory as _sync_session_factory + +log = logging.getLogger(__name__) + +_BATCH = 500 +_PROGRESS_TICK = 100 +_MAX_MATCHED = 50_000 + +_RULES = { + "transparency": transparency.evaluate, + "single_color": single_color.evaluate, +} + + +@celery.task( + name="backend.app.tasks.library_audit.scan_library_for_rule", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=5, + retry_backoff_max=60, + retry_jitter=True, + max_retries=3, + soft_time_limit=7200, + time_limit=7500, +) +def scan_library_for_rule(self, audit_id: int) -> dict: + """See module docstring. Returns a small summary dict for eager-mode + test assertions (real workers ignore the return value).""" + SessionLocal = _sync_session_factory() + try: + with SessionLocal() as session: + audit = session.get(LibraryAuditRun, audit_id) + if audit is None: + return {"audit_id": audit_id, "status": "missing"} + evaluate = _RULES.get(audit.rule) + if evaluate is None: + _mark_error(session, audit_id, f"unknown rule {audit.rule!r}") + return {"audit_id": audit_id, "status": "error"} + params = dict(audit.params or {}) + matched: list[int] = [] + scanned = 0 + last_id = 0 + while True: + # Cancellation check between batches. + current_status = session.execute( + select(LibraryAuditRun.status) + .where(LibraryAuditRun.id == audit_id) + ).scalar_one() + if current_status == "cancelled": + return {"audit_id": audit_id, "status": "cancelled"} + rows = session.execute( + select(ImageRecord.id, ImageRecord.path) + .where(ImageRecord.id > last_id) + .where(ImageRecord.mime.like("image/%")) + .order_by(ImageRecord.id.asc()) + .limit(_BATCH) + ).all() + if not rows: + break + for image_id, image_path in rows: + last_id = image_id + scanned += 1 + try: + with Image.open(image_path) as im: + try: + if evaluate(im, **params): + matched.append(image_id) + except Exception as exc: # noqa: BLE001 + log.warning( + "audit %s: rule evaluate failed on %s: %s", + audit_id, image_path, exc, + ) + except FileNotFoundError: + log.warning( + "audit %s: image_record %s file missing at %s; skipping", + audit_id, image_id, image_path, + ) + except OSError as exc: + log.warning( + "audit %s: PIL load failed for %s: %s", + audit_id, image_path, exc, + ) + if len(matched) > _MAX_MATCHED: + _mark_error( + session, audit_id, + f"matched > {_MAX_MATCHED} images; " + "tighten threshold and re-run", + ) + return {"audit_id": audit_id, "status": "error"} + if scanned % _PROGRESS_TICK == 0: + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values(scanned_count=scanned) + ) + session.commit() + # Final state. + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values( + scanned_count=scanned, + matched_count=len(matched), + matched_ids=matched, + status="ready", + finished_at=datetime.now(UTC), + ) + ) + session.commit() + return { + "audit_id": audit_id, + "status": "ready", + "scanned": scanned, + "matched": len(matched), + } + except SoftTimeLimitExceeded: + with SessionLocal() as session: + _mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)") + raise + except (OperationalError, DBAPIError): + # Retryable per the decorator; leave row in 'running' and let + # autoretry try again. Recovery sweep catches if all retries fail. + raise + except Exception: # noqa: BLE001 + tb = traceback.format_exc() + with SessionLocal() as session: + _mark_error(session, audit_id, tb) + raise + + +def _mark_error(session, audit_id: int, error_msg: str) -> None: + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values( + status="error", + error=error_msg, + finished_at=datetime.now(UTC), + ) + ) + session.commit() diff --git a/backend/app/utils/phash.py b/backend/app/utils/phash.py index 8d00cb3..a610e96 100644 --- a/backend/app/utils/phash.py +++ b/backend/app/utils/phash.py @@ -13,8 +13,21 @@ HASH_SIZE = 8 def compute_phash(pil_image) -> str | None: """Perceptual hash of an opened PIL image, as a hex string. None on any - failure (videos/unreadable/non-image).""" + failure (videos/unreadable/non-image). + + For animated images (multi-frame WebP/GIF/APNG), explicitly seek to + frame 0 first. Without this, some PIL operations downstream of + imagehash.phash (convert("L"), resize) can iterate all frames and + blow past Celery's hard time limit on large animations + (operator-flagged 2026-05-26 against animated WebPs). The pHash of + frame 0 is the conventional choice for animated content. + """ try: + if getattr(pil_image, "is_animated", False): + try: + pil_image.seek(0) + except Exception: + pass return str(imagehash.phash(pil_image, hash_size=HASH_SIZE)) except Exception: return None diff --git a/tests/test_audits_single_color.py b/tests/test_audits_single_color.py new file mode 100644 index 0000000..e133bab --- /dev/null +++ b/tests/test_audits_single_color.py @@ -0,0 +1,42 @@ +"""Tests for the single-color audit rule. + +The rule downsamples + measures the fraction of pixels within `tolerance` +(Euclidean RGB distance) of the dominant color. Matches if that fraction +exceeds `threshold`. Single-color content is typically uploaded by +mistake (placeholder/error/preview images) and should be flagged. +""" + +from PIL import Image + +from backend.app.services.audits import single_color + + +def test_single_color_evaluate_true_for_uniform_image(): + im = Image.new("RGB", (50, 50), (128, 64, 200)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True + + +def test_single_color_evaluate_false_for_diverse_image(): + # Half black, half white — no single color dominates. + im = Image.new("RGB", (50, 50), (0, 0, 0)) + for x in range(25): + for y in range(50): + im.putpixel((x, y), (255, 255, 255)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is False + + +def test_single_color_evaluate_respects_tolerance_widening(): + # Gradient image: pixels span 0..50 in R channel. Tight tolerance + # rejects (no concentration), wide tolerance accepts (all near 25). + im = Image.new("RGB", (50, 50), (0, 0, 0)) + for x in range(50): + for y in range(50): + im.putpixel((x, y), (x, 0, 0)) + assert single_color.evaluate(im, threshold=0.9, tolerance=5) is False + assert single_color.evaluate(im, threshold=0.9, tolerance=50) is True + + +def test_single_color_evaluate_handles_rgba_input(): + # Alpha channel should be ignored — only RGB matters for the rule. + im = Image.new("RGBA", (50, 50), (100, 100, 100, 128)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True diff --git a/tests/test_audits_transparency.py b/tests/test_audits_transparency.py new file mode 100644 index 0000000..44d6abb --- /dev/null +++ b/tests/test_audits_transparency.py @@ -0,0 +1,46 @@ +"""Tests for the transparency audit rule. + +The rule mirrors `Importer._transparency_pct` semantics for retroactive +enforcement: returns True iff the fraction of fully-transparent pixels +exceeds the threshold. Animated images short-circuit to False to avoid +the multi-frame PIL decode that triggered SoftTimeLimitExceeded +2026-05-26 against animated WebPs. +""" + +from PIL import Image + +from backend.app.services.audits import transparency + + +def test_transparency_evaluate_true_when_fully_transparent(): + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + assert transparency.evaluate(im, threshold=0.5) is True + + +def test_transparency_evaluate_false_when_fully_opaque(): + im = Image.new("RGBA", (10, 10), (200, 100, 50, 255)) + assert transparency.evaluate(im, threshold=0.5) is False + + +def test_transparency_evaluate_respects_threshold_boundary(): + # Half-transparent image: 50% alpha=0 pixels, 50% alpha=255. + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + for x in range(5): + for y in range(10): + im.putpixel((x, y), (0, 0, 0, 255)) + # 50% transparent. threshold=0.4 → True; threshold=0.6 → False. + assert transparency.evaluate(im, threshold=0.4) is True + assert transparency.evaluate(im, threshold=0.6) is False + + +def test_transparency_evaluate_false_for_rgb_image_without_alpha(): + im = Image.new("RGB", (10, 10), (128, 128, 128)) + assert transparency.evaluate(im, threshold=0.5) is False + + +def test_transparency_evaluate_false_for_animated_image(): + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + # Mark as animated (mimics PIL's WebP/GIF multi-frame attribute). + im.is_animated = True # type: ignore[attr-defined] + im.n_frames = 5 # type: ignore[attr-defined] + assert transparency.evaluate(im, threshold=0.5) is False diff --git a/tests/test_cleanup_service_audit.py b/tests/test_cleanup_service_audit.py new file mode 100644 index 0000000..ec488d7 --- /dev/null +++ b/tests/test_cleanup_service_audit.py @@ -0,0 +1,175 @@ +"""Tests for cleanup_service's library-audit additions. + +Covers: + - project_min_dimension_violations: SQL-only query against width/height + - delete_min_dimension_violations: routes through existing delete_images + - audit lifecycle (start_audit_run / apply_audit_run / cancel_audit_run) + +Tests assert via column selects per reference-async-coredml-test-assertions +(post-DML ORM entity access via session.get() raises MissingGreenlet on +async sessions; we use db_sync here but the convention is preserved for +consistency). +""" + +from pathlib import Path + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ImageRecord, LibraryAuditRun +from backend.app.services import cleanup_service + +pytestmark = pytest.mark.integration + + +def _make_image_record(db_sync, tmp_path, *, width, height, color): + """Helper: write a real PIL file to a UNIQUE path (reference-image-record-path-unique) + and insert an ImageRecord row with all required NOT-NULL columns set.""" + path = tmp_path / f"img_{width}x{height}_{color[0]}.png" + Image.new("RGB", (width, height), color).save(path) + # sha256 column is varchar(64) — use a deterministic 64-char pseudo + # hash built from the unique inputs. + sha = f"{width:04d}{height:04d}{color[0]:03d}".ljust(64, "0")[:64] + rec = ImageRecord( + path=str(path), + sha256=sha, + size_bytes=path.stat().st_size, + mime="image/png", # reference-image-record-required-columns + width=width, + height=height, + origin="imported_filesystem", # feedback-check-existing-enums + integrity_status="ok", + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def test_project_min_dimension_violations_returns_count_and_samples(db_sync, tmp_path): + _make_image_record(db_sync, tmp_path, width=100, height=100, color=(10, 0, 0)) + _make_image_record(db_sync, tmp_path, width=50, height=50, color=(20, 0, 0)) + _make_image_record(db_sync, tmp_path, width=400, height=400, color=(30, 0, 0)) + db_sync.commit() + + result = cleanup_service.project_min_dimension_violations( + db_sync, min_width=200, min_height=200, + ) + assert result["count"] == 2 # 100x100 and 50x50 violate + assert len(result["sample_ids"]) == 2 + + +def test_delete_min_dimension_violations_unlinks_and_cascades(db_sync, tmp_path): + rec_small = _make_image_record(db_sync, tmp_path, width=50, height=50, color=(40, 0, 0)) + rec_big = _make_image_record(db_sync, tmp_path, width=500, height=500, color=(50, 0, 0)) + small_path = Path(rec_small.path) + db_sync.commit() + + # images_root is tmp_path here because the test fixtures stored files there; + # delete_images() uses it to unlink originals + thumbs from the right tree. + deleted = cleanup_service.delete_min_dimension_violations( + db_sync, min_width=200, min_height=200, images_root=tmp_path, + ) + assert deleted == 1 + # Verify via column selects per banked rule. + remaining_ids = db_sync.execute( + select(ImageRecord.id).order_by(ImageRecord.id) + ).scalars().all() + assert remaining_ids == [rec_big.id] + assert not small_path.exists() # file unlinked + + +# --- Audit lifecycle tests (Task 5) --- + +import backend.app.tasks.library_audit # noqa: F401, E402 — celery registration + + +def test_start_audit_run_creates_row_and_dispatches(db_sync, monkeypatch): + dispatched = [] + from backend.app.tasks import library_audit as la_mod + monkeypatch.setattr( + la_mod.scan_library_for_rule, "delay", + lambda audit_id: dispatched.append(audit_id), + ) + audit_id = cleanup_service.start_audit_run( + db_sync, rule="transparency", params={"threshold": 0.9}, + ) + db_sync.commit() + row_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + assert row_status == "running" + assert dispatched == [audit_id] + + +def test_start_audit_run_rejects_when_another_is_running(db_sync, monkeypatch): + from backend.app.tasks import library_audit as la_mod + monkeypatch.setattr( + la_mod.scan_library_for_rule, "delay", lambda audit_id: None, + ) + cleanup_service.start_audit_run( + db_sync, rule="transparency", params={"threshold": 0.9}, + ) + db_sync.commit() + with pytest.raises(cleanup_service.AuditAlreadyRunning): + cleanup_service.start_audit_run( + db_sync, rule="single_color", + params={"threshold": 0.95, "tolerance": 30}, + ) + + +def test_apply_audit_run_with_correct_token_deletes_matched(db_sync, tmp_path): + rec = _make_image_record( + db_sync, tmp_path, width=100, height=100, color=(60, 0, 0), + ) + db_sync.flush() + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", scanned_count=1, matched_count=1, + matched_ids=[rec.id], + ) + db_sync.add(audit) + db_sync.commit() + + deleted = cleanup_service.apply_audit_run( + db_sync, audit_id=audit.id, + confirm_token=f"apply-audit-{audit.id}", + images_root=tmp_path, + ) + assert deleted == 1 + remaining_count = db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() + assert remaining_count == 0 + new_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id) + ).scalar_one() + assert new_status == "applied" + + +def test_apply_audit_run_with_wrong_token_raises(db_sync, tmp_path): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + with pytest.raises(cleanup_service.ConfirmTokenMismatch): + cleanup_service.apply_audit_run( + db_sync, audit_id=audit.id, + confirm_token="wrong-token", images_root=tmp_path, + ) + + +def test_cancel_audit_run_flips_status(db_sync): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + cleanup_service.cancel_audit_run(db_sync, audit_id=audit.id) + new_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id) + ).scalar_one() + assert new_status == "cancelled" diff --git a/tests/test_tasks_library_audit.py b/tests/test_tasks_library_audit.py new file mode 100644 index 0000000..b10c35b --- /dev/null +++ b/tests/test_tasks_library_audit.py @@ -0,0 +1,100 @@ +"""Tests for scan_library_for_rule Celery task. + +Eager mode is used so the task runs synchronously in-test and we can +assert state via column selects (post-DML ORM access banned per +reference-async-coredml-test-assertions).""" + +import pytest +from PIL import Image +from sqlalchemy import select + +import backend.app.tasks.library_audit # noqa: F401 — celery registration +from backend.app import celery_app +from backend.app.models import ImageRecord, LibraryAuditRun + +pytestmark = pytest.mark.integration + + +def _mk_image(db_sync, tmp_path, *, mode, color, name): + path = tmp_path / name + Image.new(mode, (10, 10), color).save(path) + # sha256 column is varchar(64) — pad/truncate a per-file pseudo hash + # exactly to 64 chars. Each test fixture file must have a unique + # sha256 (reference-image-record-path-unique peer constraint). + sha = f"audit-{name}".ljust(64, "x")[:64] + rec = ImageRecord( + path=str(path), + sha256=sha, + size_bytes=path.stat().st_size, + mime="image/png", + width=10, height=10, + origin="imported_filesystem", + integrity_status="ok", + ) + db_sync.add(rec) + db_sync.flush() + return rec, path + + +def test_scan_library_for_rule_populates_matched_ids_for_transparency( + db_sync, tmp_path, monkeypatch, +): + transparent_rec, _ = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="trans.png", + ) + opaque_rec, _ = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(200, 0, 0, 255), name="opaque.png", + ) + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.5}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + audit_id = audit.id + + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True) + from backend.app.tasks.library_audit import scan_library_for_rule + scan_library_for_rule.run(audit_id) + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False) + + matched = db_sync.execute( + select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + assert transparent_rec.id in matched + assert opaque_rec.id not in matched + assert status == "ready" + + +def test_scan_library_for_rule_skips_missing_files_gracefully( + db_sync, tmp_path, monkeypatch, +): + rec, path = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="ghost.png", + ) + path.unlink() # delete the file but leave the DB row + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.5}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + audit_id = audit.id + + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True) + from backend.app.tasks.library_audit import scan_library_for_rule + scan_library_for_rule.run(audit_id) + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False) + + status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + matched = db_sync.execute( + select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + # Missing file is skipped (warning logged), audit completes successfully. + assert status == "ready" + assert rec.id not in matched