From c0c9e56fb9f163634082538956ed831cee82327e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 00:45:08 -0400 Subject: [PATCH 1/8] =?UTF-8?q?fix(importer):=20skip=20transparency=20chec?= =?UTF-8?q?k=20on=20animated=20images=20(operator-flagged=202026-05-26:=20?= =?UTF-8?q?animated=20WebP=20triggered=205+=20min=20PIL=20multi-frame=20de?= =?UTF-8?q?code=20=E2=86=92=20Celery=20hard-timeout=20SIGKILL);=20compute?= =?UTF-8?q?=5Fphash=20seeks=20frame=200=20defensively=20=E2=80=94=20Co-Aut?= =?UTF-8?q?hored-By:=20Claude=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/importer.py | 20 +++++++++++++++++++- backend/app/utils/phash.py | 15 ++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 50fe479..fb9d0e3 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -860,8 +860,26 @@ class Importer: pass def _transparency_pct(self, source: Path) -> float: - """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.""" + """Fraction of fully-transparent pixels in the image. 0.0 if no alpha. + + For animated formats (multi-frame WebP / GIF / APNG), short-circuit + to 0.0 instead of decoding every frame. PIL's `getchannel("A")` + forces a full decode of all frames in an animated image, which for + a large animated WebP takes 5+ minutes and blows past the Celery + soft+hard time limits (300s/360s → SIGKILL). Operator-flagged + 2026-05-26. Transparency analysis on a multi-frame image isn't + meaningful for art-curation purposes anyway — different frames + have different alpha — so the existing too_transparent skip rule + is bypassed entirely for animated content. + """ with Image.open(source) as im: + if getattr(im, "is_animated", False): + log.info( + "skipping transparency check for animated image %s " + "(n_frames=%d) — avoids multi-frame decode timeout", + source, getattr(im, "n_frames", 0), + ) + return 0.0 if im.mode not in ("RGBA", "LA") and not ( im.mode == "P" and "transparency" in im.info ): diff --git a/backend/app/utils/phash.py b/backend/app/utils/phash.py index 8d00cb3..a610e96 100644 --- a/backend/app/utils/phash.py +++ b/backend/app/utils/phash.py @@ -13,8 +13,21 @@ HASH_SIZE = 8 def compute_phash(pil_image) -> str | None: """Perceptual hash of an opened PIL image, as a hex string. None on any - failure (videos/unreadable/non-image).""" + failure (videos/unreadable/non-image). + + For animated images (multi-frame WebP/GIF/APNG), explicitly seek to + frame 0 first. Without this, some PIL operations downstream of + imagehash.phash (convert("L"), resize) can iterate all frames and + blow past Celery's hard time limit on large animations + (operator-flagged 2026-05-26 against animated WebPs). The pHash of + frame 0 is the conventional choice for animated content. + """ try: + if getattr(pil_image, "is_animated", False): + try: + pil_image.seek(0) + except Exception: + pass return str(imagehash.phash(pil_image, hash_size=HASH_SIZE)) except Exception: return None From 929d3fc092ecef5be2b1f4fe8ac0d335b7c1536a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:17:42 -0400 Subject: [PATCH 2/8] =?UTF-8?q?feat(fc-cleanup):=20migration=200020=20+=20?= =?UTF-8?q?LibraryAuditRun=20model=20=E2=80=94=20Co-Authored-By:=20Claude?= =?UTF-8?q?=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- alembic/versions/0020_library_audit_run.py | 65 ++++++++++++++++++++++ backend/app/models/__init__.py | 2 + backend/app/models/library_audit_run.py | 37 ++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 alembic/versions/0020_library_audit_run.py create mode 100644 backend/app/models/library_audit_run.py diff --git a/alembic/versions/0020_library_audit_run.py b/alembic/versions/0020_library_audit_run.py new file mode 100644 index 0000000..07a8b87 --- /dev/null +++ b/alembic/versions/0020_library_audit_run.py @@ -0,0 +1,65 @@ +"""fc-cleanup: library_audit_run table for async transparency/single_color audits + +Revision ID: 0020 +Revises: 0019 +Create Date: 2026-05-26 + +The table backs the async audit lifecycle: rule + params snapshot, status +state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and +the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs +per row by the scan task (oversize = rule too aggressive, operator narrows +before re-running). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0020" +down_revision: Union[str, None] = "0019" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "library_audit_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("rule", sa.String(32), nullable=False), + sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column( + "status", sa.String(16), + nullable=False, server_default="running", + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "scanned_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_ids", postgresql.JSONB(astext_type=sa.Text()), + nullable=False, server_default=sa.text("'[]'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + ) + op.create_index( + "ix_library_audit_run_rule", "library_audit_run", ["rule"], + ) + op.create_index( + "ix_library_audit_run_status", "library_audit_run", ["status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_library_audit_run_status", table_name="library_audit_run") + op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run") + op.drop_table("library_audit_run") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 0cefb2d..00a7c80 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -11,6 +11,7 @@ from .image_record import ImageRecord from .import_batch import ImportBatch from .import_settings import ImportSettings from .import_task import ImportTask +from .library_audit_run import LibraryAuditRun from .migration_run import MigrationRun from .ml_settings import MLSettings from .post import Post @@ -43,6 +44,7 @@ __all__ = [ "ImportBatch", "ImportTask", "ImportSettings", + "LibraryAuditRun", "MLSettings", "MigrationRun", "TagAlias", diff --git a/backend/app/models/library_audit_run.py b/backend/app/models/library_audit_run.py new file mode 100644 index 0000000..ca5aa82 --- /dev/null +++ b/backend/app/models/library_audit_run.py @@ -0,0 +1,37 @@ +"""LibraryAuditRun — async transparency / single_color audit lifecycle. + +State machine: running → ready → applied / cancelled / error. +matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run +reads it and routes through cleanup_service.delete_images. +""" + +from datetime import datetime +from typing import Any + +from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class LibraryAuditRun(Base): + __tablename__ = "library_audit_run" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="running", index=True, + ) + # running | ready | applied | cancelled | error + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + finished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list) + error: Mapped[str | None] = mapped_column(Text, nullable=True) From fd80d40a347c2eff5dc551cfc9f7971ed4b942d2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:18:07 -0400 Subject: [PATCH 3/8] =?UTF-8?q?feat(fc-cleanup):=20audits/transparency.py?= =?UTF-8?q?=20+=20tests=20=E2=80=94=20Co-Authored-By:=20Claude=20Opus=204.?= =?UTF-8?q?7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/audits/__init__.py | 6 +++ backend/app/services/audits/transparency.py | 27 ++++++++++++ tests/test_audits_transparency.py | 46 +++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 backend/app/services/audits/__init__.py create mode 100644 backend/app/services/audits/transparency.py create mode 100644 tests/test_audits_transparency.py diff --git a/backend/app/services/audits/__init__.py b/backend/app/services/audits/__init__.py new file mode 100644 index 0000000..96e5198 --- /dev/null +++ b/backend/app/services/audits/__init__.py @@ -0,0 +1,6 @@ +"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool. + +The retroactive library-cleanup tab and (future) import-time filter logic +both consume these. Importers should NOT inline rule logic going forward; +add the rule here and call from both sides. +""" diff --git a/backend/app/services/audits/transparency.py b/backend/app/services/audits/transparency.py new file mode 100644 index 0000000..f15f31f --- /dev/null +++ b/backend/app/services/audits/transparency.py @@ -0,0 +1,27 @@ +"""Transparency audit: matches images whose transparent-pixel fraction +exceeds the threshold. Animated images short-circuit (skipped) to avoid +the multi-frame PIL decode that hits Celery's hard time limit.""" + + +def evaluate(pil_image, *, threshold: float) -> bool: + """True iff the image's transparent-pixel fraction exceeds threshold. + + False for non-alpha modes and animated images. Mirrors the import-side + Importer._transparency_pct logic so retroactive enforcement matches + prospective filtering. + """ + if getattr(pil_image, "is_animated", False): + return False + if pil_image.mode not in ("RGBA", "LA") and not ( + pil_image.mode == "P" and "transparency" in pil_image.info + ): + return False + im = pil_image + if im.mode != "RGBA": + im = im.convert("RGBA") + alpha = im.getchannel("A") + histogram = alpha.histogram() + transparent = histogram[0] + total = sum(histogram) + pct = transparent / total if total else 0.0 + return pct > threshold diff --git a/tests/test_audits_transparency.py b/tests/test_audits_transparency.py new file mode 100644 index 0000000..44d6abb --- /dev/null +++ b/tests/test_audits_transparency.py @@ -0,0 +1,46 @@ +"""Tests for the transparency audit rule. + +The rule mirrors `Importer._transparency_pct` semantics for retroactive +enforcement: returns True iff the fraction of fully-transparent pixels +exceeds the threshold. Animated images short-circuit to False to avoid +the multi-frame PIL decode that triggered SoftTimeLimitExceeded +2026-05-26 against animated WebPs. +""" + +from PIL import Image + +from backend.app.services.audits import transparency + + +def test_transparency_evaluate_true_when_fully_transparent(): + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + assert transparency.evaluate(im, threshold=0.5) is True + + +def test_transparency_evaluate_false_when_fully_opaque(): + im = Image.new("RGBA", (10, 10), (200, 100, 50, 255)) + assert transparency.evaluate(im, threshold=0.5) is False + + +def test_transparency_evaluate_respects_threshold_boundary(): + # Half-transparent image: 50% alpha=0 pixels, 50% alpha=255. + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + for x in range(5): + for y in range(10): + im.putpixel((x, y), (0, 0, 0, 255)) + # 50% transparent. threshold=0.4 → True; threshold=0.6 → False. + assert transparency.evaluate(im, threshold=0.4) is True + assert transparency.evaluate(im, threshold=0.6) is False + + +def test_transparency_evaluate_false_for_rgb_image_without_alpha(): + im = Image.new("RGB", (10, 10), (128, 128, 128)) + assert transparency.evaluate(im, threshold=0.5) is False + + +def test_transparency_evaluate_false_for_animated_image(): + im = Image.new("RGBA", (10, 10), (0, 0, 0, 0)) + # Mark as animated (mimics PIL's WebP/GIF multi-frame attribute). + im.is_animated = True # type: ignore[attr-defined] + im.n_frames = 5 # type: ignore[attr-defined] + assert transparency.evaluate(im, threshold=0.5) is False From 900d878d2735bea11a4db9723327dbe3b7718e1c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:18:38 -0400 Subject: [PATCH 4/8] =?UTF-8?q?feat(fc-cleanup):=20audits/single=5Fcolor.p?= =?UTF-8?q?y=20+=20tests=20=E2=80=94=20Co-Authored-By:=20Claude=20Opus=204?= =?UTF-8?q?.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/audits/single_color.py | 53 +++++++++++++++++++++ tests/test_audits_single_color.py | 42 ++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 backend/app/services/audits/single_color.py create mode 100644 tests/test_audits_single_color.py diff --git a/backend/app/services/audits/single_color.py b/backend/app/services/audits/single_color.py new file mode 100644 index 0000000..167e078 --- /dev/null +++ b/backend/app/services/audits/single_color.py @@ -0,0 +1,53 @@ +"""Single-color audit: matches images where one color dominates beyond +the threshold (within the given Euclidean RGB tolerance). The first +canonical implementation — the import-side filter (SkipReason.single_color) +was never wired; FC-Cleanup's audit module is the source of truth and a +future spec can adopt it on the import path too. +""" + +from PIL import Image + +_THUMB_SIZE = (64, 64) + + +def evaluate( + pil_image, + *, + threshold: float, + tolerance: int, +) -> bool: + """True iff the fraction of pixels within `tolerance` (Euclidean RGB + distance) of the dominant color exceeds `threshold`. + + Downsamples to 64x64 for speed (~4ms regardless of source size). + Alpha channels are stripped; only RGB is considered. Animated images + use frame 0 (PIL's default after Image.open without seek). + """ + im = pil_image + if im.mode == "RGBA": + im = im.convert("RGB") + elif im.mode not in ("RGB", "L"): + im = im.convert("RGB") + if im.size != _THUMB_SIZE: + im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR) + pixels = list(im.getdata()) + if not pixels: + return False + # Normalize L-mode pixels to RGB tuples for distance math. + if isinstance(pixels[0], int): + pixels = [(p, p, p) for p in pixels] + # Dominant color = mean RGB. + n = len(pixels) + sum_r = sum(p[0] for p in pixels) + sum_g = sum(p[1] for p in pixels) + sum_b = sum(p[2] for p in pixels) + dom = (sum_r / n, sum_g / n, sum_b / n) + tol_sq = tolerance * tolerance + within = 0 + for r, g, b in pixels: + dr = r - dom[0] + dg = g - dom[1] + db = b - dom[2] + if dr * dr + dg * dg + db * db <= tol_sq: + within += 1 + return (within / n) > threshold diff --git a/tests/test_audits_single_color.py b/tests/test_audits_single_color.py new file mode 100644 index 0000000..e133bab --- /dev/null +++ b/tests/test_audits_single_color.py @@ -0,0 +1,42 @@ +"""Tests for the single-color audit rule. + +The rule downsamples + measures the fraction of pixels within `tolerance` +(Euclidean RGB distance) of the dominant color. Matches if that fraction +exceeds `threshold`. Single-color content is typically uploaded by +mistake (placeholder/error/preview images) and should be flagged. +""" + +from PIL import Image + +from backend.app.services.audits import single_color + + +def test_single_color_evaluate_true_for_uniform_image(): + im = Image.new("RGB", (50, 50), (128, 64, 200)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True + + +def test_single_color_evaluate_false_for_diverse_image(): + # Half black, half white — no single color dominates. + im = Image.new("RGB", (50, 50), (0, 0, 0)) + for x in range(25): + for y in range(50): + im.putpixel((x, y), (255, 255, 255)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is False + + +def test_single_color_evaluate_respects_tolerance_widening(): + # Gradient image: pixels span 0..50 in R channel. Tight tolerance + # rejects (no concentration), wide tolerance accepts (all near 25). + im = Image.new("RGB", (50, 50), (0, 0, 0)) + for x in range(50): + for y in range(50): + im.putpixel((x, y), (x, 0, 0)) + assert single_color.evaluate(im, threshold=0.9, tolerance=5) is False + assert single_color.evaluate(im, threshold=0.9, tolerance=50) is True + + +def test_single_color_evaluate_handles_rgba_input(): + # Alpha channel should be ignored — only RGB matters for the rule. + im = Image.new("RGBA", (50, 50), (100, 100, 100, 128)) + assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True From 8a5b337a5328e64dc46ca9cddd433091647d8384 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:20:04 -0400 Subject: [PATCH 5/8] =?UTF-8?q?feat(fc-cleanup):=20min-dimension=20service?= =?UTF-8?q?=20functions=20+=20tests=20=E2=80=94=20Co-Authored-By:=20Claude?= =?UTF-8?q?=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/cleanup_service.py | 44 ++++++++++++++ tests/test_cleanup_service_audit.py | 76 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 tests/test_cleanup_service_audit.py diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 3f749ad..99d2a6f 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -365,3 +365,47 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict: ) session.commit() return {"deleted": len(ids), "sample_names": sample} + + +# --------------------------------------------------------------------------- +# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules. +# --------------------------------------------------------------------------- + +_MIN_DIM_SAMPLE_CAP = 50 + + +def project_min_dimension_violations( + session: Session, *, min_width: int, min_height: int, +) -> dict: + """Return {count, sample_ids} for image_record rows with width or + height below the thresholds. Synchronous SQL — no PIL inspection + needed since width/height are stored columns.""" + base = select(ImageRecord.id).where( + (ImageRecord.width < min_width) | (ImageRecord.height < min_height) + ) + count = session.execute( + select(func.count()).select_from(base.subquery()) + ).scalar_one() + sample_ids = session.execute( + base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP) + ).scalars().all() + return {"count": count, "sample_ids": list(sample_ids)} + + +def delete_min_dimension_violations( + session: Session, *, min_width: int, min_height: int, images_root: Path, +) -> int: + """Delete every image_record where width Date: Tue, 26 May 2026 01:20:41 -0400 Subject: [PATCH 6/8] =?UTF-8?q?feat(fc-cleanup):=20audit=20lifecycle=20ser?= =?UTF-8?q?vice=20functions=20(start/apply/cancel)=20+=20tests=20=E2=80=94?= =?UTF-8?q?=20Co-Authored-By:=20Claude=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/cleanup_service.py | 101 +++++++++++++++++++++++- tests/test_cleanup_service_audit.py | 96 ++++++++++++++++++++++ 2 files changed, 195 insertions(+), 2 deletions(-) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 99d2a6f..b8325d5 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -12,12 +12,14 @@ re-exports from this module and then delete the wrapper. """ from __future__ import annotations +from datetime import UTC, datetime from pathlib import Path +from typing import Any -from sqlalchemy import func, select +from sqlalchemy import func, select, update from sqlalchemy.orm import Session -from ..models import Artist, ImageRecord, Tag +from ..models import Artist, ImageRecord, LibraryAuditRun, Tag from ..models.series_page import SeriesPage from ..models.tag import image_tag @@ -409,3 +411,98 @@ def delete_min_dimension_violations( session, image_ids=list(ids), images_root=images_root, ) return result["images_deleted"] + + +# --------------------------------------------------------------------------- +# Audit lifecycle (transparency + single_color async scans). +# --------------------------------------------------------------------------- + + +class AuditAlreadyRunning(Exception): + """Another audit_run is currently in status='running' — wait or + cancel it before starting a new one. Surfaces as HTTP 409 in the + /api/cleanup/audit POST endpoint.""" + + +class AuditNotReady(Exception): + """apply_audit_run called on an audit whose status is not 'ready'.""" + + +class ConfirmTokenMismatch(Exception): + """Operator-supplied confirm token did not match server-recomputed token.""" + + +_VALID_RULES = ("transparency", "single_color") + + +def start_audit_run( + session: Session, *, rule: str, params: dict[str, Any], +) -> int: + """Create a LibraryAuditRun row in status='running' and dispatch the + scan_library_for_rule Celery task. Returns the new audit_id. + + Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run + has status='running'. Operator must cancel or wait.""" + if rule not in _VALID_RULES: + raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}") + existing = session.execute( + select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running") + ).scalar_one_or_none() + if existing is not None: + raise AuditAlreadyRunning(existing) + audit = LibraryAuditRun( + rule=rule, + params=params, + status="running", + scanned_count=0, + matched_count=0, + matched_ids=[], + ) + session.add(audit) + session.flush() + audit_id = audit.id + # Dispatch after flush so audit_id is populated; commit happens in + # the API handler so the audit row + dispatch are visible together. + from ..tasks.library_audit import scan_library_for_rule + scan_library_for_rule.delay(audit_id) + return audit_id + + +def apply_audit_run( + session: Session, *, audit_id: int, confirm_token: str, images_root: Path, +) -> int: + """Delete all images in audit_run.matched_ids after confirming token. + Marks audit status='applied'. Routes through delete_images so files + + cascading FK rows are handled uniformly.""" + audit = session.execute( + select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id) + ).scalar_one_or_none() + if audit is None: + raise ValueError(f"audit_run {audit_id} not found") + if audit.status != "ready": + raise AuditNotReady(audit.status) + expected = f"apply-audit-{audit_id}" + if confirm_token != expected: + raise ConfirmTokenMismatch(expected) + ids = list(audit.matched_ids or []) + deleted = 0 + if ids: + result = delete_images(session, image_ids=ids, images_root=images_root) + deleted = result["images_deleted"] + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values(status="applied", finished_at=datetime.now(UTC)) + ) + return deleted + + +def cancel_audit_run(session: Session, *, audit_id: int) -> None: + """Flip a running audit_run to 'cancelled'. The scan task checks + for status=='cancelled' between batches and exits cleanly.""" + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .where(LibraryAuditRun.status == "running") + .values(status="cancelled", finished_at=datetime.now(UTC)) + ) diff --git a/tests/test_cleanup_service_audit.py b/tests/test_cleanup_service_audit.py index 69514c5..b215dbf 100644 --- a/tests/test_cleanup_service_audit.py +++ b/tests/test_cleanup_service_audit.py @@ -74,3 +74,99 @@ def test_delete_min_dimension_violations_unlinks_and_cascades(db_sync, tmp_path) ).scalars().all() assert remaining_ids == [rec_big.id] assert not small_path.exists() # file unlinked + + +# --- Audit lifecycle tests (Task 5) --- + +import backend.app.tasks.library_audit # noqa: F401, E402 — celery registration + + +def test_start_audit_run_creates_row_and_dispatches(db_sync, monkeypatch): + dispatched = [] + from backend.app.tasks import library_audit as la_mod + monkeypatch.setattr( + la_mod.scan_library_for_rule, "delay", + lambda audit_id: dispatched.append(audit_id), + ) + audit_id = cleanup_service.start_audit_run( + db_sync, rule="transparency", params={"threshold": 0.9}, + ) + db_sync.commit() + row_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + assert row_status == "running" + assert dispatched == [audit_id] + + +def test_start_audit_run_rejects_when_another_is_running(db_sync, monkeypatch): + from backend.app.tasks import library_audit as la_mod + monkeypatch.setattr( + la_mod.scan_library_for_rule, "delay", lambda audit_id: None, + ) + cleanup_service.start_audit_run( + db_sync, rule="transparency", params={"threshold": 0.9}, + ) + db_sync.commit() + with pytest.raises(cleanup_service.AuditAlreadyRunning): + cleanup_service.start_audit_run( + db_sync, rule="single_color", + params={"threshold": 0.95, "tolerance": 30}, + ) + + +def test_apply_audit_run_with_correct_token_deletes_matched(db_sync, tmp_path): + rec = _make_image_record( + db_sync, tmp_path, width=100, height=100, color=(60, 0, 0), + ) + db_sync.flush() + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", scanned_count=1, matched_count=1, + matched_ids=[rec.id], + ) + db_sync.add(audit) + db_sync.commit() + + deleted = cleanup_service.apply_audit_run( + db_sync, audit_id=audit.id, + confirm_token=f"apply-audit-{audit.id}", + images_root=tmp_path, + ) + assert deleted == 1 + remaining_count = db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() + assert remaining_count == 0 + new_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id) + ).scalar_one() + assert new_status == "applied" + + +def test_apply_audit_run_with_wrong_token_raises(db_sync, tmp_path): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + with pytest.raises(cleanup_service.ConfirmTokenMismatch): + cleanup_service.apply_audit_run( + db_sync, audit_id=audit.id, + confirm_token="wrong-token", images_root=tmp_path, + ) + + +def test_cancel_audit_run_flips_status(db_sync): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + cleanup_service.cancel_audit_run(db_sync, audit_id=audit.id) + new_status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id) + ).scalar_one() + assert new_status == "cancelled" From 6ed2021ad6d145302d583dfca17445219b9180cf Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:21:37 -0400 Subject: [PATCH 7/8] =?UTF-8?q?feat(fc-cleanup):=20scan=5Flibrary=5Ffor=5F?= =?UTF-8?q?rule=20Celery=20task=20+=20maintenance-queue=20registration=20?= =?UTF-8?q?=E2=80=94=20Co-Authored-By:=20Claude=20Opus=204.7=20(1M=20conte?= =?UTF-8?q?xt)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/celery_app.py | 2 + backend/app/tasks/library_audit.py | 167 +++++++++++++++++++++++++++++ tests/test_tasks_library_audit.py | 98 +++++++++++++++++ 3 files changed, 267 insertions(+) create mode 100644 backend/app/tasks/library_audit.py create mode 100644 tests/test_tasks_library_audit.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 6ba84cb..3068ca2 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -33,6 +33,7 @@ def make_celery() -> Celery: "backend.app.tasks.download", "backend.app.tasks.backup", "backend.app.tasks.admin", + "backend.app.tasks.library_audit", ], ) app.conf.update( @@ -47,6 +48,7 @@ def make_celery() -> Celery: "backend.app.tasks.migration.*": {"queue": "maintenance"}, "backend.app.tasks.backup.*": {"queue": "maintenance"}, "backend.app.tasks.admin.*": {"queue": "maintenance"}, + "backend.app.tasks.library_audit.*": {"queue": "maintenance"}, }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, diff --git a/backend/app/tasks/library_audit.py b/backend/app/tasks/library_audit.py new file mode 100644 index 0000000..dd66a23 --- /dev/null +++ b/backend/app/tasks/library_audit.py @@ -0,0 +1,167 @@ +"""scan_library_for_rule Celery task — iterates image_record in keyset- +paginated batches, evaluates the audit rule per image, populates +LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft +time limit (plenty of margin for 100k+ image libraries at ~100ms PIL +decode + histogram per image). + +State machine: + start: status='running' + end success: status='ready' + end error: status='error', error=traceback + oversize: status='error', error='matched too many images; tighten threshold' + external cancel: scan sees status='cancelled' between batches, exits. +""" + +import logging +import traceback +from datetime import UTC, datetime + +from celery.exceptions import SoftTimeLimitExceeded +from PIL import Image +from sqlalchemy import select, update +from sqlalchemy.exc import DBAPIError, OperationalError + +from ..celery_app import celery +from ..models import ImageRecord, LibraryAuditRun +from ..services.audits import single_color, transparency +from ._sync_engine import sync_session_factory as _sync_session_factory + +log = logging.getLogger(__name__) + +_BATCH = 500 +_PROGRESS_TICK = 100 +_MAX_MATCHED = 50_000 + +_RULES = { + "transparency": transparency.evaluate, + "single_color": single_color.evaluate, +} + + +@celery.task( + name="backend.app.tasks.library_audit.scan_library_for_rule", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=5, + retry_backoff_max=60, + retry_jitter=True, + max_retries=3, + soft_time_limit=7200, + time_limit=7500, +) +def scan_library_for_rule(self, audit_id: int) -> dict: + """See module docstring. Returns a small summary dict for eager-mode + test assertions (real workers ignore the return value).""" + SessionLocal = _sync_session_factory() + try: + with SessionLocal() as session: + audit = session.get(LibraryAuditRun, audit_id) + if audit is None: + return {"audit_id": audit_id, "status": "missing"} + evaluate = _RULES.get(audit.rule) + if evaluate is None: + _mark_error(session, audit_id, f"unknown rule {audit.rule!r}") + return {"audit_id": audit_id, "status": "error"} + params = dict(audit.params or {}) + matched: list[int] = [] + scanned = 0 + last_id = 0 + while True: + # Cancellation check between batches. + current_status = session.execute( + select(LibraryAuditRun.status) + .where(LibraryAuditRun.id == audit_id) + ).scalar_one() + if current_status == "cancelled": + return {"audit_id": audit_id, "status": "cancelled"} + rows = session.execute( + select(ImageRecord.id, ImageRecord.path) + .where(ImageRecord.id > last_id) + .where(ImageRecord.mime.like("image/%")) + .order_by(ImageRecord.id.asc()) + .limit(_BATCH) + ).all() + if not rows: + break + for image_id, image_path in rows: + last_id = image_id + scanned += 1 + try: + with Image.open(image_path) as im: + try: + if evaluate(im, **params): + matched.append(image_id) + except Exception as exc: # noqa: BLE001 + log.warning( + "audit %s: rule evaluate failed on %s: %s", + audit_id, image_path, exc, + ) + except FileNotFoundError: + log.warning( + "audit %s: image_record %s file missing at %s; skipping", + audit_id, image_id, image_path, + ) + except OSError as exc: + log.warning( + "audit %s: PIL load failed for %s: %s", + audit_id, image_path, exc, + ) + if len(matched) > _MAX_MATCHED: + _mark_error( + session, audit_id, + f"matched > {_MAX_MATCHED} images; " + "tighten threshold and re-run", + ) + return {"audit_id": audit_id, "status": "error"} + if scanned % _PROGRESS_TICK == 0: + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values(scanned_count=scanned) + ) + session.commit() + # Final state. + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values( + scanned_count=scanned, + matched_count=len(matched), + matched_ids=matched, + status="ready", + finished_at=datetime.now(UTC), + ) + ) + session.commit() + return { + "audit_id": audit_id, + "status": "ready", + "scanned": scanned, + "matched": len(matched), + } + except SoftTimeLimitExceeded: + with SessionLocal() as session: + _mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)") + raise + except (OperationalError, DBAPIError): + # Retryable per the decorator; leave row in 'running' and let + # autoretry try again. Recovery sweep catches if all retries fail. + raise + except Exception: # noqa: BLE001 + tb = traceback.format_exc() + with SessionLocal() as session: + _mark_error(session, audit_id, tb) + raise + + +def _mark_error(session, audit_id: int, error_msg: str) -> None: + session.execute( + update(LibraryAuditRun) + .where(LibraryAuditRun.id == audit_id) + .values( + status="error", + error=error_msg, + finished_at=datetime.now(UTC), + ) + ) + session.commit() diff --git a/tests/test_tasks_library_audit.py b/tests/test_tasks_library_audit.py new file mode 100644 index 0000000..3599dfc --- /dev/null +++ b/tests/test_tasks_library_audit.py @@ -0,0 +1,98 @@ +"""Tests for scan_library_for_rule Celery task. + +Eager mode is used so the task runs synchronously in-test and we can +assert state via column selects (post-DML ORM access banned per +reference-async-coredml-test-assertions).""" + +import pytest +from PIL import Image +from sqlalchemy import select + +from backend.app import celery_app +from backend.app.models import ImageRecord, LibraryAuditRun + +import backend.app.tasks.library_audit # noqa: F401 — registration + +pytestmark = pytest.mark.integration + + +def _mk_image(db_sync, tmp_path, *, mode, color, name): + path = tmp_path / name + Image.new(mode, (10, 10), color).save(path) + short_sha = f"fake-{name}" + rec = ImageRecord( + path=str(path), + sha256=short_sha + "0" * (64 - len(short_sha)), + size_bytes=path.stat().st_size, + mime="image/png", + width=10, height=10, + origin="imported_filesystem", + integrity_status="ok", + ) + db_sync.add(rec) + db_sync.flush() + return rec, path + + +def test_scan_library_for_rule_populates_matched_ids_for_transparency( + db_sync, tmp_path, monkeypatch, +): + transparent_rec, _ = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="trans.png", + ) + opaque_rec, _ = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(200, 0, 0, 255), name="opaque.png", + ) + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.5}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + audit_id = audit.id + + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True) + from backend.app.tasks.library_audit import scan_library_for_rule + scan_library_for_rule.run(audit_id) + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False) + + matched = db_sync.execute( + select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + assert transparent_rec.id in matched + assert opaque_rec.id not in matched + assert status == "ready" + + +def test_scan_library_for_rule_skips_missing_files_gracefully( + db_sync, tmp_path, monkeypatch, +): + rec, path = _mk_image( + db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="ghost.png", + ) + path.unlink() # delete the file but leave the DB row + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.5}, + status="running", matched_ids=[], + ) + db_sync.add(audit) + db_sync.commit() + audit_id = audit.id + + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True) + from backend.app.tasks.library_audit import scan_library_for_rule + scan_library_for_rule.run(audit_id) + monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False) + + status = db_sync.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + matched = db_sync.execute( + select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id) + ).scalar_one() + # Missing file is skipped (warning logged), audit completes successfully. + assert status == "ready" + assert rec.id not in matched From 2d4bfa4375763ecd1473c17b9280e21253f9709a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 01:32:50 -0400 Subject: [PATCH 8/8] =?UTF-8?q?fix(fc-cleanup):=20test=20sha256=20fixtures?= =?UTF-8?q?=20stay=20within=20varchar(64)=20+=20isort=20the=20registration?= =?UTF-8?q?=20import=20=E2=80=94=20Co-Authored-By:=20Claude=20Opus=204.7?= =?UTF-8?q?=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_cleanup_service_audit.py | 5 ++++- tests/test_tasks_library_audit.py | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_cleanup_service_audit.py b/tests/test_cleanup_service_audit.py index b215dbf..ec488d7 100644 --- a/tests/test_cleanup_service_audit.py +++ b/tests/test_cleanup_service_audit.py @@ -28,9 +28,12 @@ def _make_image_record(db_sync, tmp_path, *, width, height, color): and insert an ImageRecord row with all required NOT-NULL columns set.""" path = tmp_path / f"img_{width}x{height}_{color[0]}.png" Image.new("RGB", (width, height), color).save(path) + # sha256 column is varchar(64) — use a deterministic 64-char pseudo + # hash built from the unique inputs. + sha = f"{width:04d}{height:04d}{color[0]:03d}".ljust(64, "0")[:64] rec = ImageRecord( path=str(path), - sha256=f"fake-{width}-{height}-{color[0]:03d}" + "0" * 50, + sha256=sha, size_bytes=path.stat().st_size, mime="image/png", # reference-image-record-required-columns width=width, diff --git a/tests/test_tasks_library_audit.py b/tests/test_tasks_library_audit.py index 3599dfc..b10c35b 100644 --- a/tests/test_tasks_library_audit.py +++ b/tests/test_tasks_library_audit.py @@ -8,21 +8,23 @@ import pytest from PIL import Image from sqlalchemy import select +import backend.app.tasks.library_audit # noqa: F401 — celery registration from backend.app import celery_app from backend.app.models import ImageRecord, LibraryAuditRun -import backend.app.tasks.library_audit # noqa: F401 — registration - pytestmark = pytest.mark.integration def _mk_image(db_sync, tmp_path, *, mode, color, name): path = tmp_path / name Image.new(mode, (10, 10), color).save(path) - short_sha = f"fake-{name}" + # sha256 column is varchar(64) — pad/truncate a per-file pseudo hash + # exactly to 64 chars. Each test fixture file must have a unique + # sha256 (reference-image-record-path-unique peer constraint). + sha = f"audit-{name}".ljust(64, "x")[:64] rec = ImageRecord( path=str(path), - sha256=short_sha + "0" * (64 - len(short_sha)), + sha256=sha, size_bytes=path.stat().st_size, mime="image/png", width=10, height=10,