Merge pull request 'Release v26.05.25.7 — animated-WebP worker fix + FC-Cleanup backend' (#21) from dev into main
This commit was merged in pull request #21.
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
"""fc-cleanup: library_audit_run table for async transparency/single_color audits
|
||||
|
||||
Revision ID: 0020
|
||||
Revises: 0019
|
||||
Create Date: 2026-05-26
|
||||
|
||||
The table backs the async audit lifecycle: rule + params snapshot, status
|
||||
state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and
|
||||
the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs
|
||||
per row by the scan task (oversize = rule too aggressive, operator narrows
|
||||
before re-running).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0020"
|
||||
down_revision: Union[str, None] = "0019"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"library_audit_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("rule", sa.String(32), nullable=False),
|
||||
sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(16),
|
||||
nullable=False, server_default="running",
|
||||
),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"scanned_count", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
sa.Column(
|
||||
"matched_count", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
sa.Column(
|
||||
"matched_ids", postgresql.JSONB(astext_type=sa.Text()),
|
||||
nullable=False, server_default=sa.text("'[]'::jsonb"),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_audit_run_rule", "library_audit_run", ["rule"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_audit_run_status", "library_audit_run", ["status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_library_audit_run_status", table_name="library_audit_run")
|
||||
op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run")
|
||||
op.drop_table("library_audit_run")
|
||||
@@ -33,6 +33,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.download",
|
||||
"backend.app.tasks.backup",
|
||||
"backend.app.tasks.admin",
|
||||
"backend.app.tasks.library_audit",
|
||||
],
|
||||
)
|
||||
app.conf.update(
|
||||
@@ -47,6 +48,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||
},
|
||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||
task_acks_late=True,
|
||||
|
||||
@@ -11,6 +11,7 @@ from .image_record import ImageRecord
|
||||
from .import_batch import ImportBatch
|
||||
from .import_settings import ImportSettings
|
||||
from .import_task import ImportTask
|
||||
from .library_audit_run import LibraryAuditRun
|
||||
from .migration_run import MigrationRun
|
||||
from .ml_settings import MLSettings
|
||||
from .post import Post
|
||||
@@ -43,6 +44,7 @@ __all__ = [
|
||||
"ImportBatch",
|
||||
"ImportTask",
|
||||
"ImportSettings",
|
||||
"LibraryAuditRun",
|
||||
"MLSettings",
|
||||
"MigrationRun",
|
||||
"TagAlias",
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""LibraryAuditRun — async transparency / single_color audit lifecycle.
|
||||
|
||||
State machine: running → ready → applied / cancelled / error.
|
||||
matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run
|
||||
reads it and routes through cleanup_service.delete_images.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class LibraryAuditRun(Base):
|
||||
__tablename__ = "library_audit_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
)
|
||||
# running | ready | applied | cancelled | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool.
|
||||
|
||||
The retroactive library-cleanup tab and (future) import-time filter logic
|
||||
both consume these. Importers should NOT inline rule logic going forward;
|
||||
add the rule here and call from both sides.
|
||||
"""
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Single-color audit: matches images where one color dominates beyond
|
||||
the threshold (within the given Euclidean RGB tolerance). The first
|
||||
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||
future spec can adopt it on the import path too.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
_THUMB_SIZE = (64, 64)
|
||||
|
||||
|
||||
def evaluate(
|
||||
pil_image,
|
||||
*,
|
||||
threshold: float,
|
||||
tolerance: int,
|
||||
) -> bool:
|
||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||
distance) of the dominant color exceeds `threshold`.
|
||||
|
||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||
Alpha channels are stripped; only RGB is considered. Animated images
|
||||
use frame 0 (PIL's default after Image.open without seek).
|
||||
"""
|
||||
im = pil_image
|
||||
if im.mode == "RGBA":
|
||||
im = im.convert("RGB")
|
||||
elif im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
if im.size != _THUMB_SIZE:
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||
pixels = list(im.getdata())
|
||||
if not pixels:
|
||||
return False
|
||||
# Normalize L-mode pixels to RGB tuples for distance math.
|
||||
if isinstance(pixels[0], int):
|
||||
pixels = [(p, p, p) for p in pixels]
|
||||
# Dominant color = mean RGB.
|
||||
n = len(pixels)
|
||||
sum_r = sum(p[0] for p in pixels)
|
||||
sum_g = sum(p[1] for p in pixels)
|
||||
sum_b = sum(p[2] for p in pixels)
|
||||
dom = (sum_r / n, sum_g / n, sum_b / n)
|
||||
tol_sq = tolerance * tolerance
|
||||
within = 0
|
||||
for r, g, b in pixels:
|
||||
dr = r - dom[0]
|
||||
dg = g - dom[1]
|
||||
db = b - dom[2]
|
||||
if dr * dr + dg * dg + db * db <= tol_sq:
|
||||
within += 1
|
||||
return (within / n) > threshold
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Transparency audit: matches images whose transparent-pixel fraction
|
||||
exceeds the threshold. Animated images short-circuit (skipped) to avoid
|
||||
the multi-frame PIL decode that hits Celery's hard time limit."""
|
||||
|
||||
|
||||
def evaluate(pil_image, *, threshold: float) -> bool:
|
||||
"""True iff the image's transparent-pixel fraction exceeds threshold.
|
||||
|
||||
False for non-alpha modes and animated images. Mirrors the import-side
|
||||
Importer._transparency_pct logic so retroactive enforcement matches
|
||||
prospective filtering.
|
||||
"""
|
||||
if getattr(pil_image, "is_animated", False):
|
||||
return False
|
||||
if pil_image.mode not in ("RGBA", "LA") and not (
|
||||
pil_image.mode == "P" and "transparency" in pil_image.info
|
||||
):
|
||||
return False
|
||||
im = pil_image
|
||||
if im.mode != "RGBA":
|
||||
im = im.convert("RGBA")
|
||||
alpha = im.getchannel("A")
|
||||
histogram = alpha.histogram()
|
||||
transparent = histogram[0]
|
||||
total = sum(histogram)
|
||||
pct = transparent / total if total else 0.0
|
||||
return pct > threshold
|
||||
@@ -12,12 +12,14 @@ re-exports from this module and then delete the wrapper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, ImageRecord, Tag
|
||||
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||
from ..models.series_page import SeriesPage
|
||||
from ..models.tag import image_tag
|
||||
|
||||
@@ -365,3 +367,142 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||
)
|
||||
session.commit()
|
||||
return {"deleted": len(ids), "sample_names": sample}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MIN_DIM_SAMPLE_CAP = 50
|
||||
|
||||
|
||||
def project_min_dimension_violations(
|
||||
session: Session, *, min_width: int, min_height: int,
|
||||
) -> dict:
|
||||
"""Return {count, sample_ids} for image_record rows with width or
|
||||
height below the thresholds. Synchronous SQL — no PIL inspection
|
||||
needed since width/height are stored columns."""
|
||||
base = select(ImageRecord.id).where(
|
||||
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||
)
|
||||
count = session.execute(
|
||||
select(func.count()).select_from(base.subquery())
|
||||
).scalar_one()
|
||||
sample_ids = session.execute(
|
||||
base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP)
|
||||
).scalars().all()
|
||||
return {"count": count, "sample_ids": list(sample_ids)}
|
||||
|
||||
|
||||
def delete_min_dimension_violations(
|
||||
session: Session, *, min_width: int, min_height: int, images_root: Path,
|
||||
) -> int:
|
||||
"""Delete every image_record where width<min_w OR height<min_h.
|
||||
Routes through delete_images so file-unlink + cascading FKs
|
||||
(image_tag / image_provenance / etc.) are handled uniformly."""
|
||||
ids = session.execute(
|
||||
select(ImageRecord.id).where(
|
||||
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||
)
|
||||
).scalars().all()
|
||||
if not ids:
|
||||
return 0
|
||||
result = delete_images(
|
||||
session, image_ids=list(ids), images_root=images_root,
|
||||
)
|
||||
return result["images_deleted"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audit lifecycle (transparency + single_color async scans).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AuditAlreadyRunning(Exception):
|
||||
"""Another audit_run is currently in status='running' — wait or
|
||||
cancel it before starting a new one. Surfaces as HTTP 409 in the
|
||||
/api/cleanup/audit POST endpoint."""
|
||||
|
||||
|
||||
class AuditNotReady(Exception):
|
||||
"""apply_audit_run called on an audit whose status is not 'ready'."""
|
||||
|
||||
|
||||
class ConfirmTokenMismatch(Exception):
|
||||
"""Operator-supplied confirm token did not match server-recomputed token."""
|
||||
|
||||
|
||||
_VALID_RULES = ("transparency", "single_color")
|
||||
|
||||
|
||||
def start_audit_run(
|
||||
session: Session, *, rule: str, params: dict[str, Any],
|
||||
) -> int:
|
||||
"""Create a LibraryAuditRun row in status='running' and dispatch the
|
||||
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||
|
||||
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||
has status='running'. Operator must cancel or wait."""
|
||||
if rule not in _VALID_RULES:
|
||||
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||
existing = session.execute(
|
||||
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise AuditAlreadyRunning(existing)
|
||||
audit = LibraryAuditRun(
|
||||
rule=rule,
|
||||
params=params,
|
||||
status="running",
|
||||
scanned_count=0,
|
||||
matched_count=0,
|
||||
matched_ids=[],
|
||||
)
|
||||
session.add(audit)
|
||||
session.flush()
|
||||
audit_id = audit.id
|
||||
# Dispatch after flush so audit_id is populated; commit happens in
|
||||
# the API handler so the audit row + dispatch are visible together.
|
||||
from ..tasks.library_audit import scan_library_for_rule
|
||||
scan_library_for_rule.delay(audit_id)
|
||||
return audit_id
|
||||
|
||||
|
||||
def apply_audit_run(
|
||||
session: Session, *, audit_id: int, confirm_token: str, images_root: Path,
|
||||
) -> int:
|
||||
"""Delete all images in audit_run.matched_ids after confirming token.
|
||||
Marks audit status='applied'. Routes through delete_images so files
|
||||
+ cascading FK rows are handled uniformly."""
|
||||
audit = session.execute(
|
||||
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one_or_none()
|
||||
if audit is None:
|
||||
raise ValueError(f"audit_run {audit_id} not found")
|
||||
if audit.status != "ready":
|
||||
raise AuditNotReady(audit.status)
|
||||
expected = f"apply-audit-{audit_id}"
|
||||
if confirm_token != expected:
|
||||
raise ConfirmTokenMismatch(expected)
|
||||
ids = list(audit.matched_ids or [])
|
||||
deleted = 0
|
||||
if ids:
|
||||
result = delete_images(session, image_ids=ids, images_root=images_root)
|
||||
deleted = result["images_deleted"]
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(status="applied", finished_at=datetime.now(UTC))
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||
"""Flip a running audit_run to 'cancelled'. The scan task checks
|
||||
for status=='cancelled' between batches and exits cleanly."""
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.where(LibraryAuditRun.status == "running")
|
||||
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||
)
|
||||
|
||||
@@ -860,8 +860,26 @@ class Importer:
|
||||
pass
|
||||
|
||||
def _transparency_pct(self, source: Path) -> float:
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||
|
||||
For animated formats (multi-frame WebP / GIF / APNG), short-circuit
|
||||
to 0.0 instead of decoding every frame. PIL's `getchannel("A")`
|
||||
forces a full decode of all frames in an animated image, which for
|
||||
a large animated WebP takes 5+ minutes and blows past the Celery
|
||||
soft+hard time limits (300s/360s → SIGKILL). Operator-flagged
|
||||
2026-05-26. Transparency analysis on a multi-frame image isn't
|
||||
meaningful for art-curation purposes anyway — different frames
|
||||
have different alpha — so the existing too_transparent skip rule
|
||||
is bypassed entirely for animated content.
|
||||
"""
|
||||
with Image.open(source) as im:
|
||||
if getattr(im, "is_animated", False):
|
||||
log.info(
|
||||
"skipping transparency check for animated image %s "
|
||||
"(n_frames=%d) — avoids multi-frame decode timeout",
|
||||
source, getattr(im, "n_frames", 0),
|
||||
)
|
||||
return 0.0
|
||||
if im.mode not in ("RGBA", "LA") and not (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
):
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
"""scan_library_for_rule Celery task — iterates image_record in keyset-
|
||||
paginated batches, evaluates the audit rule per image, populates
|
||||
LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft
|
||||
time limit (plenty of margin for 100k+ image libraries at ~100ms PIL
|
||||
decode + histogram per image).
|
||||
|
||||
State machine:
|
||||
start: status='running'
|
||||
end success: status='ready'
|
||||
end error: status='error', error=traceback
|
||||
oversize: status='error', error='matched too many images; tighten threshold'
|
||||
external cancel: scan sees status='cancelled' between batches, exits.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from PIL import Image
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImageRecord, LibraryAuditRun
|
||||
from ..services.audits import single_color, transparency
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BATCH = 500
|
||||
_PROGRESS_TICK = 100
|
||||
_MAX_MATCHED = 50_000
|
||||
|
||||
_RULES = {
|
||||
"transparency": transparency.evaluate,
|
||||
"single_color": single_color.evaluate,
|
||||
}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.library_audit.scan_library_for_rule",
|
||||
bind=True,
|
||||
autoretry_for=(OperationalError, DBAPIError),
|
||||
retry_backoff=5,
|
||||
retry_backoff_max=60,
|
||||
retry_jitter=True,
|
||||
max_retries=3,
|
||||
soft_time_limit=7200,
|
||||
time_limit=7500,
|
||||
)
|
||||
def scan_library_for_rule(self, audit_id: int) -> dict:
|
||||
"""See module docstring. Returns a small summary dict for eager-mode
|
||||
test assertions (real workers ignore the return value)."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
try:
|
||||
with SessionLocal() as session:
|
||||
audit = session.get(LibraryAuditRun, audit_id)
|
||||
if audit is None:
|
||||
return {"audit_id": audit_id, "status": "missing"}
|
||||
evaluate = _RULES.get(audit.rule)
|
||||
if evaluate is None:
|
||||
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
||||
return {"audit_id": audit_id, "status": "error"}
|
||||
params = dict(audit.params or {})
|
||||
matched: list[int] = []
|
||||
scanned = 0
|
||||
last_id = 0
|
||||
while True:
|
||||
# Cancellation check between batches.
|
||||
current_status = session.execute(
|
||||
select(LibraryAuditRun.status)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
if current_status == "cancelled":
|
||||
return {"audit_id": audit_id, "status": "cancelled"}
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.where(ImageRecord.mime.like("image/%"))
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(_BATCH)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
for image_id, image_path in rows:
|
||||
last_id = image_id
|
||||
scanned += 1
|
||||
try:
|
||||
with Image.open(image_path) as im:
|
||||
try:
|
||||
if evaluate(im, **params):
|
||||
matched.append(image_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"audit %s: rule evaluate failed on %s: %s",
|
||||
audit_id, image_path, exc,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
log.warning(
|
||||
"audit %s: image_record %s file missing at %s; skipping",
|
||||
audit_id, image_id, image_path,
|
||||
)
|
||||
except OSError as exc:
|
||||
log.warning(
|
||||
"audit %s: PIL load failed for %s: %s",
|
||||
audit_id, image_path, exc,
|
||||
)
|
||||
if len(matched) > _MAX_MATCHED:
|
||||
_mark_error(
|
||||
session, audit_id,
|
||||
f"matched > {_MAX_MATCHED} images; "
|
||||
"tighten threshold and re-run",
|
||||
)
|
||||
return {"audit_id": audit_id, "status": "error"}
|
||||
if scanned % _PROGRESS_TICK == 0:
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(scanned_count=scanned)
|
||||
)
|
||||
session.commit()
|
||||
# Final state.
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(
|
||||
scanned_count=scanned,
|
||||
matched_count=len(matched),
|
||||
matched_ids=matched,
|
||||
status="ready",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return {
|
||||
"audit_id": audit_id,
|
||||
"status": "ready",
|
||||
"scanned": scanned,
|
||||
"matched": len(matched),
|
||||
}
|
||||
except SoftTimeLimitExceeded:
|
||||
with SessionLocal() as session:
|
||||
_mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)")
|
||||
raise
|
||||
except (OperationalError, DBAPIError):
|
||||
# Retryable per the decorator; leave row in 'running' and let
|
||||
# autoretry try again. Recovery sweep catches if all retries fail.
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
tb = traceback.format_exc()
|
||||
with SessionLocal() as session:
|
||||
_mark_error(session, audit_id, tb)
|
||||
raise
|
||||
|
||||
|
||||
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
||||
session.execute(
|
||||
update(LibraryAuditRun)
|
||||
.where(LibraryAuditRun.id == audit_id)
|
||||
.values(
|
||||
status="error",
|
||||
error=error_msg,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
@@ -13,8 +13,21 @@ HASH_SIZE = 8
|
||||
|
||||
def compute_phash(pil_image) -> str | None:
|
||||
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
||||
failure (videos/unreadable/non-image)."""
|
||||
failure (videos/unreadable/non-image).
|
||||
|
||||
For animated images (multi-frame WebP/GIF/APNG), explicitly seek to
|
||||
frame 0 first. Without this, some PIL operations downstream of
|
||||
imagehash.phash (convert("L"), resize) can iterate all frames and
|
||||
blow past Celery's hard time limit on large animations
|
||||
(operator-flagged 2026-05-26 against animated WebPs). The pHash of
|
||||
frame 0 is the conventional choice for animated content.
|
||||
"""
|
||||
try:
|
||||
if getattr(pil_image, "is_animated", False):
|
||||
try:
|
||||
pil_image.seek(0)
|
||||
except Exception:
|
||||
pass
|
||||
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Tests for the single-color audit rule.
|
||||
|
||||
The rule downsamples + measures the fraction of pixels within `tolerance`
|
||||
(Euclidean RGB distance) of the dominant color. Matches if that fraction
|
||||
exceeds `threshold`. Single-color content is typically uploaded by
|
||||
mistake (placeholder/error/preview images) and should be flagged.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from backend.app.services.audits import single_color
|
||||
|
||||
|
||||
def test_single_color_evaluate_true_for_uniform_image():
|
||||
im = Image.new("RGB", (50, 50), (128, 64, 200))
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
|
||||
|
||||
|
||||
def test_single_color_evaluate_false_for_diverse_image():
|
||||
# Half black, half white — no single color dominates.
|
||||
im = Image.new("RGB", (50, 50), (0, 0, 0))
|
||||
for x in range(25):
|
||||
for y in range(50):
|
||||
im.putpixel((x, y), (255, 255, 255))
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is False
|
||||
|
||||
|
||||
def test_single_color_evaluate_respects_tolerance_widening():
|
||||
# Gradient image: pixels span 0..50 in R channel. Tight tolerance
|
||||
# rejects (no concentration), wide tolerance accepts (all near 25).
|
||||
im = Image.new("RGB", (50, 50), (0, 0, 0))
|
||||
for x in range(50):
|
||||
for y in range(50):
|
||||
im.putpixel((x, y), (x, 0, 0))
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=5) is False
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=50) is True
|
||||
|
||||
|
||||
def test_single_color_evaluate_handles_rgba_input():
|
||||
# Alpha channel should be ignored — only RGB matters for the rule.
|
||||
im = Image.new("RGBA", (50, 50), (100, 100, 100, 128))
|
||||
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Tests for the transparency audit rule.
|
||||
|
||||
The rule mirrors `Importer._transparency_pct` semantics for retroactive
|
||||
enforcement: returns True iff the fraction of fully-transparent pixels
|
||||
exceeds the threshold. Animated images short-circuit to False to avoid
|
||||
the multi-frame PIL decode that triggered SoftTimeLimitExceeded
|
||||
2026-05-26 against animated WebPs.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from backend.app.services.audits import transparency
|
||||
|
||||
|
||||
def test_transparency_evaluate_true_when_fully_transparent():
|
||||
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||
assert transparency.evaluate(im, threshold=0.5) is True
|
||||
|
||||
|
||||
def test_transparency_evaluate_false_when_fully_opaque():
|
||||
im = Image.new("RGBA", (10, 10), (200, 100, 50, 255))
|
||||
assert transparency.evaluate(im, threshold=0.5) is False
|
||||
|
||||
|
||||
def test_transparency_evaluate_respects_threshold_boundary():
|
||||
# Half-transparent image: 50% alpha=0 pixels, 50% alpha=255.
|
||||
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||
for x in range(5):
|
||||
for y in range(10):
|
||||
im.putpixel((x, y), (0, 0, 0, 255))
|
||||
# 50% transparent. threshold=0.4 → True; threshold=0.6 → False.
|
||||
assert transparency.evaluate(im, threshold=0.4) is True
|
||||
assert transparency.evaluate(im, threshold=0.6) is False
|
||||
|
||||
|
||||
def test_transparency_evaluate_false_for_rgb_image_without_alpha():
|
||||
im = Image.new("RGB", (10, 10), (128, 128, 128))
|
||||
assert transparency.evaluate(im, threshold=0.5) is False
|
||||
|
||||
|
||||
def test_transparency_evaluate_false_for_animated_image():
|
||||
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||
# Mark as animated (mimics PIL's WebP/GIF multi-frame attribute).
|
||||
im.is_animated = True # type: ignore[attr-defined]
|
||||
im.n_frames = 5 # type: ignore[attr-defined]
|
||||
assert transparency.evaluate(im, threshold=0.5) is False
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Tests for cleanup_service's library-audit additions.
|
||||
|
||||
Covers:
|
||||
- project_min_dimension_violations: SQL-only query against width/height
|
||||
- delete_min_dimension_violations: routes through existing delete_images
|
||||
- audit lifecycle (start_audit_run / apply_audit_run / cancel_audit_run)
|
||||
|
||||
Tests assert via column selects per reference-async-coredml-test-assertions
|
||||
(post-DML ORM entity access via session.get() raises MissingGreenlet on
|
||||
async sessions; we use db_sync here but the convention is preserved for
|
||||
consistency).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import ImageRecord, LibraryAuditRun
|
||||
from backend.app.services import cleanup_service
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _make_image_record(db_sync, tmp_path, *, width, height, color):
|
||||
"""Helper: write a real PIL file to a UNIQUE path (reference-image-record-path-unique)
|
||||
and insert an ImageRecord row with all required NOT-NULL columns set."""
|
||||
path = tmp_path / f"img_{width}x{height}_{color[0]}.png"
|
||||
Image.new("RGB", (width, height), color).save(path)
|
||||
# sha256 column is varchar(64) — use a deterministic 64-char pseudo
|
||||
# hash built from the unique inputs.
|
||||
sha = f"{width:04d}{height:04d}{color[0]:03d}".ljust(64, "0")[:64]
|
||||
rec = ImageRecord(
|
||||
path=str(path),
|
||||
sha256=sha,
|
||||
size_bytes=path.stat().st_size,
|
||||
mime="image/png", # reference-image-record-required-columns
|
||||
width=width,
|
||||
height=height,
|
||||
origin="imported_filesystem", # feedback-check-existing-enums
|
||||
integrity_status="ok",
|
||||
)
|
||||
db_sync.add(rec)
|
||||
db_sync.flush()
|
||||
return rec
|
||||
|
||||
|
||||
def test_project_min_dimension_violations_returns_count_and_samples(db_sync, tmp_path):
|
||||
_make_image_record(db_sync, tmp_path, width=100, height=100, color=(10, 0, 0))
|
||||
_make_image_record(db_sync, tmp_path, width=50, height=50, color=(20, 0, 0))
|
||||
_make_image_record(db_sync, tmp_path, width=400, height=400, color=(30, 0, 0))
|
||||
db_sync.commit()
|
||||
|
||||
result = cleanup_service.project_min_dimension_violations(
|
||||
db_sync, min_width=200, min_height=200,
|
||||
)
|
||||
assert result["count"] == 2 # 100x100 and 50x50 violate
|
||||
assert len(result["sample_ids"]) == 2
|
||||
|
||||
|
||||
def test_delete_min_dimension_violations_unlinks_and_cascades(db_sync, tmp_path):
|
||||
rec_small = _make_image_record(db_sync, tmp_path, width=50, height=50, color=(40, 0, 0))
|
||||
rec_big = _make_image_record(db_sync, tmp_path, width=500, height=500, color=(50, 0, 0))
|
||||
small_path = Path(rec_small.path)
|
||||
db_sync.commit()
|
||||
|
||||
# images_root is tmp_path here because the test fixtures stored files there;
|
||||
# delete_images() uses it to unlink originals + thumbs from the right tree.
|
||||
deleted = cleanup_service.delete_min_dimension_violations(
|
||||
db_sync, min_width=200, min_height=200, images_root=tmp_path,
|
||||
)
|
||||
assert deleted == 1
|
||||
# Verify via column selects per banked rule.
|
||||
remaining_ids = db_sync.execute(
|
||||
select(ImageRecord.id).order_by(ImageRecord.id)
|
||||
).scalars().all()
|
||||
assert remaining_ids == [rec_big.id]
|
||||
assert not small_path.exists() # file unlinked
|
||||
|
||||
|
||||
# --- Audit lifecycle tests (Task 5) ---
|
||||
|
||||
import backend.app.tasks.library_audit # noqa: F401, E402 — celery registration
|
||||
|
||||
|
||||
def test_start_audit_run_creates_row_and_dispatches(db_sync, monkeypatch):
|
||||
dispatched = []
|
||||
from backend.app.tasks import library_audit as la_mod
|
||||
monkeypatch.setattr(
|
||||
la_mod.scan_library_for_rule, "delay",
|
||||
lambda audit_id: dispatched.append(audit_id),
|
||||
)
|
||||
audit_id = cleanup_service.start_audit_run(
|
||||
db_sync, rule="transparency", params={"threshold": 0.9},
|
||||
)
|
||||
db_sync.commit()
|
||||
row_status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
assert row_status == "running"
|
||||
assert dispatched == [audit_id]
|
||||
|
||||
|
||||
def test_start_audit_run_rejects_when_another_is_running(db_sync, monkeypatch):
|
||||
from backend.app.tasks import library_audit as la_mod
|
||||
monkeypatch.setattr(
|
||||
la_mod.scan_library_for_rule, "delay", lambda audit_id: None,
|
||||
)
|
||||
cleanup_service.start_audit_run(
|
||||
db_sync, rule="transparency", params={"threshold": 0.9},
|
||||
)
|
||||
db_sync.commit()
|
||||
with pytest.raises(cleanup_service.AuditAlreadyRunning):
|
||||
cleanup_service.start_audit_run(
|
||||
db_sync, rule="single_color",
|
||||
params={"threshold": 0.95, "tolerance": 30},
|
||||
)
|
||||
|
||||
|
||||
def test_apply_audit_run_with_correct_token_deletes_matched(db_sync, tmp_path):
|
||||
rec = _make_image_record(
|
||||
db_sync, tmp_path, width=100, height=100, color=(60, 0, 0),
|
||||
)
|
||||
db_sync.flush()
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.9},
|
||||
status="ready", scanned_count=1, matched_count=1,
|
||||
matched_ids=[rec.id],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
|
||||
deleted = cleanup_service.apply_audit_run(
|
||||
db_sync, audit_id=audit.id,
|
||||
confirm_token=f"apply-audit-{audit.id}",
|
||||
images_root=tmp_path,
|
||||
)
|
||||
assert deleted == 1
|
||||
remaining_count = db_sync.execute(
|
||||
select(func.count()).select_from(ImageRecord)
|
||||
).scalar_one()
|
||||
assert remaining_count == 0
|
||||
new_status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id)
|
||||
).scalar_one()
|
||||
assert new_status == "applied"
|
||||
|
||||
|
||||
def test_apply_audit_run_with_wrong_token_raises(db_sync, tmp_path):
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.9},
|
||||
status="ready", matched_ids=[],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
with pytest.raises(cleanup_service.ConfirmTokenMismatch):
|
||||
cleanup_service.apply_audit_run(
|
||||
db_sync, audit_id=audit.id,
|
||||
confirm_token="wrong-token", images_root=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
def test_cancel_audit_run_flips_status(db_sync):
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.9},
|
||||
status="running", matched_ids=[],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
cleanup_service.cancel_audit_run(db_sync, audit_id=audit.id)
|
||||
new_status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id)
|
||||
).scalar_one()
|
||||
assert new_status == "cancelled"
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Tests for scan_library_for_rule Celery task.
|
||||
|
||||
Eager mode is used so the task runs synchronously in-test and we can
|
||||
assert state via column selects (post-DML ORM access banned per
|
||||
reference-async-coredml-test-assertions)."""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
|
||||
import backend.app.tasks.library_audit # noqa: F401 — celery registration
|
||||
from backend.app import celery_app
|
||||
from backend.app.models import ImageRecord, LibraryAuditRun
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _mk_image(db_sync, tmp_path, *, mode, color, name):
|
||||
path = tmp_path / name
|
||||
Image.new(mode, (10, 10), color).save(path)
|
||||
# sha256 column is varchar(64) — pad/truncate a per-file pseudo hash
|
||||
# exactly to 64 chars. Each test fixture file must have a unique
|
||||
# sha256 (reference-image-record-path-unique peer constraint).
|
||||
sha = f"audit-{name}".ljust(64, "x")[:64]
|
||||
rec = ImageRecord(
|
||||
path=str(path),
|
||||
sha256=sha,
|
||||
size_bytes=path.stat().st_size,
|
||||
mime="image/png",
|
||||
width=10, height=10,
|
||||
origin="imported_filesystem",
|
||||
integrity_status="ok",
|
||||
)
|
||||
db_sync.add(rec)
|
||||
db_sync.flush()
|
||||
return rec, path
|
||||
|
||||
|
||||
def test_scan_library_for_rule_populates_matched_ids_for_transparency(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
transparent_rec, _ = _mk_image(
|
||||
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="trans.png",
|
||||
)
|
||||
opaque_rec, _ = _mk_image(
|
||||
db_sync, tmp_path, mode="RGBA", color=(200, 0, 0, 255), name="opaque.png",
|
||||
)
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.5},
|
||||
status="running", matched_ids=[],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
audit_id = audit.id
|
||||
|
||||
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
|
||||
from backend.app.tasks.library_audit import scan_library_for_rule
|
||||
scan_library_for_rule.run(audit_id)
|
||||
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
|
||||
|
||||
matched = db_sync.execute(
|
||||
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
assert transparent_rec.id in matched
|
||||
assert opaque_rec.id not in matched
|
||||
assert status == "ready"
|
||||
|
||||
|
||||
def test_scan_library_for_rule_skips_missing_files_gracefully(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
rec, path = _mk_image(
|
||||
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="ghost.png",
|
||||
)
|
||||
path.unlink() # delete the file but leave the DB row
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.5},
|
||||
status="running", matched_ids=[],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
audit_id = audit.id
|
||||
|
||||
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
|
||||
from backend.app.tasks.library_audit import scan_library_for_rule
|
||||
scan_library_for_rule.run(audit_id)
|
||||
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
|
||||
|
||||
status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
matched = db_sync.execute(
|
||||
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
# Missing file is skipped (warning logged), audit completes successfully.
|
||||
assert status == "ready"
|
||||
assert rec.id not in matched
|
||||
Reference in New Issue
Block a user