Compare commits
20 Commits
ext-1.0.3
...
v26.05.26.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 5d4f223b71 | |||
| 2505b197ae | |||
| 0d0b236ac3 | |||
| a06ada4c9b | |||
| ebd985990c | |||
| 4da8d1d774 | |||
| 05090c6e85 | |||
| 2d4bfa4375 | |||
| 6ed2021ad6 | |||
| 4f2ceaaf31 | |||
| 8a5b337a53 | |||
| 900d878d27 | |||
| fd80d40a34 | |||
| 929d3fc092 | |||
| c0c9e56fb9 | |||
| 3a577d5ade | |||
| 06a2f60c08 | |||
| 0978fbac66 | |||
| f4fe02e346 | |||
| efb142239d |
@@ -68,17 +68,11 @@ jobs:
|
|||||||
echo "No release named ext-$VERSION; will sign via AMO and upload"
|
echo "No release named ext-$VERSION; will sign via AMO and upload"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Download cached signed XPI (cache hit)
|
# No "download cached XPI in sign-extension" step: build-web
|
||||||
if: steps.cache.outputs.cached == 'true'
|
# fetches directly from the Forgejo ext-<version> release asset
|
||||||
env:
|
# (removed 2026-05-26 alongside the actions/upload-artifact
|
||||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
# removal — sign-extension's job is just to ensure the cache
|
||||||
run: |
|
# exists on Forgejo; the build-web side reads it independently).
|
||||||
set -eu
|
|
||||||
mkdir -p extension/web-ext-artifacts
|
|
||||||
curl -sL -H "Authorization: token $TOKEN" \
|
|
||||||
-o "extension/web-ext-artifacts/fabledcurator-${{ steps.extver.outputs.version }}.xpi" \
|
|
||||||
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/assets/${{ steps.cache.outputs.asset_id }}"
|
|
||||||
ls -la extension/web-ext-artifacts/
|
|
||||||
|
|
||||||
- name: Sign via AMO (cache miss)
|
- name: Sign via AMO (cache miss)
|
||||||
if: steps.cache.outputs.cached != 'true'
|
if: steps.cache.outputs.cached != 'true'
|
||||||
@@ -148,12 +142,11 @@ jobs:
|
|||||||
trap - EXIT
|
trap - EXIT
|
||||||
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
|
echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
|
||||||
|
|
||||||
- name: Upload XPI as Actions artifact (handoff to build-web)
|
# No actions/upload-artifact step: Forgejo Actions (and our
|
||||||
uses: actions/upload-artifact@v4
|
# act_runner) doesn't support upload-artifact@v4+ (GHES limitation
|
||||||
with:
|
# surfaced 2026-05-26). Instead build-web reads the signed XPI
|
||||||
name: signed-extension
|
# straight from the ext-<version> Forgejo release we just uploaded
|
||||||
path: extension/web-ext-artifacts/fabledcurator-*.xpi
|
# to. Same source of truth; no double-store.
|
||||||
retention-days: 1
|
|
||||||
|
|
||||||
build-web:
|
build-web:
|
||||||
needs: [sign-extension]
|
needs: [sign-extension]
|
||||||
@@ -165,22 +158,43 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Download signed extension (main only)
|
- name: Download signed XPI from Forgejo release asset (main only)
|
||||||
if: github.ref == 'refs/heads/main'
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
name: signed-extension
|
|
||||||
path: extension/web-ext-artifacts/
|
|
||||||
|
|
||||||
- name: Place signed XPI in build context (main only)
|
|
||||||
if: github.ref == 'refs/heads/main'
|
if: github.ref == 'refs/heads/main'
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
|
||||||
XPI=$(ls extension/web-ext-artifacts/*.xpi | head -1)
|
# Look up the ext-<version> release; extract the .xpi asset's
|
||||||
|
# browser_download_url (Forgejo's /releases/assets/<id> endpoint
|
||||||
|
# returns ASSET METADATA, not the binary blob — operator-flagged
|
||||||
|
# 2026-05-26: my prior code curl'd the metadata endpoint without
|
||||||
|
# -f and wrote the resulting 404-page-not-found text into
|
||||||
|
# fabledcurator-*.xpi, which Firefox then rejected as "corrupt").
|
||||||
|
# browser_download_url is the canonical binary endpoint and is
|
||||||
|
# also publicly accessible (no token needed) but we pass the
|
||||||
|
# token anyway for symmetry with private-repo support.
|
||||||
|
curl -sf -H "Authorization: token $TOKEN" \
|
||||||
|
"https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \
|
||||||
|
-o release.json
|
||||||
|
DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])")
|
||||||
|
test -n "$DOWNLOAD_URL"
|
||||||
|
echo "Downloading XPI from: $DOWNLOAD_URL"
|
||||||
mkdir -p frontend/public/extension
|
mkdir -p frontend/public/extension
|
||||||
cp "$XPI" "frontend/public/extension/fabledcurator-$VERSION.xpi"
|
DEST="frontend/public/extension/fabledcurator-$VERSION.xpi"
|
||||||
cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
|
# -f = fail on HTTP error (prevents silent corruption like the
|
||||||
|
# 2026-05-26 incident); -L = follow redirects.
|
||||||
|
curl -sfL -H "Authorization: token $TOKEN" -o "$DEST" "$DOWNLOAD_URL"
|
||||||
|
# Sanity check: the binary should start with the ZIP magic (PK\x03\x04).
|
||||||
|
# If it's anything else, the next docker build will ship a corrupt XPI.
|
||||||
|
MAGIC=$(head -c 2 "$DEST" | od -An -c | tr -d ' \n')
|
||||||
|
if [ "$MAGIC" != "PK" ]; then
|
||||||
|
echo "ERROR: downloaded XPI does not start with ZIP magic 'PK' (got '$MAGIC')"
|
||||||
|
echo "File contents preview:"
|
||||||
|
head -c 200 "$DEST"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi"
|
||||||
ls -la frontend/public/extension/
|
ls -la frontend/public/extension/
|
||||||
|
|
||||||
- name: Determine tag
|
- name: Determine tag
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""fc-cleanup: library_audit_run table for async transparency/single_color audits
|
||||||
|
|
||||||
|
Revision ID: 0020
|
||||||
|
Revises: 0019
|
||||||
|
Create Date: 2026-05-26
|
||||||
|
|
||||||
|
The table backs the async audit lifecycle: rule + params snapshot, status
|
||||||
|
state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and
|
||||||
|
the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs
|
||||||
|
per row by the scan task (oversize = rule too aggressive, operator narrows
|
||||||
|
before re-running).
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "0020"
|
||||||
|
down_revision: Union[str, None] = "0019"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"library_audit_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("rule", sa.String(32), nullable=False),
|
||||||
|
sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(16),
|
||||||
|
nullable=False, server_default="running",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True),
|
||||||
|
nullable=False, server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"scanned_count", sa.Integer(),
|
||||||
|
nullable=False, server_default="0",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"matched_count", sa.Integer(),
|
||||||
|
nullable=False, server_default="0",
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"matched_ids", postgresql.JSONB(astext_type=sa.Text()),
|
||||||
|
nullable=False, server_default=sa.text("'[]'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_audit_run_rule", "library_audit_run", ["rule"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_library_audit_run_status", "library_audit_run", ["status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_library_audit_run_status", table_name="library_audit_run")
|
||||||
|
op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run")
|
||||||
|
op.drop_table("library_audit_run")
|
||||||
@@ -20,6 +20,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .artist import artist_bp
|
from .artist import artist_bp
|
||||||
from .artists import artists_bp
|
from .artists import artists_bp
|
||||||
from .attachments import attachments_bp
|
from .attachments import attachments_bp
|
||||||
|
from .cleanup import cleanup_bp
|
||||||
from .credentials import credentials_bp
|
from .credentials import credentials_bp
|
||||||
from .downloads import downloads_bp
|
from .downloads import downloads_bp
|
||||||
from .extension import extension_bp
|
from .extension import extension_bp
|
||||||
@@ -50,6 +51,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
system_activity_bp,
|
system_activity_bp,
|
||||||
system_backup_bp,
|
system_backup_bp,
|
||||||
admin_bp,
|
admin_bp,
|
||||||
|
cleanup_bp,
|
||||||
import_admin_bp,
|
import_admin_bp,
|
||||||
migrate_bp,
|
migrate_bp,
|
||||||
suggestions_bp,
|
suggestions_bp,
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
"""FC-Cleanup: /api/cleanup/* — retroactive enforcement of import filters.
|
||||||
|
|
||||||
|
Endpoints:
|
||||||
|
POST /min-dimension/preview synchronous SQL audit
|
||||||
|
POST /min-dimension/delete synchronous SQL delete (Tier-C token)
|
||||||
|
POST /audit async transparency / single_color start
|
||||||
|
GET /audit list recent audit_run rows
|
||||||
|
GET /audit/<id> single audit_run row
|
||||||
|
POST /audit/<id>/apply apply matched_ids deletes (Tier-C token)
|
||||||
|
POST /audit/<id>/cancel flip running audit to cancelled
|
||||||
|
|
||||||
|
Unused-tags retroactive prune intentionally NOT in this namespace —
|
||||||
|
TagMaintenanceCard (Maintenance tab → moved to Cleanup tab in v26.05.25.7)
|
||||||
|
uses the existing /api/admin/tags/prune-unused endpoint via the admin
|
||||||
|
store. No duplicate route here.
|
||||||
|
|
||||||
|
Confirm-token format matches modal/DestructiveConfirmModal.vue convention:
|
||||||
|
`delete-min-dim-<sha8(w,h)>` for min-dim delete
|
||||||
|
`delete-audit-<id>` for audit apply
|
||||||
|
(Modal hardcodes action ∈ {'restore', 'delete'}; "apply audit" is semantically a delete of the matched images, so we use `delete-audit-<id>`.)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify, request
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import LibraryAuditRun
|
||||||
|
from ..services import cleanup_service
|
||||||
|
|
||||||
|
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
||||||
|
|
||||||
|
IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
def _bad(error: str, *, status: int = 400, **extra):
|
||||||
|
body = {"error": error}
|
||||||
|
body.update(extra)
|
||||||
|
return jsonify(body), status
|
||||||
|
|
||||||
|
|
||||||
|
def _min_dim_token(min_w: int, min_h: int) -> str:
|
||||||
|
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
||||||
|
# sides use SHA-256 truncated to 8 hex chars.
|
||||||
|
canon = f"{min_w}x{min_h}"
|
||||||
|
return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}"
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_audit_run(audit: LibraryAuditRun) -> dict:
|
||||||
|
return {
|
||||||
|
"id": audit.id,
|
||||||
|
"rule": audit.rule,
|
||||||
|
"params": audit.params,
|
||||||
|
"status": audit.status,
|
||||||
|
"started_at": audit.started_at.isoformat() if audit.started_at else None,
|
||||||
|
"finished_at": audit.finished_at.isoformat() if audit.finished_at else None,
|
||||||
|
"scanned_count": audit.scanned_count,
|
||||||
|
"matched_count": audit.matched_count,
|
||||||
|
"matched_ids": audit.matched_ids,
|
||||||
|
"error": audit.error,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/min-dimension/preview", methods=["POST"])
|
||||||
|
async def min_dim_preview():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
min_w = int(body.get("min_width", 0))
|
||||||
|
min_h = int(body.get("min_height", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
if min_w < 0 or min_h < 0:
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
async with get_session() as session:
|
||||||
|
projection = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.project_min_dimension_violations(
|
||||||
|
s, min_width=min_w, min_height=min_h,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return jsonify(projection)
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/min-dimension/delete", methods=["POST"])
|
||||||
|
async def min_dim_delete():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
min_w = int(body.get("min_width", 0))
|
||||||
|
min_h = int(body.get("min_height", 0))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
if min_w < 0 or min_h < 0:
|
||||||
|
return _bad("invalid_dimensions")
|
||||||
|
supplied = body.get("confirm", "")
|
||||||
|
expected = _min_dim_token(min_w, min_h)
|
||||||
|
if supplied != expected:
|
||||||
|
return _bad("confirm_mismatch", expected=expected)
|
||||||
|
async with get_session() as session:
|
||||||
|
deleted = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.delete_min_dimension_violations(
|
||||||
|
s, min_width=min_w, min_height=min_h, images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"deleted": deleted})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit", methods=["POST"])
|
||||||
|
async def audit_create():
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
rule = body.get("rule")
|
||||||
|
params = body.get("params") or {}
|
||||||
|
if rule not in ("transparency", "single_color"):
|
||||||
|
return _bad("invalid_rule")
|
||||||
|
if not isinstance(params, dict):
|
||||||
|
return _bad("invalid_params")
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
audit_id = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.start_audit_run(
|
||||||
|
s, rule=rule, params=params,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except cleanup_service.AuditAlreadyRunning as running_id:
|
||||||
|
return _bad(
|
||||||
|
"audit_already_running", status=409,
|
||||||
|
running_id=int(str(running_id)),
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return _bad(str(exc))
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"audit_id": audit_id, "status": "running"}), 202
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>", methods=["GET"])
|
||||||
|
async def audit_get(audit_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
audit = (await session.execute(
|
||||||
|
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if audit is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
return jsonify(_serialize_audit_run(audit))
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit", methods=["GET"])
|
||||||
|
async def audit_history():
|
||||||
|
try:
|
||||||
|
limit = min(int(request.args.get("limit", "20")), 100)
|
||||||
|
except ValueError:
|
||||||
|
return _bad("invalid_limit")
|
||||||
|
async with get_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(LibraryAuditRun)
|
||||||
|
.order_by(LibraryAuditRun.id.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)).scalars().all()
|
||||||
|
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>/apply", methods=["POST"])
|
||||||
|
async def audit_apply(audit_id: int):
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
confirm = body.get("confirm", "")
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
deleted = await session.run_sync(
|
||||||
|
lambda s: cleanup_service.apply_audit_run(
|
||||||
|
s, audit_id=audit_id, confirm_token=confirm,
|
||||||
|
images_root=IMAGES_ROOT,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except cleanup_service.AuditNotReady as exc:
|
||||||
|
return _bad("audit_not_ready", current_status=str(exc))
|
||||||
|
except cleanup_service.ConfirmTokenMismatch as exc:
|
||||||
|
return _bad("confirm_mismatch", expected=str(exc))
|
||||||
|
except ValueError as exc:
|
||||||
|
return _bad("not_found", status=404, detail=str(exc))
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"deleted": deleted})
|
||||||
|
|
||||||
|
|
||||||
|
@cleanup_bp.route("/audit/<int:audit_id>/cancel", methods=["POST"])
|
||||||
|
async def audit_cancel(audit_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
await session.run_sync(
|
||||||
|
lambda s: cleanup_service.cancel_audit_run(s, audit_id=audit_id)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"cancelled": True})
|
||||||
@@ -93,10 +93,20 @@ def _read_manifest_sync() -> dict | None:
|
|||||||
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
||||||
if not XPI_DIR.is_dir():
|
if not XPI_DIR.is_dir():
|
||||||
return None
|
return None
|
||||||
xpis = sorted(XPI_DIR.glob("fabledcurator-*.xpi"), key=lambda p: p.stat().st_mtime)
|
# Exclude the `fabledcurator-latest.xpi` alias when picking the file to
|
||||||
if not xpis:
|
# extract a version from — it's a copy of the latest versioned XPI,
|
||||||
|
# written at the same mtime by build.yml, and would otherwise tie or
|
||||||
|
# win the sort (operator-flagged 2026-05-26: UI displayed "v latest"
|
||||||
|
# because `_extract_version("fabledcurator-latest.xpi")` returns
|
||||||
|
# the literal "latest"). The alias still serves as `latest_url`.
|
||||||
|
versioned = [
|
||||||
|
p for p in XPI_DIR.glob("fabledcurator-*.xpi")
|
||||||
|
if p.name != "fabledcurator-latest.xpi"
|
||||||
|
]
|
||||||
|
if not versioned:
|
||||||
return None
|
return None
|
||||||
latest = xpis[-1]
|
versioned.sort(key=lambda p: p.stat().st_mtime)
|
||||||
|
latest = versioned[-1]
|
||||||
return {
|
return {
|
||||||
"installed": True,
|
"installed": True,
|
||||||
"version": _extract_version(latest.name),
|
"version": _extract_version(latest.name),
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.download",
|
"backend.app.tasks.download",
|
||||||
"backend.app.tasks.backup",
|
"backend.app.tasks.backup",
|
||||||
"backend.app.tasks.admin",
|
"backend.app.tasks.admin",
|
||||||
|
"backend.app.tasks.library_audit",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
app.conf.update(
|
app.conf.update(
|
||||||
@@ -47,6 +48,7 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||||
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||||
},
|
},
|
||||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from .image_record import ImageRecord
|
|||||||
from .import_batch import ImportBatch
|
from .import_batch import ImportBatch
|
||||||
from .import_settings import ImportSettings
|
from .import_settings import ImportSettings
|
||||||
from .import_task import ImportTask
|
from .import_task import ImportTask
|
||||||
|
from .library_audit_run import LibraryAuditRun
|
||||||
from .migration_run import MigrationRun
|
from .migration_run import MigrationRun
|
||||||
from .ml_settings import MLSettings
|
from .ml_settings import MLSettings
|
||||||
from .post import Post
|
from .post import Post
|
||||||
@@ -43,6 +44,7 @@ __all__ = [
|
|||||||
"ImportBatch",
|
"ImportBatch",
|
||||||
"ImportTask",
|
"ImportTask",
|
||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
|
"LibraryAuditRun",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
"MigrationRun",
|
"MigrationRun",
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""LibraryAuditRun — async transparency / single_color audit lifecycle.
|
||||||
|
|
||||||
|
State machine: running → ready → applied / cancelled / error.
|
||||||
|
matched_ids JSONB is appended-to by scan_library_for_rule; apply_audit_run
|
||||||
|
reads it and routes through cleanup_service.delete_images.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class LibraryAuditRun(Base):
|
||||||
|
__tablename__ = "library_audit_run"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
rule: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||||
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
)
|
||||||
|
# running | ready | applied | cancelled | error
|
||||||
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True,
|
||||||
|
)
|
||||||
|
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||||
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
"""Audit rule modules. Each module exposes evaluate(pil_image, **params) -> bool.
|
||||||
|
|
||||||
|
The retroactive library-cleanup tab and (future) import-time filter logic
|
||||||
|
both consume these. Importers should NOT inline rule logic going forward;
|
||||||
|
add the rule here and call from both sides.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""Single-color audit: matches images where one color dominates beyond
|
||||||
|
the threshold (within the given Euclidean RGB tolerance). The first
|
||||||
|
canonical implementation — the import-side filter (SkipReason.single_color)
|
||||||
|
was never wired; FC-Cleanup's audit module is the source of truth and a
|
||||||
|
future spec can adopt it on the import path too.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
_THUMB_SIZE = (64, 64)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(
|
||||||
|
pil_image,
|
||||||
|
*,
|
||||||
|
threshold: float,
|
||||||
|
tolerance: int,
|
||||||
|
) -> bool:
|
||||||
|
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||||
|
distance) of the dominant color exceeds `threshold`.
|
||||||
|
|
||||||
|
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||||
|
Alpha channels are stripped; only RGB is considered. Animated images
|
||||||
|
use frame 0 (PIL's default after Image.open without seek).
|
||||||
|
"""
|
||||||
|
im = pil_image
|
||||||
|
if im.mode == "RGBA":
|
||||||
|
im = im.convert("RGB")
|
||||||
|
elif im.mode not in ("RGB", "L"):
|
||||||
|
im = im.convert("RGB")
|
||||||
|
if im.size != _THUMB_SIZE:
|
||||||
|
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||||
|
pixels = list(im.getdata())
|
||||||
|
if not pixels:
|
||||||
|
return False
|
||||||
|
# Normalize L-mode pixels to RGB tuples for distance math.
|
||||||
|
if isinstance(pixels[0], int):
|
||||||
|
pixels = [(p, p, p) for p in pixels]
|
||||||
|
# Dominant color = mean RGB.
|
||||||
|
n = len(pixels)
|
||||||
|
sum_r = sum(p[0] for p in pixels)
|
||||||
|
sum_g = sum(p[1] for p in pixels)
|
||||||
|
sum_b = sum(p[2] for p in pixels)
|
||||||
|
dom = (sum_r / n, sum_g / n, sum_b / n)
|
||||||
|
tol_sq = tolerance * tolerance
|
||||||
|
within = 0
|
||||||
|
for r, g, b in pixels:
|
||||||
|
dr = r - dom[0]
|
||||||
|
dg = g - dom[1]
|
||||||
|
db = b - dom[2]
|
||||||
|
if dr * dr + dg * dg + db * db <= tol_sq:
|
||||||
|
within += 1
|
||||||
|
return (within / n) > threshold
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""Transparency audit: matches images whose transparent-pixel fraction
|
||||||
|
exceeds the threshold. Animated images short-circuit (skipped) to avoid
|
||||||
|
the multi-frame PIL decode that hits Celery's hard time limit."""
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(pil_image, *, threshold: float) -> bool:
|
||||||
|
"""True iff the image's transparent-pixel fraction exceeds threshold.
|
||||||
|
|
||||||
|
False for non-alpha modes and animated images. Mirrors the import-side
|
||||||
|
Importer._transparency_pct logic so retroactive enforcement matches
|
||||||
|
prospective filtering.
|
||||||
|
"""
|
||||||
|
if getattr(pil_image, "is_animated", False):
|
||||||
|
return False
|
||||||
|
if pil_image.mode not in ("RGBA", "LA") and not (
|
||||||
|
pil_image.mode == "P" and "transparency" in pil_image.info
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
im = pil_image
|
||||||
|
if im.mode != "RGBA":
|
||||||
|
im = im.convert("RGBA")
|
||||||
|
alpha = im.getchannel("A")
|
||||||
|
histogram = alpha.histogram()
|
||||||
|
transparent = histogram[0]
|
||||||
|
total = sum(histogram)
|
||||||
|
pct = transparent / total if total else 0.0
|
||||||
|
return pct > threshold
|
||||||
@@ -12,12 +12,14 @@ re-exports from this module and then delete the wrapper.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import Artist, ImageRecord, Tag
|
from ..models import Artist, ImageRecord, LibraryAuditRun, Tag
|
||||||
from ..models.series_page import SeriesPage
|
from ..models.series_page import SeriesPage
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
|
|
||||||
@@ -365,3 +367,146 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
|||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
return {"deleted": len(ids), "sample_names": sample}
|
return {"deleted": len(ids), "sample_names": sample}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FC-Cleanup additions (2026-05-26): retroactive audit of import-filter rules.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_MIN_DIM_SAMPLE_CAP = 50
|
||||||
|
|
||||||
|
|
||||||
|
def project_min_dimension_violations(
|
||||||
|
session: Session, *, min_width: int, min_height: int,
|
||||||
|
) -> dict:
|
||||||
|
"""Return {count, sample_ids} for image_record rows with width or
|
||||||
|
height below the thresholds. Synchronous SQL — no PIL inspection
|
||||||
|
needed since width/height are stored columns."""
|
||||||
|
base = select(ImageRecord.id).where(
|
||||||
|
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||||
|
)
|
||||||
|
count = session.execute(
|
||||||
|
select(func.count()).select_from(base.subquery())
|
||||||
|
).scalar_one()
|
||||||
|
sample_ids = session.execute(
|
||||||
|
base.order_by(ImageRecord.id).limit(_MIN_DIM_SAMPLE_CAP)
|
||||||
|
).scalars().all()
|
||||||
|
return {"count": count, "sample_ids": list(sample_ids)}
|
||||||
|
|
||||||
|
|
||||||
|
def delete_min_dimension_violations(
|
||||||
|
session: Session, *, min_width: int, min_height: int, images_root: Path,
|
||||||
|
) -> int:
|
||||||
|
"""Delete every image_record where width<min_w OR height<min_h.
|
||||||
|
Routes through delete_images so file-unlink + cascading FKs
|
||||||
|
(image_tag / image_provenance / etc.) are handled uniformly."""
|
||||||
|
ids = session.execute(
|
||||||
|
select(ImageRecord.id).where(
|
||||||
|
(ImageRecord.width < min_width) | (ImageRecord.height < min_height)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
if not ids:
|
||||||
|
return 0
|
||||||
|
result = delete_images(
|
||||||
|
session, image_ids=list(ids), images_root=images_root,
|
||||||
|
)
|
||||||
|
return result["images_deleted"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Audit lifecycle (transparency + single_color async scans).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class AuditAlreadyRunning(Exception):
|
||||||
|
"""Another audit_run is currently in status='running' — wait or
|
||||||
|
cancel it before starting a new one. Surfaces as HTTP 409 in the
|
||||||
|
/api/cleanup/audit POST endpoint."""
|
||||||
|
|
||||||
|
|
||||||
|
class AuditNotReady(Exception):
|
||||||
|
"""apply_audit_run called on an audit whose status is not 'ready'."""
|
||||||
|
|
||||||
|
|
||||||
|
class ConfirmTokenMismatch(Exception):
|
||||||
|
"""Operator-supplied confirm token did not match server-recomputed token."""
|
||||||
|
|
||||||
|
|
||||||
|
_VALID_RULES = ("transparency", "single_color")
|
||||||
|
|
||||||
|
|
||||||
|
def start_audit_run(
|
||||||
|
session: Session, *, rule: str, params: dict[str, Any],
|
||||||
|
) -> int:
|
||||||
|
"""Create a LibraryAuditRun row in status='running' and dispatch the
|
||||||
|
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||||
|
|
||||||
|
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||||
|
has status='running'. Operator must cancel or wait."""
|
||||||
|
if rule not in _VALID_RULES:
|
||||||
|
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||||
|
existing = session.execute(
|
||||||
|
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
raise AuditAlreadyRunning(existing)
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule=rule,
|
||||||
|
params=params,
|
||||||
|
status="running",
|
||||||
|
scanned_count=0,
|
||||||
|
matched_count=0,
|
||||||
|
matched_ids=[],
|
||||||
|
)
|
||||||
|
session.add(audit)
|
||||||
|
session.flush()
|
||||||
|
audit_id = audit.id
|
||||||
|
# Dispatch after flush so audit_id is populated; commit happens in
|
||||||
|
# the API handler so the audit row + dispatch are visible together.
|
||||||
|
from ..tasks.library_audit import scan_library_for_rule
|
||||||
|
scan_library_for_rule.delay(audit_id)
|
||||||
|
return audit_id
|
||||||
|
|
||||||
|
|
||||||
|
def apply_audit_run(
|
||||||
|
session: Session, *, audit_id: int, confirm_token: str, images_root: Path,
|
||||||
|
) -> int:
|
||||||
|
"""Delete all images in audit_run.matched_ids after confirming token.
|
||||||
|
Marks audit status='applied'. Routes through delete_images so files
|
||||||
|
+ cascading FK rows are handled uniformly."""
|
||||||
|
audit = session.execute(
|
||||||
|
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if audit is None:
|
||||||
|
raise ValueError(f"audit_run {audit_id} not found")
|
||||||
|
if audit.status != "ready":
|
||||||
|
raise AuditNotReady(audit.status)
|
||||||
|
# Token format matches modal/DestructiveConfirmModal.vue convention:
|
||||||
|
# ${action}-${kind}-${runId}. The modal hardcodes action ∈ {'restore',
|
||||||
|
# 'delete'}; "apply audit" is semantically a delete of the matched
|
||||||
|
# images, so we use 'delete-audit-<id>' (not 'apply-audit-<id>').
|
||||||
|
expected = f"delete-audit-{audit_id}"
|
||||||
|
if confirm_token != expected:
|
||||||
|
raise ConfirmTokenMismatch(expected)
|
||||||
|
ids = list(audit.matched_ids or [])
|
||||||
|
deleted = 0
|
||||||
|
if ids:
|
||||||
|
result = delete_images(session, image_ids=ids, images_root=images_root)
|
||||||
|
deleted = result["images_deleted"]
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(status="applied", finished_at=datetime.now(UTC))
|
||||||
|
)
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_audit_run(session: Session, *, audit_id: int) -> None:
|
||||||
|
"""Flip a running audit_run to 'cancelled'. The scan task checks
|
||||||
|
for status=='cancelled' between batches and exits cleanly."""
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.where(LibraryAuditRun.status == "running")
|
||||||
|
.values(status="cancelled", finished_at=datetime.now(UTC))
|
||||||
|
)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import (
|
from ..models import (
|
||||||
@@ -202,6 +203,80 @@ class Importer:
|
|||||||
(phash, width or 0, height or 0, image_id)
|
(phash, width or 0, height or 0, image_id)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _find_or_create_source(
|
||||||
|
self, *, artist_id: int, platform: str, url: str,
|
||||||
|
) -> Source:
|
||||||
|
"""Race-safe find-or-create on `source` keyed by
|
||||||
|
(artist_id, platform, url) — the same key as the
|
||||||
|
`uq_source_artist_platform_url` constraint.
|
||||||
|
|
||||||
|
Two concurrent workers processing different files in the same
|
||||||
|
post can both find no existing Source row then both INSERT,
|
||||||
|
which trips the unique constraint and poisons the session with
|
||||||
|
`psycopg.errors.UniqueViolation`. Operator-flagged 2026-05-26.
|
||||||
|
|
||||||
|
Pattern: select; if absent, open a savepoint and INSERT.
|
||||||
|
On IntegrityError, roll the savepoint back (NOT the outer
|
||||||
|
transaction, which would lose the surrounding scan's progress)
|
||||||
|
and re-select — the concurrent op just created the row we
|
||||||
|
wanted, so the second select will find it.
|
||||||
|
"""
|
||||||
|
existing = self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = Source(artist_id=artist_id, platform=platform, url=url)
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.artist_id == artist_id,
|
||||||
|
Source.platform == platform,
|
||||||
|
Source.url == url,
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
def _find_or_create_post(
|
||||||
|
self, *, source_id: int, external_post_id: str,
|
||||||
|
) -> Post:
|
||||||
|
"""Race-safe find-or-create on `post` keyed by
|
||||||
|
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||||
|
— same savepoint + IntegrityError-recovery pattern."""
|
||||||
|
existing = self.session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if existing is not None:
|
||||||
|
return existing
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
|
row = Post(source_id=source_id, external_post_id=external_post_id)
|
||||||
|
self.session.add(row)
|
||||||
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
return row
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
return self.session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
def import_one(self, source: Path) -> ImportResult:
|
def import_one(self, source: Path) -> ImportResult:
|
||||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||||
media members (one Post via the archive-adjacent sidecar) and
|
media members (one Post via the archive-adjacent sidecar) and
|
||||||
@@ -241,29 +316,13 @@ class Importer:
|
|||||||
sd = parse_sidecar(data)
|
sd = parse_sidecar(data)
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
url = sd.post_url or f"sidecar:{platform}"
|
url = sd.post_url or f"sidecar:{platform}"
|
||||||
src = self.session.execute(
|
src = self._find_or_create_source(
|
||||||
select(Source).where(
|
artist_id=artist.id, platform=platform, url=url,
|
||||||
Source.artist_id == artist.id,
|
)
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if src is None:
|
|
||||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
||||||
self.session.add(src)
|
|
||||||
self.session.flush()
|
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
post = self.session.execute(
|
return self._find_or_create_post(
|
||||||
select(Post).where(
|
source_id=src.id, external_post_id=epid,
|
||||||
Post.source_id == src.id,
|
)
|
||||||
Post.external_post_id == epid,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if post is None:
|
|
||||||
post = Post(source_id=src.id, external_post_id=epid)
|
|
||||||
self.session.add(post)
|
|
||||||
self.session.flush()
|
|
||||||
return post
|
|
||||||
|
|
||||||
def _capture_attachment(
|
def _capture_attachment(
|
||||||
self, source: Path, *, post: Post | None = None,
|
self, source: Path, *, post: Post | None = None,
|
||||||
@@ -705,29 +764,14 @@ class Importer:
|
|||||||
else:
|
else:
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
url = sd.post_url or f"sidecar:{platform}"
|
url = sd.post_url or f"sidecar:{platform}"
|
||||||
src = self.session.execute(
|
src = self._find_or_create_source(
|
||||||
select(Source).where(
|
artist_id=artist.id, platform=platform, url=url,
|
||||||
Source.artist_id == artist.id,
|
)
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if src is None:
|
|
||||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
|
||||||
self.session.add(src)
|
|
||||||
self.session.flush()
|
|
||||||
|
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
post = self.session.execute(
|
post = self._find_or_create_post(
|
||||||
select(Post).where(
|
source_id=src.id, external_post_id=epid,
|
||||||
Post.source_id == src.id,
|
)
|
||||||
Post.external_post_id == epid,
|
|
||||||
)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
if post is None:
|
|
||||||
post = Post(source_id=src.id, external_post_id=epid)
|
|
||||||
self.session.add(post)
|
|
||||||
self.session.flush()
|
|
||||||
if sd.post_url is not None:
|
if sd.post_url is not None:
|
||||||
post.post_url = sd.post_url
|
post.post_url = sd.post_url
|
||||||
if sd.post_title is not None:
|
if sd.post_title is not None:
|
||||||
@@ -860,8 +904,26 @@ class Importer:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def _transparency_pct(self, source: Path) -> float:
|
def _transparency_pct(self, source: Path) -> float:
|
||||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha.
|
||||||
|
|
||||||
|
For animated formats (multi-frame WebP / GIF / APNG), short-circuit
|
||||||
|
to 0.0 instead of decoding every frame. PIL's `getchannel("A")`
|
||||||
|
forces a full decode of all frames in an animated image, which for
|
||||||
|
a large animated WebP takes 5+ minutes and blows past the Celery
|
||||||
|
soft+hard time limits (300s/360s → SIGKILL). Operator-flagged
|
||||||
|
2026-05-26. Transparency analysis on a multi-frame image isn't
|
||||||
|
meaningful for art-curation purposes anyway — different frames
|
||||||
|
have different alpha — so the existing too_transparent skip rule
|
||||||
|
is bypassed entirely for animated content.
|
||||||
|
"""
|
||||||
with Image.open(source) as im:
|
with Image.open(source) as im:
|
||||||
|
if getattr(im, "is_animated", False):
|
||||||
|
log.info(
|
||||||
|
"skipping transparency check for animated image %s "
|
||||||
|
"(n_frames=%d) — avoids multi-frame decode timeout",
|
||||||
|
source, getattr(im, "n_frames", 0),
|
||||||
|
)
|
||||||
|
return 0.0
|
||||||
if im.mode not in ("RGBA", "LA") and not (
|
if im.mode not in ("RGBA", "LA") and not (
|
||||||
im.mode == "P" and "transparency" in im.info
|
im.mode == "P" and "transparency" in im.info
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
"""scan_library_for_rule Celery task — iterates image_record in keyset-
|
||||||
|
paginated batches, evaluates the audit rule per image, populates
|
||||||
|
LibraryAuditRun.matched_ids. Runs on the maintenance queue with a 2h soft
|
||||||
|
time limit (plenty of margin for 100k+ image libraries at ~100ms PIL
|
||||||
|
decode + histogram per image).
|
||||||
|
|
||||||
|
State machine:
|
||||||
|
start: status='running'
|
||||||
|
end success: status='ready'
|
||||||
|
end error: status='error', error=traceback
|
||||||
|
oversize: status='error', error='matched too many images; tighten threshold'
|
||||||
|
external cancel: scan sees status='cancelled' between batches, exits.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import traceback
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
from sqlalchemy.exc import DBAPIError, OperationalError
|
||||||
|
|
||||||
|
from ..celery_app import celery
|
||||||
|
from ..models import ImageRecord, LibraryAuditRun
|
||||||
|
from ..services.audits import single_color, transparency
|
||||||
|
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BATCH = 500
|
||||||
|
_PROGRESS_TICK = 100
|
||||||
|
_MAX_MATCHED = 50_000
|
||||||
|
|
||||||
|
_RULES = {
|
||||||
|
"transparency": transparency.evaluate,
|
||||||
|
"single_color": single_color.evaluate,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.library_audit.scan_library_for_rule",
|
||||||
|
bind=True,
|
||||||
|
autoretry_for=(OperationalError, DBAPIError),
|
||||||
|
retry_backoff=5,
|
||||||
|
retry_backoff_max=60,
|
||||||
|
retry_jitter=True,
|
||||||
|
max_retries=3,
|
||||||
|
soft_time_limit=7200,
|
||||||
|
time_limit=7500,
|
||||||
|
)
|
||||||
|
def scan_library_for_rule(self, audit_id: int) -> dict:
|
||||||
|
"""See module docstring. Returns a small summary dict for eager-mode
|
||||||
|
test assertions (real workers ignore the return value)."""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
try:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
audit = session.get(LibraryAuditRun, audit_id)
|
||||||
|
if audit is None:
|
||||||
|
return {"audit_id": audit_id, "status": "missing"}
|
||||||
|
evaluate = _RULES.get(audit.rule)
|
||||||
|
if evaluate is None:
|
||||||
|
_mark_error(session, audit_id, f"unknown rule {audit.rule!r}")
|
||||||
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
|
params = dict(audit.params or {})
|
||||||
|
matched: list[int] = []
|
||||||
|
scanned = 0
|
||||||
|
last_id = 0
|
||||||
|
while True:
|
||||||
|
# Cancellation check between batches.
|
||||||
|
current_status = session.execute(
|
||||||
|
select(LibraryAuditRun.status)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
if current_status == "cancelled":
|
||||||
|
return {"audit_id": audit_id, "status": "cancelled"}
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.path)
|
||||||
|
.where(ImageRecord.id > last_id)
|
||||||
|
.where(ImageRecord.mime.like("image/%"))
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(_BATCH)
|
||||||
|
).all()
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
for image_id, image_path in rows:
|
||||||
|
last_id = image_id
|
||||||
|
scanned += 1
|
||||||
|
try:
|
||||||
|
with Image.open(image_path) as im:
|
||||||
|
try:
|
||||||
|
if evaluate(im, **params):
|
||||||
|
matched.append(image_id)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.warning(
|
||||||
|
"audit %s: rule evaluate failed on %s: %s",
|
||||||
|
audit_id, image_path, exc,
|
||||||
|
)
|
||||||
|
except FileNotFoundError:
|
||||||
|
log.warning(
|
||||||
|
"audit %s: image_record %s file missing at %s; skipping",
|
||||||
|
audit_id, image_id, image_path,
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning(
|
||||||
|
"audit %s: PIL load failed for %s: %s",
|
||||||
|
audit_id, image_path, exc,
|
||||||
|
)
|
||||||
|
if len(matched) > _MAX_MATCHED:
|
||||||
|
_mark_error(
|
||||||
|
session, audit_id,
|
||||||
|
f"matched > {_MAX_MATCHED} images; "
|
||||||
|
"tighten threshold and re-run",
|
||||||
|
)
|
||||||
|
return {"audit_id": audit_id, "status": "error"}
|
||||||
|
if scanned % _PROGRESS_TICK == 0:
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(scanned_count=scanned)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
# Final state.
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(
|
||||||
|
scanned_count=scanned,
|
||||||
|
matched_count=len(matched),
|
||||||
|
matched_ids=matched,
|
||||||
|
status="ready",
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {
|
||||||
|
"audit_id": audit_id,
|
||||||
|
"status": "ready",
|
||||||
|
"scanned": scanned,
|
||||||
|
"matched": len(matched),
|
||||||
|
}
|
||||||
|
except SoftTimeLimitExceeded:
|
||||||
|
with SessionLocal() as session:
|
||||||
|
_mark_error(session, audit_id, "soft_time_limit exceeded (>7200s)")
|
||||||
|
raise
|
||||||
|
except (OperationalError, DBAPIError):
|
||||||
|
# Retryable per the decorator; leave row in 'running' and let
|
||||||
|
# autoretry try again. Recovery sweep catches if all retries fail.
|
||||||
|
raise
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
_mark_error(session, audit_id, tb)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _mark_error(session, audit_id: int, error_msg: str) -> None:
|
||||||
|
session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.id == audit_id)
|
||||||
|
.values(
|
||||||
|
status="error",
|
||||||
|
error=error_msg,
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
@@ -13,8 +13,21 @@ HASH_SIZE = 8
|
|||||||
|
|
||||||
def compute_phash(pil_image) -> str | None:
|
def compute_phash(pil_image) -> str | None:
|
||||||
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
"""Perceptual hash of an opened PIL image, as a hex string. None on any
|
||||||
failure (videos/unreadable/non-image)."""
|
failure (videos/unreadable/non-image).
|
||||||
|
|
||||||
|
For animated images (multi-frame WebP/GIF/APNG), explicitly seek to
|
||||||
|
frame 0 first. Without this, some PIL operations downstream of
|
||||||
|
imagehash.phash (convert("L"), resize) can iterate all frames and
|
||||||
|
blow past Celery's hard time limit on large animations
|
||||||
|
(operator-flagged 2026-05-26 against animated WebPs). The pHash of
|
||||||
|
frame 0 is the conventional choice for animated content.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
|
if getattr(pil_image, "is_animated", False):
|
||||||
|
try:
|
||||||
|
pil_image.seek(0)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
return str(imagehash.phash(pil_image, hash_size=HASH_SIZE))
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ No per-platform branching: a small common key set with fallbacks; the
|
|||||||
full JSON is kept in raw so anything unmapped is recoverable later.
|
full JSON is kept in raw so anything unmapped is recoverable later.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -21,10 +22,28 @@ class SidecarData:
|
|||||||
raw: dict
|
raw: dict
|
||||||
|
|
||||||
|
|
||||||
|
# gallery-dl prefixes media filenames with `NN_` for in-post ordering
|
||||||
|
# (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar
|
||||||
|
# under the attachment's stem WITHOUT that ordering prefix
|
||||||
|
# (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars.
|
||||||
|
# Confirmed against real Patreon downloads 2026-05-26 — without this
|
||||||
|
# strip, every gallery-dl post-level sidecar was invisible to FC since
|
||||||
|
# FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB).
|
||||||
|
_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
|
||||||
|
|
||||||
|
|
||||||
def find_sidecar(media: Path) -> Path | None:
|
def find_sidecar(media: Path) -> Path | None:
|
||||||
|
# Attachment-level sidecars (image.jpg.json, image.json).
|
||||||
for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
|
for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
|
||||||
if cand.is_file():
|
if cand.is_file():
|
||||||
return cand
|
return cand
|
||||||
|
# gallery-dl post-numbered convention: strip the `NN_` prefix from
|
||||||
|
# the stem and look for that.json in the same directory.
|
||||||
|
m = _NUMBERING_PREFIX.match(media.stem)
|
||||||
|
if m:
|
||||||
|
cand = media.parent / f"{m.group(1)}.json"
|
||||||
|
if cand.is_file():
|
||||||
|
return cand
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-image-size-select-small" size="small" />
|
||||||
|
<span>Minimum dimensions</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Find and delete images smaller than the threshold. Mirrors the
|
||||||
|
import-time <code>min_width</code> / <code>min_height</code>
|
||||||
|
filter, applied retroactively to the existing library.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-row dense>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="minW" label="Min width (px)" type="number"
|
||||||
|
min="0" density="compact" hide-details
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="minH" label="Min height (px)" type="number"
|
||||||
|
min="0" density="compact" hide-details
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<div class="d-flex align-center mt-3" style="gap: 10px;">
|
||||||
|
<v-btn
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onPreview"
|
||||||
|
>Preview</v-btn>
|
||||||
|
|
||||||
|
<span v-if="preview" class="text-body-2">
|
||||||
|
<strong>{{ preview.count }}</strong> image(s) would be deleted.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="preview && preview.count > 0"
|
||||||
|
class="mt-3"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onDeleteClick"
|
||||||
|
>Delete {{ preview.count }} matching...</v-btn>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="min-dim"
|
||||||
|
:run-id="tokenSha8"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
:description="`Width < ${minW} OR height < ${minH}`"
|
||||||
|
@confirm="onConfirmedDelete"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const minW = ref(0)
|
||||||
|
const minH = ref(0)
|
||||||
|
const preview = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const tokenSha8 = ref('')
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
minW.value = store.defaults.min_width
|
||||||
|
minH.value = store.defaults.min_height
|
||||||
|
})
|
||||||
|
|
||||||
|
// SHA-256 truncated to 8 hex chars — matches the backend's
|
||||||
|
// _min_dim_token() exactly. Web Crypto rejects MD5 as insecure.
|
||||||
|
async function sha8(canon) {
|
||||||
|
const enc = new TextEncoder()
|
||||||
|
const buf = await crypto.subtle.digest('SHA-256', enc.encode(canon))
|
||||||
|
const hex = Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('')
|
||||||
|
return hex.slice(0, 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onPreview() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
preview.value = await store.previewMinDim(minW.value, minH.value)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Preview failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onDeleteClick() {
|
||||||
|
tokenSha8.value = await sha8(`${minW.value}x${minH.value}`)
|
||||||
|
projectedCounts.value = { 'Images to delete': preview.value.count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedDelete(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.deleteMinDim(minW.value, minH.value, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
preview.value = null
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Delete failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-palette-swatch" size="small" />
|
||||||
|
<span>Single-color audit</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Scan library for images dominated by one color within the
|
||||||
|
tolerance. Catches placeholder / solid-fill / error-page images
|
||||||
|
that slipped through the import filter. Same background-scan
|
||||||
|
cadence as the transparency audit.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-row dense>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="threshold" label="Threshold (0–1)"
|
||||||
|
type="number" min="0" max="1" step="0.01"
|
||||||
|
density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="6">
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="tolerance" label="Color tolerance (0–441)"
|
||||||
|
type="number" min="0" max="441"
|
||||||
|
density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
/>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="!audit || audit.status !== 'running'"
|
||||||
|
class="mt-3"
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify-scan"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onStart"
|
||||||
|
>Scan library</v-btn>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||||
|
<v-progress-linear indeterminate color="accent" />
|
||||||
|
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||||
|
<span>
|
||||||
|
Scanning… {{ audit.scanned_count }} checked,
|
||||||
|
{{ audit.matched_count }} matched
|
||||||
|
</span>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" color="warning" rounded="pill"
|
||||||
|
@click="onCancel"
|
||||||
|
>Cancel</v-btn>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||||
|
image(s) match.
|
||||||
|
</p>
|
||||||
|
<v-btn
|
||||||
|
v-if="audit.matched_count > 0"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onApplyClick"
|
||||||
|
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'error'"
|
||||||
|
type="error" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Scan failed: {{ audit.error }}</v-alert>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'applied'"
|
||||||
|
type="success" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Applied — matched images deleted.</v-alert>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="audit"
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="audit"
|
||||||
|
:run-id="audit.id"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
description="Permanently deletes images matched by the single-color scan."
|
||||||
|
@confirm="onConfirmedApply"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const threshold = ref(0.95)
|
||||||
|
const tolerance = ref(30)
|
||||||
|
const audit = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
let pollTimer = null
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
threshold.value = store.defaults.single_color_threshold
|
||||||
|
tolerance.value = store.defaults.single_color_tolerance
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => stopPoll())
|
||||||
|
|
||||||
|
function startPoll(id) {
|
||||||
|
stopPoll()
|
||||||
|
pollTimer = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const fresh = await store.getAudit(id)
|
||||||
|
audit.value = fresh
|
||||||
|
if (fresh.status !== 'running') stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
stopPoll()
|
||||||
|
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}, 5000)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPoll() {
|
||||||
|
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onStart() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
const res = await store.startAudit('single_color', {
|
||||||
|
threshold: threshold.value, tolerance: tolerance.value,
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
|
startPoll(res.audit_id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCancel() {
|
||||||
|
if (!audit.value) return
|
||||||
|
try {
|
||||||
|
await store.cancelAudit(audit.value.id)
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onApplyClick() {
|
||||||
|
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedApply(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
<template>
|
||||||
|
<v-card class="fc-clean-card">
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 10px;">
|
||||||
|
<v-icon icon="mdi-checkerboard" size="small" />
|
||||||
|
<span>Transparency audit</span>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
|
Scan library for images whose transparent-pixel fraction exceeds
|
||||||
|
the threshold. Animated WebPs / GIFs are skipped (the import-side
|
||||||
|
rule does the same). Runs as a background task — ~50ms per image,
|
||||||
|
so a 57k library takes ~50 minutes.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-text-field
|
||||||
|
v-model.number="threshold" label="Transparency threshold (0–1)"
|
||||||
|
type="number" min="0" max="1" step="0.01" density="compact" hide-details
|
||||||
|
:disabled="audit && audit.status === 'running'"
|
||||||
|
class="mb-3"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<v-btn
|
||||||
|
v-if="!audit || audit.status !== 'running'"
|
||||||
|
color="accent" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-magnify-scan"
|
||||||
|
:loading="busy"
|
||||||
|
@click="onStart"
|
||||||
|
>Scan library</v-btn>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'running'" class="mt-3">
|
||||||
|
<v-progress-linear indeterminate color="accent" />
|
||||||
|
<div class="text-body-2 mt-2 d-flex align-center" style="gap: 10px;">
|
||||||
|
<span>
|
||||||
|
Scanning… {{ audit.scanned_count }} checked,
|
||||||
|
{{ audit.matched_count }} matched
|
||||||
|
</span>
|
||||||
|
<v-btn
|
||||||
|
variant="text" size="small" color="warning" rounded="pill"
|
||||||
|
@click="onCancel"
|
||||||
|
>Cancel</v-btn>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="audit && audit.status === 'ready'" class="mt-3">
|
||||||
|
<p class="text-body-2 mb-2">
|
||||||
|
Scan complete. <strong>{{ audit.matched_count }}</strong>
|
||||||
|
image(s) match.
|
||||||
|
</p>
|
||||||
|
<v-btn
|
||||||
|
v-if="audit.matched_count > 0"
|
||||||
|
color="error" variant="flat" rounded="pill"
|
||||||
|
prepend-icon="mdi-delete"
|
||||||
|
@click="onApplyClick"
|
||||||
|
>Delete {{ audit.matched_count }} matching...</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'error'"
|
||||||
|
type="error" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Scan failed: {{ audit.error }}</v-alert>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="audit && audit.status === 'applied'"
|
||||||
|
type="success" variant="tonal" density="compact" class="mt-3"
|
||||||
|
>Applied — matched images deleted.</v-alert>
|
||||||
|
</v-card-text>
|
||||||
|
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="audit"
|
||||||
|
v-model="showModal"
|
||||||
|
action="delete"
|
||||||
|
kind="audit"
|
||||||
|
:run-id="audit.id"
|
||||||
|
tier="C"
|
||||||
|
:projected-counts="projectedCounts"
|
||||||
|
description="Permanently deletes images matched by the transparency scan."
|
||||||
|
@confirm="onConfirmedApply"
|
||||||
|
/>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
|
const store = useCleanupStore()
|
||||||
|
const threshold = ref(0.9)
|
||||||
|
const audit = ref(null)
|
||||||
|
const busy = ref(false)
|
||||||
|
const showModal = ref(false)
|
||||||
|
const projectedCounts = ref({})
|
||||||
|
let pollTimer = null
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
await store.loadDefaults()
|
||||||
|
threshold.value = store.defaults.transparency_threshold
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => stopPoll())
|
||||||
|
|
||||||
|
function startPoll(id) {
|
||||||
|
stopPoll()
|
||||||
|
pollTimer = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const fresh = await store.getAudit(id)
|
||||||
|
audit.value = fresh
|
||||||
|
if (fresh.status !== 'running') stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
stopPoll()
|
||||||
|
window.__fcToast?.({ text: `Audit poll failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}, 5000)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPoll() {
|
||||||
|
if (pollTimer) { clearInterval(pollTimer); pollTimer = null }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onStart() {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
const res = await store.startAudit('transparency', { threshold: threshold.value })
|
||||||
|
audit.value = await store.getAudit(res.audit_id)
|
||||||
|
startPoll(res.audit_id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Scan start failed: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCancel() {
|
||||||
|
if (!audit.value) return
|
||||||
|
try {
|
||||||
|
await store.cancelAudit(audit.value.id)
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
stopPoll()
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Cancel failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function onApplyClick() {
|
||||||
|
projectedCounts.value = { 'Images to delete': audit.value.matched_count }
|
||||||
|
showModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onConfirmedApply(token) {
|
||||||
|
try {
|
||||||
|
const res = await store.applyAudit(audit.value.id, token)
|
||||||
|
window.__fcToast?.({
|
||||||
|
text: `Deleted ${res.deleted} image(s)`, type: 'success',
|
||||||
|
})
|
||||||
|
audit.value = await store.getAudit(audit.value.id)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Apply failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-clean-card { border-radius: 8px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
<template>
|
||||||
|
<v-dialog :model-value="modelValue" max-width="900"
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title class="d-flex align-center" style="gap: 12px;">
|
||||||
|
<v-icon icon="mdi-alert-circle-outline" color="error" />
|
||||||
|
<span>{{ title }}</span>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn icon variant="text" size="small" @click="close">
|
||||||
|
<v-icon>mdi-close</v-icon>
|
||||||
|
</v-btn>
|
||||||
|
</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<pre class="fc-err-pre">{{ message || '(no error message)' }}</pre>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-btn
|
||||||
|
variant="text" rounded="pill" size="small"
|
||||||
|
:prepend-icon="copied ? 'mdi-check' : 'mdi-content-copy'"
|
||||||
|
@click="onCopy"
|
||||||
|
>{{ copied ? 'Copied' : 'Copy' }}</v-btn>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" rounded="pill" @click="close">Close</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref, watch } from 'vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
modelValue: { type: Boolean, default: false },
|
||||||
|
title: { type: String, default: 'Error details' },
|
||||||
|
message: { type: String, default: '' },
|
||||||
|
})
|
||||||
|
|
||||||
|
const emit = defineEmits(['update:modelValue'])
|
||||||
|
|
||||||
|
const copied = ref(false)
|
||||||
|
let copiedTimer = null
|
||||||
|
|
||||||
|
watch(() => props.modelValue, (open) => {
|
||||||
|
if (!open) {
|
||||||
|
copied.value = false
|
||||||
|
if (copiedTimer) { clearTimeout(copiedTimer); copiedTimer = null }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
function close() {
|
||||||
|
emit('update:modelValue', false)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onCopy() {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(props.message || '')
|
||||||
|
copied.value = true
|
||||||
|
if (copiedTimer) clearTimeout(copiedTimer)
|
||||||
|
copiedTimer = setTimeout(() => { copied.value = false }, 1500)
|
||||||
|
} catch (e) {
|
||||||
|
window.__fcToast?.({ text: `Copy failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
/* The full error often contains a SQLAlchemy statement + parameters
|
||||||
|
block + multi-line traceback. Pre-wrap keeps long lines readable;
|
||||||
|
monospace + tabular layout keeps the structure scannable. The
|
||||||
|
max-height + overflow-auto prevents a 50-line traceback from
|
||||||
|
pushing the Close button off-screen. Operator-flagged 2026-05-26:
|
||||||
|
the prior :title="..." tooltip was unusable for content this long. */
|
||||||
|
.fc-err-pre {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.5;
|
||||||
|
white-space: pre-wrap;
|
||||||
|
word-break: break-word;
|
||||||
|
background: rgb(var(--v-theme-surface-variant, 38 36 41));
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
padding: 12px 14px;
|
||||||
|
border-radius: 6px;
|
||||||
|
max-height: 60vh;
|
||||||
|
overflow: auto;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -34,11 +34,18 @@
|
|||||||
|
|
||||||
<template v-else-if="manifest?.installed">
|
<template v-else-if="manifest?.installed">
|
||||||
<div class="fc-ext-install mt-3">
|
<div class="fc-ext-install mt-3">
|
||||||
|
<!-- Install button: direct :href anchor click (no programmatic
|
||||||
|
window.location.assign). Firefox's XPI-install gesture
|
||||||
|
requires a user-clicked anchor pointing at an
|
||||||
|
application/x-xpinstall response; programmatic navigation
|
||||||
|
sometimes triggered nothing instead of the install dialog
|
||||||
|
(operator-flagged 2026-05-26). No `download` attribute —
|
||||||
|
that would force a save dialog instead of install. -->
|
||||||
<v-btn
|
<v-btn
|
||||||
v-if="isFirefox"
|
v-if="isFirefox"
|
||||||
color="accent" variant="flat" rounded="pill"
|
color="accent" variant="flat" rounded="pill"
|
||||||
prepend-icon="mdi-firefox"
|
prepend-icon="mdi-firefox"
|
||||||
@click="installXpi"
|
:href="manifest.latest_url"
|
||||||
>Install Firefox extension</v-btn>
|
>Install Firefox extension</v-btn>
|
||||||
|
|
||||||
<v-btn
|
<v-btn
|
||||||
@@ -146,11 +153,6 @@ async function loadKey() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function installXpi() {
|
|
||||||
if (!manifest.value?.latest_url) return
|
|
||||||
window.location.assign(manifest.value.latest_url)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function rotateKey() {
|
async function rotateKey() {
|
||||||
rotating.value = true
|
rotating.value = true
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -45,9 +45,11 @@
|
|||||||
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
<template #item.size_bytes="{ item }">{{ formatBytes(item.size_bytes) }}</template>
|
||||||
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
<template #item.created_at="{ item }">{{ formatDate(item.created_at) }}</template>
|
||||||
<template #item.error="{ item }">
|
<template #item.error="{ item }">
|
||||||
<span v-if="item.error" :title="item.error" class="text-caption">
|
<button
|
||||||
{{ shorten(item.error, 60) }}
|
v-if="item.error" type="button" class="fc-err-link text-caption"
|
||||||
</span>
|
@click="openError(`Task ${item.id} failed`, item.error)"
|
||||||
|
title="Click for full error"
|
||||||
|
>{{ shorten(item.error, 60) }}</button>
|
||||||
</template>
|
</template>
|
||||||
</v-data-table-virtual>
|
</v-data-table-virtual>
|
||||||
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
<div v-if="store.hasMore" class="d-flex justify-center py-3">
|
||||||
@@ -100,16 +102,35 @@
|
|||||||
</v-card-actions>
|
</v-card-actions>
|
||||||
</v-card>
|
</v-card>
|
||||||
</v-dialog>
|
</v-dialog>
|
||||||
|
|
||||||
|
<ErrorDetailModal
|
||||||
|
v-model="showErrorModal"
|
||||||
|
:title="errorModalTitle"
|
||||||
|
:message="errorModalMessage"
|
||||||
|
/>
|
||||||
</v-card>
|
</v-card>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import { useImportStore } from '../../stores/import.js'
|
import { useImportStore } from '../../stores/import.js'
|
||||||
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
|
|
||||||
const store = useImportStore()
|
const store = useImportStore()
|
||||||
const statusFilter = ref(null)
|
const statusFilter = ref(null)
|
||||||
const clearDialog = ref(false)
|
const clearDialog = ref(false)
|
||||||
|
// Click-to-open modal for full error text (operator-flagged 2026-05-26
|
||||||
|
// — the prior :title="..." tooltip cramped multi-line SQLAlchemy
|
||||||
|
// tracebacks into an unusable popup with no copy-paste affordance).
|
||||||
|
const showErrorModal = ref(false)
|
||||||
|
const errorModalTitle = ref('')
|
||||||
|
const errorModalMessage = ref('')
|
||||||
|
|
||||||
|
function openError(title, message) {
|
||||||
|
errorModalTitle.value = title
|
||||||
|
errorModalMessage.value = message || ''
|
||||||
|
showErrorModal.value = true
|
||||||
|
}
|
||||||
const clearAgeDays = ref(7)
|
const clearAgeDays = ref(7)
|
||||||
const clearStuckDialog = ref(false)
|
const clearStuckDialog = ref(false)
|
||||||
|
|
||||||
@@ -179,3 +200,21 @@ async function onClearStuckConfirm() {
|
|||||||
clearStuckDialog.value = false
|
clearStuckDialog.value = false
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-err-link {
|
||||||
|
/* Truncated error preview as a clickable button — opens
|
||||||
|
ErrorDetailModal with the full text. Inherits the row's font
|
||||||
|
sizing so it doesn't visually drift from the prior tooltip-bearing
|
||||||
|
span. */
|
||||||
|
color: rgb(var(--v-theme-error, 220 80 80));
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
font: inherit;
|
||||||
|
text-align: left;
|
||||||
|
text-decoration: underline dotted;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-err-link:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
|
|||||||
@@ -13,7 +13,9 @@
|
|||||||
<AllowlistTable class="mt-4" />
|
<AllowlistTable class="mt-4" />
|
||||||
<AliasTable class="mt-4" />
|
<AliasTable class="mt-4" />
|
||||||
<BackupCard class="mt-6" />
|
<BackupCard class="mt-6" />
|
||||||
<TagMaintenanceCard class="mt-6" />
|
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||||
|
operates on the existing library which fits the Cleanup-tab
|
||||||
|
theme, and clusters with the other audit cards. -->
|
||||||
<LegacyMigrationCard class="mt-6" />
|
<LegacyMigrationCard class="mt-6" />
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -25,7 +27,6 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
|
|||||||
import AllowlistTable from './AllowlistTable.vue'
|
import AllowlistTable from './AllowlistTable.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import BackupCard from './BackupCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
import TagMaintenanceCard from './TagMaintenanceCard.vue'
|
|
||||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -59,8 +59,12 @@
|
|||||||
<td>{{ r.queue }}</td>
|
<td>{{ r.queue }}</td>
|
||||||
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
<td><code>{{ shortTaskName(r.task_name) }}</code></td>
|
||||||
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
<td class="fc-tabular">{{ r.target_id ?? '—' }}</td>
|
||||||
<td class="fc-err" :title="r.error_message">
|
<td>
|
||||||
{{ r.error_type }}
|
<button
|
||||||
|
type="button" class="fc-err-link"
|
||||||
|
@click="openError(r.error_type, r.error_message)"
|
||||||
|
:title="'Click for full error'"
|
||||||
|
>{{ r.error_type }}</button>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr v-if="!filteredFailures.length">
|
<tr v-if="!filteredFailures.length">
|
||||||
@@ -138,6 +142,12 @@
|
|||||||
</div>
|
</div>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
|
|
||||||
|
<ErrorDetailModal
|
||||||
|
v-model="showErrorModal"
|
||||||
|
:title="errorModalTitle"
|
||||||
|
:message="errorModalMessage"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -145,8 +155,23 @@
|
|||||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||||
import QueuesTable from './QueuesTable.vue'
|
import QueuesTable from './QueuesTable.vue'
|
||||||
|
|
||||||
|
// Click-to-open modal for full error text. Replaces the unusable
|
||||||
|
// :title="..." tooltip (operator-flagged 2026-05-26: SQLAlchemy
|
||||||
|
// rollback + traceback content rendered as a cramped browser tooltip
|
||||||
|
// you couldn't copy from or scroll within).
|
||||||
|
const showErrorModal = ref(false)
|
||||||
|
const errorModalTitle = ref('')
|
||||||
|
const errorModalMessage = ref('')
|
||||||
|
|
||||||
|
function openError(title, message) {
|
||||||
|
errorModalTitle.value = title || 'Error details'
|
||||||
|
errorModalMessage.value = message || ''
|
||||||
|
showErrorModal.value = true
|
||||||
|
}
|
||||||
|
|
||||||
const store = useSystemActivityStore()
|
const store = useSystemActivityStore()
|
||||||
|
|
||||||
const filterQueue = ref(null)
|
const filterQueue = ref(null)
|
||||||
@@ -276,5 +301,18 @@ function formatRelative(iso) {
|
|||||||
font-feature-settings: 'tnum';
|
font-feature-settings: 'tnum';
|
||||||
}
|
}
|
||||||
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
.fc-err { color: rgb(var(--v-theme-error, 220 80 80)); }
|
||||||
|
.fc-err-link {
|
||||||
|
/* Styled as a text-only button so the error_type cell stays
|
||||||
|
visually identical to the prior tooltip-bearing row, but is
|
||||||
|
now a real clickable target with hover affordance. */
|
||||||
|
color: rgb(var(--v-theme-error, 220 80 80));
|
||||||
|
background: transparent;
|
||||||
|
border: 0;
|
||||||
|
padding: 0;
|
||||||
|
font: inherit;
|
||||||
|
text-decoration: underline dotted;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-err-link:hover { text-decoration: underline; }
|
||||||
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
export const useCleanupStore = defineStore('cleanup', () => {
|
||||||
|
const api = useApi()
|
||||||
|
|
||||||
|
// Defaults sourced from ImportSettings on mount. Cards pre-fill from
|
||||||
|
// these so the common case ("apply current import filters
|
||||||
|
// retroactively") is one click; operator can override per-audit.
|
||||||
|
const defaults = ref({
|
||||||
|
min_width: 0,
|
||||||
|
min_height: 0,
|
||||||
|
transparency_threshold: 0.9,
|
||||||
|
single_color_threshold: 0.95,
|
||||||
|
single_color_tolerance: 30,
|
||||||
|
})
|
||||||
|
|
||||||
|
const recentRuns = ref([])
|
||||||
|
|
||||||
|
async function loadDefaults() {
|
||||||
|
const s = await api.get('/api/settings/import')
|
||||||
|
defaults.value = {
|
||||||
|
min_width: s.min_width ?? 0,
|
||||||
|
min_height: s.min_height ?? 0,
|
||||||
|
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||||
|
single_color_threshold: s.single_color_threshold ?? 0.95,
|
||||||
|
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function previewMinDim(min_width, min_height) {
|
||||||
|
return await api.post('/api/cleanup/min-dimension/preview', {
|
||||||
|
body: { min_width, min_height },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteMinDim(min_width, min_height, confirm) {
|
||||||
|
return await api.post('/api/cleanup/min-dimension/delete', {
|
||||||
|
body: { min_width, min_height, confirm },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startAudit(rule, params) {
|
||||||
|
return await api.post('/api/cleanup/audit', { body: { rule, params } })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAudit(id) {
|
||||||
|
return await api.get(`/api/cleanup/audit/${id}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadHistory(limit = 20) {
|
||||||
|
const body = await api.get(`/api/cleanup/audit?limit=${limit}`)
|
||||||
|
recentRuns.value = body.runs
|
||||||
|
return body.runs
|
||||||
|
}
|
||||||
|
|
||||||
|
async function applyAudit(id, confirm) {
|
||||||
|
return await api.post(`/api/cleanup/audit/${id}/apply`, { body: { confirm } })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function cancelAudit(id) {
|
||||||
|
return await api.post(`/api/cleanup/audit/${id}/cancel`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
defaults, recentRuns,
|
||||||
|
loadDefaults,
|
||||||
|
previewMinDim, deleteMinDim,
|
||||||
|
startAudit, getAudit, loadHistory, applyAudit, cancelAudit,
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-cleanup">
|
||||||
|
<p class="fc-muted text-body-2 mb-4">
|
||||||
|
Retroactive enforcement of import-filter rules. Each card scans the
|
||||||
|
existing library for content that the current import filters would
|
||||||
|
now exclude. Destructive — typed-token confirmation required.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<MinDimensionCard class="mb-4" />
|
||||||
|
<TransparencyAuditCard class="mb-4" />
|
||||||
|
<SingleColorAuditCard class="mb-4" />
|
||||||
|
<TagMaintenanceCard />
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import MinDimensionCard from '../components/cleanup/MinDimensionCard.vue'
|
||||||
|
import TransparencyAuditCard from '../components/cleanup/TransparencyAuditCard.vue'
|
||||||
|
import SingleColorAuditCard from '../components/cleanup/SingleColorAuditCard.vue'
|
||||||
|
// Reuse existing TagMaintenanceCard (FC-3k) as-is — it already handles
|
||||||
|
// preview + commit of prune-unused-tags via the admin store. Operator
|
||||||
|
// confirmed 2026-05-26: don't duplicate into a new UnusedTagsCard.
|
||||||
|
// MaintenancePanel drops its TagMaintenanceCard reference in the
|
||||||
|
// SettingsView edit (Task 16) so this is now the sole rendering site.
|
||||||
|
import TagMaintenanceCard from '../components/settings/TagMaintenanceCard.vue'
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-cleanup { max-width: 900px; }
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
<v-tab value="overview">Overview</v-tab>
|
<v-tab value="overview">Overview</v-tab>
|
||||||
<v-tab value="activity">Activity</v-tab>
|
<v-tab value="activity">Activity</v-tab>
|
||||||
<v-tab value="import">Import</v-tab>
|
<v-tab value="import">Import</v-tab>
|
||||||
|
<v-tab value="cleanup">Cleanup</v-tab>
|
||||||
<v-tab value="maintenance">Maintenance</v-tab>
|
<v-tab value="maintenance">Maintenance</v-tab>
|
||||||
</v-tabs>
|
</v-tabs>
|
||||||
|
|
||||||
@@ -53,6 +54,10 @@
|
|||||||
<ImportFiltersForm />
|
<ImportFiltersForm />
|
||||||
</v-window-item>
|
</v-window-item>
|
||||||
|
|
||||||
|
<v-window-item value="cleanup">
|
||||||
|
<CleanupView />
|
||||||
|
</v-window-item>
|
||||||
|
|
||||||
<v-window-item value="maintenance">
|
<v-window-item value="maintenance">
|
||||||
<MaintenancePanel />
|
<MaintenancePanel />
|
||||||
</v-window-item>
|
</v-window-item>
|
||||||
@@ -72,6 +77,7 @@ import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
|
|||||||
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
|
||||||
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
import ImportTaskList from '../components/settings/ImportTaskList.vue'
|
||||||
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
||||||
|
import CleanupView from './CleanupView.vue'
|
||||||
import { useMLStore } from '../stores/ml.js'
|
import { useMLStore } from '../stores/ml.js'
|
||||||
|
|
||||||
const tab = ref('overview')
|
const tab = ref('overview')
|
||||||
|
|||||||
@@ -0,0 +1,215 @@
|
|||||||
|
"""API tests for the /api/cleanup/* blueprint.
|
||||||
|
|
||||||
|
Per reference-async-coredml-test-assertions, post-DML state checks go
|
||||||
|
via column selects, not ORM entity access.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app import create_app
|
||||||
|
from backend.app.celery_app import celery
|
||||||
|
from backend.app.models import ImageRecord, LibraryAuditRun
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def disable_celery_eager(monkeypatch):
|
||||||
|
monkeypatch.setattr(celery.conf, "task_always_eager", False)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def app():
|
||||||
|
return create_app()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def client(app):
|
||||||
|
async with app.test_client() as c:
|
||||||
|
yield c
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_min_dim_token(min_w: int, min_h: int) -> str:
|
||||||
|
canon = f"{min_w}x{min_h}"
|
||||||
|
return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}"
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_image(db, tmp_path, *, w, h, name):
|
||||||
|
path = tmp_path / name
|
||||||
|
Image.new("RGB", (w, h), (w % 256, h % 256, 0)).save(path)
|
||||||
|
sha = f"api-cleanup-{name}".ljust(64, "x")[:64]
|
||||||
|
rec = ImageRecord(
|
||||||
|
path=str(path), sha256=sha,
|
||||||
|
size_bytes=path.stat().st_size, mime="image/png",
|
||||||
|
width=w, height=h, origin="imported_filesystem",
|
||||||
|
integrity_status="ok",
|
||||||
|
)
|
||||||
|
db.add(rec)
|
||||||
|
await db.flush()
|
||||||
|
return rec
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_min_dimension_preview_returns_count(client, db, tmp_path):
|
||||||
|
await _seed_image(db, tmp_path, w=50, h=50, name="small.png")
|
||||||
|
await _seed_image(db, tmp_path, w=500, h=500, name="big.png")
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/min-dimension/preview",
|
||||||
|
json={"min_width": 200, "min_height": 200},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_min_dimension_delete_with_token_removes_rows(client, db, tmp_path):
|
||||||
|
await _seed_image(db, tmp_path, w=50, h=50, name="s2.png")
|
||||||
|
await _seed_image(db, tmp_path, w=500, h=500, name="b2.png")
|
||||||
|
await db.commit()
|
||||||
|
token = _sha256_min_dim_token(200, 200)
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/min-dimension/delete",
|
||||||
|
json={"min_width": 200, "min_height": 200, "confirm": token},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["deleted"] == 1
|
||||||
|
remaining = await db.execute(select(func.count()).select_from(ImageRecord))
|
||||||
|
assert remaining.scalar_one() == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_min_dimension_delete_with_bad_token_returns_400(client, db, tmp_path):
|
||||||
|
await _seed_image(db, tmp_path, w=50, h=50, name="s3.png")
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/min-dimension/delete",
|
||||||
|
json={"min_width": 200, "min_height": 200, "confirm": "nope"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["error"] == "confirm_mismatch"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_create_returns_id_and_running_status(
|
||||||
|
client, db, monkeypatch,
|
||||||
|
):
|
||||||
|
from backend.app.tasks import library_audit
|
||||||
|
monkeypatch.setattr(
|
||||||
|
library_audit.scan_library_for_rule, "delay", lambda audit_id: None,
|
||||||
|
)
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/audit",
|
||||||
|
json={"rule": "transparency", "params": {"threshold": 0.9}},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 202
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["status"] == "running"
|
||||||
|
assert isinstance(body["audit_id"], int)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_create_returns_409_when_another_is_running(client, db):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db.add(audit)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
"/api/cleanup/audit",
|
||||||
|
json={"rule": "single_color", "params": {"threshold": 0.95, "tolerance": 30}},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 409
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_get_by_id_returns_full_row(client, db):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.85},
|
||||||
|
status="ready", scanned_count=100, matched_count=3,
|
||||||
|
matched_ids=[1, 2, 3],
|
||||||
|
)
|
||||||
|
db.add(audit)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.get(f"/api/cleanup/audit/{audit.id}")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["rule"] == "transparency"
|
||||||
|
assert body["matched_count"] == 3
|
||||||
|
assert body["matched_ids"] == [1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_history_returns_recent_runs(client, db):
|
||||||
|
for _ in range(3):
|
||||||
|
db.add(LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="applied", matched_ids=[],
|
||||||
|
finished_at=datetime.now(UTC),
|
||||||
|
))
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.get("/api/cleanup/audit?limit=5")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert len(body["runs"]) >= 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_apply_with_token_deletes(client, db, tmp_path):
|
||||||
|
rec = await _seed_image(db, tmp_path, w=100, h=100, name="apply.png")
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="ready", scanned_count=1, matched_count=1,
|
||||||
|
matched_ids=[rec.id],
|
||||||
|
)
|
||||||
|
db.add(audit)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/cleanup/audit/{audit.id}/apply",
|
||||||
|
json={"confirm": f"delete-audit-{audit.id}"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["deleted"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_apply_with_bad_token_returns_400(client, db):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="ready", matched_ids=[],
|
||||||
|
)
|
||||||
|
db.add(audit)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(
|
||||||
|
f"/api/cleanup/audit/{audit.id}/apply",
|
||||||
|
json={"confirm": "wrong-token"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["error"] == "confirm_mismatch"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_audit_cancel_flips_status(client, db):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db.add(audit)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(f"/api/cleanup/audit/{audit.id}/cancel")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
new_status = await db.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id)
|
||||||
|
)
|
||||||
|
assert new_status.scalar_one() == "cancelled"
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""Tests for the single-color audit rule.
|
||||||
|
|
||||||
|
The rule downsamples + measures the fraction of pixels within `tolerance`
|
||||||
|
(Euclidean RGB distance) of the dominant color. Matches if that fraction
|
||||||
|
exceeds `threshold`. Single-color content is typically uploaded by
|
||||||
|
mistake (placeholder/error/preview images) and should be flagged.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from backend.app.services.audits import single_color
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_color_evaluate_true_for_uniform_image():
|
||||||
|
im = Image.new("RGB", (50, 50), (128, 64, 200))
|
||||||
|
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_color_evaluate_false_for_diverse_image():
|
||||||
|
# Half black, half white — no single color dominates.
|
||||||
|
im = Image.new("RGB", (50, 50), (0, 0, 0))
|
||||||
|
for x in range(25):
|
||||||
|
for y in range(50):
|
||||||
|
im.putpixel((x, y), (255, 255, 255))
|
||||||
|
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_color_evaluate_respects_tolerance_widening():
|
||||||
|
# Gradient image: pixels span 0..50 in R channel. Tight tolerance
|
||||||
|
# rejects (no concentration), wide tolerance accepts (all near 25).
|
||||||
|
im = Image.new("RGB", (50, 50), (0, 0, 0))
|
||||||
|
for x in range(50):
|
||||||
|
for y in range(50):
|
||||||
|
im.putpixel((x, y), (x, 0, 0))
|
||||||
|
assert single_color.evaluate(im, threshold=0.9, tolerance=5) is False
|
||||||
|
assert single_color.evaluate(im, threshold=0.9, tolerance=50) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_single_color_evaluate_handles_rgba_input():
|
||||||
|
# Alpha channel should be ignored — only RGB matters for the rule.
|
||||||
|
im = Image.new("RGBA", (50, 50), (100, 100, 100, 128))
|
||||||
|
assert single_color.evaluate(im, threshold=0.9, tolerance=10) is True
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
"""Tests for the transparency audit rule.
|
||||||
|
|
||||||
|
The rule mirrors `Importer._transparency_pct` semantics for retroactive
|
||||||
|
enforcement: returns True iff the fraction of fully-transparent pixels
|
||||||
|
exceeds the threshold. Animated images short-circuit to False to avoid
|
||||||
|
the multi-frame PIL decode that triggered SoftTimeLimitExceeded
|
||||||
|
2026-05-26 against animated WebPs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from backend.app.services.audits import transparency
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparency_evaluate_true_when_fully_transparent():
|
||||||
|
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||||
|
assert transparency.evaluate(im, threshold=0.5) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparency_evaluate_false_when_fully_opaque():
|
||||||
|
im = Image.new("RGBA", (10, 10), (200, 100, 50, 255))
|
||||||
|
assert transparency.evaluate(im, threshold=0.5) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparency_evaluate_respects_threshold_boundary():
|
||||||
|
# Half-transparent image: 50% alpha=0 pixels, 50% alpha=255.
|
||||||
|
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||||
|
for x in range(5):
|
||||||
|
for y in range(10):
|
||||||
|
im.putpixel((x, y), (0, 0, 0, 255))
|
||||||
|
# 50% transparent. threshold=0.4 → True; threshold=0.6 → False.
|
||||||
|
assert transparency.evaluate(im, threshold=0.4) is True
|
||||||
|
assert transparency.evaluate(im, threshold=0.6) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparency_evaluate_false_for_rgb_image_without_alpha():
|
||||||
|
im = Image.new("RGB", (10, 10), (128, 128, 128))
|
||||||
|
assert transparency.evaluate(im, threshold=0.5) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_transparency_evaluate_false_for_animated_image():
|
||||||
|
im = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
|
||||||
|
# Mark as animated (mimics PIL's WebP/GIF multi-frame attribute).
|
||||||
|
im.is_animated = True # type: ignore[attr-defined]
|
||||||
|
im.n_frames = 5 # type: ignore[attr-defined]
|
||||||
|
assert transparency.evaluate(im, threshold=0.5) is False
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
"""Tests for cleanup_service's library-audit additions.
|
||||||
|
|
||||||
|
Covers:
|
||||||
|
- project_min_dimension_violations: SQL-only query against width/height
|
||||||
|
- delete_min_dimension_violations: routes through existing delete_images
|
||||||
|
- audit lifecycle (start_audit_run / apply_audit_run / cancel_audit_run)
|
||||||
|
|
||||||
|
Tests assert via column selects per reference-async-coredml-test-assertions
|
||||||
|
(post-DML ORM entity access via session.get() raises MissingGreenlet on
|
||||||
|
async sessions; we use db_sync here but the convention is preserved for
|
||||||
|
consistency).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from backend.app.models import ImageRecord, LibraryAuditRun
|
||||||
|
from backend.app.services import cleanup_service
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _make_image_record(db_sync, tmp_path, *, width, height, color):
|
||||||
|
"""Helper: write a real PIL file to a UNIQUE path (reference-image-record-path-unique)
|
||||||
|
and insert an ImageRecord row with all required NOT-NULL columns set."""
|
||||||
|
path = tmp_path / f"img_{width}x{height}_{color[0]}.png"
|
||||||
|
Image.new("RGB", (width, height), color).save(path)
|
||||||
|
# sha256 column is varchar(64) — use a deterministic 64-char pseudo
|
||||||
|
# hash built from the unique inputs.
|
||||||
|
sha = f"{width:04d}{height:04d}{color[0]:03d}".ljust(64, "0")[:64]
|
||||||
|
rec = ImageRecord(
|
||||||
|
path=str(path),
|
||||||
|
sha256=sha,
|
||||||
|
size_bytes=path.stat().st_size,
|
||||||
|
mime="image/png", # reference-image-record-required-columns
|
||||||
|
width=width,
|
||||||
|
height=height,
|
||||||
|
origin="imported_filesystem", # feedback-check-existing-enums
|
||||||
|
integrity_status="ok",
|
||||||
|
)
|
||||||
|
db_sync.add(rec)
|
||||||
|
db_sync.flush()
|
||||||
|
return rec
|
||||||
|
|
||||||
|
|
||||||
|
def test_project_min_dimension_violations_returns_count_and_samples(db_sync, tmp_path):
|
||||||
|
_make_image_record(db_sync, tmp_path, width=100, height=100, color=(10, 0, 0))
|
||||||
|
_make_image_record(db_sync, tmp_path, width=50, height=50, color=(20, 0, 0))
|
||||||
|
_make_image_record(db_sync, tmp_path, width=400, height=400, color=(30, 0, 0))
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
result = cleanup_service.project_min_dimension_violations(
|
||||||
|
db_sync, min_width=200, min_height=200,
|
||||||
|
)
|
||||||
|
assert result["count"] == 2 # 100x100 and 50x50 violate
|
||||||
|
assert len(result["sample_ids"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_min_dimension_violations_unlinks_and_cascades(db_sync, tmp_path):
|
||||||
|
rec_small = _make_image_record(db_sync, tmp_path, width=50, height=50, color=(40, 0, 0))
|
||||||
|
rec_big = _make_image_record(db_sync, tmp_path, width=500, height=500, color=(50, 0, 0))
|
||||||
|
small_path = Path(rec_small.path)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
# images_root is tmp_path here because the test fixtures stored files there;
|
||||||
|
# delete_images() uses it to unlink originals + thumbs from the right tree.
|
||||||
|
deleted = cleanup_service.delete_min_dimension_violations(
|
||||||
|
db_sync, min_width=200, min_height=200, images_root=tmp_path,
|
||||||
|
)
|
||||||
|
assert deleted == 1
|
||||||
|
# Verify via column selects per banked rule.
|
||||||
|
remaining_ids = db_sync.execute(
|
||||||
|
select(ImageRecord.id).order_by(ImageRecord.id)
|
||||||
|
).scalars().all()
|
||||||
|
assert remaining_ids == [rec_big.id]
|
||||||
|
assert not small_path.exists() # file unlinked
|
||||||
|
|
||||||
|
|
||||||
|
# --- Audit lifecycle tests (Task 5) ---
|
||||||
|
|
||||||
|
import backend.app.tasks.library_audit # noqa: F401, E402 — celery registration
|
||||||
|
|
||||||
|
|
||||||
|
def test_start_audit_run_creates_row_and_dispatches(db_sync, monkeypatch):
|
||||||
|
dispatched = []
|
||||||
|
from backend.app.tasks import library_audit as la_mod
|
||||||
|
monkeypatch.setattr(
|
||||||
|
la_mod.scan_library_for_rule, "delay",
|
||||||
|
lambda audit_id: dispatched.append(audit_id),
|
||||||
|
)
|
||||||
|
audit_id = cleanup_service.start_audit_run(
|
||||||
|
db_sync, rule="transparency", params={"threshold": 0.9},
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
row_status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert row_status == "running"
|
||||||
|
assert dispatched == [audit_id]
|
||||||
|
|
||||||
|
|
||||||
|
def test_start_audit_run_rejects_when_another_is_running(db_sync, monkeypatch):
|
||||||
|
from backend.app.tasks import library_audit as la_mod
|
||||||
|
monkeypatch.setattr(
|
||||||
|
la_mod.scan_library_for_rule, "delay", lambda audit_id: None,
|
||||||
|
)
|
||||||
|
cleanup_service.start_audit_run(
|
||||||
|
db_sync, rule="transparency", params={"threshold": 0.9},
|
||||||
|
)
|
||||||
|
db_sync.commit()
|
||||||
|
with pytest.raises(cleanup_service.AuditAlreadyRunning):
|
||||||
|
cleanup_service.start_audit_run(
|
||||||
|
db_sync, rule="single_color",
|
||||||
|
params={"threshold": 0.95, "tolerance": 30},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_audit_run_with_correct_token_deletes_matched(db_sync, tmp_path):
|
||||||
|
rec = _make_image_record(
|
||||||
|
db_sync, tmp_path, width=100, height=100, color=(60, 0, 0),
|
||||||
|
)
|
||||||
|
db_sync.flush()
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="ready", scanned_count=1, matched_count=1,
|
||||||
|
matched_ids=[rec.id],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
|
||||||
|
deleted = cleanup_service.apply_audit_run(
|
||||||
|
db_sync, audit_id=audit.id,
|
||||||
|
confirm_token=f"delete-audit-{audit.id}",
|
||||||
|
images_root=tmp_path,
|
||||||
|
)
|
||||||
|
assert deleted == 1
|
||||||
|
remaining_count = db_sync.execute(
|
||||||
|
select(func.count()).select_from(ImageRecord)
|
||||||
|
).scalar_one()
|
||||||
|
assert remaining_count == 0
|
||||||
|
new_status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id)
|
||||||
|
).scalar_one()
|
||||||
|
assert new_status == "applied"
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_audit_run_with_wrong_token_raises(db_sync, tmp_path):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="ready", matched_ids=[],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
with pytest.raises(cleanup_service.ConfirmTokenMismatch):
|
||||||
|
cleanup_service.apply_audit_run(
|
||||||
|
db_sync, audit_id=audit.id,
|
||||||
|
confirm_token="wrong-token", images_root=tmp_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_cancel_audit_run_flips_status(db_sync):
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.9},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
cleanup_service.cancel_audit_run(db_sync, audit_id=audit.id)
|
||||||
|
new_status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id)
|
||||||
|
).scalar_one()
|
||||||
|
assert new_status == "cancelled"
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
"""Tests for Importer._find_or_create_source / _find_or_create_post —
|
||||||
|
the race-safe savepoint-based helpers that replaced the previous
|
||||||
|
check-then-insert pattern.
|
||||||
|
|
||||||
|
Operator-flagged 2026-05-26: concurrent workers processing different
|
||||||
|
files in the same post both found no existing Source row, then both
|
||||||
|
INSERTed, tripping uq_source_artist_platform_url and poisoning the
|
||||||
|
session with `psycopg.errors.UniqueViolation`. The new helpers wrap
|
||||||
|
the INSERT in a savepoint and recover from IntegrityError by
|
||||||
|
re-selecting the row that the concurrent op committed.
|
||||||
|
|
||||||
|
Tests cover:
|
||||||
|
- idempotent return: same (artist_id, platform, url) → same Source row
|
||||||
|
- idempotent return for Post: same (source_id, external_post_id) → same Post
|
||||||
|
- IntegrityError recovery: monkeypatched flush raises once, helper
|
||||||
|
finds the row a concurrent op committed
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
|
from backend.app.models import Artist, ImportSettings, Post, Source
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
from backend.app.services.thumbnailer import Thumbnailer
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def importer(db_sync, tmp_path):
|
||||||
|
import_root = tmp_path / "import"
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
import_root.mkdir()
|
||||||
|
images_root.mkdir()
|
||||||
|
settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
return Importer(
|
||||||
|
session=db_sync,
|
||||||
|
images_root=images_root,
|
||||||
|
import_root=import_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root),
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def artist_row(db_sync):
|
||||||
|
a = Artist(name="TestArtist", slug="testartist")
|
||||||
|
db_sync.add(a)
|
||||||
|
db_sync.flush()
|
||||||
|
return a
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_or_create_source_creates_then_returns_existing(importer, artist_row):
|
||||||
|
s1 = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/posts/test-1",
|
||||||
|
)
|
||||||
|
s2 = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/posts/test-1",
|
||||||
|
)
|
||||||
|
assert s1.id == s2.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_or_create_source_distinct_urls_yield_distinct_rows(
|
||||||
|
importer, artist_row,
|
||||||
|
):
|
||||||
|
a = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/posts/a",
|
||||||
|
)
|
||||||
|
b = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/posts/b",
|
||||||
|
)
|
||||||
|
assert a.id != b.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_or_create_post_idempotent(importer, artist_row, db_sync):
|
||||||
|
src = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon",
|
||||||
|
url="https://www.patreon.com/posts/post-test",
|
||||||
|
)
|
||||||
|
p1 = importer._find_or_create_post(
|
||||||
|
source_id=src.id, external_post_id="ext-001",
|
||||||
|
)
|
||||||
|
p2 = importer._find_or_create_post(
|
||||||
|
source_id=src.id, external_post_id="ext-001",
|
||||||
|
)
|
||||||
|
assert p1.id == p2.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_or_create_source_recovers_from_integrity_error(
|
||||||
|
importer, artist_row, db_sync, monkeypatch,
|
||||||
|
):
|
||||||
|
"""Simulate the race: another worker has already inserted a Source row
|
||||||
|
matching our (artist_id, platform, url) just before our flush would
|
||||||
|
have. Our flush raises IntegrityError; the helper rolls back the
|
||||||
|
savepoint and re-selects, returning the row the concurrent op created.
|
||||||
|
"""
|
||||||
|
canonical_url = "https://www.patreon.com/posts/race-141226276"
|
||||||
|
pre_existing = Source(
|
||||||
|
artist_id=artist_row.id, platform="patreon", url=canonical_url,
|
||||||
|
)
|
||||||
|
db_sync.add(pre_existing)
|
||||||
|
db_sync.flush()
|
||||||
|
|
||||||
|
# Force a fresh select within the helper to MISS the existing row by
|
||||||
|
# detaching it from the identity map; SQLAlchemy's first-level cache
|
||||||
|
# would otherwise return the pre_existing row immediately.
|
||||||
|
# Easier: monkeypatch the FIRST select inside the helper to return
|
||||||
|
# None on first call, real result on subsequent. We do that by
|
||||||
|
# patching session.execute with a single-shot wrapper.
|
||||||
|
real_execute = db_sync.execute
|
||||||
|
skip_count = [0]
|
||||||
|
|
||||||
|
def execute_with_first_select_miss(stmt, *args, **kwargs):
|
||||||
|
# Strip-down heuristic: the first SELECT issued by the helper is
|
||||||
|
# the existence check. Force it to return a "no row" result.
|
||||||
|
result = real_execute(stmt, *args, **kwargs)
|
||||||
|
if skip_count[0] == 0:
|
||||||
|
skip_count[0] += 1
|
||||||
|
# Wrap result so .scalar_one_or_none() returns None for this
|
||||||
|
# one call, then unwrap on subsequent uses.
|
||||||
|
class _ForcedMiss:
|
||||||
|
def scalar_one_or_none(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def scalar_one(self):
|
||||||
|
return result.scalar_one()
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
return getattr(result, name)
|
||||||
|
return _ForcedMiss()
|
||||||
|
return result
|
||||||
|
|
||||||
|
monkeypatch.setattr(db_sync, "execute", execute_with_first_select_miss)
|
||||||
|
|
||||||
|
recovered = importer._find_or_create_source(
|
||||||
|
artist_id=artist_row.id, platform="patreon", url=canonical_url,
|
||||||
|
)
|
||||||
|
assert recovered.id == pre_existing.id
|
||||||
@@ -23,6 +23,51 @@ def test_find_sidecar_none(tmp_path):
|
|||||||
assert find_sidecar(media) is None
|
assert find_sidecar(media) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_sidecar_gallerydl_numbered_prefix(tmp_path):
|
||||||
|
"""gallery-dl prefixes media filenames with NN_ for in-post ordering
|
||||||
|
(e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under
|
||||||
|
the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`).
|
||||||
|
Confirmed against real Patreon downloads 2026-05-26 — operator's deep
|
||||||
|
scan produced 24 refresh calls but 0 Posts because the unprefixed
|
||||||
|
sidecar was invisible to find_sidecar."""
|
||||||
|
media = tmp_path / "01_HOLLOW-ICHIGO.png"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
sc = tmp_path / "HOLLOW-ICHIGO.json"
|
||||||
|
sc.write_text("{}")
|
||||||
|
assert find_sidecar(media) == sc
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_sidecar_multidigit_prefix(tmp_path):
|
||||||
|
"""Numbering prefix can be wider than 2 digits (`001_...`); the strip
|
||||||
|
handles any \\d+_ form."""
|
||||||
|
media = tmp_path / "001_mirko-sketch.png"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
sc = tmp_path / "mirko-sketch.json"
|
||||||
|
sc.write_text("{}")
|
||||||
|
assert find_sidecar(media) == sc
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path):
|
||||||
|
"""If BOTH a per-attachment sidecar and a post-level sidecar exist,
|
||||||
|
the attachment-level one wins (it's more specific)."""
|
||||||
|
media = tmp_path / "01_image.png"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
per_attachment = tmp_path / "01_image.json"
|
||||||
|
per_attachment.write_text('{"specific": true}')
|
||||||
|
post_level = tmp_path / "image.json"
|
||||||
|
post_level.write_text('{"specific": false}')
|
||||||
|
assert find_sidecar(media) == per_attachment
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path):
|
||||||
|
"""`01.png` (just digits, no underscore-separated stem) shouldn't
|
||||||
|
match. The regex requires NN_<something>."""
|
||||||
|
media = tmp_path / "01.png"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
(tmp_path / ".json").write_text("{}") # would be matched only if buggy
|
||||||
|
assert find_sidecar(media) is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_empty_dict_all_none():
|
def test_parse_empty_dict_all_none():
|
||||||
sd = parse_sidecar({})
|
sd = parse_sidecar({})
|
||||||
assert isinstance(sd, SidecarData)
|
assert isinstance(sd, SidecarData)
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
"""Tests for scan_library_for_rule Celery task.
|
||||||
|
|
||||||
|
Eager mode is used so the task runs synchronously in-test and we can
|
||||||
|
assert state via column selects (post-DML ORM access banned per
|
||||||
|
reference-async-coredml-test-assertions)."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
import backend.app.tasks.library_audit # noqa: F401 — celery registration
|
||||||
|
from backend.app import celery_app
|
||||||
|
from backend.app.models import ImageRecord, LibraryAuditRun
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _mk_image(db_sync, tmp_path, *, mode, color, name):
|
||||||
|
path = tmp_path / name
|
||||||
|
Image.new(mode, (10, 10), color).save(path)
|
||||||
|
# sha256 column is varchar(64) — pad/truncate a per-file pseudo hash
|
||||||
|
# exactly to 64 chars. Each test fixture file must have a unique
|
||||||
|
# sha256 (reference-image-record-path-unique peer constraint).
|
||||||
|
sha = f"audit-{name}".ljust(64, "x")[:64]
|
||||||
|
rec = ImageRecord(
|
||||||
|
path=str(path),
|
||||||
|
sha256=sha,
|
||||||
|
size_bytes=path.stat().st_size,
|
||||||
|
mime="image/png",
|
||||||
|
width=10, height=10,
|
||||||
|
origin="imported_filesystem",
|
||||||
|
integrity_status="ok",
|
||||||
|
)
|
||||||
|
db_sync.add(rec)
|
||||||
|
db_sync.flush()
|
||||||
|
return rec, path
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_library_for_rule_populates_matched_ids_for_transparency(
|
||||||
|
db_sync, tmp_path, monkeypatch,
|
||||||
|
):
|
||||||
|
transparent_rec, _ = _mk_image(
|
||||||
|
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="trans.png",
|
||||||
|
)
|
||||||
|
opaque_rec, _ = _mk_image(
|
||||||
|
db_sync, tmp_path, mode="RGBA", color=(200, 0, 0, 255), name="opaque.png",
|
||||||
|
)
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.5},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
audit_id = audit.id
|
||||||
|
|
||||||
|
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
|
||||||
|
from backend.app.tasks.library_audit import scan_library_for_rule
|
||||||
|
scan_library_for_rule.run(audit_id)
|
||||||
|
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
|
||||||
|
|
||||||
|
matched = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
assert transparent_rec.id in matched
|
||||||
|
assert opaque_rec.id not in matched
|
||||||
|
assert status == "ready"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_library_for_rule_skips_missing_files_gracefully(
|
||||||
|
db_sync, tmp_path, monkeypatch,
|
||||||
|
):
|
||||||
|
rec, path = _mk_image(
|
||||||
|
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="ghost.png",
|
||||||
|
)
|
||||||
|
path.unlink() # delete the file but leave the DB row
|
||||||
|
audit = LibraryAuditRun(
|
||||||
|
rule="transparency", params={"threshold": 0.5},
|
||||||
|
status="running", matched_ids=[],
|
||||||
|
)
|
||||||
|
db_sync.add(audit)
|
||||||
|
db_sync.commit()
|
||||||
|
audit_id = audit.id
|
||||||
|
|
||||||
|
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", True)
|
||||||
|
from backend.app.tasks.library_audit import scan_library_for_rule
|
||||||
|
scan_library_for_rule.run(audit_id)
|
||||||
|
monkeypatch.setattr(celery_app.celery.conf, "task_always_eager", False)
|
||||||
|
|
||||||
|
status = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
matched = db_sync.execute(
|
||||||
|
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||||
|
).scalar_one()
|
||||||
|
# Missing file is skipped (warning logged), audit completes successfully.
|
||||||
|
assert status == "ready"
|
||||||
|
assert rec.id not in matched
|
||||||
Reference in New Issue
Block a user