From 0d0b236ac37f6b6f7de30aa88f981bd6b4cbdbd3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 08:13:53 -0400 Subject: [PATCH] =?UTF-8?q?feat(fc-cleanup):=20api/cleanup.py=20blueprint?= =?UTF-8?q?=20(9=20endpoints)=20+=20register=20+=20delete-audit-=20tok?= =?UTF-8?q?en=20(matches=20modal=20convention)=20=E2=80=94=20Co-Authored-B?= =?UTF-8?q?y:=20Claude=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/__init__.py | 2 + backend/app/api/cleanup.py | 193 +++++++++++++++++++++ backend/app/services/cleanup_service.py | 6 +- tests/test_api_cleanup.py | 215 ++++++++++++++++++++++++ tests/test_cleanup_service_audit.py | 2 +- 5 files changed, 416 insertions(+), 2 deletions(-) create mode 100644 backend/app/api/cleanup.py create mode 100644 tests/test_api_cleanup.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index a2d743b..b5b6afa 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -16,6 +16,7 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) def all_blueprints() -> list[Blueprint]: from .admin import admin_bp from .aliases import aliases_bp + from .cleanup import cleanup_bp from .allowlist import allowlist_bp from .artist import artist_bp from .artists import artists_bp @@ -50,6 +51,7 @@ def all_blueprints() -> list[Blueprint]: system_activity_bp, system_backup_bp, admin_bp, + cleanup_bp, import_admin_bp, migrate_bp, suggestions_bp, diff --git a/backend/app/api/cleanup.py b/backend/app/api/cleanup.py new file mode 100644 index 0000000..386c3e5 --- /dev/null +++ b/backend/app/api/cleanup.py @@ -0,0 +1,193 @@ +"""FC-Cleanup: /api/cleanup/* — retroactive enforcement of import filters. + +Endpoints: + POST /min-dimension/preview synchronous SQL audit + POST /min-dimension/delete synchronous SQL delete (Tier-C token) + POST /audit async transparency / single_color start + GET /audit list recent audit_run rows + GET /audit/ single audit_run row + POST /audit//apply apply matched_ids deletes (Tier-C token) + POST /audit//cancel flip running audit to cancelled + +Unused-tags retroactive prune intentionally NOT in this namespace — +TagMaintenanceCard (Maintenance tab → moved to Cleanup tab in v26.05.25.7) +uses the existing /api/admin/tags/prune-unused endpoint via the admin +store. No duplicate route here. + +Confirm-token format matches modal/DestructiveConfirmModal.vue convention: + `delete-min-dim-` for min-dim delete + `delete-audit-` for audit apply +(Modal hardcodes action ∈ {'restore', 'delete'}; "apply audit" is semantically a delete of the matched images, so we use `delete-audit-`.) +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import LibraryAuditRun +from ..services import cleanup_service + +cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup") + +IMAGES_ROOT = Path("/images") + + +def _bad(error: str, *, status: int = 400, **extra): + body = {"error": error} + body.update(extra) + return jsonify(body), status + + +def _min_dim_token(min_w: int, min_h: int) -> str: + # SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both + # sides use SHA-256 truncated to 8 hex chars. + canon = f"{min_w}x{min_h}" + return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}" + + +def _serialize_audit_run(audit: LibraryAuditRun) -> dict: + return { + "id": audit.id, + "rule": audit.rule, + "params": audit.params, + "status": audit.status, + "started_at": audit.started_at.isoformat() if audit.started_at else None, + "finished_at": audit.finished_at.isoformat() if audit.finished_at else None, + "scanned_count": audit.scanned_count, + "matched_count": audit.matched_count, + "matched_ids": audit.matched_ids, + "error": audit.error, + } + + +@cleanup_bp.route("/min-dimension/preview", methods=["POST"]) +async def min_dim_preview(): + body = await request.get_json(silent=True) or {} + try: + min_w = int(body.get("min_width", 0)) + min_h = int(body.get("min_height", 0)) + except (TypeError, ValueError): + return _bad("invalid_dimensions") + if min_w < 0 or min_h < 0: + return _bad("invalid_dimensions") + async with get_session() as session: + projection = await session.run_sync( + lambda s: cleanup_service.project_min_dimension_violations( + s, min_width=min_w, min_height=min_h, + ) + ) + return jsonify(projection) + + +@cleanup_bp.route("/min-dimension/delete", methods=["POST"]) +async def min_dim_delete(): + body = await request.get_json(silent=True) or {} + try: + min_w = int(body.get("min_width", 0)) + min_h = int(body.get("min_height", 0)) + except (TypeError, ValueError): + return _bad("invalid_dimensions") + if min_w < 0 or min_h < 0: + return _bad("invalid_dimensions") + supplied = body.get("confirm", "") + expected = _min_dim_token(min_w, min_h) + if supplied != expected: + return _bad("confirm_mismatch", expected=expected) + async with get_session() as session: + deleted = await session.run_sync( + lambda s: cleanup_service.delete_min_dimension_violations( + s, min_width=min_w, min_height=min_h, images_root=IMAGES_ROOT, + ) + ) + await session.commit() + return jsonify({"deleted": deleted}) + + +@cleanup_bp.route("/audit", methods=["POST"]) +async def audit_create(): + body = await request.get_json(silent=True) or {} + rule = body.get("rule") + params = body.get("params") or {} + if rule not in ("transparency", "single_color"): + return _bad("invalid_rule") + if not isinstance(params, dict): + return _bad("invalid_params") + async with get_session() as session: + try: + audit_id = await session.run_sync( + lambda s: cleanup_service.start_audit_run( + s, rule=rule, params=params, + ) + ) + except cleanup_service.AuditAlreadyRunning as running_id: + return _bad( + "audit_already_running", status=409, + running_id=int(str(running_id)), + ) + except ValueError as exc: + return _bad(str(exc)) + await session.commit() + return jsonify({"audit_id": audit_id, "status": "running"}), 202 + + +@cleanup_bp.route("/audit/", methods=["GET"]) +async def audit_get(audit_id: int): + async with get_session() as session: + audit = (await session.execute( + select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id) + )).scalar_one_or_none() + if audit is None: + return _bad("not_found", status=404) + return jsonify(_serialize_audit_run(audit)) + + +@cleanup_bp.route("/audit", methods=["GET"]) +async def audit_history(): + try: + limit = min(int(request.args.get("limit", "20")), 100) + except ValueError: + return _bad("invalid_limit") + async with get_session() as session: + rows = (await session.execute( + select(LibraryAuditRun) + .order_by(LibraryAuditRun.id.desc()) + .limit(limit) + )).scalars().all() + return jsonify({"runs": [_serialize_audit_run(r) for r in rows]}) + + +@cleanup_bp.route("/audit//apply", methods=["POST"]) +async def audit_apply(audit_id: int): + body = await request.get_json(silent=True) or {} + confirm = body.get("confirm", "") + async with get_session() as session: + try: + deleted = await session.run_sync( + lambda s: cleanup_service.apply_audit_run( + s, audit_id=audit_id, confirm_token=confirm, + images_root=IMAGES_ROOT, + ) + ) + except cleanup_service.AuditNotReady as exc: + return _bad("audit_not_ready", current_status=str(exc)) + except cleanup_service.ConfirmTokenMismatch as exc: + return _bad("confirm_mismatch", expected=str(exc)) + except ValueError as exc: + return _bad("not_found", status=404, detail=str(exc)) + await session.commit() + return jsonify({"deleted": deleted}) + + +@cleanup_bp.route("/audit//cancel", methods=["POST"]) +async def audit_cancel(audit_id: int): + async with get_session() as session: + await session.run_sync( + lambda s: cleanup_service.cancel_audit_run(s, audit_id=audit_id) + ) + await session.commit() + return jsonify({"cancelled": True}) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index b8325d5..7025b8b 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -481,7 +481,11 @@ def apply_audit_run( raise ValueError(f"audit_run {audit_id} not found") if audit.status != "ready": raise AuditNotReady(audit.status) - expected = f"apply-audit-{audit_id}" + # Token format matches modal/DestructiveConfirmModal.vue convention: + # ${action}-${kind}-${runId}. The modal hardcodes action ∈ {'restore', + # 'delete'}; "apply audit" is semantically a delete of the matched + # images, so we use 'delete-audit-' (not 'apply-audit-'). + expected = f"delete-audit-{audit_id}" if confirm_token != expected: raise ConfirmTokenMismatch(expected) ids = list(audit.matched_ids or []) diff --git a/tests/test_api_cleanup.py b/tests/test_api_cleanup.py new file mode 100644 index 0000000..2f48960 --- /dev/null +++ b/tests/test_api_cleanup.py @@ -0,0 +1,215 @@ +"""API tests for the /api/cleanup/* blueprint. + +Per reference-async-coredml-test-assertions, post-DML state checks go +via column selects, not ORM entity access. +""" + +import hashlib +from datetime import UTC, datetime + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app import create_app +from backend.app.celery_app import celery +from backend.app.models import ImageRecord, LibraryAuditRun + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +def disable_celery_eager(monkeypatch): + monkeypatch.setattr(celery.conf, "task_always_eager", False) + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +def _sha256_min_dim_token(min_w: int, min_h: int) -> str: + canon = f"{min_w}x{min_h}" + return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}" + + +async def _seed_image(db, tmp_path, *, w, h, name): + path = tmp_path / name + Image.new("RGB", (w, h), (w % 256, h % 256, 0)).save(path) + sha = f"api-cleanup-{name}".ljust(64, "x")[:64] + rec = ImageRecord( + path=str(path), sha256=sha, + size_bytes=path.stat().st_size, mime="image/png", + width=w, height=h, origin="imported_filesystem", + integrity_status="ok", + ) + db.add(rec) + await db.flush() + return rec + + +@pytest.mark.asyncio +async def test_min_dimension_preview_returns_count(client, db, tmp_path): + await _seed_image(db, tmp_path, w=50, h=50, name="small.png") + await _seed_image(db, tmp_path, w=500, h=500, name="big.png") + await db.commit() + resp = await client.post( + "/api/cleanup/min-dimension/preview", + json={"min_width": 200, "min_height": 200}, + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["count"] == 1 + + +@pytest.mark.asyncio +async def test_min_dimension_delete_with_token_removes_rows(client, db, tmp_path): + await _seed_image(db, tmp_path, w=50, h=50, name="s2.png") + await _seed_image(db, tmp_path, w=500, h=500, name="b2.png") + await db.commit() + token = _sha256_min_dim_token(200, 200) + resp = await client.post( + "/api/cleanup/min-dimension/delete", + json={"min_width": 200, "min_height": 200, "confirm": token}, + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["deleted"] == 1 + remaining = await db.execute(select(func.count()).select_from(ImageRecord)) + assert remaining.scalar_one() == 1 + + +@pytest.mark.asyncio +async def test_min_dimension_delete_with_bad_token_returns_400(client, db, tmp_path): + await _seed_image(db, tmp_path, w=50, h=50, name="s3.png") + await db.commit() + resp = await client.post( + "/api/cleanup/min-dimension/delete", + json={"min_width": 200, "min_height": 200, "confirm": "nope"}, + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "confirm_mismatch" + + +@pytest.mark.asyncio +async def test_audit_create_returns_id_and_running_status( + client, db, monkeypatch, +): + from backend.app.tasks import library_audit + monkeypatch.setattr( + library_audit.scan_library_for_rule, "delay", lambda audit_id: None, + ) + resp = await client.post( + "/api/cleanup/audit", + json={"rule": "transparency", "params": {"threshold": 0.9}}, + ) + assert resp.status_code == 202 + body = await resp.get_json() + assert body["status"] == "running" + assert isinstance(body["audit_id"], int) + + +@pytest.mark.asyncio +async def test_audit_create_returns_409_when_another_is_running(client, db): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="running", matched_ids=[], + ) + db.add(audit) + await db.commit() + resp = await client.post( + "/api/cleanup/audit", + json={"rule": "single_color", "params": {"threshold": 0.95, "tolerance": 30}}, + ) + assert resp.status_code == 409 + + +@pytest.mark.asyncio +async def test_audit_get_by_id_returns_full_row(client, db): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.85}, + status="ready", scanned_count=100, matched_count=3, + matched_ids=[1, 2, 3], + ) + db.add(audit) + await db.commit() + resp = await client.get(f"/api/cleanup/audit/{audit.id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["rule"] == "transparency" + assert body["matched_count"] == 3 + assert body["matched_ids"] == [1, 2, 3] + + +@pytest.mark.asyncio +async def test_audit_history_returns_recent_runs(client, db): + for i in range(3): + db.add(LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="applied", matched_ids=[], + finished_at=datetime.now(UTC), + )) + await db.commit() + resp = await client.get("/api/cleanup/audit?limit=5") + assert resp.status_code == 200 + body = await resp.get_json() + assert len(body["runs"]) >= 3 + + +@pytest.mark.asyncio +async def test_audit_apply_with_token_deletes(client, db, tmp_path): + rec = await _seed_image(db, tmp_path, w=100, h=100, name="apply.png") + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", scanned_count=1, matched_count=1, + matched_ids=[rec.id], + ) + db.add(audit) + await db.commit() + resp = await client.post( + f"/api/cleanup/audit/{audit.id}/apply", + json={"confirm": f"delete-audit-{audit.id}"}, + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["deleted"] == 1 + + +@pytest.mark.asyncio +async def test_audit_apply_with_bad_token_returns_400(client, db): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="ready", matched_ids=[], + ) + db.add(audit) + await db.commit() + resp = await client.post( + f"/api/cleanup/audit/{audit.id}/apply", + json={"confirm": "wrong-token"}, + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "confirm_mismatch" + + +@pytest.mark.asyncio +async def test_audit_cancel_flips_status(client, db): + audit = LibraryAuditRun( + rule="transparency", params={"threshold": 0.9}, + status="running", matched_ids=[], + ) + db.add(audit) + await db.commit() + resp = await client.post(f"/api/cleanup/audit/{audit.id}/cancel") + assert resp.status_code == 200 + new_status = await db.execute( + select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit.id) + ) + assert new_status.scalar_one() == "cancelled" diff --git a/tests/test_cleanup_service_audit.py b/tests/test_cleanup_service_audit.py index ec488d7..f9100b0 100644 --- a/tests/test_cleanup_service_audit.py +++ b/tests/test_cleanup_service_audit.py @@ -133,7 +133,7 @@ def test_apply_audit_run_with_correct_token_deletes_matched(db_sync, tmp_path): deleted = cleanup_service.apply_audit_run( db_sync, audit_id=audit.id, - confirm_token=f"apply-audit-{audit.id}", + confirm_token=f"delete-audit-{audit.id}", images_root=tmp_path, ) assert deleted == 1