bvandeusen
91b0145bc8
Wipe every general + character tag so the operator can re-tag from scratch via
the Camie auto-suggest, while PRESERVING fandoms, series (+ series_page order),
and each image's stored tagger_predictions (so suggestions repopulate
immediately). One set-based DELETE FROM tag WHERE kind IN ('general','character')
— the five tag-referencing tables all cascade, so applications + aliases +
allowlist + rejections + centroids clear automatically; series tags aren't
deleted so series survive; Tag.fandom_id is SET NULL so fandoms are untouched.
Reuses the established dry-run-preview -> confirm pattern: cleanup_service.
reset_content_tagging() + POST /api/admin/tags/reset-content +
TagMaintenanceCard section with a backup-first warning and a red confirm
showing exact counts (tags by kind + image applications). Irreversible except
via DB backup restore; the wipe only fires when the operator confirms.
Tests: service dry-run counts + live delete preserves fandom/series/series_page
while content tags + their image_tag cascade away; API dry-run wiring.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
396 lines
13 KiB
Python
396 lines
13 KiB
Python
"""FC-3k: cleanup_service unit tests.
|
|
|
|
Mutations go against real Postgres (db_sync fixture). File-system
|
|
side effects use tmp_path. Assertions on mutated rows use COLUMN
|
|
SELECTS per reference_async_coredml_test_assertions — never
|
|
re-read ORM attributes after a service mutates and re-fetches.
|
|
"""
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
|
|
from backend.app.models import Artist, ImageRecord, Tag, TagKind
|
|
from backend.app.models.series_page import SeriesPage
|
|
from backend.app.models.tag import image_tag
|
|
from backend.app.services import cleanup_service
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _make_image(db_sync, *, artist, path, sha256, size=1000, thumb=None):
|
|
img = ImageRecord(
|
|
artist_id=artist.id,
|
|
path=path,
|
|
sha256=sha256,
|
|
size_bytes=size,
|
|
mime="image/jpeg",
|
|
origin="imported_filesystem",
|
|
thumbnail_path=thumb,
|
|
)
|
|
db_sync.add(img)
|
|
db_sync.flush()
|
|
return img
|
|
|
|
|
|
def _make_artist(db_sync, *, slug="aria", name="Aria"):
|
|
a = Artist(name=name, slug=slug)
|
|
db_sync.add(a)
|
|
db_sync.flush()
|
|
return a
|
|
|
|
|
|
def _make_tag(db_sync, *, name, kind=TagKind.general):
|
|
t = Tag(name=name, kind=kind)
|
|
db_sync.add(t)
|
|
db_sync.flush()
|
|
return t
|
|
|
|
|
|
# --- project_artist_cascade -----------------------------------------
|
|
|
|
|
|
def test_project_artist_cascade_returns_zeroes_for_empty_artist(db_sync):
|
|
_make_artist(db_sync, slug="empty")
|
|
db_sync.commit()
|
|
result = cleanup_service.project_artist_cascade(db_sync, slug="empty")
|
|
assert result["artist"]["slug"] == "empty"
|
|
assert result["projected"] == {
|
|
"images": 0,
|
|
"sources": 0,
|
|
"thumbs": 0,
|
|
"import_tasks": 0,
|
|
"bytes_on_disk": 0,
|
|
}
|
|
|
|
|
|
def test_project_artist_cascade_counts_images_and_thumbs_and_bytes(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="counted")
|
|
_make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "a.jpg"),
|
|
sha256="a" * 64, size=1000, thumb=str(tmp_path / "a.thumb"),
|
|
)
|
|
_make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "b.jpg"),
|
|
sha256="b" * 64, size=2500, thumb=None,
|
|
)
|
|
db_sync.commit()
|
|
result = cleanup_service.project_artist_cascade(db_sync, slug="counted")
|
|
assert result["projected"]["images"] == 2
|
|
assert result["projected"]["thumbs"] == 1
|
|
assert result["projected"]["bytes_on_disk"] == 3500
|
|
|
|
|
|
def test_project_artist_cascade_raises_on_unknown_slug(db_sync):
|
|
with pytest.raises(LookupError):
|
|
cleanup_service.project_artist_cascade(db_sync, slug="nope")
|
|
|
|
|
|
# --- project_bulk_image_delete --------------------------------------
|
|
|
|
|
|
def test_project_bulk_image_delete_separates_found_from_missing(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="bd")
|
|
i1 = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "1.jpg"),
|
|
sha256="1" * 64, size=10, thumb="t1",
|
|
)
|
|
i2 = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "2.jpg"),
|
|
sha256="2" * 64, size=20, thumb=None,
|
|
)
|
|
db_sync.commit()
|
|
result = cleanup_service.project_bulk_image_delete(
|
|
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
|
)
|
|
assert result["images_found"] == 2
|
|
assert result["thumbs_to_unlink"] == 1
|
|
assert result["bytes_on_disk"] == 30
|
|
assert result["missing_ids"] == [9_999_999]
|
|
|
|
|
|
def test_project_bulk_image_delete_empty_input(db_sync):
|
|
result = cleanup_service.project_bulk_image_delete(db_sync, image_ids=[])
|
|
assert result == {
|
|
"images_found": 0,
|
|
"thumbs_to_unlink": 0,
|
|
"bytes_on_disk": 0,
|
|
"missing_ids": [],
|
|
}
|
|
|
|
|
|
# --- count_tag_associations / find_unused_tags ----------------------
|
|
|
|
|
|
def test_count_tag_associations_counts_image_tag_rows(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="ta")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="c" * 64,
|
|
)
|
|
tag = _make_tag(db_sync, name="cyberpunk")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id,
|
|
))
|
|
db_sync.commit()
|
|
assert cleanup_service.count_tag_associations(db_sync, tag_id=tag.id) == 1
|
|
|
|
|
|
def test_find_unused_tags_returns_only_unreferenced(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="fu")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "y.jpg"), sha256="d" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="used")
|
|
_make_tag(db_sync, name="aaa-unused")
|
|
_make_tag(db_sync, name="zzz-unused")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
result = cleanup_service.find_unused_tags(db_sync)
|
|
names = [t.name for t in result]
|
|
assert "used" not in names
|
|
assert "aaa-unused" in names
|
|
assert "zzz-unused" in names
|
|
# Sorted by name ascending.
|
|
assert names.index("aaa-unused") < names.index("zzz-unused")
|
|
|
|
|
|
# --- unlink_image_files ---------------------------------------------
|
|
|
|
|
|
def test_unlink_image_files_removes_original_and_thumbnail(db_sync, tmp_path):
|
|
original = tmp_path / "orig.jpg"
|
|
original.write_bytes(b"x")
|
|
custom_thumb = tmp_path / "custom.thumb"
|
|
custom_thumb.write_bytes(b"x")
|
|
conv_thumb = tmp_path / "thumbs" / "aaa" / ("a" * 64 + ".jpg")
|
|
conv_thumb.parent.mkdir(parents=True)
|
|
conv_thumb.write_bytes(b"x")
|
|
|
|
img = ImageRecord(
|
|
artist_id=None, path=str(original), sha256="a" * 64,
|
|
size_bytes=1, thumbnail_path=str(custom_thumb),
|
|
)
|
|
result = cleanup_service.unlink_image_files(img, tmp_path)
|
|
assert result == {"original": True, "thumbnail": True}
|
|
assert not original.exists()
|
|
assert not custom_thumb.exists()
|
|
assert not conv_thumb.exists()
|
|
|
|
|
|
def test_unlink_image_files_missing_files_count_as_success(tmp_path):
|
|
img = ImageRecord(
|
|
artist_id=None, path=str(tmp_path / "nope.jpg"),
|
|
sha256="b" * 64, size_bytes=1, thumbnail_path=None,
|
|
)
|
|
result = cleanup_service.unlink_image_files(img, tmp_path)
|
|
assert result == {"original": True, "thumbnail": False}
|
|
|
|
|
|
# --- delete_artist_cascade ------------------------------------------
|
|
|
|
|
|
def test_delete_artist_cascade_removes_images_and_artist_row(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="cas")
|
|
for i in range(3):
|
|
f = tmp_path / f"img{i}.jpg"
|
|
f.write_bytes(b"x")
|
|
_make_image(
|
|
db_sync, artist=a, path=str(f),
|
|
sha256=f"{i:064x}", size=10,
|
|
)
|
|
db_sync.commit()
|
|
artist_id = a.id
|
|
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=artist_id, images_root=tmp_path,
|
|
)
|
|
assert result["artist"]["slug"] == "cas"
|
|
assert result["summary"]["images_deleted"] == 3
|
|
assert result["summary"]["files_deleted"] == 3
|
|
|
|
# Column-select assertions per reference_async_coredml_test_assertions.
|
|
surviving_artist = db_sync.execute(
|
|
select(func.count(Artist.id)).where(Artist.id == artist_id)
|
|
).scalar_one()
|
|
surviving_images = db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
.where(ImageRecord.artist_id == artist_id)
|
|
).scalar_one()
|
|
assert surviving_artist == 0
|
|
assert surviving_images == 0
|
|
|
|
|
|
def test_delete_artist_cascade_idempotent_on_missing(db_sync, tmp_path):
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=9_999_999, images_root=tmp_path,
|
|
)
|
|
assert result["artist"] is None
|
|
assert result["summary"]["images_deleted"] == 0
|
|
|
|
|
|
# --- delete_images --------------------------------------------------
|
|
|
|
|
|
def test_delete_images_removes_rows_and_files(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="di")
|
|
f1 = tmp_path / "1.jpg"
|
|
f1.write_bytes(b"x")
|
|
f2 = tmp_path / "2.jpg"
|
|
f2.write_bytes(b"x")
|
|
i1 = _make_image(db_sync, artist=a, path=str(f1), sha256="1" * 64)
|
|
i2 = _make_image(db_sync, artist=a, path=str(f2), sha256="2" * 64)
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.delete_images(
|
|
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
|
images_root=tmp_path,
|
|
)
|
|
assert result["images_deleted"] == 2
|
|
assert result["files_deleted"] == 2
|
|
assert result["missing_ids"] == [9_999_999]
|
|
|
|
surviving = db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
.where(ImageRecord.id.in_([i1.id, i2.id]))
|
|
).scalar_one()
|
|
assert surviving == 0
|
|
|
|
|
|
# --- delete_tag -----------------------------------------------------
|
|
|
|
|
|
def test_delete_tag_cascades_associations(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="dt")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="d" * 64,
|
|
)
|
|
tag = _make_tag(db_sync, name="doomed")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id,
|
|
))
|
|
db_sync.commit()
|
|
tag_id = tag.id
|
|
|
|
result = cleanup_service.delete_tag(db_sync, tag_id=tag_id)
|
|
assert result["deleted"]["id"] == tag_id
|
|
assert result["associations_removed"] == 1
|
|
|
|
surviving_tag = db_sync.execute(
|
|
select(func.count(Tag.id)).where(Tag.id == tag_id)
|
|
).scalar_one()
|
|
surviving_assoc = db_sync.execute(
|
|
select(func.count())
|
|
.select_from(image_tag).where(image_tag.c.tag_id == tag_id)
|
|
).scalar_one()
|
|
assert surviving_tag == 0
|
|
assert surviving_assoc == 0
|
|
|
|
|
|
def test_delete_tag_raises_on_unknown_id(db_sync):
|
|
with pytest.raises(LookupError):
|
|
cleanup_service.delete_tag(db_sync, tag_id=9_999_999)
|
|
|
|
|
|
# --- prune_unused_tags ----------------------------------------------
|
|
|
|
|
|
def test_prune_unused_tags_dry_run_returns_count_and_names(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="pu")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "z.jpg"), sha256="e" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="kept")
|
|
_make_tag(db_sync, name="prune-me-1")
|
|
_make_tag(db_sync, name="prune-me-2")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.prune_unused_tags(db_sync, dry_run=True)
|
|
assert result["count"] == 2
|
|
assert "prune-me-1" in result["sample_names"]
|
|
assert "prune-me-2" in result["sample_names"]
|
|
# Nothing actually deleted.
|
|
surviving = db_sync.execute(select(func.count(Tag.id))).scalar_one()
|
|
assert surviving == 3
|
|
|
|
|
|
def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="pu2")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "k.jpg"), sha256="f" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="kept")
|
|
_make_tag(db_sync, name="bye")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.prune_unused_tags(db_sync, dry_run=False)
|
|
assert result["deleted"] == 1
|
|
|
|
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
|
|
assert "kept" in surviving_names
|
|
assert "bye" not in surviving_names
|
|
|
|
|
|
# --- reset_content_tagging ------------------------------------------
|
|
|
|
|
|
def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="rc")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "r.jpg"), sha256="1" * 64,
|
|
)
|
|
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
|
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
|
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
|
_make_tag(db_sync, name="my-series", kind=TagKind.series)
|
|
db_sync.execute(image_tag.insert().values([
|
|
{"image_record_id": img.id, "tag_id": g.id},
|
|
{"image_record_id": img.id, "tag_id": c.id},
|
|
]))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.reset_content_tagging(db_sync, dry_run=True)
|
|
assert result["count"] == 2
|
|
assert result["by_kind"] == {"general": 1, "character": 1}
|
|
assert result["applications"] == 2
|
|
# Nothing deleted — all 4 tags still present.
|
|
assert db_sync.execute(select(func.count(Tag.id))).scalar_one() == 4
|
|
|
|
|
|
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="rc2")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "r2.jpg"), sha256="2" * 64,
|
|
)
|
|
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
|
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
|
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
|
s = _make_tag(db_sync, name="my-series", kind=TagKind.series)
|
|
db_sync.execute(image_tag.insert().values([
|
|
{"image_record_id": img.id, "tag_id": g.id},
|
|
{"image_record_id": img.id, "tag_id": c.id},
|
|
{"image_record_id": img.id, "tag_id": s.id}, # series membership
|
|
]))
|
|
db_sync.add(SeriesPage(series_tag_id=s.id, image_id=img.id, page_number=1))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
|
|
assert result["deleted"] == 2
|
|
|
|
# general + character gone; fandom + series kept.
|
|
kinds = db_sync.execute(select(Tag.kind)).scalars().all()
|
|
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
|
|
assert kind_vals == {"fandom", "series"}
|
|
# series_page ordering survived.
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(SeriesPage)
|
|
).scalar_one() == 1
|
|
# Only the series image_tag association survived (content ones cascaded).
|
|
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
|
|
assert remaining == [s.id]
|