bvandeusenandClaude Opus 5
2e0f8f8c61
PostAttachment's two FKs are both ON DELETE SET NULL, so a deleted post or artist left the row behind rather than taking it. Nothing ever pruned those rows, and nothing in the repo had ever unlinked a file under the attachment store — so both rows and bytes accumulated permanently, invisible to every existing diagnostic. Why a disk->DB reconciliation rather than a row sweep: the store is sha-addressed and idempotent, so ONE blob backs MANY rows. Deleting a row does not free its blob, and since the artist cascade (#3066) now deletes its attachment rows outright, a freed blob has no DB pointer left to find it by. Walking the store and asking "does any row still reference this sha?" catches orphans from every cause, including ones no future delete path will think to report. Preview and apply share `_orphan_attachment_conditions` (rule 93). The dry-run derives its surviving-sha set by NEGATING that same predicate, so it is honest about blobs the delete would free rather than counting them as still-referenced — the one place this was easy to get backwards, so it has its own parity test. Guards, each with a reason: - A blob is written before its row commits, so a just-stored file legitimately has no referencing row. Files under 6h are never judged — same guard and reasoning as ORPHAN_TEMP_MIN_AGE_HOURS. - `.partial` staging files belong to cleanup_orphaned_temp_files; skipped rather than raced. - The sha is parsed as the first 64 chars, not via Path.stem: store() takes the extension from the source filename, and a URL-encoded basename yields a multi-dot suffix that would make stem eat part of the sha. - A 900s walk budget reports partial=True instead of running to the task's hard limit (rule 89). - TASK_STUCK_THRESHOLD_MINUTES override at 30 (= time_limit 25 + 5). Without it a healthy 20-minute walk is phantom-flagged 'RecoverySweep' at the bare 5-min default — the #883 failure class; its invariant test is mirrored here. Defaults to the safe preview at both the task and the route, unlike the other maintenance triggers: this apply unlinks files. Operator-triggered only, never on a beat. Ships with its UI (rule 27): AttachmentReclaimCard in Cleanup → Duplicates & leftovers, built on the existing useMaintenanceTask/MaintenanceTile shapes, so a run survives navigating away. Surfaces files_failed and partial explicitly, since both change what the numbers mean. Also promotes humanBytes to utils/bytes.js — it was byte-identical in VideoDedupCard and GatedPurgeCard and this card would have been the third copy. The three divergent `formatBytes` helpers are deliberately left alone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1258 lines
47 KiB
Python
1258 lines
47 KiB
Python
"""FC-3k: cleanup_service unit tests.
|
|
|
|
Mutations go against real Postgres (db_sync fixture). File-system
|
|
side effects use tmp_path. Assertions on mutated rows use COLUMN
|
|
SELECTS per reference_async_coredml_test_assertions — never
|
|
re-read ORM attributes after a service mutates and re-fetches.
|
|
"""
|
|
import os
|
|
from datetime import UTC, datetime
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
|
|
from backend.app.models import (
|
|
Artist,
|
|
ImageProvenance,
|
|
ImageRecord,
|
|
PatreonSeenMedia,
|
|
Post,
|
|
PostAttachment,
|
|
Source,
|
|
Tag,
|
|
TagKind,
|
|
)
|
|
from backend.app.models.series_chapter import SeriesChapter
|
|
from backend.app.models.series_page import SeriesPage
|
|
from backend.app.models.tag import image_tag
|
|
from backend.app.services import cleanup_service
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _make_image(db_sync, *, artist, path, sha256, size=1000, thumb=None):
|
|
img = ImageRecord(
|
|
artist_id=artist.id,
|
|
path=path,
|
|
sha256=sha256,
|
|
size_bytes=size,
|
|
mime="image/jpeg",
|
|
origin="imported_filesystem",
|
|
thumbnail_path=thumb,
|
|
)
|
|
db_sync.add(img)
|
|
db_sync.flush()
|
|
return img
|
|
|
|
|
|
def _make_artist(db_sync, *, slug="aria", name="Aria"):
|
|
a = Artist(name=name, slug=slug)
|
|
db_sync.add(a)
|
|
db_sync.flush()
|
|
return a
|
|
|
|
|
|
def _make_tag(db_sync, *, name, kind=TagKind.general):
|
|
t = Tag(name=name, kind=kind)
|
|
db_sync.add(t)
|
|
db_sync.flush()
|
|
return t
|
|
|
|
|
|
# --- project_artist_cascade -----------------------------------------
|
|
|
|
|
|
def test_project_artist_cascade_returns_zeroes_for_empty_artist(db_sync):
|
|
_make_artist(db_sync, slug="empty")
|
|
db_sync.commit()
|
|
result = cleanup_service.project_artist_cascade(db_sync, slug="empty")
|
|
assert result["artist"]["slug"] == "empty"
|
|
assert result["projected"] == {
|
|
"images": 0,
|
|
"posts": 0,
|
|
"attachments": 0,
|
|
"sources": 0,
|
|
"thumbs": 0,
|
|
"import_tasks": 0,
|
|
"bytes_on_disk": 0,
|
|
}
|
|
|
|
|
|
def test_project_artist_cascade_counts_images_and_thumbs_and_bytes(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="counted")
|
|
_make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "a.jpg"),
|
|
sha256="a" * 64, size=1000, thumb=str(tmp_path / "a.thumb"),
|
|
)
|
|
_make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "b.jpg"),
|
|
sha256="b" * 64, size=2500, thumb=None,
|
|
)
|
|
db_sync.commit()
|
|
result = cleanup_service.project_artist_cascade(db_sync, slug="counted")
|
|
assert result["projected"]["images"] == 2
|
|
assert result["projected"]["thumbs"] == 1
|
|
assert result["projected"]["bytes_on_disk"] == 3500
|
|
|
|
|
|
def test_project_artist_cascade_counts_posts_and_attachments(db_sync, tmp_path):
|
|
"""The body-only artist: zero images, but posts and attachments that the
|
|
apply destroys. Previewing this as `images: 0` alone is what made a
|
|
content-only artist read as an empty one (#3067)."""
|
|
a = _make_artist(db_sync, slug="bodyonly")
|
|
p1 = Post(artist_id=a.id, external_post_id="bo-1", description="a body")
|
|
p2 = Post(artist_id=a.id, external_post_id="bo-2", description="another")
|
|
db_sync.add_all([p1, p2])
|
|
db_sync.flush()
|
|
db_sync.add(PostAttachment(
|
|
post_id=p1.id, artist_id=a.id, sha256="b0d1".ljust(64, "0"),
|
|
path="/store/b0d1/a.pdf", original_filename="a.pdf",
|
|
ext=".pdf", size_bytes=5,
|
|
))
|
|
# artist_id NULL, reachable only through its post — the second arm of
|
|
# _artist_attachments_conditions.
|
|
db_sync.add(PostAttachment(
|
|
post_id=p2.id, artist_id=None, sha256="b0d2".ljust(64, "0"),
|
|
path="/store/b0d2/b.pdf", original_filename="b.pdf",
|
|
ext=".pdf", size_bytes=5,
|
|
))
|
|
db_sync.commit()
|
|
|
|
projected = cleanup_service.project_artist_cascade(
|
|
db_sync, slug="bodyonly",
|
|
)["projected"]
|
|
assert projected["images"] == 0
|
|
assert projected["posts"] == 2
|
|
assert projected["attachments"] == 2
|
|
|
|
|
|
def test_artist_cascade_preview_matches_apply(db_sync, tmp_path):
|
|
"""Rule 93: the preview's numbers must be what the apply actually does.
|
|
|
|
Guards the drift directly rather than trusting that both halves happen to
|
|
use the same predicate — the preview and apply are asserted against each
|
|
other on one artist carrying all three row kinds.
|
|
"""
|
|
a = _make_artist(db_sync, slug="parity")
|
|
for i in range(3):
|
|
f = tmp_path / f"par{i}.jpg"
|
|
f.write_bytes(b"x")
|
|
_make_image(
|
|
db_sync, artist=a, path=str(f), sha256=f"{i:064x}", size=10,
|
|
)
|
|
posts = [
|
|
Post(artist_id=a.id, external_post_id=f"par-{i}") for i in range(4)
|
|
]
|
|
db_sync.add_all(posts)
|
|
db_sync.flush()
|
|
for i, p in enumerate(posts[:2]):
|
|
db_sync.add(PostAttachment(
|
|
post_id=p.id, artist_id=a.id, sha256=f"par{i}".ljust(64, "0"),
|
|
path=f"/store/par{i}/f.zip", original_filename="f.zip",
|
|
ext=".zip", size_bytes=9,
|
|
))
|
|
db_sync.commit()
|
|
artist_id = a.id
|
|
|
|
projected = cleanup_service.project_artist_cascade(
|
|
db_sync, slug="parity",
|
|
)["projected"]
|
|
summary = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=artist_id, images_root=tmp_path,
|
|
)["summary"]
|
|
|
|
assert projected["images"] == summary["images_deleted"] == 3
|
|
assert projected["posts"] == summary["posts_deleted"] == 4
|
|
assert projected["attachments"] == summary["attachments_deleted"] == 2
|
|
|
|
# And the apply really did remove them — a matching pair of numbers is
|
|
# worth nothing if neither half touched the DB.
|
|
assert db_sync.execute(
|
|
select(func.count(Post.id)).where(Post.artist_id == artist_id)
|
|
).scalar_one() == 0
|
|
assert db_sync.execute(
|
|
select(func.count(PostAttachment.id))
|
|
.where(PostAttachment.artist_id == artist_id)
|
|
).scalar_one() == 0
|
|
|
|
|
|
def test_project_artist_cascade_raises_on_unknown_slug(db_sync):
|
|
with pytest.raises(LookupError):
|
|
cleanup_service.project_artist_cascade(db_sync, slug="nope")
|
|
|
|
|
|
# --- project_bulk_image_delete --------------------------------------
|
|
|
|
|
|
def test_project_bulk_image_delete_separates_found_from_missing(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="bd")
|
|
i1 = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "1.jpg"),
|
|
sha256="1" * 64, size=10, thumb="t1",
|
|
)
|
|
i2 = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "2.jpg"),
|
|
sha256="2" * 64, size=20, thumb=None,
|
|
)
|
|
db_sync.commit()
|
|
result = cleanup_service.project_bulk_image_delete(
|
|
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
|
)
|
|
assert result["images_found"] == 2
|
|
assert result["thumbs_to_unlink"] == 1
|
|
assert result["bytes_on_disk"] == 30
|
|
assert result["missing_ids"] == [9_999_999]
|
|
|
|
|
|
def test_project_bulk_image_delete_empty_input(db_sync):
|
|
result = cleanup_service.project_bulk_image_delete(db_sync, image_ids=[])
|
|
assert result == {
|
|
"images_found": 0,
|
|
"thumbs_to_unlink": 0,
|
|
"bytes_on_disk": 0,
|
|
"missing_ids": [],
|
|
}
|
|
|
|
|
|
# --- count_tag_associations / find_unused_tags ----------------------
|
|
|
|
|
|
def test_count_tag_associations_counts_image_tag_rows(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="ta")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="c" * 64,
|
|
)
|
|
tag = _make_tag(db_sync, name="cyberpunk")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id,
|
|
))
|
|
db_sync.commit()
|
|
assert cleanup_service.count_tag_associations(db_sync, tag_id=tag.id) == 1
|
|
|
|
|
|
def test_count_tag_associations_fandom_includes_character_images(db_sync, tmp_path):
|
|
# Deleting a fandom affects images carrying its characters too, so the
|
|
# Tier-B blast-radius prompt must count them — not just direct image_tag
|
|
# rows on the fandom (a fandom usually has zero of those). DISTINCT: an
|
|
# image with both the fandom and its character counts once.
|
|
a = _make_artist(db_sync, slug="fc")
|
|
img0 = _make_image(db_sync, artist=a, path=str(tmp_path / "0.jpg"), sha256="e" * 64)
|
|
img1 = _make_image(db_sync, artist=a, path=str(tmp_path / "1.jpg"), sha256="f" * 64)
|
|
fandom = _make_tag(db_sync, name="Creux", kind=TagKind.fandom)
|
|
char = _make_tag(db_sync, name="OcChar", kind=TagKind.character)
|
|
char.fandom_id = fandom.id
|
|
db_sync.flush()
|
|
db_sync.execute(image_tag.insert().values([
|
|
{"image_record_id": img0.id, "tag_id": char.id},
|
|
{"image_record_id": img1.id, "tag_id": char.id},
|
|
{"image_record_id": img1.id, "tag_id": fandom.id},
|
|
]))
|
|
db_sync.commit()
|
|
assert cleanup_service.count_tag_associations(db_sync, tag_id=fandom.id) == 2
|
|
|
|
|
|
def test_find_unused_tags_returns_only_unreferenced(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="fu")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "y.jpg"), sha256="d" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="used")
|
|
_make_tag(db_sync, name="aaa-unused")
|
|
_make_tag(db_sync, name="zzz-unused")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
result = cleanup_service.find_unused_tags(db_sync)
|
|
names = [t.name for t in result]
|
|
assert "used" not in names
|
|
assert "aaa-unused" in names
|
|
assert "zzz-unused" in names
|
|
# Sorted by name ascending.
|
|
assert names.index("aaa-unused") < names.index("zzz-unused")
|
|
|
|
|
|
def test_find_unused_tags_excludes_fandom_referenced_by_character(db_sync):
|
|
# A fandom is never on an image — a character carries it via fandom_id — so
|
|
# an assigned fandom must NOT be flagged unused (deleting it SET-NULLs the
|
|
# fandom off its characters). Operator-flagged 2026-06-08.
|
|
fandom = _make_tag(db_sync, name="Creux", kind=TagKind.fandom)
|
|
char = _make_tag(db_sync, name="OcChar", kind=TagKind.character)
|
|
char.fandom_id = fandom.id
|
|
_make_tag(db_sync, name="NobodysFandom", kind=TagKind.fandom)
|
|
db_sync.commit()
|
|
|
|
names = [t.name for t in cleanup_service.find_unused_tags(db_sync)]
|
|
assert "Creux" not in names # referenced by a character → kept
|
|
assert "NobodysFandom" in names # genuinely orphaned → still swept
|
|
|
|
|
|
def test_find_unused_tags_excludes_series_with_a_divider(db_sync):
|
|
# A series with a chapter divider (which anchors to a page) is in use.
|
|
a = _make_artist(db_sync)
|
|
series = _make_tag(db_sync, name="DivSeries", kind=TagKind.series)
|
|
img = _make_image(
|
|
db_sync, artist=a, path="/tmp/fc_divser.jpg", sha256="e" * 64,
|
|
)
|
|
page = SeriesPage(series_tag_id=series.id, image_id=img.id, page_number=1)
|
|
db_sync.add(page)
|
|
db_sync.flush()
|
|
db_sync.add(SeriesChapter(series_tag_id=series.id, anchor_page_id=page.id))
|
|
db_sync.commit()
|
|
|
|
names = [t.name for t in cleanup_service.find_unused_tags(db_sync)]
|
|
assert "DivSeries" not in names
|
|
|
|
|
|
# --- unlink_image_files ---------------------------------------------
|
|
|
|
|
|
def test_unlink_image_files_removes_original_and_thumbnail(db_sync, tmp_path):
|
|
original = tmp_path / "orig.jpg"
|
|
original.write_bytes(b"x")
|
|
custom_thumb = tmp_path / "custom.thumb"
|
|
custom_thumb.write_bytes(b"x")
|
|
conv_thumb = tmp_path / "thumbs" / "aaa" / ("a" * 64 + ".jpg")
|
|
conv_thumb.parent.mkdir(parents=True)
|
|
conv_thumb.write_bytes(b"x")
|
|
|
|
img = ImageRecord(
|
|
artist_id=None, path=str(original), sha256="a" * 64,
|
|
size_bytes=1, thumbnail_path=str(custom_thumb),
|
|
)
|
|
result = cleanup_service.unlink_image_files(img, tmp_path)
|
|
assert result == {"original": True, "thumbnail": True}
|
|
assert not original.exists()
|
|
assert not custom_thumb.exists()
|
|
assert not conv_thumb.exists()
|
|
|
|
|
|
def test_unlink_image_files_missing_files_count_as_success(tmp_path):
|
|
img = ImageRecord(
|
|
artist_id=None, path=str(tmp_path / "nope.jpg"),
|
|
sha256="b" * 64, size_bytes=1, thumbnail_path=None,
|
|
)
|
|
result = cleanup_service.unlink_image_files(img, tmp_path)
|
|
assert result == {"original": True, "thumbnail": False}
|
|
|
|
|
|
# --- delete_artist_cascade ------------------------------------------
|
|
|
|
|
|
def test_delete_artist_cascade_removes_images_and_artist_row(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="cas")
|
|
for i in range(3):
|
|
f = tmp_path / f"img{i}.jpg"
|
|
f.write_bytes(b"x")
|
|
_make_image(
|
|
db_sync, artist=a, path=str(f),
|
|
sha256=f"{i:064x}", size=10,
|
|
)
|
|
db_sync.commit()
|
|
artist_id = a.id
|
|
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=artist_id, images_root=tmp_path,
|
|
)
|
|
assert result["artist"]["slug"] == "cas"
|
|
assert result["summary"]["images_deleted"] == 3
|
|
assert result["summary"]["files_deleted"] == 3
|
|
|
|
# Column-select assertions per reference_async_coredml_test_assertions.
|
|
surviving_artist = db_sync.execute(
|
|
select(func.count(Artist.id)).where(Artist.id == artist_id)
|
|
).scalar_one()
|
|
surviving_images = db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
.where(ImageRecord.artist_id == artist_id)
|
|
).scalar_one()
|
|
assert surviving_artist == 0
|
|
assert surviving_images == 0
|
|
|
|
|
|
def test_delete_artist_cascade_idempotent_on_missing(db_sync, tmp_path):
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=9_999_999, images_root=tmp_path,
|
|
)
|
|
assert result["artist"] is None
|
|
assert result["summary"]["images_deleted"] == 0
|
|
|
|
|
|
def test_delete_artist_cascade_survives_same_sha_on_two_posts(db_sync, tmp_path):
|
|
"""Same file attached to two of the artist's posts must not abort the delete.
|
|
|
|
Left to the cascade this raises: artist delete CASCADEs to Post, which SET
|
|
NULLs post_attachment.post_id, and `uq_post_attachment_null_post_sha`
|
|
(sha256 alone, WHERE post_id IS NULL) then rejects the second row. That's an
|
|
ordinary shape — _capture_attachment writes one row per post over one
|
|
sha-addressed blob by design. Also covers the NULL-artist_id arm of the
|
|
delete predicate: the second row has no artist_id, only a post that does.
|
|
"""
|
|
a = _make_artist(db_sync, slug="casatt")
|
|
p1 = Post(artist_id=a.id, external_post_id="att-p1")
|
|
p2 = Post(artist_id=a.id, external_post_id="att-p2")
|
|
db_sync.add_all([p1, p2])
|
|
db_sync.flush()
|
|
|
|
shared_sha = "ca5a".ljust(64, "0")
|
|
db_sync.add(PostAttachment(
|
|
post_id=p1.id, artist_id=a.id, sha256=shared_sha,
|
|
path="/store/ca5a/bundle.zip", original_filename="bundle.zip",
|
|
ext=".zip", size_bytes=7,
|
|
))
|
|
db_sync.add(PostAttachment(
|
|
post_id=p2.id, artist_id=None, sha256=shared_sha,
|
|
path="/store/ca5a/bundle.zip", original_filename="bundle.zip",
|
|
ext=".zip", size_bytes=7,
|
|
))
|
|
db_sync.commit()
|
|
artist_id = a.id
|
|
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=artist_id, images_root=tmp_path,
|
|
)
|
|
|
|
assert result["summary"]["attachments_deleted"] == 2
|
|
assert db_sync.execute(
|
|
select(func.count(Artist.id)).where(Artist.id == artist_id)
|
|
).scalar_one() == 0
|
|
assert db_sync.execute(
|
|
select(func.count(PostAttachment.id))
|
|
.where(PostAttachment.sha256 == shared_sha)
|
|
).scalar_one() == 0
|
|
|
|
|
|
def test_delete_artist_cascade_keeps_unrelated_null_post_attachment(
|
|
db_sync, tmp_path,
|
|
):
|
|
"""A filesystem-import row (post_id NULL) sharing the sha is the other way
|
|
this collides — and it must SURVIVE: it belongs to no artist, so the
|
|
cascade has no claim on it."""
|
|
a = _make_artist(db_sync, slug="casorph")
|
|
p = Post(artist_id=a.id, external_post_id="orph-p1")
|
|
db_sync.add(p)
|
|
db_sync.flush()
|
|
|
|
sha = "0rfa".ljust(64, "0")
|
|
standalone = PostAttachment(
|
|
post_id=None, artist_id=None, sha256=sha,
|
|
path="/store/0rfa/manual.pdf", original_filename="manual.pdf",
|
|
ext=".pdf", size_bytes=3,
|
|
)
|
|
db_sync.add(standalone)
|
|
db_sync.add(PostAttachment(
|
|
post_id=p.id, artist_id=a.id, sha256=sha,
|
|
path="/store/0rfa/manual.pdf", original_filename="manual.pdf",
|
|
ext=".pdf", size_bytes=3,
|
|
))
|
|
db_sync.commit()
|
|
artist_id, standalone_id = a.id, standalone.id
|
|
|
|
result = cleanup_service.delete_artist_cascade(
|
|
db_sync, artist_id=artist_id, images_root=tmp_path,
|
|
)
|
|
|
|
assert result["summary"]["attachments_deleted"] == 1
|
|
surviving = db_sync.execute(
|
|
select(PostAttachment.id, PostAttachment.post_id)
|
|
.where(PostAttachment.sha256 == sha)
|
|
).all()
|
|
assert surviving == [(standalone_id, None)]
|
|
|
|
|
|
# --- delete_images --------------------------------------------------
|
|
|
|
|
|
def test_delete_images_removes_rows_and_files(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="di")
|
|
f1 = tmp_path / "1.jpg"
|
|
f1.write_bytes(b"x")
|
|
f2 = tmp_path / "2.jpg"
|
|
f2.write_bytes(b"x")
|
|
i1 = _make_image(db_sync, artist=a, path=str(f1), sha256="1" * 64)
|
|
i2 = _make_image(db_sync, artist=a, path=str(f2), sha256="2" * 64)
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.delete_images(
|
|
db_sync, image_ids=[i1.id, i2.id, 9_999_999],
|
|
images_root=tmp_path,
|
|
)
|
|
assert result["images_deleted"] == 2
|
|
assert result["files_deleted"] == 2
|
|
assert result["missing_ids"] == [9_999_999]
|
|
|
|
surviving = db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
.where(ImageRecord.id.in_([i1.id, i2.id]))
|
|
).scalar_one()
|
|
assert surviving == 0
|
|
|
|
|
|
# --- delete_tag -----------------------------------------------------
|
|
|
|
|
|
def test_delete_tag_cascades_associations(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="dt")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="d" * 64,
|
|
)
|
|
tag = _make_tag(db_sync, name="doomed")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=tag.id,
|
|
))
|
|
db_sync.commit()
|
|
tag_id = tag.id
|
|
|
|
result = cleanup_service.delete_tag(db_sync, tag_id=tag_id)
|
|
assert result["deleted"]["id"] == tag_id
|
|
assert result["associations_removed"] == 1
|
|
|
|
surviving_tag = db_sync.execute(
|
|
select(func.count(Tag.id)).where(Tag.id == tag_id)
|
|
).scalar_one()
|
|
surviving_assoc = db_sync.execute(
|
|
select(func.count())
|
|
.select_from(image_tag).where(image_tag.c.tag_id == tag_id)
|
|
).scalar_one()
|
|
assert surviving_tag == 0
|
|
assert surviving_assoc == 0
|
|
|
|
|
|
def test_delete_tag_raises_on_unknown_id(db_sync):
|
|
with pytest.raises(LookupError):
|
|
cleanup_service.delete_tag(db_sync, tag_id=9_999_999)
|
|
|
|
|
|
# --- prune_unused_tags ----------------------------------------------
|
|
|
|
|
|
def test_prune_unused_tags_dry_run_returns_count_and_names(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="pu")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "z.jpg"), sha256="e" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="kept")
|
|
_make_tag(db_sync, name="prune-me-1")
|
|
_make_tag(db_sync, name="prune-me-2")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.prune_unused_tags(db_sync, dry_run=True)
|
|
assert result["count"] == 2
|
|
assert "prune-me-1" in result["sample_names"]
|
|
assert "prune-me-2" in result["sample_names"]
|
|
# Nothing actually deleted. Non-system only — the seeded hygiene tags
|
|
# (#128) are exempt from pruning and would inflate a whole-table count.
|
|
surviving = db_sync.execute(
|
|
select(func.count(Tag.id)).where(Tag.is_system.is_(False))
|
|
).scalar_one()
|
|
assert surviving == 3
|
|
|
|
|
|
def test_prune_unused_tags_commit_deletes_them(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="pu2")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "k.jpg"), sha256="f" * 64,
|
|
)
|
|
used = _make_tag(db_sync, name="kept")
|
|
_make_tag(db_sync, name="bye")
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=used.id,
|
|
))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.prune_unused_tags(db_sync, dry_run=False)
|
|
assert result["deleted"] == 1
|
|
|
|
surviving_names = db_sync.execute(select(Tag.name)).scalars().all()
|
|
assert "kept" in surviving_names
|
|
assert "bye" not in surviving_names
|
|
|
|
|
|
def test_prune_unused_tags_commit_spares_fandom_and_chaptered_series(
|
|
db_sync, tmp_path,
|
|
):
|
|
# The LIVE delete (not just the preview) must use the same predicate — it
|
|
# previously had only the image_tag/series_page checks and deleted every
|
|
# fandom the preview correctly excluded (operator-flagged 2026-06-08, real
|
|
# data loss). Both dry-run count and the commit must spare referenced tags.
|
|
fandom = _make_tag(db_sync, name="Creux", kind=TagKind.fandom)
|
|
char = _make_tag(db_sync, name="OcChar", kind=TagKind.character)
|
|
char.fandom_id = fandom.id
|
|
# The character itself is used (tagged on an image) — that is the real-world
|
|
# shape: characters survive on their image_tag rows, and the fandom they
|
|
# point at must survive with them.
|
|
a = _make_artist(db_sync, slug="psf")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "c.jpg"), sha256="d" * 64,
|
|
)
|
|
db_sync.execute(image_tag.insert().values(
|
|
image_record_id=img.id, tag_id=char.id,
|
|
))
|
|
series = _make_tag(db_sync, name="EmptySeries", kind=TagKind.series)
|
|
simg = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "s.jpg"), sha256="e" * 64,
|
|
)
|
|
db_sync.add(SeriesPage(
|
|
series_tag_id=series.id, image_id=simg.id, page_number=1
|
|
))
|
|
_make_tag(db_sync, name="GenuinelyUnused")
|
|
db_sync.commit()
|
|
|
|
# dry-run count agrees with the delete: only the 1 truly-unused tag.
|
|
assert cleanup_service.prune_unused_tags(db_sync, dry_run=True)["count"] == 1
|
|
result = cleanup_service.prune_unused_tags(db_sync, dry_run=False)
|
|
assert result["deleted"] == 1
|
|
|
|
surviving = db_sync.execute(select(Tag.name)).scalars().all()
|
|
assert "OcChar" in surviving # used character
|
|
assert "Creux" in surviving # fandom referenced by a character
|
|
assert "EmptySeries" in surviving # series referenced by a chapter
|
|
assert "GenuinelyUnused" not in surviving
|
|
|
|
|
|
def test_prune_bare_posts_spares_linked_and_deletes_bare(db_sync, tmp_path):
|
|
# The LIVE delete (not just the preview) shares _bare_post_conditions, so a
|
|
# post is spared if it has ANY link — a primary image, a provenance (cross-
|
|
# posted/duplicate) image, or an attachment — and only the truly-bare shell
|
|
# is deleted (the empty-post flood, operator-flagged 2026-06-08).
|
|
a = _make_artist(db_sync, slug="pb")
|
|
img = _make_image(db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="a" * 64)
|
|
|
|
bare = Post(artist_id=a.id, external_post_id="bare1")
|
|
has_primary = Post(artist_id=a.id, external_post_id="prim")
|
|
has_prov = Post(artist_id=a.id, external_post_id="prov")
|
|
has_att = Post(artist_id=a.id, external_post_id="att")
|
|
db_sync.add_all([bare, has_primary, has_prov, has_att])
|
|
db_sync.flush()
|
|
|
|
img.primary_post_id = has_primary.id
|
|
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=has_prov.id))
|
|
db_sync.add(PostAttachment(
|
|
post_id=has_att.id, artist_id=a.id, sha256="b" * 64,
|
|
path="/store/b", original_filename="f.pdf", ext=".pdf", size_bytes=1,
|
|
))
|
|
db_sync.commit()
|
|
|
|
# dry-run count agrees with the delete: only the 1 truly-bare post.
|
|
assert cleanup_service.prune_bare_posts(db_sync, dry_run=True)["count"] == 1
|
|
result = cleanup_service.prune_bare_posts(db_sync, dry_run=False)
|
|
assert result["deleted"] == 1
|
|
|
|
surviving = db_sync.execute(select(Post.external_post_id)).scalars().all()
|
|
assert "bare1" not in surviving
|
|
assert set(surviving) == {"prim", "prov", "att"}
|
|
|
|
|
|
# --- reset_content_tagging ------------------------------------------
|
|
|
|
|
|
def test_reset_content_tagging_dry_run_counts_without_deleting(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="rc")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "r.jpg"), sha256="1" * 64,
|
|
)
|
|
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
|
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
|
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
|
_make_tag(db_sync, name="my-series", kind=TagKind.series)
|
|
db_sync.execute(image_tag.insert().values([
|
|
{"image_record_id": img.id, "tag_id": g.id},
|
|
{"image_record_id": img.id, "tag_id": c.id},
|
|
]))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.reset_content_tagging(db_sync, dry_run=True)
|
|
assert result["count"] == 2
|
|
assert result["by_kind"] == {"general": 1, "character": 1}
|
|
assert result["applications"] == 2
|
|
# Nothing deleted — all 4 fixture tags still present (non-system count;
|
|
# the seeded hygiene tags #128 also survive, by design).
|
|
assert db_sync.execute(
|
|
select(func.count(Tag.id)).where(Tag.is_system.is_(False))
|
|
).scalar_one() == 4
|
|
|
|
|
|
def test_reset_content_tagging_deletes_content_keeps_fandom_series(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="rc2")
|
|
img = _make_image(
|
|
db_sync, artist=a, path=str(tmp_path / "r2.jpg"), sha256="2" * 64,
|
|
)
|
|
g = _make_tag(db_sync, name="solo", kind=TagKind.general)
|
|
c = _make_tag(db_sync, name="naruto", kind=TagKind.character)
|
|
_make_tag(db_sync, name="Naruto", kind=TagKind.fandom)
|
|
s = _make_tag(db_sync, name="my-series", kind=TagKind.series)
|
|
db_sync.execute(image_tag.insert().values([
|
|
{"image_record_id": img.id, "tag_id": g.id},
|
|
{"image_record_id": img.id, "tag_id": c.id},
|
|
{"image_record_id": img.id, "tag_id": s.id}, # series membership
|
|
]))
|
|
db_sync.add(SeriesPage(
|
|
series_tag_id=s.id, image_id=img.id, page_number=1
|
|
))
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.reset_content_tagging(db_sync, dry_run=False)
|
|
assert result["deleted"] == 2
|
|
|
|
# general + character gone; fandom + series kept. Non-system only — the
|
|
# seeded hygiene tags (#128, kind=general) survive a reset by design.
|
|
kinds = db_sync.execute(
|
|
select(Tag.kind).where(Tag.is_system.is_(False))
|
|
).scalars().all()
|
|
kind_vals = {k.value if hasattr(k, "value") else str(k) for k in kinds}
|
|
assert kind_vals == {"fandom", "series"}
|
|
# series_page ordering survived.
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(SeriesPage)
|
|
).scalar_one() == 1
|
|
# Only the series image_tag association survived (content ones cascaded).
|
|
remaining = db_sync.execute(select(image_tag.c.tag_id)).scalars().all()
|
|
assert remaining == [s.id]
|
|
|
|
|
|
# ---- Tier-1 video dedup (#871) ------------------------------------------
|
|
|
|
|
|
def _make_video(db_sync, *, artist, path, sha256, duration, w, h, size=2000):
|
|
img = ImageRecord(
|
|
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
|
|
mime="video/mp4", origin="downloaded",
|
|
width=w, height=h, duration_seconds=duration,
|
|
)
|
|
db_sync.add(img)
|
|
db_sync.flush()
|
|
return img
|
|
|
|
|
|
def test_dedup_videos_collapses_and_relinks(db_sync, tmp_path):
|
|
"""Apply: a same-artist, same-duration, same-aspect video pair collapses to the
|
|
higher-res keeper, and the loser's post is re-pointed to the keeper first so it
|
|
doesn't lose the video."""
|
|
a = _make_artist(db_sync, slug="vd", name="VD")
|
|
keeper = _make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="k" * 64,
|
|
duration=120.0, w=1920, h=1080,
|
|
)
|
|
loser = _make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="l" * 64,
|
|
duration=120.3, w=1280, h=720, # within 1s, same 16:9, smaller
|
|
)
|
|
(tmp_path / "k.mp4").write_bytes(b"k")
|
|
(tmp_path / "l.mp4").write_bytes(b"l")
|
|
pa = Post(artist_id=a.id, external_post_id="pa")
|
|
pb = Post(artist_id=a.id, external_post_id="pb")
|
|
db_sync.add_all([pa, pb])
|
|
db_sync.flush()
|
|
db_sync.add(ImageProvenance(image_record_id=keeper.id, post_id=pa.id))
|
|
db_sync.add(ImageProvenance(image_record_id=loser.id, post_id=pb.id))
|
|
db_sync.commit()
|
|
keeper_id, loser_id, pb_id = keeper.id, loser.id, pb.id
|
|
|
|
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=False)
|
|
assert out["groups"] == 1
|
|
assert out["deleted"] == 1
|
|
assert out["relinked_posts"] == 1
|
|
|
|
db_sync.expire_all()
|
|
assert db_sync.get(ImageRecord, loser_id) is None
|
|
assert db_sync.get(ImageRecord, keeper_id) is not None
|
|
linked = db_sync.execute(
|
|
select(ImageProvenance.post_id)
|
|
.where(ImageProvenance.image_record_id == keeper_id)
|
|
).scalars().all()
|
|
assert pb_id in set(linked) # keeper inherited the loser's post
|
|
assert not (tmp_path / "l.mp4").exists() # redundant file removed
|
|
|
|
|
|
def test_dedup_videos_dry_run_keeps_everything(db_sync, tmp_path):
|
|
"""Preview reports the group without deleting (rule 93 shared predicate)."""
|
|
a = _make_artist(db_sync, slug="vd2", name="VD2")
|
|
_make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "k.mp4"), sha256="a" * 64,
|
|
duration=90.0, w=1920, h=1080,
|
|
)
|
|
_make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "l.mp4"), sha256="b" * 64,
|
|
duration=90.2, w=1280, h=720,
|
|
)
|
|
db_sync.commit()
|
|
|
|
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
|
|
assert out["groups"] == 1
|
|
assert out["redundant"] == 1
|
|
assert "deleted" not in out
|
|
assert db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
).scalar_one() == 2
|
|
|
|
|
|
def test_dedup_videos_distinct_durations_not_grouped(db_sync, tmp_path):
|
|
"""False-merge guard: clearly different durations stay separate."""
|
|
a = _make_artist(db_sync, slug="vd3", name="VD3")
|
|
_make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "x.mp4"), sha256="c" * 64,
|
|
duration=60.0, w=1920, h=1080,
|
|
)
|
|
_make_video(
|
|
db_sync, artist=a, path=str(tmp_path / "y.mp4"), sha256="d" * 64,
|
|
duration=200.0, w=1920, h=1080,
|
|
)
|
|
db_sync.commit()
|
|
|
|
out = cleanup_service.dedup_videos(db_sync, images_root=tmp_path, dry_run=True)
|
|
assert out["groups"] == 0
|
|
assert db_sync.execute(
|
|
select(func.count(ImageRecord.id))
|
|
).scalar_one() == 2
|
|
|
|
|
|
# ---- Gated-post blurred-preview cleanup (#874 follow-up) ------------------
|
|
|
|
|
|
class _GatedFakeClient:
|
|
"""Stub PatreonClient for collect_gated_previews: `posts` is a list of
|
|
(post_id, gated_bool, [filehashes])."""
|
|
|
|
def __init__(self, posts):
|
|
self._posts = posts
|
|
|
|
@staticmethod
|
|
def post_is_gated(post):
|
|
return (post.get("attributes") or {}).get("current_user_can_view") is False
|
|
|
|
def iter_posts(self, campaign_id, cursor=None):
|
|
for pid, gated, hashes in self._posts:
|
|
yield (
|
|
{
|
|
"id": pid,
|
|
"attributes": {"current_user_can_view": not gated},
|
|
"_h": hashes,
|
|
},
|
|
{},
|
|
None,
|
|
)
|
|
|
|
def extract_media(self, post, included):
|
|
from backend.app.services.patreon_client import MediaItem
|
|
return [
|
|
MediaItem(
|
|
url=f"https://cdn.patreon.com/{h}/x.jpg", filename="x.jpg",
|
|
kind="images", filehash=h, post_id=str(post["id"]),
|
|
)
|
|
for h in post["_h"]
|
|
]
|
|
|
|
|
|
def test_collect_gated_previews_only_gated_posts():
|
|
client = _GatedFakeClient([
|
|
("g1", True, ["aa", "bb"]),
|
|
("open1", False, ["cc"]),
|
|
("g2", True, []), # gated but feed served no media
|
|
])
|
|
out = cleanup_service.collect_gated_previews(client, "camp")
|
|
assert out == {"g1": ["aa", "bb"], "g2": []} # accessible post excluded
|
|
|
|
|
|
def _make_src(db_sync, artist):
|
|
s = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/x", enabled=True, config_overrides={},
|
|
)
|
|
db_sync.add(s)
|
|
db_sync.flush()
|
|
return s
|
|
|
|
|
|
def _make_dl_image(db_sync, *, artist, path, sha256, fh, size=500):
|
|
img = ImageRecord(
|
|
artist_id=artist.id, path=path, sha256=sha256, size_bytes=size,
|
|
mime="image/jpeg", origin="downloaded", source_filehash=fh,
|
|
)
|
|
db_sync.add(img)
|
|
db_sync.flush()
|
|
return img
|
|
|
|
|
|
def test_purge_gated_previews_deletes_only_hash_matched(db_sync, tmp_path):
|
|
"""#874: hash-matched blurred previews are deleted; a real file with a
|
|
different source_filehash (downloaded when access existed) and a
|
|
null-source_filehash image are both spared. A post left bare is removed; a post
|
|
that still has real content is kept. Ledger rows for the blurred hash are
|
|
cleared (so real media can re-ingest) while the real hash stays."""
|
|
a = _make_artist(db_sync, slug="gp", name="GP")
|
|
s = _make_src(db_sync, a)
|
|
blurred = _make_dl_image(
|
|
db_sync, artist=a, path=str(tmp_path / "blur.jpg"), sha256="1" * 64,
|
|
fh="b" * 32, size=500,
|
|
)
|
|
real = _make_dl_image(
|
|
db_sync, artist=a, path=str(tmp_path / "real.jpg"), sha256="2" * 64,
|
|
fh="r" * 32, size=999,
|
|
)
|
|
nohash = _make_dl_image(
|
|
db_sync, artist=a, path=str(tmp_path / "nh.jpg"), sha256="3" * 64,
|
|
fh=None, size=100,
|
|
)
|
|
blurred2 = _make_dl_image(
|
|
db_sync, artist=a, path=str(tmp_path / "blur2.jpg"), sha256="4" * 64,
|
|
fh="c" * 32, size=700,
|
|
)
|
|
for name in ("blur.jpg", "real.jpg", "nh.jpg", "blur2.jpg"):
|
|
(tmp_path / name).write_bytes(b"x")
|
|
p1 = Post(artist_id=a.id, source_id=s.id, external_post_id="g1")
|
|
p2 = Post(artist_id=a.id, source_id=s.id, external_post_id="g2")
|
|
db_sync.add_all([p1, p2])
|
|
db_sync.flush()
|
|
# g1: blurred + real + nohash; g2: only blurred2.
|
|
for img in (blurred, real, nohash):
|
|
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=p1.id, source_id=s.id))
|
|
db_sync.add(ImageProvenance(image_record_id=blurred2.id, post_id=p2.id, source_id=s.id))
|
|
db_sync.add_all([
|
|
PatreonSeenMedia(source_id=s.id, filehash="b" * 32, post_id="g1"),
|
|
PatreonSeenMedia(source_id=s.id, filehash="r" * 32, post_id="g1"),
|
|
PatreonSeenMedia(source_id=s.id, filehash="c" * 32, post_id="g2"),
|
|
])
|
|
db_sync.commit()
|
|
ids = {
|
|
"blurred": blurred.id, "real": real.id, "nohash": nohash.id,
|
|
"blurred2": blurred2.id, "p1": p1.id, "p2": p2.id,
|
|
}
|
|
|
|
# The gated feed now serves the blurred hashes for g1 + g2.
|
|
gated_map = {s.id: {"g1": ["b" * 32], "g2": ["c" * 32]}}
|
|
|
|
# Preview (rule 93): reports matches without deleting.
|
|
prev = cleanup_service.purge_gated_previews(
|
|
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=True,
|
|
)
|
|
assert prev["matched"] == 2
|
|
assert prev["gated_posts"] == 2
|
|
assert prev["unverifiable"] == 1 # the null-source_filehash image on g1
|
|
assert prev["reclaim_bytes"] == 500 + 700
|
|
assert db_sync.get(ImageRecord, ids["blurred"]) is not None # dry-run kept
|
|
|
|
# Apply.
|
|
out = cleanup_service.purge_gated_previews(
|
|
db_sync, gated_map=gated_map, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert out["deleted"] == 2
|
|
assert out["posts_deleted"] == 1 # g2 went bare; g1 still has real content
|
|
assert out["ledger_cleared"] == 2 # b* and c* cleared; r* retained
|
|
|
|
db_sync.expire_all()
|
|
assert db_sync.get(ImageRecord, ids["blurred"]) is None # blurred deleted
|
|
assert db_sync.get(ImageRecord, ids["blurred2"]) is None
|
|
assert db_sync.get(ImageRecord, ids["real"]) is not None # real spared
|
|
assert db_sync.get(ImageRecord, ids["nohash"]) is not None # unverifiable kept
|
|
assert not (tmp_path / "blur.jpg").exists()
|
|
assert (tmp_path / "real.jpg").exists()
|
|
# Ledger: blurred hashes cleared so the real media can re-ingest; real kept.
|
|
remaining = db_sync.execute(
|
|
select(PatreonSeenMedia.filehash).where(PatreonSeenMedia.source_id == s.id)
|
|
).scalars().all()
|
|
assert set(remaining) == {"r" * 32}
|
|
assert db_sync.get(Post, ids["p1"]) is not None # still has real + nohash
|
|
assert db_sync.get(Post, ids["p2"]) is None # bare → removed
|
|
|
|
|
|
def test_purge_gated_previews_no_gated_posts_is_noop(db_sync, tmp_path):
|
|
out = cleanup_service.purge_gated_previews(
|
|
db_sync, gated_map={}, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert out["matched"] == 0
|
|
assert out["deleted"] == 0
|
|
assert out["gated_posts"] == 0
|
|
|
|
|
|
# ---- duplicate-post reconciliation (gallery-dl → native, milestone #73) ----
|
|
|
|
|
|
def test_reconcile_merges_attachmentid_and_postid_dupes(db_sync, tmp_path):
|
|
# gallery-dl row (keyed by attachment id, holds the image + body/date) +
|
|
# native row (keyed by the real post id, bare) → unified onto ONE post-id-
|
|
# keyed keeper, image moved over, keeper backfilled from the legacy row.
|
|
a = _make_artist(db_sync, slug="rc1")
|
|
img = _make_image(db_sync, artist=a, path=str(tmp_path / "x.jpg"), sha256="c" * 64)
|
|
legacy = Post(
|
|
artist_id=a.id, external_post_id="711509",
|
|
raw_metadata={"post_id": 1923726, "category": "subscribestar"},
|
|
description="real body", post_title="T",
|
|
post_date=datetime(2025, 6, 20, tzinfo=UTC),
|
|
)
|
|
native = Post(
|
|
artist_id=a.id, external_post_id="1923726",
|
|
raw_metadata={"post_id": 1923726, "category": "subscribestar"},
|
|
)
|
|
db_sync.add_all([legacy, native])
|
|
db_sync.flush()
|
|
img.primary_post_id = legacy.id
|
|
img_id, native_id, legacy_id = img.id, native.id, legacy.id
|
|
db_sync.commit()
|
|
|
|
prev = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=True)
|
|
assert prev["groups"] == 1
|
|
assert prev["posts_to_merge"] == 1
|
|
res = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
|
|
assert res["merged"] == 1
|
|
|
|
rows = db_sync.execute(select(Post.id, Post.external_post_id)).all()
|
|
assert len(rows) == 1
|
|
keeper_id, keeper_epid = rows[0]
|
|
assert keeper_id == native_id # native (post-id-keyed) row survives
|
|
assert keeper_epid == "1923726"
|
|
# image moved to keeper; legacy row gone; keeper backfilled.
|
|
assert db_sync.execute(
|
|
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
|
|
).scalar_one() == native_id
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(Post).where(Post.id == legacy_id)
|
|
).scalar_one() == 0
|
|
desc, pdate = db_sync.execute(
|
|
select(Post.description, Post.post_date).where(Post.id == native_id)
|
|
).one()
|
|
assert desc == "real body"
|
|
assert pdate is not None
|
|
|
|
|
|
def test_reconcile_noop_when_posts_unique(db_sync):
|
|
a = _make_artist(db_sync, slug="rc2")
|
|
db_sync.add_all([
|
|
Post(artist_id=a.id, external_post_id="100", raw_metadata={"post_id": 100}),
|
|
Post(artist_id=a.id, external_post_id="200", raw_metadata={"post_id": 200}),
|
|
])
|
|
db_sync.commit()
|
|
assert cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=True)["groups"] == 0
|
|
res = cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
|
|
assert res["merged"] == 0
|
|
assert db_sync.execute(select(func.count()).select_from(Post)).scalar_one() == 2
|
|
|
|
|
|
def test_reconcile_dedups_provenance_collision(db_sync, tmp_path):
|
|
# Both dup rows link the SAME image via provenance; merging must DROP the
|
|
# colliding loser row (unique image_record_id+post_id), not raise.
|
|
a = _make_artist(db_sync, slug="rc3")
|
|
img = _make_image(db_sync, artist=a, path=str(tmp_path / "y.jpg"), sha256="d" * 64)
|
|
legacy = Post(artist_id=a.id, external_post_id="9", raw_metadata={"post_id": 55})
|
|
native = Post(artist_id=a.id, external_post_id="55", raw_metadata={"post_id": 55})
|
|
db_sync.add_all([legacy, native])
|
|
db_sync.flush()
|
|
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=legacy.id))
|
|
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=native.id))
|
|
img_id, native_id = img.id, native.id
|
|
db_sync.commit()
|
|
|
|
cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
|
|
prov = db_sync.execute(
|
|
select(ImageProvenance.post_id).where(ImageProvenance.image_record_id == img_id)
|
|
).scalars().all()
|
|
assert prov == [native_id]
|
|
|
|
|
|
def test_reconcile_preserves_from_attachment_on_provenance_collision(db_sync, tmp_path):
|
|
# The gallery-dl loser row recorded which archive the image came out of
|
|
# (#87); the native keeper row didn't. Merging drops the loser provenance row
|
|
# on the (image, post) collision — but must first carry its from_attachment_id
|
|
# onto the keeper so the containing-archive linkage survives.
|
|
a = _make_artist(db_sync, slug="rc4")
|
|
img = _make_image(db_sync, artist=a, path=str(tmp_path / "z.jpg"), sha256="e" * 64)
|
|
legacy = Post(artist_id=a.id, external_post_id="9", raw_metadata={"post_id": 77})
|
|
native = Post(artist_id=a.id, external_post_id="77", raw_metadata={"post_id": 77})
|
|
db_sync.add_all([legacy, native])
|
|
db_sync.flush()
|
|
att = PostAttachment(
|
|
post_id=legacy.id, artist_id=a.id, sha256="f" * 64,
|
|
path=str(tmp_path / "bundle.cbz"), original_filename="bundle.cbz",
|
|
ext=".cbz", size_bytes=9,
|
|
)
|
|
db_sync.add(att)
|
|
db_sync.flush()
|
|
db_sync.add(ImageProvenance(
|
|
image_record_id=img.id, post_id=legacy.id, from_attachment_id=att.id,
|
|
))
|
|
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=native.id))
|
|
img_id, native_id, att_id = img.id, native.id, att.id
|
|
db_sync.commit()
|
|
|
|
cleanup_service.reconcile_duplicate_posts(db_sync, dry_run=False)
|
|
rows = db_sync.execute(
|
|
select(ImageProvenance.post_id, ImageProvenance.from_attachment_id)
|
|
.where(ImageProvenance.image_record_id == img_id)
|
|
).all()
|
|
assert rows == [(native_id, att_id)]
|
|
|
|
|
|
# --- reclaim_orphaned_attachments -----------------------------------
|
|
|
|
|
|
def _store_blob(root, sha, *, ext=".pdf", age_hours=48, data=b"blob"):
|
|
"""Write a file into the sha-addressed attachment store, aged past the
|
|
min-age guard by default."""
|
|
d = root / "attachments" / sha[:3]
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
p = d / f"{sha}{ext}"
|
|
p.write_bytes(data)
|
|
old = datetime.now(UTC).timestamp() - age_hours * 3600
|
|
os.utime(p, (old, old))
|
|
return p
|
|
|
|
|
|
def _attachment(db_sync, *, sha, post=None, artist=None):
|
|
att = PostAttachment(
|
|
post_id=post.id if post else None,
|
|
artist_id=artist.id if artist else None,
|
|
sha256=sha, path=f"/store/{sha[:3]}/f.pdf",
|
|
original_filename="f.pdf", ext=".pdf", size_bytes=4,
|
|
)
|
|
db_sync.add(att)
|
|
db_sync.flush()
|
|
return att
|
|
|
|
|
|
def test_reclaim_attachments_dry_run_projects_without_mutating(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="recl-dry")
|
|
p = Post(artist_id=a.id, external_post_id="rd-1")
|
|
db_sync.add(p)
|
|
db_sync.flush()
|
|
kept_sha, orphan_sha = "aa11".ljust(64, "0"), "bb22".ljust(64, "0")
|
|
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
|
_attachment(db_sync, sha=orphan_sha) # both FKs NULL → orphan
|
|
db_sync.commit()
|
|
kept_blob = _store_blob(tmp_path, kept_sha)
|
|
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=True,
|
|
)
|
|
assert result["rows"] == 1
|
|
assert result["files"] == 1
|
|
assert result["bytes"] == orphan_blob.stat().st_size
|
|
|
|
# Nothing actually happened.
|
|
assert kept_blob.exists() and orphan_blob.exists()
|
|
assert db_sync.execute(
|
|
select(func.count(PostAttachment.id))
|
|
).scalar_one() == 2
|
|
|
|
|
|
def test_reclaim_attachments_apply_deletes_rows_and_unlinks_blobs(db_sync, tmp_path):
|
|
a = _make_artist(db_sync, slug="recl-apply")
|
|
p = Post(artist_id=a.id, external_post_id="ra-1")
|
|
db_sync.add(p)
|
|
db_sync.flush()
|
|
kept_sha, orphan_sha = "cc33".ljust(64, "0"), "dd44".ljust(64, "0")
|
|
_attachment(db_sync, sha=kept_sha, post=p, artist=a)
|
|
_attachment(db_sync, sha=orphan_sha)
|
|
db_sync.commit()
|
|
kept_blob = _store_blob(tmp_path, kept_sha)
|
|
orphan_blob = _store_blob(tmp_path, orphan_sha)
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert result["rows"] == 1
|
|
assert result["files"] == 1
|
|
|
|
assert kept_blob.exists() # still referenced
|
|
assert not orphan_blob.exists() # nothing points at it any more
|
|
surviving = db_sync.execute(select(PostAttachment.sha256)).scalars().all()
|
|
assert surviving == [kept_sha]
|
|
|
|
|
|
def test_reclaim_attachments_preview_matches_apply(db_sync, tmp_path):
|
|
"""Rule 93 — the dry-run's numbers are what the apply does. The projection
|
|
has to negate the orphan predicate to be honest about blobs the delete is
|
|
about to free, so this is the assertion that catches getting that backwards.
|
|
"""
|
|
orphan_sha = "ee55".ljust(64, "0")
|
|
_attachment(db_sync, sha=orphan_sha)
|
|
db_sync.commit()
|
|
_store_blob(tmp_path, orphan_sha)
|
|
|
|
projected = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=True,
|
|
)
|
|
applied = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
for key in ("rows", "files", "bytes"):
|
|
assert projected[key] == applied[key], key
|
|
assert applied["rows"] == 1 and applied["files"] == 1
|
|
|
|
|
|
def test_reclaim_attachments_keeps_shared_blob_while_any_row_remains(db_sync, tmp_path):
|
|
"""The refcount case this whole sweep exists for: one sha-addressed blob
|
|
backs several rows, so deleting SOME of them must not free the file."""
|
|
a = _make_artist(db_sync, slug="recl-shared")
|
|
p = Post(artist_id=a.id, external_post_id="rs-1")
|
|
db_sync.add(p)
|
|
db_sync.flush()
|
|
sha = "ff66".ljust(64, "0")
|
|
_attachment(db_sync, sha=sha, post=p, artist=a) # attributed — survives
|
|
_attachment(db_sync, sha=sha) # orphan — deleted
|
|
db_sync.commit()
|
|
blob = _store_blob(tmp_path, sha)
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert result["rows"] == 1 # the orphan row went
|
|
assert result["files"] == 0 # the blob did NOT
|
|
assert blob.exists()
|
|
|
|
|
|
def test_reclaim_attachments_spares_filesystem_import_rows(db_sync, tmp_path):
|
|
"""post_id NULL with an artist_id is the deliberate filesystem-import shape
|
|
(importer._capture_attachment), not an orphan — it is still attributed."""
|
|
a = _make_artist(db_sync, slug="recl-fsimport")
|
|
sha = "1177".ljust(64, "0")
|
|
_attachment(db_sync, sha=sha, artist=a) # post NULL, artist set
|
|
db_sync.commit()
|
|
blob = _store_blob(tmp_path, sha)
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert result["rows"] == 0
|
|
assert result["files"] == 0
|
|
assert blob.exists()
|
|
assert db_sync.execute(
|
|
select(func.count(PostAttachment.id))
|
|
).scalar_one() == 1
|
|
|
|
|
|
def test_reclaim_attachments_skips_recent_and_staging_files(db_sync, tmp_path):
|
|
"""A blob is written BEFORE its row commits, so a just-stored file with no
|
|
row is in-flight, not orphaned. `.partial` staging files belong to
|
|
cleanup_orphaned_temp_files and must be left alone either way."""
|
|
fresh_sha, staged_sha = "2288".ljust(64, "0"), "3399".ljust(64, "0")
|
|
fresh = _store_blob(tmp_path, fresh_sha, age_hours=0)
|
|
staged = _store_blob(tmp_path, staged_sha, ext=".pdf.partial")
|
|
db_sync.commit()
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert result["files"] == 0
|
|
assert result["skipped_recent"] == 1
|
|
assert fresh.exists() and staged.exists()
|
|
|
|
|
|
def test_reclaim_attachments_ignores_non_sha_named_files(db_sync, tmp_path):
|
|
"""The walk must only judge files it can identify as store blobs — anything
|
|
else under the root is none of its business."""
|
|
d = tmp_path / "attachments" / "zzz"
|
|
d.mkdir(parents=True)
|
|
stray = d / "notes.txt"
|
|
stray.write_text("not a blob")
|
|
old = datetime.now(UTC).timestamp() - 48 * 3600
|
|
os.utime(stray, (old, old))
|
|
|
|
result = cleanup_service.reclaim_orphaned_attachments(
|
|
db_sync, images_root=tmp_path, dry_run=False,
|
|
)
|
|
assert result["files"] == 0
|
|
assert stray.exists()
|