"""The Discord repair (#3999): forget the broken downloads, delete them, backfill again. The danger in a repair like this is reach. It deletes files and clears gallery-dl's memory of what it fetched, so most of these tests pin what it must NOT touch: other platforms' archive entries, Discord server assets, images outside a `discord/None/` folder, and a correctly named Discord file that happens to share a folder name. """ import sqlite3 import pytest from sqlalchemy import func, select from backend.app.models import Artist, ImageRecord, Source from backend.app.services import discord_repair from backend.app.services.gallery_dl import archive_path from backend.app.services.source_service import BACKFILL_MAX_CHUNKS def _archive(tmp_path, entries): path = archive_path(tmp_path) path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(path) # The table gallery-dl itself creates (gallery_dl/archive.py). conn.execute("CREATE TABLE archive (entry TEXT PRIMARY KEY) WITHOUT ROWID") conn.executemany("INSERT INTO archive (entry) VALUES (?)", [(e,) for e in entries]) conn.commit() conn.close() return path def _entries(path): conn = sqlite3.connect(path) try: return sorted(r[0] for r in conn.execute("SELECT entry FROM archive")) finally: conn.close() # --- the archive ------------------------------------------------------------ def test_only_discord_message_entries_are_forgotten(tmp_path): path = _archive(tmp_path, [ "discord300000000000000003_1", "discord300000000000000004_2", "discordasset_400000000000000004_9", "hentaifoundry12345", ]) assert discord_repair.count_archive_entries(path) == 2 assert discord_repair.forget_archive_entries(path) == 2 assert _entries(path) == ["discordasset_400000000000000004_9", "hentaifoundry12345"] def test_counting_changes_nothing(tmp_path): path = _archive(tmp_path, ["discord1_1", "discord2_1"]) discord_repair.count_archive_entries(path) assert _entries(path) == ["discord1_1", "discord2_1"] def test_a_missing_archive_or_table_is_zero_not_an_error(tmp_path): assert discord_repair.count_archive_entries(tmp_path / "nope.sqlite3") == 0 empty = tmp_path / "empty.sqlite3" sqlite3.connect(empty).close() assert discord_repair.forget_archive_entries(empty) == 0 # --- the folders ------------------------------------------------------------ def test_broken_directories_finds_every_artist_folder_including_case_twins(tmp_path): for artist in ("Conto", "conto", "knuxy"): (tmp_path / artist / "discord" / "None").mkdir(parents=True) (tmp_path / "conto" / "discord" / "general").mkdir(parents=True) (tmp_path / "conto" / "patreon" / "None").mkdir(parents=True) found = [ p.relative_to(tmp_path).as_posix() for p in discord_repair.broken_directories(tmp_path) ] assert found == ["Conto/discord/None", "conto/discord/None", "knuxy/discord/None"] def test_sweep_removes_leftover_sidecars_and_the_empty_folder(tmp_path): d = tmp_path / "conto" / "discord" / "None" d.mkdir(parents=True) (d / "image.json").write_text("{}") (d / "20240716_None_rejected.png").write_bytes(b"x") removed, gone = discord_repair._sweep_directory(d) assert (removed, gone) == (2, True) assert not d.exists() # --- the whole repair, against the database ----------------------------------- def _image(db_sync, artist, path, sha): img = ImageRecord( artist_id=artist.id, path=str(path), sha256=sha * 64, size_bytes=100, mime="image/png", origin="downloaded", ) db_sync.add(img) return img @pytest.mark.integration def test_repair_deletes_only_broken_discord_images_and_rearms_discord_backfills( db_sync, tmp_path, ): artist = Artist(name="Conto", slug="conto") db_sync.add(artist) db_sync.flush() discord = Source( artist_id=artist.id, platform="discord", url="https://discord.com/channels/1/2", enabled=True, config_overrides={"_backfill_state": "complete", "_backfill_cursor": "x"}, ) patreon = Source( artist_id=artist.id, platform="patreon", url="https://www.patreon.com/conto", enabled=True, config_overrides={}, ) db_sync.add_all([discord, patreon]) broken_dir = tmp_path / "conto" / "discord" / "None" good_dir = tmp_path / "conto" / "discord" / "general" patreon_dir = tmp_path / "conto" / "patreon" / "None" for d in (broken_dir, good_dir, patreon_dir): d.mkdir(parents=True) broken = broken_dir / "20240716_None_image.png" good = good_dir / "20240716_300000000000000003_01_image.png" # A `None` folder under another platform, and a `_None_` name outside one. # Neither is the bug's pair, so neither may be touched. elsewhere = patreon_dir / "20240716_None_image.png" for f in (broken, good, elsewhere): f.write_bytes(b"x") (broken_dir / "image.json").write_text("{}") _image(db_sync, artist, broken, "a") _image(db_sync, artist, good, "b") _image(db_sync, artist, elsewhere, "c") db_sync.commit() _archive(tmp_path, ["discord300000000000000003_1", "hentaifoundry1"]) preview = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=True) assert preview["images"] == 1 assert preview["archive_entries"] == 1 assert broken.exists() result = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=False) assert result["images_deleted"] == 1 assert result["remaining"] == 0 assert result["backfills_started"] == 1 assert not broken.exists() and not broken_dir.exists() assert good.exists() and elsewhere.exists() assert _entries(archive_path(tmp_path)) == ["hentaifoundry1"] kept = db_sync.execute(select(func.count(ImageRecord.id))).scalar_one() assert kept == 2 rows = { platform: (runs, overrides) for platform, runs, overrides in db_sync.execute( select(Source.platform, Source.backfill_runs_remaining, Source.config_overrides) ).all() } runs, overrides = rows["discord"] assert runs == BACKFILL_MAX_CHUNKS assert overrides["_backfill_state"] == "running" assert "_backfill_cursor" not in overrides # Only Discord sources are re-armed. assert rows["patreon"][1] == {}