feat: Settings can re-download the Discord images the None naming broke (3999)
CI / lint (push) Successful in 3s
Build images / sign-extension (push) Successful in 4s
CI / extension-version (push) Successful in 2s
Build images / build-agent (push) Successful in 6s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 33s
Build images / build-web (push) Successful in 1m9s
Build images / smoke-web (push) Skipped
Build images / build-ml (push) Successful in 2m3s
Build images / promote (push) Skipped
CI / integration (push) Successful in 2m35s
CI / lint (push) Successful in 3s
Build images / sign-extension (push) Successful in 4s
CI / extension-version (push) Successful in 2s
Build images / build-agent (push) Successful in 6s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 33s
Build images / build-web (push) Successful in 1m9s
Build images / smoke-web (push) Skipped
Build images / build-ml (push) Successful in 2m3s
Build images / promote (push) Skipped
CI / integration (push) Successful in 2m35s
The operator chose a clean re-download over relinking in place. The ~1,600 broken files can't be relinked reliably: their message ids are gone, and their sidecars collided.
Settings → Maintenance → "Repair Discord downloads" previews, then applies:
- Deletes every image whose path is `…/discord/None/<8 digits>_None_…`. Both the folder and the name are required, because that pair is only what the bug produced. It reuses cleanup_service.delete_images for the record and file deletes.
- Sweeps the leftover collided sidecars from those folders and removes the emptied folders.
- Only then clears gallery-dl's archive rows `discord%`, excluding `discordasset_%`. Upstream keys message attachments as `discord{message_id}_{num}`. Since the broken files lost their message ids, per-source forgetting is impossible. Every pre-fix Discord download is broken, and files fetched after the fix still exist on disk, so gallery-dl's `skip` won't re-fetch them.
- Arms a fresh backfill on every Discord source.
The apply defaults to preview at both the route and the task, runs on maintenance_long, and is never on a beat. The card uses the confirm-dialog pattern of AttachmentReclaimCard.
Supporting refactors, with no behaviour change:
- gallery_dl.archive_path() is the single definition of the archive location.
- source_service.arm_backfill() is the mutation start_backfill already did, now shared with the sync repair.
Tests (tests/test_discord_repair.py):
- The archive clear leaves other platforms and Discord assets alone, and counting mutates nothing.
- Case-twin artist folders are both found.
- The folder sweep works.
- An integration run shows only the broken image goes. A correctly named Discord file and a `None` folder under Patreon survive, and only Discord sources are re-armed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
"""The Discord repair (#3999): forget the broken downloads, delete them, backfill again.
|
||||
|
||||
The danger in a repair like this is reach. It deletes files and clears
|
||||
gallery-dl's memory of what it fetched, so most of these tests pin what it must
|
||||
NOT touch: other platforms' archive entries, Discord server assets, images
|
||||
outside a `discord/None/` folder, and a correctly named Discord file that
|
||||
happens to share a folder name.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import Artist, ImageRecord, Source
|
||||
from backend.app.services import discord_repair
|
||||
from backend.app.services.gallery_dl import archive_path
|
||||
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
|
||||
def _archive(tmp_path, entries):
|
||||
path = archive_path(tmp_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path)
|
||||
# The table gallery-dl itself creates (gallery_dl/archive.py).
|
||||
conn.execute("CREATE TABLE archive (entry TEXT PRIMARY KEY) WITHOUT ROWID")
|
||||
conn.executemany("INSERT INTO archive (entry) VALUES (?)", [(e,) for e in entries])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return path
|
||||
|
||||
|
||||
def _entries(path):
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
return sorted(r[0] for r in conn.execute("SELECT entry FROM archive"))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# --- the archive ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_only_discord_message_entries_are_forgotten(tmp_path):
|
||||
path = _archive(tmp_path, [
|
||||
"discord300000000000000003_1",
|
||||
"discord300000000000000004_2",
|
||||
"discordasset_400000000000000004_9",
|
||||
"hentaifoundry12345",
|
||||
])
|
||||
assert discord_repair.count_archive_entries(path) == 2
|
||||
assert discord_repair.forget_archive_entries(path) == 2
|
||||
assert _entries(path) == ["discordasset_400000000000000004_9", "hentaifoundry12345"]
|
||||
|
||||
|
||||
def test_counting_changes_nothing(tmp_path):
|
||||
path = _archive(tmp_path, ["discord1_1", "discord2_1"])
|
||||
discord_repair.count_archive_entries(path)
|
||||
assert _entries(path) == ["discord1_1", "discord2_1"]
|
||||
|
||||
|
||||
def test_a_missing_archive_or_table_is_zero_not_an_error(tmp_path):
|
||||
assert discord_repair.count_archive_entries(tmp_path / "nope.sqlite3") == 0
|
||||
empty = tmp_path / "empty.sqlite3"
|
||||
sqlite3.connect(empty).close()
|
||||
assert discord_repair.forget_archive_entries(empty) == 0
|
||||
|
||||
|
||||
# --- the folders ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_broken_directories_finds_every_artist_folder_including_case_twins(tmp_path):
|
||||
for artist in ("Conto", "conto", "knuxy"):
|
||||
(tmp_path / artist / "discord" / "None").mkdir(parents=True)
|
||||
(tmp_path / "conto" / "discord" / "general").mkdir(parents=True)
|
||||
(tmp_path / "conto" / "patreon" / "None").mkdir(parents=True)
|
||||
found = [
|
||||
p.relative_to(tmp_path).as_posix()
|
||||
for p in discord_repair.broken_directories(tmp_path)
|
||||
]
|
||||
assert found == ["Conto/discord/None", "conto/discord/None", "knuxy/discord/None"]
|
||||
|
||||
|
||||
def test_sweep_removes_leftover_sidecars_and_the_empty_folder(tmp_path):
|
||||
d = tmp_path / "conto" / "discord" / "None"
|
||||
d.mkdir(parents=True)
|
||||
(d / "image.json").write_text("{}")
|
||||
(d / "20240716_None_rejected.png").write_bytes(b"x")
|
||||
removed, gone = discord_repair._sweep_directory(d)
|
||||
assert (removed, gone) == (2, True)
|
||||
assert not d.exists()
|
||||
|
||||
|
||||
# --- the whole repair, against the database -----------------------------------
|
||||
|
||||
|
||||
def _image(db_sync, artist, path, sha):
|
||||
img = ImageRecord(
|
||||
artist_id=artist.id, path=str(path), sha256=sha * 64, size_bytes=100,
|
||||
mime="image/png", origin="downloaded",
|
||||
)
|
||||
db_sync.add(img)
|
||||
return img
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_repair_deletes_only_broken_discord_images_and_rearms_discord_backfills(
|
||||
db_sync, tmp_path,
|
||||
):
|
||||
artist = Artist(name="Conto", slug="conto")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
discord = Source(
|
||||
artist_id=artist.id, platform="discord", url="https://discord.com/channels/1/2",
|
||||
enabled=True, config_overrides={"_backfill_state": "complete", "_backfill_cursor": "x"},
|
||||
)
|
||||
patreon = Source(
|
||||
artist_id=artist.id, platform="patreon", url="https://www.patreon.com/conto",
|
||||
enabled=True, config_overrides={},
|
||||
)
|
||||
db_sync.add_all([discord, patreon])
|
||||
|
||||
broken_dir = tmp_path / "conto" / "discord" / "None"
|
||||
good_dir = tmp_path / "conto" / "discord" / "general"
|
||||
patreon_dir = tmp_path / "conto" / "patreon" / "None"
|
||||
for d in (broken_dir, good_dir, patreon_dir):
|
||||
d.mkdir(parents=True)
|
||||
broken = broken_dir / "20240716_None_image.png"
|
||||
good = good_dir / "20240716_300000000000000003_01_image.png"
|
||||
# A `None` folder under another platform, and a `_None_` name outside one.
|
||||
# Neither is the bug's pair, so neither may be touched.
|
||||
elsewhere = patreon_dir / "20240716_None_image.png"
|
||||
for f in (broken, good, elsewhere):
|
||||
f.write_bytes(b"x")
|
||||
(broken_dir / "image.json").write_text("{}")
|
||||
_image(db_sync, artist, broken, "a")
|
||||
_image(db_sync, artist, good, "b")
|
||||
_image(db_sync, artist, elsewhere, "c")
|
||||
db_sync.commit()
|
||||
_archive(tmp_path, ["discord300000000000000003_1", "hentaifoundry1"])
|
||||
|
||||
preview = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=True)
|
||||
assert preview["images"] == 1
|
||||
assert preview["archive_entries"] == 1
|
||||
assert broken.exists()
|
||||
|
||||
result = discord_repair.repair_discord_downloads(db_sync, images_root=tmp_path, dry_run=False)
|
||||
assert result["images_deleted"] == 1
|
||||
assert result["remaining"] == 0
|
||||
assert result["backfills_started"] == 1
|
||||
|
||||
assert not broken.exists() and not broken_dir.exists()
|
||||
assert good.exists() and elsewhere.exists()
|
||||
assert _entries(archive_path(tmp_path)) == ["hentaifoundry1"]
|
||||
|
||||
kept = db_sync.execute(select(func.count(ImageRecord.id))).scalar_one()
|
||||
assert kept == 2
|
||||
|
||||
rows = {
|
||||
platform: (runs, overrides)
|
||||
for platform, runs, overrides in db_sync.execute(
|
||||
select(Source.platform, Source.backfill_runs_remaining, Source.config_overrides)
|
||||
).all()
|
||||
}
|
||||
runs, overrides = rows["discord"]
|
||||
assert runs == BACKFILL_MAX_CHUNKS
|
||||
assert overrides["_backfill_state"] == "running"
|
||||
assert "_backfill_cursor" not in overrides
|
||||
# Only Discord sources are re-armed.
|
||||
assert rows["patreon"][1] == {}
|
||||
Reference in New Issue
Block a user