"""Repair the Discord downloads made before the naming fix (issue #3999). Until dc840fe, gallery-dl's Discord patterns asked for keys the extractor never emits, so every Discord download landed as `/discord/None/_None_`, next to a sidecar named after the attachment's ORIGINAL name. That broke two things: * **No Post, no date.** `find_sidecar` can never pair those names, so these files were imported as loose images with no Post, and a card shows the download time. * **No trustworthy metadata to relink from.** Every `image.png` in a channel wrote the same `image.json`, so the surviving sidecar describes whichever message was written last. The message id is gone from the filename too. The operator chose a clean re-download (2026-09-13) over relinking in place: delete the broken files and their records, make gallery-dl forget it fetched them, and backfill every Discord source again under the fixed naming. ## Why the archive is cleared for ALL of Discord gallery-dl records a download as `discord{message_id}_{num}` (upstream `DiscordExtractor.archive_fmt`, prefixed with the category). The broken files lost their message ids, so there is no way to forget one source's entries and not another's. Every Discord download made before the fix is broken, so forgetting all of them is exactly right. Anything downloaded AFTER the fix still exists on disk under its correct name, and gallery-dl's `skip` sees the file and does not fetch it again. ## What it does not touch Discord posts FC grouped itself (#388 E2) are built from Posts, and these files never had one, so there is nothing grouped to unwind. Images outside a `discord/None/` directory are never selected: the path filter requires both the `None` directory and the `_None_` filename, the pair only the bug produced. Operator-triggered only (Settings, preview first). Never on a beat. """ from __future__ import annotations import logging import sqlite3 from pathlib import Path from sqlalchemy import func, select from sqlalchemy.orm import Session from ..models import ImageRecord, Source from .cleanup_service import delete_images from .gallery_dl import archive_path from .source_service import arm_backfill log = logging.getLogger(__name__) # `%` and `_` are LIKE wildcards, so the literal underscores around None are # escaped. `________` is the eight-digit date prefix the old pattern wrote. _BROKEN_PATH_LIKE = r"%/discord/None/________\_None\_%" # Upstream keys asset downloads as `asset_{server_id}_{id}`. FC never fetches # server assets, but excluding them keeps this to exactly the message # attachments the bug mangled. _ARCHIVE_SQL_MATCH = r"entry LIKE 'discord%' AND entry NOT LIKE 'discordasset\_%' ESCAPE '\'" _COUNT_SQL = "SELECT COUNT(*) FROM archive WHERE " + _ARCHIVE_SQL_MATCH _DELETE_SQL = "DELETE FROM archive WHERE " + _ARCHIVE_SQL_MATCH def broken_directories(images_root: Path) -> list[Path]: """Every `/discord/None` directory. Artist folders that differ only by case (`Conto` and `conto`) are separate directories and both are found.""" return sorted(d for d in Path(images_root).glob("*/discord/None") if d.is_dir()) def count_archive_entries(archive: Path) -> int: return _archive(archive, delete=False) def forget_archive_entries(archive: Path) -> int: return _archive(archive, delete=True) def _archive(archive: Path, *, delete: bool) -> int: if not archive.is_file(): return 0 # A download running at the same moment holds this file briefly. Waiting # 30s for its lock beats failing the repair over a transient contention. conn = sqlite3.connect(str(archive), timeout=30) try: has_table = conn.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name='archive'" ).fetchone() if not has_table: return 0 if not delete: return conn.execute(_COUNT_SQL).fetchone()[0] cur = conn.execute(_DELETE_SQL) conn.commit() return cur.rowcount finally: conn.close() def _sweep_directory(directory: Path) -> tuple[int, bool]: """Remove what the record deletes left behind: the collided sidecars, plus any file that never became a record (a quarantined or rejected download). Then the directory itself, if it is empty. Returns (files removed, dir removed).""" removed = 0 for f in directory.iterdir(): if f.is_file(): try: f.unlink() removed += 1 except OSError as exc: log.warning("discord repair: could not remove %s: %s", f, exc) try: directory.rmdir() return removed, True except OSError: return removed, False def repair_discord_downloads( session: Session, *, images_root: Path, dry_run: bool, ) -> dict: images_root = Path(images_root) archive = archive_path(images_root) broken = select(ImageRecord.id, ImageRecord.size_bytes).where( ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\") ) rows = session.execute(broken).all() image_ids = [r.id for r in rows] directories = broken_directories(images_root) sources = session.execute( select(Source).where(Source.platform == "discord") ).scalars().all() summary = { "images": len(image_ids), "bytes": sum(r.size_bytes or 0 for r in rows), "directories": len(directories), "sources": len(sources), "enabled_sources": sum(1 for s in sources if s.enabled), } if dry_run: summary["archive_entries"] = count_archive_entries(archive) return summary deleted = delete_images(session, image_ids=image_ids, images_root=images_root) swept = 0 directories_removed = 0 for d in directories: n, gone = _sweep_directory(d) swept += n directories_removed += int(gone) # Only after the files are gone. Forgetting first and failing half way would # leave gallery-dl free to re-fetch into a directory still full of the old # copies. forgotten = forget_archive_entries(archive) for source in sources: arm_backfill(source) session.commit() remaining = session.execute( select(func.count(ImageRecord.id)).where( ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\") ) ).scalar_one() summary.update( images_deleted=deleted["images_deleted"], files_failed=deleted["files_failed"], leftover_files_removed=swept, directories_removed=directories_removed, archive_entries=forgotten, backfills_started=len(sources), remaining=remaining, ) log.info("discord repair applied: %s", summary) return summary