diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 9f676be..41b865f 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -475,6 +475,20 @@ async def trigger_reclaim_attachments(): return _queued(async_result) +@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"]) +async def trigger_repair_discord_downloads(): + """Clean re-download of the Discord files broken by the `None` naming + (#3999). Body {"dry_run": bool}; dry_run is the DEFAULT, because the apply + deletes files and makes gallery-dl forget every Discord download. Returns the + Celery task id — poll /maintenance/task-result/ for the summary.""" + from ..tasks.admin import repair_discord_downloads_task + + body = await request.get_json(silent=True) or {} + dry_run = bool(body.get("dry_run", True)) + async_result = repair_discord_downloads_task.delay(dry_run=dry_run) + return _queued(async_result) + + @admin_bp.route("/maintenance/dedup-videos", methods=["POST"]) async def trigger_dedup_videos(): """Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews diff --git a/backend/app/services/discord_repair.py b/backend/app/services/discord_repair.py new file mode 100644 index 0000000..891e13a --- /dev/null +++ b/backend/app/services/discord_repair.py @@ -0,0 +1,183 @@ +"""Repair the Discord downloads made before the naming fix (issue #3999). + +Until dc840fe, gallery-dl's Discord patterns asked for keys the extractor never +emits, so every Discord download landed as +`/discord/None/_None_`, next to a sidecar named +after the attachment's ORIGINAL name. That broke two things: + +* **No Post, no date.** `find_sidecar` can never pair those names, so these + files were imported as loose images with no Post, and a card shows the + download time. +* **No trustworthy metadata to relink from.** Every `image.png` in a channel + wrote the same `image.json`, so the surviving sidecar describes whichever + message was written last. The message id is gone from the filename too. + +The operator chose a clean re-download (2026-09-13) over relinking in place: +delete the broken files and their records, make gallery-dl forget it fetched +them, and backfill every Discord source again under the fixed naming. + +## Why the archive is cleared for ALL of Discord + +gallery-dl records a download as `discord{message_id}_{num}` (upstream +`DiscordExtractor.archive_fmt`, prefixed with the category). The broken files +lost their message ids, so there is no way to forget one source's entries and +not another's. Every Discord download made before the fix is broken, so +forgetting all of them is exactly right. Anything downloaded AFTER the fix still +exists on disk under its correct name, and gallery-dl's `skip` sees the file and +does not fetch it again. + +## What it does not touch + +Discord posts FC grouped itself (#388 E2) are built from Posts, and these files +never had one, so there is nothing grouped to unwind. Images outside a +`discord/None/` directory are never selected: the path filter requires both the +`None` directory and the `_None_` filename, the pair only the bug produced. + +Operator-triggered only (Settings, preview first). Never on a beat. +""" + +from __future__ import annotations + +import logging +import sqlite3 +from pathlib import Path + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ..models import ImageRecord, Source +from .cleanup_service import delete_images +from .gallery_dl import archive_path +from .source_service import arm_backfill + +log = logging.getLogger(__name__) + +# `%` and `_` are LIKE wildcards, so the literal underscores around None are +# escaped. `________` is the eight-digit date prefix the old pattern wrote. +_BROKEN_PATH_LIKE = r"%/discord/None/________\_None\_%" + +# Upstream keys asset downloads as `asset_{server_id}_{id}`. FC never fetches +# server assets, but excluding them keeps this to exactly the message +# attachments the bug mangled. +_ARCHIVE_SQL_MATCH = r"entry LIKE 'discord%' AND entry NOT LIKE 'discordasset\_%' ESCAPE '\'" +_COUNT_SQL = "SELECT COUNT(*) FROM archive WHERE " + _ARCHIVE_SQL_MATCH +_DELETE_SQL = "DELETE FROM archive WHERE " + _ARCHIVE_SQL_MATCH + + +def broken_directories(images_root: Path) -> list[Path]: + """Every `/discord/None` directory. Artist folders that differ only by + case (`Conto` and `conto`) are separate directories and both are found.""" + return sorted(d for d in Path(images_root).glob("*/discord/None") if d.is_dir()) + + +def count_archive_entries(archive: Path) -> int: + return _archive(archive, delete=False) + + +def forget_archive_entries(archive: Path) -> int: + return _archive(archive, delete=True) + + +def _archive(archive: Path, *, delete: bool) -> int: + if not archive.is_file(): + return 0 + # A download running at the same moment holds this file briefly. Waiting + # 30s for its lock beats failing the repair over a transient contention. + conn = sqlite3.connect(str(archive), timeout=30) + try: + has_table = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='archive'" + ).fetchone() + if not has_table: + return 0 + if not delete: + return conn.execute(_COUNT_SQL).fetchone()[0] + cur = conn.execute(_DELETE_SQL) + conn.commit() + return cur.rowcount + finally: + conn.close() + + +def _sweep_directory(directory: Path) -> tuple[int, bool]: + """Remove what the record deletes left behind: the collided sidecars, plus any + file that never became a record (a quarantined or rejected download). Then + the directory itself, if it is empty. Returns (files removed, dir removed).""" + removed = 0 + for f in directory.iterdir(): + if f.is_file(): + try: + f.unlink() + removed += 1 + except OSError as exc: + log.warning("discord repair: could not remove %s: %s", f, exc) + try: + directory.rmdir() + return removed, True + except OSError: + return removed, False + + +def repair_discord_downloads( + session: Session, *, images_root: Path, dry_run: bool, +) -> dict: + images_root = Path(images_root) + archive = archive_path(images_root) + + broken = select(ImageRecord.id, ImageRecord.size_bytes).where( + ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\") + ) + rows = session.execute(broken).all() + image_ids = [r.id for r in rows] + directories = broken_directories(images_root) + sources = session.execute( + select(Source).where(Source.platform == "discord") + ).scalars().all() + + summary = { + "images": len(image_ids), + "bytes": sum(r.size_bytes or 0 for r in rows), + "directories": len(directories), + "sources": len(sources), + "enabled_sources": sum(1 for s in sources if s.enabled), + } + + if dry_run: + summary["archive_entries"] = count_archive_entries(archive) + return summary + + deleted = delete_images(session, image_ids=image_ids, images_root=images_root) + + swept = 0 + directories_removed = 0 + for d in directories: + n, gone = _sweep_directory(d) + swept += n + directories_removed += int(gone) + + # Only after the files are gone. Forgetting first and failing half way would + # leave gallery-dl free to re-fetch into a directory still full of the old + # copies. + forgotten = forget_archive_entries(archive) + + for source in sources: + arm_backfill(source) + session.commit() + + remaining = session.execute( + select(func.count(ImageRecord.id)).where( + ImageRecord.path.like(_BROKEN_PATH_LIKE, escape="\\") + ) + ).scalar_one() + + summary.update( + images_deleted=deleted["images_deleted"], + files_failed=deleted["files_failed"], + leftover_files_removed=swept, + directories_removed=directories_removed, + archive_entries=forgotten, + backfills_started=len(sources), + remaining=remaining, + ) + log.info("discord repair applied: %s", summary) + return summary diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index ddaf9f7..b07fb1b 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -94,6 +94,58 @@ BACKFILL_CHUNK_SECONDS = 600 _DEFAULT_GDL_TIMEOUT_SECONDS = 870 +# --- Discord naming --------------------------------------------------------- +# +# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory +# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits +# for an attachment: `channel` is a plain STRING (the channel's name), the +# message is `message_id`, the attachment's position in it is `num`, and there +# is NO `id` key at all. +# +# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as +# "None", so every Discord download since the platform was added landed in a +# directory called `None` as `_None_`. Worse, the sidecar was +# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_ +# sidecar` can never pair with `_None_.png`, so no Discord file ever +# got a Post or a post date, and (b) collides: every `image.png` in a channel +# overwrote the same `image.json`, so the one sidecar that survived described +# whichever message happened to be written last. +# +# The fix names the sidecar EXACTLY like the media minus its extension, so +# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match +# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders +# these patterns against a sanitized copy of the real sidecar, so a key that +# does not exist fails CI instead of silently becoming "None". +DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}" +DISCORD_DIRECTORY = ["{channel}"] + + +def sidecar_name_for(media_pattern: str) -> str | None: + """The metadata filename pattern that names a sidecar exactly like its media. + + Returns None for a pattern that does not end in `.{extension}`, since then + there is no media stem to mirror and the caller must fall back. + """ + suffix = ".{extension}" + if not media_pattern.endswith(suffix): + return None + return media_pattern[: -len(suffix)] + ".json" + + +def metadata_postprocessor(filename: str) -> dict: + return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename} + + +def archive_path(images_root: Path) -> Path: + """gallery-dl's download archive: the record of what it has already fetched. + + One definition, because the Discord repair (services/discord_repair.py) has + to find the same file the downloader writes, without constructing a service + whose __init__ creates directories. + """ + return Path(images_root) / ".gallery-dl" / "archive.sqlite3" + + @dataclass class SourceConfig: """Per-source overrides loaded from Source.config_overrides JSON. @@ -376,8 +428,11 @@ class GalleryDLService: }, "discord": { "content_types": ["all"], - "directory": ["{channel[name]}"], - "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", + "directory": DISCORD_DIRECTORY, + "filename": DISCORD_FILENAME, + # Overrides the global `{filename}.json` sidecar for this extractor + # only — see the Discord naming note above. + "postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))], "embeds": "all", "stickers": True, "reactions": False, @@ -402,7 +457,7 @@ class GalleryDLService: config = { "extractor": { "base-directory": str(self.images_root), - "archive": str(self._config_dir / "archive.sqlite3"), + "archive": str(archive_path(self.images_root)), "skip": True, "sleep": self._rate_limit, "sleep-request": max(0.5, self._rate_limit / 4), @@ -497,6 +552,17 @@ class GalleryDLService: if source_config.filename_pattern: platform_section["filename"] = source_config.filename_pattern + # A platform that names its sidecar after its media must keep doing so + # under a per-source filename override, or the pairing breaks exactly the + # way Discord's did. No metadata wanted means no platform postprocessor + # either — the global list was already dropped above. + if "postprocessors" in platform_section: + mirrored = sidecar_name_for(platform_section.get("filename") or "") + if not source_config.save_metadata or mirrored is None: + platform_section.pop("postprocessors") + else: + platform_section["postprocessors"] = [metadata_postprocessor(mirrored)] + platform_section["metadata"] = source_config.save_metadata return config diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py index 22a49ba..034eeb5 100644 --- a/backend/app/services/source_service.py +++ b/backend/app/services/source_service.py @@ -158,6 +158,19 @@ def _is_app_managed(key: str) -> bool: BACKFILL_MAX_CHUNKS = 200 +def arm_backfill(source: Source) -> None: + """Arm a fresh run-until-done backfill on `source` (plan #693). Mutation + only — the caller commits. Shared by `SourceService.start_backfill` and the + sync Discord repair, so both clear exactly the same resume state.""" + co = dict(source.config_overrides or {}) + co["_backfill_state"] = "running" + for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", + "_backfill_posts"): + co.pop(k, None) + source.config_overrides = co + source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + + class SourceService: def __init__(self, session: AsyncSession): self.session = session @@ -473,13 +486,7 @@ class SourceService: )).scalar_one_or_none() if source is None: raise LookupError(f"source id={source_id} not found") - co = dict(source.config_overrides or {}) - co["_backfill_state"] = "running" - for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks", - "_backfill_posts"): - co.pop(k, None) - source.config_overrides = co - source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS + arm_backfill(source) await self.session.commit() return await self._row_to_record(source) diff --git a/backend/app/tasks/admin.py b/backend/app/tasks/admin.py index 73ed1d7..571da4a 100644 --- a/backend/app/tasks/admin.py +++ b/backend/app/tasks/admin.py @@ -122,6 +122,28 @@ def prune_missing_file_records_task(self) -> dict: return {"checked": checked, "missing": len(missing_ids), "deleted": deleted} +@celery.task( + name="backend.app.tasks.admin.repair_discord_downloads_task", + bind=True, + autoretry_for=(OperationalError, DBAPIError), + retry_backoff=15, retry_backoff_max=180, max_retries=1, + soft_time_limit=1800, time_limit=2400, # 30 min / 40 min +) +def repair_discord_downloads_task(self, dry_run: bool = True) -> dict: + """Clean re-download of the Discord files broken by the `None` naming + (#3999). dry_run (the default) returns the projection; apply deletes the + broken images and their files, makes gallery-dl forget every Discord + download, and restarts every Discord source's backfill. Defaults to the SAFE + preview because the apply deletes files. Operator-triggered only.""" + from ..services.discord_repair import repair_discord_downloads + + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + return repair_discord_downloads( + session, images_root=IMAGES_ROOT, dry_run=dry_run, + ) + + @celery.task( name="backend.app.tasks.admin.dedup_videos_task", bind=True, diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index f3b7216..8a8cfdc 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -1,5 +1,8 @@