feat: finish the Discord switchover — recapture on every native source, backfills that run, gallery-dl's Discord config retired (milestone 428)

- Recover and Recapture show on every native source. The menu gated them on
  a copied platform list ('patreon', 'subscribestar') that went stale when
  Discord moved over. Sources now carry `native_ingester` from the backend's
  own predicate.
- A running backfill is due on every scheduler tick. Nothing queued a
  backfill's next chunk: each one waited for the source's regular interval,
  so an armed backfill sat idle until the next check (8h at the default) and
  a five-chunk walk took most of two days. The in-flight guard and the
  platform lock keep one chunk at a time. A failing source falls back to its
  backoff, and a stalled or out-of-budget walk stops being due. It also runs
  when the artist has auto-check off, since the operator started it by hand.
- gallery-dl no longer carries Discord: its naming constants, platform
  defaults, sidecar-mirroring postprocessor and token injection are gone.
  The naming test moves to the native downloader and still renders against
  the real gallery-dl sidecar fixture. That is the guard that the files
  gallery-dl wrote are found on disk rather than fetched again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

backend/app/services/discord_downloader.py

@@ -6,9 +6,11 @@ existing file on disk (`skipped_disk`) instead of fetching it again:
 
     <images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
 
-That is FC's gallery-dl config (`gallery_dl.DISCORD_DIRECTORY` /
-`DISCORD_FILENAME`) under the per-source base directory
-`<images>/<artist>/<platform>`. The name is cleaned the way gallery-dl cleans it
+That is what FC's gallery-dl config produced (directory `{channel}`, filename
+`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
+per-source base directory `<images>/<artist>/<platform>`), retired from that
+config once Discord moved here; tests/test_discord_naming.py pins the match
+against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
 on Linux — `/` becomes `_` and control characters are removed, nothing else
 (`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
 whose Windows set would turn a `:` in a channel or file name into `_` and miss

backend/app/services/gallery_dl.py

@@ -95,48 +95,6 @@ BACKFILL_CHUNK_SECONDS = 600
 _DEFAULT_GDL_TIMEOUT_SECONDS = 870
 
 
-# --- Discord naming ---------------------------------------------------------
-#
-# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
-# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
-# for an attachment: `channel` is a plain STRING (the channel's name), the
-# message is `message_id`, the attachment's position in it is `num`, and there
-# is NO `id` key at all.
-#
-# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
-# "None", so every Discord download since the platform was added landed in a
-# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
-# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
-# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
-# got a Post or a post date, and (b) collides: every `image.png` in a channel
-# overwrote the same `image.json`, so the one sidecar that survived described
-# whichever message happened to be written last.
-#
-# The fix names the sidecar EXACTLY like the media minus its extension, so
-# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
-# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
-# these patterns against a sanitized copy of the real sidecar, so a key that
-# does not exist fails CI instead of silently becoming "None".
-DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
-DISCORD_DIRECTORY = ["{channel}"]
-
-
-def sidecar_name_for(media_pattern: str) -> str | None:
-    """The metadata filename pattern that names a sidecar exactly like its media.
-
-    Returns None for a pattern that does not end in `.{extension}`, since then
-    there is no media stem to mirror and the caller must fall back.
-    """
-    suffix = ".{extension}"
-    if not media_pattern.endswith(suffix):
-        return None
-    return media_pattern[: -len(suffix)] + ".json"
-
-
-def metadata_postprocessor(filename: str) -> dict:
-    return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
-
-
 def archive_path(images_root: Path) -> Path:
     """gallery-dl's download archive: the record of what it has already fetched.
 
@@ -425,27 +383,16 @@ class GalleryDLService:
     # (services/patreon_ingester.py), not gallery-dl.
     PLATFORM_DEFAULTS = {
         # subscribestar removed — native-ingester platform now (#71); pixiv
-        # removed likewise (#129); deviantart removed at #3069 as a dropped
-        # platform, not a migrated one. The remaining entries are the
-        # gallery-dl platforms not yet migrated.
+        # removed likewise (#129); discord likewise (milestone 428, whose
+        # downloader keeps this config's on-disk naming); deviantart removed at
+        # #3069 as a dropped platform, not a migrated one. HentaiFoundry is the
+        # one platform left here, by the operator's choice not to migrate it.
         "hentaifoundry": {
             "content_types": ["all"],
             "directory": [],
             "filename": "{category}_{index:>03}_{title[:50]}.{extension}",
             "include": "all",
         },
-        "discord": {
-            "content_types": ["all"],
-            "directory": DISCORD_DIRECTORY,
-            "filename": DISCORD_FILENAME,
-            # Overrides the global `{filename}.json` sidecar for this extractor
-            # only — see the Discord naming note above.
-            "postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
-            "embeds": "all",
-            "stickers": True,
-            "reactions": False,
-            "threads": True,
-        },
     }
 
     def __init__(
@@ -560,17 +507,6 @@ class GalleryDLService:
         if source_config.filename_pattern:
             platform_section["filename"] = source_config.filename_pattern
 
-        # A platform that names its sidecar after its media must keep doing so
-        # under a per-source filename override, or the pairing breaks exactly the
-        # way Discord's did. No metadata wanted means no platform postprocessor
-        # either — the global list was already dropped above.
-        if "postprocessors" in platform_section:
-            mirrored = sidecar_name_for(platform_section.get("filename") or "")
-            if not source_config.save_metadata or mirrored is None:
-                platform_section.pop("postprocessors")
-            else:
-                platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
-
         platform_section["metadata"] = source_config.save_metadata
 
         return config
@@ -818,9 +754,6 @@ class GalleryDLService:
 
         if cookies_path:
             config["extractor"]["cookies"] = cookies_path
-        if auth_token and platform == "discord":
-            config["extractor"].setdefault("discord", {})
-            config["extractor"]["discord"]["token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
@@ -1004,8 +937,6 @@ class GalleryDLService:
         config = self._build_config_for_source(platform, source_config, artist_slug)
         if cookies_path:
             config["extractor"]["cookies"] = cookies_path
-        if auth_token and platform == "discord":
-            config["extractor"].setdefault("discord", {})["token"] = auth_token
 
         with tempfile.NamedTemporaryFile(
             mode="w", suffix=".json", delete=False, dir=str(self._config_dir),

backend/app/services/scheduler_service.py

@@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime
     return active
 
 
+def backfill_ready(source: Source) -> bool:
+    """A deep walk the operator started, with budget left and no failure
+    backing it off — due NOW rather than at its next scheduled check.
+
+    A backfill runs one time-boxed chunk per download (plan #693), and nothing
+    queued the next chunk: each waited for the source's regular interval. At
+    the 8-hour default a freshly armed backfill sat untouched until the next
+    check (the operator armed one on 2026-09-25 and saw nothing happen) and a
+    five-chunk walk took most of two days. The tick's in-flight guard keeps
+    one chunk at a time per source and the platform lock one walk per
+    platform, so "due every tick" means "next chunk as soon as the last one
+    ends".
+
+    The failure gate is what keeps a broken source from retrying every
+    minute: any failed chunk raises `consecutive_failures`, which drops the
+    source back onto its backed-off interval. A chunk that fails to progress
+    twice marks the walk stalled (download_service), which ends it here too.
+    """
+    co = source.config_overrides or {}
+    return (
+        co.get("_backfill_state") == "running"
+        and (source.backfill_runs_remaining or 0) > 0
+        and not (source.consecutive_failures or 0)
+    )
+
+
 async def select_due_sources(session: AsyncSession) -> list[Source]:
     """Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
 
@@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
     cooldown is the preventive half of the burst-prevention pair (per-source
     consecutive_failures backoff handles the offending source itself).
 
+    A running backfill (`backfill_ready`) is due on every tick, and whether
+    or not its artist is on auto-check — the operator started it by hand.
+
     Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
     sources go first, then the longest-since-checked, so the most overdue
     sources hit Celery's FIFO download queue first. Anti-starvation: if
@@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
         .options(selectinload(Source.artist))
         .join(Artist, Source.artist_id == Artist.id)
         .where(Source.enabled.is_(True))
-        .where(Artist.auto_check.is_(True))
         .order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
     )).scalars().all()
 
@@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
     for s in rows:
         if s.platform in cooldowns:
             continue
+        if backfill_ready(s):
+            due.append(s)
+            continue
+        if not s.artist.auto_check:
+            continue
         interval = compute_effective_interval(s, s.artist, settings)
         if s.last_checked_at is None:
             due.append(s)
@@ -161,6 +194,8 @@ def compute_next_check_at(
     source: Source, artist: Artist, settings: ImportSettings,
 ) -> datetime | None:
     """Return the projected datetime of the next check, or None if never checked."""
+    if backfill_ready(source):
+        return datetime.now(UTC)
     if source.last_checked_at is None:
         return None
     interval = compute_effective_interval(source, artist, settings)

backend/app/services/source_service.py

@@ -18,6 +18,7 @@ from ..models import (
     Source,
 )
 from .db_helpers import failing_sources_clause
+from .download_backends import uses_native_ingester
 from .gallery_dl import ErrorType
 from .membership_reconcile import KEPT_KEY, STOPPED_KEY
 from .membership_roster import gated_reasons_for_sources
@@ -125,6 +126,11 @@ class SourceRecord:
             "backfill_posts": self.backfill_posts,
             "tier_gated_count": self.tier_gated_count,
             "gated_reason": self.gated_reason,
+            # Recover / recapture exist only on the native ingester. Sent so the
+            # UI asks the backend's own predicate instead of keeping a copy of
+            # the platform list — the copy said "patreon, subscribestar" for a
+            # day after Discord went native (milestone 428).
+            "native_ingester": uses_native_ingester(self.platform),
         }
 
 
@@ -551,8 +557,8 @@ class SourceService:
         whole source); the two flags are mutually exclusive, so arming recapture
         clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
         fresh from the top. The flag is cleared on completion (download_service)
-        and on stop. Recapture is Patreon-only (the native ingester's post-record
-        capture); inert elsewhere. The UI gates the action to Patreon sources."""
+        and on stop. Recapture needs the native ingester's post-record capture,
+        so the UI offers it on native sources only (`native_ingester`)."""
         source = (await self.session.execute(
             select(Source).where(Source.id == source_id)
         )).scalar_one_or_none()