feat: finish the Discord switchover — recapture on every native source, backfills that run, gallery-dl's Discord config retired (milestone 428)

- Recover and Recapture show on every native source. The menu gated them on
  a copied platform list ('patreon', 'subscribestar') that went stale when
  Discord moved over. Sources now carry `native_ingester` from the backend's
  own predicate.
- A running backfill is due on every scheduler tick. Nothing queued a
  backfill's next chunk: each one waited for the source's regular interval,
  so an armed backfill sat idle until the next check (8h at the default) and
  a five-chunk walk took most of two days. The in-flight guard and the
  platform lock keep one chunk at a time. A failing source falls back to its
  backoff, and a stalled or out-of-budget walk stops being due. It also runs
  when the artist has auto-check off, since the operator started it by hand.
- gallery-dl no longer carries Discord: its naming constants, platform
  defaults, sidecar-mirroring postprocessor and token injection are gone.
  The naming test moves to the native downloader and still renders against
  the real gallery-dl sidecar fixture. That is the guard that the files
  gallery-dl wrote are found on disk rather than fetched again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

tests/test_discord_naming.py

@@ -0,0 +1,69 @@
+"""Discord file naming, checked against the metadata gallery-dl really emitted.
+
+Discord moved from gallery-dl to the native downloader in milestone 428. Every
+file gallery-dl wrote is on disk under gallery-dl's naming, and the native
+downloader finds it there — skips it instead of fetching it again — only if it
+builds the SAME name. This file used to pin gallery-dl's patterns (#3999: they
+named keys the extractor never set, and every file landed as
+`None/<date>_None_<name>`); it now pins the native downloader to what those
+patterns produced from the same real metadata.
+
+The fixture keeps the key set and value types of a real attachment sidecar
+from the operator's instance (2026-09-13), with every value invented.
+"""
+
+import json
+from pathlib import Path
+
+from backend.app.services.discord_client import DiscordClient
+from backend.app.services.discord_downloader import channel_dir, media_stem
+from backend.app.utils.sidecar import find_sidecar
+
+_FIXTURE = Path(__file__).parent / "fixtures" / "discord_attachment_sidecar.json"
+
+
+def _message_from_the_fixture():
+    """The API message that produced the fixture's sidecar."""
+    gdl = json.loads(_FIXTURE.read_text())
+    date, time = gdl["date"].split(" ")
+    return {
+        "id": gdl["message_id"],
+        "type": 0,
+        "timestamp": f"{date}T{time}.000000+00:00",
+        "content": gdl["message"],
+        "attachments": [{"id": "500000000000000005", "url": gdl["url"]}],
+        "embeds": [],
+        "_meta": {"channel": gdl["channel"], "channel_id": gdl["channel_id"],
+                  "server": gdl["server"], "server_id": gdl["server_id"]},
+    }
+
+
+def test_the_native_name_is_the_name_gallery_dl_wrote(tmp_path):
+    """gallery-dl's `{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`
+    in directory `{channel}`, rendered from the fixture."""
+    msg = _message_from_the_fixture()
+    [media] = DiscordClient.extract_media(msg)
+    assert media_stem(msg, media) + "." + media.extension == (
+        "20240716_300000000000000003_01_image.png"
+    )
+    assert channel_dir(tmp_path, "a", msg) == tmp_path / "a" / "discord" / "nsfw-drops"
+
+
+def test_a_gallery_dl_sidecar_on_disk_still_pairs_with_its_file(tmp_path):
+    """Files gallery-dl wrote keep their full sidecars beside them; the native
+    name must be the stem those sidecars were named after."""
+    msg = _message_from_the_fixture()
+    [media] = DiscordClient.extract_media(msg)
+    stem = media_stem(msg, media)
+    (tmp_path / f"{stem}.png").write_bytes(b"x")
+    (tmp_path / f"{stem}.json").write_text(_FIXTURE.read_text())
+    assert find_sidecar(tmp_path / f"{stem}.png") == tmp_path / f"{stem}.json"
+
+
+def test_two_files_with_the_same_original_name_get_distinct_names():
+    """`image.png` is Discord's default name; two messages must not collide."""
+    one = _message_from_the_fixture()
+    two = {**one, "id": "300000000000000099"}
+    [m1] = DiscordClient.extract_media(one)
+    [m2] = DiscordClient.extract_media(two)
+    assert media_stem(one, m1) != media_stem(two, m2)

tests/test_gallery_dl_naming.py

@@ -1,117 +0,0 @@
-"""gallery-dl naming patterns, rendered against the metadata gallery-dl really emits.
-
-gallery-dl does not fail on a format field that names a key the extractor never
-sets. It renders "None" and carries on. That is how every Discord download
-landed in a `None/` directory as `<date>_None_<name>`: the patterns asked for
-`{channel[name]}` and `{id}`, and the real metadata has a string `channel` and
-`message_id`, with no `id` at all. The mismatch also broke sidecar pairing, so
-no Discord file ever got a Post or a post date.
-
-The fixture keeps the key set and value TYPES of a real attachment sidecar from
-the operator's instance (2026-09-13), with every value invented. Rendering
-through Python's own formatter raises on a missing key or a subscript into a
-string, which is the loud failure gallery-dl does not give.
-"""
-
-import json
-import string
-from datetime import datetime
-from pathlib import Path
-
-import pytest
-
-from backend.app.services.gallery_dl import (
-    DISCORD_DIRECTORY,
-    DISCORD_FILENAME,
-    GalleryDLService,
-    SourceConfig,
-    sidecar_name_for,
-)
-from backend.app.utils.sidecar import find_sidecar
-
-_FIXTURE = Path(__file__).parent / "fixtures" / "discord_attachment_sidecar.json"
-
-
-@pytest.fixture
-def kwdict():
-    data = json.loads(_FIXTURE.read_text())
-    # gallery-dl hands the formatter a datetime; the JSON sidecar stores it as text.
-    data["date"] = datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
-    return data
-
-
-def _render(pattern, kwdict):
-    return string.Formatter().vformat(pattern, (), kwdict)
-
-
-def test_the_discord_filename_renders_from_real_keys(kwdict):
-    name = _render(DISCORD_FILENAME, kwdict)
-    assert "None" not in name
-    assert name == "20240716_300000000000000003_01_image.png"
-
-
-def test_the_discord_directory_renders_from_real_keys(kwdict):
-    assert [_render(p, kwdict) for p in DISCORD_DIRECTORY] == ["nsfw-drops"]
-
-
-@pytest.mark.parametrize(
-    "broken", ["{channel[name]}", "{date:%Y%m%d}_{id}_{filename}.{extension}"],
-)
-def test_the_patterns_that_shipped_would_have_failed_here(kwdict, broken):
-    """Positive control: the two patterns that produced `None/..._None_...`
-    must fail this renderer, or the tests above prove nothing."""
-    with pytest.raises((KeyError, TypeError)):
-        _render(broken, kwdict)
-
-
-def test_the_sidecar_is_named_exactly_like_its_media(kwdict, tmp_path):
-    """So `find_sidecar` pairs them on its first candidate, and two attachments
-    that share an original name (`image.png`) can never share a sidecar."""
-    media = tmp_path / _render(DISCORD_FILENAME, kwdict)
-    sidecar = tmp_path / _render(sidecar_name_for(DISCORD_FILENAME), kwdict)
-    media.write_bytes(b"x")
-    sidecar.write_text("{}")
-    assert find_sidecar(media) == sidecar
-
-
-def test_two_attachments_with_the_same_original_name_get_distinct_sidecars(kwdict):
-    other = {**kwdict, "message_id": "300000000000000099"}
-    pattern = sidecar_name_for(DISCORD_FILENAME)
-    assert _render(pattern, kwdict) != _render(pattern, other)
-
-
-def test_sidecar_name_for_needs_an_extension_suffix():
-    assert sidecar_name_for("{a}_{b}.{extension}") == "{a}_{b}.json"
-    assert sidecar_name_for("{a}_{b}") is None
-
-
-# --- the config gallery-dl is actually given -----------------------------------
-
-
-@pytest.fixture
-def gdl(tmp_path):
-    return GalleryDLService(images_root=tmp_path / "images", validate_files=False)
-
-
-def _discord_section(gdl, **overrides):
-    cfg = gdl._build_config_for_source(
-        platform="discord", source_config=SourceConfig(**overrides), artist_slug="a",
-    )
-    return cfg["extractor"]["discord"]
-
-
-def test_discord_config_carries_the_mirrored_sidecar(gdl):
-    section = _discord_section(gdl)
-    assert section["filename"] == DISCORD_FILENAME
-    assert section["directory"] == DISCORD_DIRECTORY
-    assert section["postprocessors"][0]["filename"] == sidecar_name_for(DISCORD_FILENAME)
-
-
-def test_a_filename_override_keeps_the_sidecar_mirrored(gdl):
-    section = _discord_section(gdl, filename_pattern="{message_id}_{num}.{extension}")
-    assert section["postprocessors"][0]["filename"] == "{message_id}_{num}.json"
-
-
-def test_no_metadata_means_no_discord_postprocessor(gdl):
-    section = _discord_section(gdl, save_metadata=False)
-    assert "postprocessors" not in section

tests/test_scheduler_service.py

@@ -11,6 +11,7 @@ import pytest
 
 from backend.app.models import Artist, ImportSettings, Source
 from backend.app.services.scheduler_service import (
+    backfill_ready,
     compute_effective_interval,
     scheduler_status,
     select_due_sources,
@@ -150,6 +151,67 @@ async def test_select_includes_past_due(db):
     assert any(s.url == "https://sel-past" for s in due)
 
 
+# --- a running backfill is due every tick (2026-09-25) ---------------------
+
+
+def _backfilling(runs=5, failures=0, state="running"):
+    src = _src(failures=failures)
+    src.config_overrides = {"_backfill_state": state}
+    src.backfill_runs_remaining = runs
+    return src
+
+
+def test_a_running_backfill_is_ready():
+    assert backfill_ready(_backfilling()) is True
+
+
+def test_a_backfill_out_of_budget_or_not_running_is_not_ready():
+    assert backfill_ready(_backfilling(runs=0)) is False
+    assert backfill_ready(_backfilling(state="stalled")) is False
+    assert backfill_ready(_src()) is False
+
+
+def test_a_failing_backfill_falls_back_to_its_backoff():
+    """Otherwise a source whose chunks keep erroring would retry every minute."""
+    assert backfill_ready(_backfilling(failures=1)) is False
+
+
+@pytest.mark.asyncio
+async def test_select_runs_a_backfill_now_not_at_its_next_check(db):
+    """Checked a minute ago on an hourly interval: not due, unless backfilling —
+    the next chunk should not wait an interval (the 2026-09-25 armed-and-idle
+    backfill)."""
+    artist = await _seed_artist(db, interval=3600, name="bf-now")
+    for url, overrides, runs in (
+        ("https://bf-running", {"_backfill_state": "running"}, 5),
+        ("https://bf-idle", {}, 0),
+    ):
+        db.add(Source(
+            artist_id=artist.id, platform="patreon", url=url, enabled=True,
+            consecutive_failures=0, config_overrides=overrides,
+            backfill_runs_remaining=runs,
+            last_checked_at=datetime.now(UTC) - timedelta(seconds=60),
+        ))
+    await db.commit()
+    urls = {s.url for s in await select_due_sources(db)}
+    assert "https://bf-running" in urls
+    assert "https://bf-idle" not in urls
+
+
+@pytest.mark.asyncio
+async def test_a_backfill_runs_even_when_its_artist_is_not_auto_checked(db):
+    """The operator started it by hand; auto-check governs the schedule only."""
+    artist = await _seed_artist(db, auto=False, name="bf-noauto")
+    db.add(Source(
+        artist_id=artist.id, platform="patreon", url="https://bf-noauto",
+        enabled=True, consecutive_failures=0,
+        config_overrides={"_backfill_state": "running"}, backfill_runs_remaining=5,
+        last_checked_at=datetime.now(UTC),
+    ))
+    await db.commit()
+    assert any(s.url == "https://bf-noauto" for s in await select_due_sources(db))
+
+
 # --- platform-rate-limit cooldown -----------------------------------------
 
 

tests/test_source_service.py

@@ -34,6 +34,24 @@ async def test_known_platforms_are_the_supported_four(db):
     assert "pixiv" not in KNOWN_PLATFORMS
 
 
+@pytest.mark.asyncio
+async def test_a_source_says_whether_it_is_on_the_native_ingester(db):
+    """The UI offers recover / recapture from this flag rather than a copied
+    platform list — the copy went stale when Discord moved (milestone 428)."""
+    artist = await _artist(db, name="Natty")
+    svc = SourceService(db)
+    discord = await svc.create(
+        artist_id=artist.id, platform="discord",
+        url="https://discord.com/channels/1/2",
+    )
+    hf = await svc.create(
+        artist_id=artist.id, platform="hentaifoundry",
+        url="https://www.hentai-foundry.com/user/Natty",
+    )
+    assert discord.to_dict()["native_ingester"] is True
+    assert hf.to_dict()["native_ingester"] is False
+
+
 @pytest.mark.asyncio
 async def test_create_flips_is_subscription_on_first_source(db):
     artist = await _artist(db)