refactor: the native downloaders share one download loop, file record and seen-ledger key; a clean walk marks its credential verified

Patreon, SubscribeStar and Discord each carried their own copy of the same
per-post download loop, file record and seen-ledger key, and every copy was
another place a fix could be missed.

- BaseNativeDownloader runs one post's files: the seen and on-disk skips,
  the mid-post time-box, pacing, validation and the per-file sidecar. Each
  platform overrides only where its layout differs: Discord's channel folder,
  file names and message-id sidecar, and Patreon's Mux video through yt-dlp.
- One MediaItem and one ledger_key in native_ingest_common replace the three
  per-client copies. Keys are unchanged, except that a SubscribeStar file with
  no gallery id is now keyed by its filename instead of an empty id.
- DownloadResult, ErrorType and the run-stats and tier-gated helpers move
  from gallery_dl.py, which now serves only HentaiFoundry, to
  download_result.py. The unused delegator methods on GalleryDLService go.
- The download lock's platform list is the native-ingester list, not a copy.
- A walk that used a credential and finished with no error type stamps that
  credential's last-verified time, so the credentials page shows when the
  account last worked. A tier-limited walk does not count: an expired session
  hides paid posts the same way.
- The extension no longer clears a retired pixiv token on startup.
- Two comments that still described every download as gallery-dl now name
  both backends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/test_discord_client.py

@@ -414,7 +414,7 @@ def test_the_request_profile_is_gallery_dls():
 # -- the adapter ---------------------------------------------------------------
 
 def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path):
-    from backend.app.services.discord_ingester import DiscordIngester, _ledger_key
+    from backend.app.services.discord_ingester import DiscordIngester
 
     ing = DiscordIngester(tmp_path, None, session_factory=None, auth_token="tok")
     assert ing.client._session.headers["Authorization"] == "tok"
@@ -423,7 +423,7 @@ def test_the_adapter_authenticates_with_the_token_and_keys_by_identity(tmp_path)
     [media] = DiscordClient.extract_media(
         _msg(9, attachments=[{"id": "300", "url": "https://cdn/a/1.png"}])
     )
-    assert _ledger_key(media) == "9:300"
+    assert ing._ledger_key(media) == "9:300"
 
 
 # -- the poster picker (#4491) -------------------------------------------------

tests/test_download_backends.py

@@ -13,7 +13,7 @@ from backend.app.services.download_backends import (
     uses_native_ingester,
     verify_source_credential,
 )
-from backend.app.services.gallery_dl import ErrorType
+from backend.app.services.download_result import ErrorType
 
 
 def test_native_platforms():

tests/test_download_service.py

@@ -101,8 +101,6 @@ def _fake_gdl_with_result(result):
         "skipped_count": 0, "per_item_failures": 0,
         "warning_count": 0, "tier_gated_count": 0,
     }
-    fake._extract_errors_warnings = lambda *a, **k: ""
-    fake._truncate_log = lambda x, **k: x
     return fake
 
 
@@ -416,7 +414,8 @@ async def test_run_download_native_unresolvable_fails_loud(
     """A campaign id we can't resolve is a loud NOT_FOUND failure, never a
     silent empty success."""
     from backend.app.services import download_backends as db_mod
-    from backend.app.services.gallery_dl import ErrorType, SourceConfig
+    from backend.app.services.download_result import ErrorType
+    from backend.app.services.gallery_dl import SourceConfig
 
     _artist, source = seed_artist_and_source
 
@@ -733,7 +732,7 @@ async def test_backfill_completes_when_every_post_was_tier_gated(
     it 'stalled', and the creator we can see least becomes the one we re-walk
     most. `walk_completed` admits informational classes for exactly this.
     """
-    from backend.app.services.gallery_dl import ErrorType
+    from backend.app.services.download_result import ErrorType
 
     _artist, source = seed_artist_and_source
     source.config_overrides = {
@@ -955,7 +954,7 @@ async def test_backfill_timeout_chunk_reclassified_to_ok(
     """A backfill chunk that hit its time-box (TIMEOUT) but advanced is
     reclassified to PARTIAL → event status 'ok' (progress, not error), and
     consecutive_failures is not bumped."""
-    from backend.app.services.gallery_dl import ErrorType
+    from backend.app.services.download_result import ErrorType
 
     _artist, source = seed_artist_and_source
     source.config_overrides = {"_backfill_state": "running"}
@@ -1059,7 +1058,7 @@ async def test_partial_error_type_maps_to_ok_status(
     (real files were downloaded before the run was cut short) is treated
     as status=ok by the orchestrator — next tick continues."""
     from backend.app.services.download_service import DownloadService
-    from backend.app.services.gallery_dl import ErrorType
+    from backend.app.services.download_result import ErrorType
     from backend.app.services.importer import Importer
 
     _artist, source = seed_artist_and_source
@@ -1244,3 +1243,54 @@ async def test_async_task_engine_uses_nullpool():
         assert isinstance(engine.sync_engine.pool, NullPool)
     finally:
         await engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_a_clean_walk_marks_the_credential_verified_and_a_tier_limited_one_does_not(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """The credentials page shows when the account last worked. A tier-limited
+    walk is not evidence: an expired session hides paid posts the same way."""
+    from backend.app.models import Credential
+    from backend.app.services.download_result import ErrorType
+    from backend.app.services.download_service import DownloadService
+    from backend.app.services.importer import Importer
+
+    _artist, source = seed_artist_and_source
+    images_root = tmp_path / "images"
+    crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
+    cred_service = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
+    await cred_service.upsert(
+        platform="patreon", credential_type="cookies",
+        data=".patreon.com\tTRUE\t/\tTRUE\t0\tsession_id\tabc",
+    )
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    importer = Importer(
+        session=db_sync, images_root=images_root, import_root=images_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
+    )
+
+    async def last_verified():
+        row = (await db.execute(
+            select(Credential).where(Credential.platform == "patreon")
+        )).scalar_one()
+        await db.refresh(row)
+        return row.last_verified
+
+    gated = _make_fake_dl_result(
+        success=True, error_type=ErrorType.TIER_LIMITED, error_message="gated",
+    )
+    svc = DownloadService(
+        async_session=db, sync_session=db_sync,
+        gdl=_fake_gdl_with_result(gated), importer=importer, cred_service=cred_service,
+    )
+    _stub_patreon_ingester(svc, gated)
+    await svc.download_source(source.id)
+    assert await last_verified() is None
+
+    clean = _make_fake_dl_result(success=True)
+    _stub_patreon_ingester(svc, clean)
+    await svc.download_source(source.id)
+    assert await last_verified() is not None

tests/test_gallery_dl_service.py

@@ -4,11 +4,12 @@ from types import SimpleNamespace
 
 import pytest
 
-from backend.app.services.gallery_dl import (
+from backend.app.services.download_result import (
     ErrorType,
-    GalleryDLService,
-    SourceConfig,
+    extract_errors_warnings,
+    truncate_log,
 )
+from backend.app.services.gallery_dl import GalleryDLService, SourceConfig
 
 
 @pytest.fixture
@@ -228,27 +229,27 @@ def test_compute_run_stats(gdl):
     assert stats["per_item_failures"] == 1
 
 
-def test_extract_errors_warnings(gdl):
+def test_extract_errors_warnings():
     stderr = (
         "[patreon][debug] foo\n[patreon][error] bar\n"
         "[urllib3][debug] baz\n[patreon][warning] qux\n"
     )
-    result = gdl._extract_errors_warnings(stderr)
+    result = extract_errors_warnings(stderr)
     assert "[error]" in result
     assert "[warning]" in result
     assert "[debug]" not in result
 
 
-def test_truncate_log_caps_large_text(gdl):
+def test_truncate_log_caps_large_text():
     huge = "line\n" * 200_000
-    result = gdl._truncate_log(huge, max_bytes=10_000)
+    result = truncate_log(huge, max_bytes=10_000)
     assert len(result.encode()) < 11_000
     assert "lines elided" in result
 
 
-def test_truncate_log_passes_short_text(gdl):
+def test_truncate_log_passes_short_text():
     text = "small\n"
-    assert gdl._truncate_log(text) == text
+    assert truncate_log(text) == text
 
 
 # --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------

tests/test_gated_reason.py

@@ -24,7 +24,7 @@ from pathlib import Path
 import pytest
 
 from backend.app.models import Artist, Source
-from backend.app.services.gallery_dl import ErrorType
+from backend.app.services.download_result import ErrorType
 from backend.app.services.membership_roster import (
     GATED_FREE,
     GATED_LAPSED,

tests/test_patreon_ingester.py

@@ -21,8 +21,9 @@ from backend.app.models import (
     PatreonSeenMedia,
     Source,
 )
-from backend.app.services.gallery_dl import ErrorType
+from backend.app.services.download_result import ErrorType
 from backend.app.services.ingest_core import _CANARY_MIN_SAMPLE
+from backend.app.services.native_ingest_common import ledger_key
 from backend.app.services.patreon_client import (
     MediaItem,
     PatreonAPIError,
@@ -30,7 +31,7 @@ from backend.app.services.patreon_client import (
     PatreonDriftError,
 )
 from backend.app.services.patreon_downloader import MediaOutcome, PostRecordOutcome
-from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
+from backend.app.services.patreon_ingester import PatreonIngester
 
 pytestmark = pytest.mark.integration
 
@@ -141,15 +142,15 @@ class _FakeDownloader:
             # disk is left alone (skipped_seen, never refetched).
             if seen and not recapture:
                 outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
-            elif _ledger_key(m) in self.on_disk:
+            elif ledger_key(m) in self.on_disk:
                 p = self.tmp_path / f"{m.post_id}_{m.filename}"
                 outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
             elif seen:
                 outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
-            elif _ledger_key(m) in self.error:
+            elif ledger_key(m) in self.error:
                 self.download_calls += 1
                 outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
-            elif _ledger_key(m) in self.quarantine:
+            elif ledger_key(m) in self.quarantine:
                 self.download_calls += 1
                 q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
                 outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
@@ -255,7 +256,7 @@ async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_
     NOT written or marked seen."""
     m1, m2 = _media("p1", 1), _media("p1", 2)
     client = _FakeClient([(None, [("p1", [m1, m2])])])
-    downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
+    downloader = _FakeDownloader(tmp_path, quarantine={ledger_key(m2)})
     ing = _ingester(sync_engine, tmp_path, client, downloader)
 
     result = ing.run(
@@ -280,7 +281,7 @@ async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
     # Pre-seed the ledger with m1's key.
     factory = sessionmaker(sync_engine, expire_on_commit=False)
     with factory() as s:
-        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
+        s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m1), post_id="p1"))
         s.commit()
 
     client = _FakeClient([(None, [("p1", [m1])])])
@@ -302,7 +303,7 @@ async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
     factory = sessionmaker(sync_engine, expire_on_commit=False)
     with factory() as s:
         for m in seen_media:
-            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
+            s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m), post_id=m.post_id))
         s.commit()
 
     pages = [(None, [(m.post_id, [m]) for m in seen_media])]
@@ -348,7 +349,7 @@ def _seed_seen_media(sync_engine, source_id, media):
     factory = sessionmaker(sync_engine, expire_on_commit=False)
     with factory() as s:
         for m in media:
-            s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
+            s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m), post_id=m.post_id))
         s.commit()
 
 
@@ -615,7 +616,7 @@ async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
     m1 = _media("p1", 1)
     factory = sessionmaker(sync_engine, expire_on_commit=False)
     with factory() as s:
-        s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
+        s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m1), post_id="p1"))
         s.commit()
 
     client = _FakeClient([(None, [("p1", [m1])])])
@@ -638,7 +639,7 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path)
     client = _FakeClient([(None, [("p1", [m1])])])
     # File still on disk (a kept image) → tier-2 spares it even under recovery.
     _seed_record(sync_engine, tmp_path / "p1_1.jpg")
-    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
+    downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
     ing = _ingester(sync_engine, tmp_path, client, downloader)
 
     result = ing.run(
@@ -665,7 +666,7 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
     itself, and a normal backfill is the backfill."""
     m1 = _media("p1", 1)
     client = _FakeClient([(None, [("p1", [m1])])])
-    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
+    downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
     ing = _ingester(sync_engine, tmp_path, client, downloader)
 
     result = ing.run(
@@ -704,13 +705,13 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     re-downloading the on-disk media."""
     m1 = _media("p1", 1)
     # Pre-seed BOTH the media key and the synthetic post key as already seen.
-    _seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
+    _seed_seen(sync_engine, source_id, ledger_key(m1), post_id="p1")
     _seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
     _seed_record(sync_engine, tmp_path / "p1_1.jpg")
 
     # 1) Plain backfill: post key is seen → gate skips body recapture.
     client = _FakeClient([(None, [("p1", [m1])])])
-    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
+    downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
     ing = _ingester(sync_engine, tmp_path, client, downloader)
     res_backfill = ing.run(
         source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -722,7 +723,7 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     # 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
     # the on-disk file is surfaced for source_filehash relink.
     client2 = _FakeClient([(None, [("p1", [m1])])])
-    downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
+    downloader2 = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
     ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
     res_recap = ing2.run(
         source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -892,7 +893,7 @@ async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
     """Recapture must NOT re-download a seen item that's absent from disk (that's
     recovery's job) — it returns skipped_seen and contributes no relink pair."""
     m1 = _media("p1", 1)
-    _seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
+    _seed_seen(sync_engine, source_id, ledger_key(m1), post_id="p1")
     client = _FakeClient([(None, [("p1", [m1])])])
     # NOT in on_disk → seen + missing.
     downloader = _FakeDownloader(tmp_path)
@@ -952,7 +953,7 @@ async def test_a_run_that_dies_before_import_leaves_its_media_unmarked(
 
     # The next walk finds the file on disk with no record, and imports it.
     ing2 = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]),
-                     _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}))
+                     _FakeDownloader(tmp_path, on_disk={ledger_key(m1)}))
     result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
                       url="https://patreon.com/ingest", mode="tick")
     assert result.written_paths == [str(tmp_path / "p1_1.jpg")]
@@ -971,9 +972,9 @@ async def test_recapture_imports_a_seen_file_nothing_imported(
     a walk that looks past it — recapture — reaches them."""
     m1, m2 = _media("p1", 1), _media("p1", 2)
     for m in (m1, m2):
-        _seed_seen(sync_engine, source_id, _ledger_key(m), post_id="p1")
+        _seed_seen(sync_engine, source_id, ledger_key(m), post_id="p1")
     _seed_record(sync_engine, tmp_path / "p1_2.jpg")   # m2 made it in; m1 did not
-    downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)})
+    downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1), ledger_key(m2)})
     ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1, m2])])]),
                     downloader)
     result = ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -1002,7 +1003,7 @@ async def test_files_phase_3_leaves_unimported_stay_unmarked(
     assert _count_ledger(sync_engine, source_id) == 3   # m1 and both post records
 
     ing2 = _ingester(sync_engine, tmp_path, _FakeClient(pages),
-                     _FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)}))
+                     _FakeDownloader(tmp_path, on_disk={ledger_key(m1), ledger_key(m2)}))
     again = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
                      url="https://patreon.com/ingest", mode="backfill")
     assert again.written_paths == [str(tmp_path / "p2_1.jpg")]
@@ -1041,7 +1042,7 @@ async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, t
         ing = _ingester(
             sync_engine, tmp_path,
             _FakeClient([(None, [("p1", [m1])])]),
-            _FakeDownloader(tmp_path, error={_ledger_key(m1)}),
+            _FakeDownloader(tmp_path, error={ledger_key(m1)}),
         )
         result = ing.run(
             source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -1070,7 +1071,7 @@ async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
     from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
 
     m1 = _media("p1", 1)
-    _seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)
+    _seed_failed(sync_engine, source_id, ledger_key(m1), DEAD_LETTER_THRESHOLD)
 
     downloader = _FakeDownloader(tmp_path)  # succeeds
     ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
@@ -1090,7 +1091,7 @@ async def test_clean_download_clears_below_threshold_failure(
     """A media with a sub-threshold failure that now downloads cleanly (tick) has
     its dead-letter row cleared."""
     m1 = _media("p1", 1)
-    _seed_failed(sync_engine, source_id, _ledger_key(m1), 1)  # not yet dead
+    _seed_failed(sync_engine, source_id, ledger_key(m1), 1)  # not yet dead
 
     downloader = _FakeDownloader(tmp_path)
     ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
@@ -1113,8 +1114,8 @@ async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
     ing = _ingester(sync_engine, tmp_path, client, downloader)
 
     # Same media downloaded twice across two runs — the second marks seen again.
-    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
-    ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
+    ing._mark_seen(source_id, [(ledger_key(m1), "p1")])
+    ing._mark_seen(source_id, [(ledger_key(m1), "p1")])
     assert _count_ledger(sync_engine, source_id) == 1
 
 
@@ -1180,7 +1181,7 @@ def test_ledger_key_video_has_no_filehash():
         url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
         kind="postfile", filehash=None, post_id="p9",
     )
-    assert _ledger_key(video) == "p9:clip.mp4"
+    assert ledger_key(video) == "p9:clip.mp4"
 
 
 # --- verify_patreon_credential (the verify counterpart, plan #697) ---------
@@ -1350,7 +1351,7 @@ def _seed_all_seen(sync_engine, source_id, media):
     the tests that use it is invisible at the point of use.
     """
     for m in media:
-        _seed_seen(sync_engine, source_id, _ledger_key(m), post_id=m.post_id)
+        _seed_seen(sync_engine, source_id, ledger_key(m), post_id=m.post_id)
 
 
 @pytest.mark.asyncio

tests/test_subscribestar_native.py

@@ -13,8 +13,8 @@ from pathlib import Path
 
 import requests
 
-from backend.app.services.gallery_dl import DownloadResult, ErrorType
-from backend.app.services.native_ingest_common import post_dir_name
+from backend.app.services.download_result import DownloadResult, ErrorType
+from backend.app.services.native_ingest_common import ledger_key, post_dir_name
 from backend.app.services.subscribestar_client import (
     MediaItem,
     SubscribeStarAPIError,
@@ -26,10 +26,7 @@ from backend.app.services.subscribestar_client import (
     _split_creator_url,
 )
 from backend.app.services.subscribestar_downloader import SubscribeStarDownloader
-from backend.app.services.subscribestar_ingester import (
-    SubscribeStarIngester,
-    _ledger_key,
-)
+from backend.app.services.subscribestar_ingester import SubscribeStarIngester
 from backend.app.utils.sidecar import find_sidecar
 
 _PNG_HEAD = b"\x89PNG\r\n\x1a\n"
@@ -489,12 +486,12 @@ def test_skip_seen_does_not_download(tmp_path):
 # -- ingester ----------------------------------------------------------------
 
 def test_ledger_key_prefers_filehash_then_post_media():
-    assert _ledger_key(_img("a.jpg")) == "111:9001"  # no filehash → post:media
+    assert ledger_key(_img("a.jpg")) == "111:9001"  # no filehash → post:media
     hashed = MediaItem(
         url="u", filename="a.jpg", kind="image", filehash="deadbeef",
         post_id="111", media_id="9001",
     )
-    assert _ledger_key(hashed) == "deadbeef"
+    assert ledger_key(hashed) == "deadbeef"
 
 
 def _ingester(tmp_path):