bvandeusen/FabledCurator
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 22 Wiki Activity

refactor: the native downloaders share one download loop, file record and seen-ledger key; a clean walk marks its credential verified
CI and images / lint (push) Failing after 2s
Details
CI and images / extension-version (push) Successful in 2s
Details
CI and images / extension-test (push) Successful in 23s
Details
CI and images / frontend-build (push) Successful in 26s
Details
CI and images / backend-lint-and-test (push) Successful in 53s
Details
CI and images / integration (push) Successful in 2m50s
Details
CI and images / sign-extension (push) Skipped
Details
CI and images / build-web (push) Skipped
Details
CI and images / smoke-web (push) Skipped
Details
CI and images / promote (push) Skipped
Details
CI and images / refresh-dev (push) Skipped
Details
CI and images / unattended-failure (push) Skipped
Details
CI and images / build-agent (push) Skipped
Details

Browse Source 
Patreon, SubscribeStar and Discord each carried their own copy of the same
per-post download loop, file record and seen-ledger key, and every copy was
another place a fix could be missed.

- BaseNativeDownloader runs one post's files: the seen and on-disk skips,
  the mid-post time-box, pacing, validation and the per-file sidecar. Each
  platform overrides only where its layout differs: Discord's channel folder,
  file names and message-id sidecar, and Patreon's Mux video through yt-dlp.
- One MediaItem and one ledger_key in native_ingest_common replace the three
  per-client copies. Keys are unchanged, except that a SubscribeStar file with
  no gallery id is now keyed by its filename instead of an empty id.
- DownloadResult, ErrorType and the run-stats and tier-gated helpers move
  from gallery_dl.py, which now serves only HentaiFoundry, to
  download_result.py. The unused delegator methods on GalleryDLService go.
- The download lock's platform list is the native-ingester list, not a copy.
- A walk that used a credential and finished with no error type stamps that
  credential's last-verified time, so the credentials page shows when the
  account last worked. A tier-limited walk does not count: an expired session
  hides paid posts the same way.
- The extension no longer clears a retired pixiv token on startup.
- Two comments that still described every download as gallery-dl now name
  both backends.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-11-08 08:03:32 -05:00
co-authored by Claude Opus 5.5
parent 4cdc851cb5
commit da98195ada
27 changed files with 602 additions and 794 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_patreon_ingester.py
+28 -27
View File
@@ -21,8 +21,9 @@ from backend.app.models import (
PatreonSeenMedia,
Source,
)
from backend.app.services.gallery_dl import ErrorType
from backend.app.services.download_result import ErrorType
from backend.app.services.ingest_core import _CANARY_MIN_SAMPLE
from backend.app.services.native_ingest_common import ledger_key
from backend.app.services.patreon_client import (
MediaItem,
PatreonAPIError,
@@ -30,7 +31,7 @@ from backend.app.services.patreon_client import (
PatreonDriftError,
)
from backend.app.services.patreon_downloader import MediaOutcome, PostRecordOutcome
from backend.app.services.patreon_ingester import PatreonIngester, _ledger_key
from backend.app.services.patreon_ingester import PatreonIngester
pytestmark = pytest.mark.integration
@@ -141,15 +142,15 @@ class _FakeDownloader:
# disk is left alone (skipped_seen, never refetched).
if seen and not recapture:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.on_disk:
elif ledger_key(m) in self.on_disk:
p = self.tmp_path / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
elif seen:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.error:
elif ledger_key(m) in self.error:
self.download_calls += 1
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
elif _ledger_key(m) in self.quarantine:
elif ledger_key(m) in self.quarantine:
self.download_calls += 1
q = self.tmp_path / "_quarantine" / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="quarantined", path=q, error="bad image"))
@@ -255,7 +256,7 @@ async def test_quarantined_media_surfaced_in_result(source_id, sync_engine, tmp_
NOT written or marked seen."""
m1, m2 = _media("p1", 1), _media("p1", 2)
client = _FakeClient([(None, [("p1", [m1, m2])])])
downloader = _FakeDownloader(tmp_path, quarantine={_ledger_key(m2)})
downloader = _FakeDownloader(tmp_path, quarantine={ledger_key(m2)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
@@ -280,7 +281,7 @@ async def test_tick_skips_seen_via_ledger(source_id, sync_engine, tmp_path):
# Pre-seed the ledger with m1's key.
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m1), post_id="p1"))
s.commit()
client = _FakeClient([(None, [("p1", [m1])])])
@@ -302,7 +303,7 @@ async def test_tick_early_out_after_threshold(source_id, sync_engine, tmp_path):
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
for m in seen_media:
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m), post_id=m.post_id))
s.commit()
pages = [(None, [(m.post_id, [m]) for m in seen_media])]
@@ -348,7 +349,7 @@ def _seed_seen_media(sync_engine, source_id, media):
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
for m in media:
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m), post_id=m.post_id))
s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m), post_id=m.post_id))
s.commit()
@@ -615,7 +616,7 @@ async def test_recovery_bypasses_seen_ledger(source_id, sync_engine, tmp_path):
m1 = _media("p1", 1)
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
s.add(PatreonSeenMedia(source_id=source_id, filehash=ledger_key(m1), post_id="p1"))
s.commit()
client = _FakeClient([(None, [("p1", [m1])])])
@@ -638,7 +639,7 @@ async def test_recovery_tier2_disk_still_skips(source_id, sync_engine, tmp_path)
client = _FakeClient([(None, [("p1", [m1])])])
# File still on disk (a kept image) → tier-2 spares it even under recovery.
_seed_record(sync_engine, tmp_path / "p1_1.jpg")
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
@@ -665,7 +666,7 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
itself, and a normal backfill is the backfill."""
m1 = _media("p1", 1)
client = _FakeClient([(None, [("p1", [m1])])])
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
@@ -704,13 +705,13 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
re-downloading the on-disk media."""
m1 = _media("p1", 1)
# Pre-seed BOTH the media key and the synthetic post key as already seen.
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
_seed_seen(sync_engine, source_id, ledger_key(m1), post_id="p1")
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
_seed_record(sync_engine, tmp_path / "p1_1.jpg")
# 1) Plain backfill: post key is seen → gate skips body recapture.
client = _FakeClient([(None, [("p1", [m1])])])
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
res_backfill = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -722,7 +723,7 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
# the on-disk file is surfaced for source_filehash relink.
client2 = _FakeClient([(None, [("p1", [m1])])])
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
downloader2 = _FakeDownloader(tmp_path, on_disk={ledger_key(m1)})
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
res_recap = ing2.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -892,7 +893,7 @@ async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
"""Recapture must NOT re-download a seen item that's absent from disk (that's
recovery's job) — it returns skipped_seen and contributes no relink pair."""
m1 = _media("p1", 1)
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
_seed_seen(sync_engine, source_id, ledger_key(m1), post_id="p1")
client = _FakeClient([(None, [("p1", [m1])])])
# NOT in on_disk → seen + missing.
downloader = _FakeDownloader(tmp_path)
@@ -952,7 +953,7 @@ async def test_a_run_that_dies_before_import_leaves_its_media_unmarked(
# The next walk finds the file on disk with no record, and imports it.
ing2 = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]),
_FakeDownloader(tmp_path, on_disk={_ledger_key(m1)}))
_FakeDownloader(tmp_path, on_disk={ledger_key(m1)}))
result = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="tick")
assert result.written_paths == [str(tmp_path / "p1_1.jpg")]
@@ -971,9 +972,9 @@ async def test_recapture_imports_a_seen_file_nothing_imported(
a walk that looks past it — recapture — reaches them."""
m1, m2 = _media("p1", 1), _media("p1", 2)
for m in (m1, m2):
_seed_seen(sync_engine, source_id, _ledger_key(m), post_id="p1")
_seed_seen(sync_engine, source_id, ledger_key(m), post_id="p1")
_seed_record(sync_engine, tmp_path / "p1_2.jpg") # m2 made it in; m1 did not
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)})
downloader = _FakeDownloader(tmp_path, on_disk={ledger_key(m1), ledger_key(m2)})
ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1, m2])])]),
downloader)
result = ing.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -1002,7 +1003,7 @@ async def test_files_phase_3_leaves_unimported_stay_unmarked(
assert _count_ledger(sync_engine, source_id) == 3 # m1 and both post records
ing2 = _ingester(sync_engine, tmp_path, _FakeClient(pages),
_FakeDownloader(tmp_path, on_disk={_ledger_key(m1), _ledger_key(m2)}))
_FakeDownloader(tmp_path, on_disk={ledger_key(m1), ledger_key(m2)}))
again = ing2.run(source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="backfill")
assert again.written_paths == [str(tmp_path / "p2_1.jpg")]
@@ -1041,7 +1042,7 @@ async def test_repeated_failures_dead_letter_then_skip(source_id, sync_engine, t
ing = _ingester(
sync_engine, tmp_path,
_FakeClient([(None, [("p1", [m1])])]),
_FakeDownloader(tmp_path, error={_ledger_key(m1)}),
_FakeDownloader(tmp_path, error={ledger_key(m1)}),
)
result = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
@@ -1070,7 +1071,7 @@ async def test_recovery_reattempts_dead_lettered_and_clears_on_success(
from backend.app.services.patreon_ingester import DEAD_LETTER_THRESHOLD
m1 = _media("p1", 1)
_seed_failed(sync_engine, source_id, _ledger_key(m1), DEAD_LETTER_THRESHOLD)
_seed_failed(sync_engine, source_id, ledger_key(m1), DEAD_LETTER_THRESHOLD)
downloader = _FakeDownloader(tmp_path) # succeeds
ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
@@ -1090,7 +1091,7 @@ async def test_clean_download_clears_below_threshold_failure(
"""A media with a sub-threshold failure that now downloads cleanly (tick) has
its dead-letter row cleared."""
m1 = _media("p1", 1)
_seed_failed(sync_engine, source_id, _ledger_key(m1), 1) # not yet dead
_seed_failed(sync_engine, source_id, ledger_key(m1), 1) # not yet dead
downloader = _FakeDownloader(tmp_path)
ing = _ingester(sync_engine, tmp_path, _FakeClient([(None, [("p1", [m1])])]), downloader)
@@ -1113,8 +1114,8 @@ async def test_mark_seen_is_idempotent(source_id, sync_engine, tmp_path):
ing = _ingester(sync_engine, tmp_path, client, downloader)
# Same media downloaded twice across two runs — the second marks seen again.
ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
ing._mark_seen(source_id, [(_ledger_key(m1), "p1")])
ing._mark_seen(source_id, [(ledger_key(m1), "p1")])
ing._mark_seen(source_id, [(ledger_key(m1), "p1")])
assert _count_ledger(sync_engine, source_id) == 1
@@ -1180,7 +1181,7 @@ def test_ledger_key_video_has_no_filehash():
url="https://stream.mux.com/abc.m3u8", filename="clip.mp4",
kind="postfile", filehash=None, post_id="p9",
)
assert _ledger_key(video) == "p9:clip.mp4"
assert ledger_key(video) == "p9:clip.mp4"
# --- verify_patreon_credential (the verify counterpart, plan #697) ---------
@@ -1350,7 +1351,7 @@ def _seed_all_seen(sync_engine, source_id, media):
the tests that use it is invisible at the point of use.
"""
for m in media:
_seed_seen(sync_engine, source_id, _ledger_key(m), post_id=m.post_id)
_seed_seen(sync_engine, source_id, ledger_key(m), post_id=m.post_id)
@pytest.mark.asyncio