feat(subscriptions): dry-run backfill preview — B4 preview (plan #708)
Owning the walk lets an operator gauge "is this source worth a backfill?" before
arming one. ingest_core.Ingester.preview walks the first few feed pages and
counts media NOT already in the seen/dead ledgers, downloading nothing
(read-only). download_backends.preview_source resolves the campaign id + runs it
(native-only, mirrors verify_source_credential / run_download); POST
/api/sources/{id}/preview returns {total_new, posts_scanned, has_more, sample[]}
(409 on auth/drift/unresolvable, 400 for gallery-dl platforms). PatreonClient
gains post_meta(post) for the sample's title/date.
UI: a Patreon-only Preview button (mdi-eye-outline) on SourceRow + SourceCard
opens PreviewDialog — self-fetches with loading / error / empty / result states
and a "Start backfill" shortcut. Store action previewSource.
Tests: preview counts new media without downloading + samples only posts with
new items; page_limit caps the walk + flags has_more.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -65,6 +65,9 @@ class _FakeClient:
|
||||
def extract_media(self, post, included_index):
|
||||
return post["_media"]
|
||||
|
||||
def post_meta(self, post):
|
||||
return {"title": post.get("id"), "date": None}
|
||||
|
||||
|
||||
class _FakeDownloader:
|
||||
"""Stub PatreonDownloader. Honors the injected is_seen (tier-1); media in
|
||||
@@ -382,6 +385,42 @@ async def test_backfill_budget_cut_returns_partial_with_progress(
|
||||
assert result.cursor == "CUR2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preview_counts_new_media_without_downloading(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""plan #708 B4: preview walks + counts media not already seen/dead, downloads
|
||||
nothing, and samples only posts that have new media."""
|
||||
m1, m2, m3 = _media("p1", 1), _media("p2", 1), _media("p2", 2)
|
||||
# Seed m1 as already-seen → only p2's two media are "new".
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
s.add(PatreonSeenMedia(source_id=source_id, filehash=_ledger_key(m1), post_id="p1"))
|
||||
s.commit()
|
||||
client = _FakeClient([(None, [("p1", [m1]), ("p2", [m2, m3])])])
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
|
||||
result = ing.preview(source_id, "c1")
|
||||
assert result["total_new"] == 2 # p2's m2 + m3 (m1 already seen)
|
||||
assert result["posts_scanned"] == 2
|
||||
assert result["has_more"] is False
|
||||
assert downloader.download_calls == 0 # dry-run — nothing downloaded
|
||||
# The sample lists only posts WITH new media (p2), not the all-seen p1.
|
||||
assert [row["new"] for row in result["sample"]] == [2]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preview_page_limit_caps_walk(source_id, sync_engine, tmp_path):
|
||||
"""plan #708 B4: preview stops after page_limit pages and flags has_more."""
|
||||
pages = [(f"CUR{i}", [(f"p{i}", [_media(f"p{i}", 1)])]) for i in range(6)]
|
||||
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
||||
result = ing.preview(source_id, "c1", page_limit=2)
|
||||
assert result["pages_scanned"] == 2
|
||||
assert result["posts_scanned"] == 2 # one post per page, 2 pages
|
||||
assert result["has_more"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_still_running_reads_backfill_state(source_id, sync_engine, tmp_path, db):
|
||||
"""plan #708 B4: _still_running reflects the source's `_backfill_state` — the
|
||||
|
||||
Reference in New Issue
Block a user