Review of the #1–#9 ingester roadmap found two real-but-small gaps; this closes both. #5 (live posts progress) shipped at per-chunk granularity — _apply_backfill_ lifecycle accumulated DownloadResult.posts_processed AFTER each chunk, so the badge didn't move during a chunk (up to ~14.5 min) and over-counted the re-walked resume page. The plan called for within-chunk live updates. Move ownership of _backfill_posts into the ingester: ingest_core writes a monotonic absolute (posts_base + net-new) via _checkpoint_posts at each page boundary and once at the end, EXCLUDING the resumed page so it no longer inflates across chunks. download_service seeds posts_base from prior chunks and stops touching the key (the lifecycle now carries the ingester's committed value forward). #8 (per-media transient/permanent retry) covered only the plain-GET path (_fetch_to_file); the Mux/video path returned None on any yt-dlp failure with no retry. Give _run_ytdlp the same split: TimeoutExpired/OSError are transient (back off + retry up to _MAX_MEDIA_RETRIES), a non-zero exit (CalledProcessError) is permanent (yt-dlp already did its own network retries) → fail fast to the per-item/dead-letter path. Tests: live-posts absolute + resume-page exclusion + tick-doesn't-persist (test_patreon_ingester); lifecycle-leaves-posts-to-ingester rewrite (test_download_service); video transient-retry + permanent-fail-fast (test_patreon_downloader). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -277,6 +277,50 @@ async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_
|
||||
assert co.get("_backfill_cursor") == "CUR3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_persists_live_posts_count(source_id, sync_engine, tmp_path, db):
|
||||
"""plan #704 #5: the ingester writes a monotonic _backfill_posts absolute
|
||||
DURING the walk (seeded from prior chunks via posts_base), EXCLUDING the
|
||||
re-walked resume page so the count doesn't inflate across chunks."""
|
||||
# Resume from CUR2 (a prior chunk left off here): its page is re-walked and
|
||||
# must NOT be re-counted. posts_base=4 stands in for the prior chunks.
|
||||
pages = [
|
||||
("CUR2", [("p1", [_media("p1", 1)])]), # the resumed page — not counted
|
||||
("CUR3", [("p2", [_media("p2", 1)])]), # net-new
|
||||
("CUR4", [("p3", [_media("p3", 1)])]), # net-new
|
||||
]
|
||||
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
||||
ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill",
|
||||
resume_cursor="CUR2", posts_base=4,
|
||||
)
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
# 4 prior + 2 net-new (p2, p3); p1 on the resumed CUR2 page is excluded.
|
||||
assert co.get("_backfill_posts") == 6
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_does_not_persist_posts(source_id, sync_engine, tmp_path, db):
|
||||
"""A tick has no resumable backfill state, so it must not write _backfill_posts
|
||||
(the badge is a backfill/recovery concept)."""
|
||||
ing = _ingester(
|
||||
sync_engine, tmp_path,
|
||||
_FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
|
||||
_FakeDownloader(tmp_path),
|
||||
)
|
||||
ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick",
|
||||
)
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
assert "_backfill_posts" not in (co or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
|
||||
"""A tick has no resumable backfill state, so it must not write a cursor."""
|
||||
|
||||
Reference in New Issue
Block a user