feat(patreon): incremental cursor checkpoint mid-walk — #705 step 1 (#6)
CI / lint (push) Successful in 2s
CI / frontend-build (push) Successful in 21s
CI / backend-lint-and-test (push) Successful in 25s
CI / integration (push) Successful in 2m59s

A worker SIGKILL (hard-time-limit backstop) mid-chunk lost the whole
chunk's walk — the cursor was only persisted at chunk boundaries by phase
3, so the next tick re-walked from the chunk start. Now the ingester
checkpoints _backfill_cursor to the DB at each page boundary (backfill/
recovery only) via an ATOMIC single-key UPDATE (config_overrides::jsonb →
jsonb_set('{_backfill_cursor}') → ::json), so it never clobbers operator
config or other backfill keys. On a crash the last mid-walk cursor
survives → the next chunk resumes near the crash, not the chunk start.
phase 3 still writes the final cursor (same value); this is the safety net.

Tests: a backfill walk leaves the last page's cursor in the DB (written by
the ingester, before any phase 3); a tick never checkpoints.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-05 23:58:16 -04:00
parent e42a86d995
commit 4bb11ce7dc
2 changed files with 73 additions and 1 deletions
+41
View File
@@ -245,6 +245,47 @@ async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine,
assert result.run_stats["downloaded_count"] == 2
@pytest.mark.asyncio
async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db):
"""plan #705 #6: the cursor is persisted to the DB at each page boundary
DURING the walk (not just at the end via phase 3), so a worker SIGKILL
mid-chunk resumes near the crash. After the walk the source carries the last
page's cursor — written by the ingester, not phase 3 (which the unit run
never reaches)."""
pages = [
(None, [("p1", [_media("p1", 1)])]),
("CUR2", [("p2", [_media("p2", 1)])]),
("CUR3", [("p3", [_media("p3", 1)])]),
]
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="backfill",
)
co = (await db.execute(
select(Source.config_overrides).where(Source.id == source_id)
)).scalar_one()
assert co.get("_backfill_cursor") == "CUR3"
@pytest.mark.asyncio
async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
"""A tick has no resumable backfill state, so it must not write a cursor."""
ing = _ingester(
sync_engine, tmp_path,
_FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
_FakeDownloader(tmp_path),
)
ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="tick",
)
co = (await db.execute(
select(Source.config_overrides).where(Source.id == source_id)
)).scalar_one()
assert "_backfill_cursor" not in (co or {})
@pytest.mark.asyncio
async def test_backfill_budget_cut_returns_partial_with_progress(
source_id, sync_engine, tmp_path, monkeypatch,