A worker SIGKILL (hard-time-limit backstop) mid-chunk lost the whole
chunk's walk — the cursor was only persisted at chunk boundaries by phase
3, so the next tick re-walked from the chunk start. Now the ingester
checkpoints _backfill_cursor to the DB at each page boundary (backfill/
recovery only) via an ATOMIC single-key UPDATE (config_overrides::jsonb →
jsonb_set('{_backfill_cursor}') → ::json), so it never clobbers operator
config or other backfill keys. On a crash the last mid-walk cursor
survives → the next chunk resumes near the crash, not the chunk start.
phase 3 still writes the final cursor (same value); this is the safety net.
Tests: a backfill walk leaves the last page's cursor in the DB (written by
the ingester, before any phase 3); a tick never checkpoints.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -245,6 +245,47 @@ async def test_backfill_reaches_bottom_signals_complete(source_id, sync_engine,
|
||||
assert result.run_stats["downloaded_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_checkpoints_cursor_mid_walk(source_id, sync_engine, tmp_path, db):
|
||||
"""plan #705 #6: the cursor is persisted to the DB at each page boundary
|
||||
DURING the walk (not just at the end via phase 3), so a worker SIGKILL
|
||||
mid-chunk resumes near the crash. After the walk the source carries the last
|
||||
page's cursor — written by the ingester, not phase 3 (which the unit run
|
||||
never reaches)."""
|
||||
pages = [
|
||||
(None, [("p1", [_media("p1", 1)])]),
|
||||
("CUR2", [("p2", [_media("p2", 1)])]),
|
||||
("CUR3", [("p3", [_media("p3", 1)])]),
|
||||
]
|
||||
ing = _ingester(sync_engine, tmp_path, _FakeClient(pages), _FakeDownloader(tmp_path))
|
||||
ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill",
|
||||
)
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_cursor") == "CUR3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_does_not_checkpoint_cursor(source_id, sync_engine, tmp_path, db):
|
||||
"""A tick has no resumable backfill state, so it must not write a cursor."""
|
||||
ing = _ingester(
|
||||
sync_engine, tmp_path,
|
||||
_FakeClient([("CUR2", [("p1", [_media("p1", 1)])])]),
|
||||
_FakeDownloader(tmp_path),
|
||||
)
|
||||
ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="tick",
|
||||
)
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
assert "_backfill_cursor" not in (co or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_budget_cut_returns_partial_with_progress(
|
||||
source_id, sync_engine, tmp_path, monkeypatch,
|
||||
|
||||
Reference in New Issue
Block a user