feat(subscriptions): live posts-processed progress on backfill/recovery — #704 step 2
The running badge only showed the chunk counter; now it shows posts walked — real walk progress. The ingester already reports posts_processed per chunk (step 1); the backfill lifecycle accumulates it into config_overrides._backfill_posts across chunks. SourceRecord exposes backfill_posts; start_backfill/start_recovery clear it (fresh walk); stop clears it too. SourceRow/SourceCard badge renders "Recovering · 45 posts" (falls back to "(N)" chunks before any posts are counted). Per-chunk accumulation (no mid-walk DB write) — simple and race-free; a small over-count from each chunk re-walking its resumed page is fine for a progress indicator. Tests: lifecycle accumulates posts_processed across chunks; start clears a prior _backfill_posts and the record exposes it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -648,6 +648,40 @@ async def test_backfill_stuck_guard_stalls_after_two_no_advance(
|
||||
assert "_backfill_cursor" not in co2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_accumulates_posts_processed(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""plan #704 (#5): the lifecycle accumulates DownloadResult.posts_processed
|
||||
into _backfill_posts across chunks for the live progress badge."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from backend.app.services.download_service import DownloadService
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_posts": 5}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
|
||||
)
|
||||
ctx = {
|
||||
"source_id": source.id, "platform": "patreon",
|
||||
"config_overrides": {"_backfill_state": "running", "_backfill_posts": 5},
|
||||
"backfill_runs_remaining": 5,
|
||||
}
|
||||
await svc._apply_backfill_lifecycle(
|
||||
ctx, _make_fake_dl_result(success=False, cursor="C1", posts_processed=10),
|
||||
)
|
||||
await db.commit()
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_posts") == 15 # 5 (prior) + 10 (this chunk)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_timeout_chunk_reclassified_to_ok(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
@@ -238,15 +238,25 @@ async def test_start_recovery_arms_bypass_flag(db):
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice",
|
||||
)
|
||||
# Simulate a prior walk's posts counter — start must clear it (plan #704).
|
||||
await db.execute(
|
||||
Source.__table__.update().where(Source.id == rec.id).values(
|
||||
config_overrides={"_backfill_posts": 42},
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
updated = await svc.start_recovery(rec.id)
|
||||
assert updated.backfill_state == "running"
|
||||
assert updated.backfill_bypass_seen is True
|
||||
assert updated.backfill_posts == 0 # cleared for a fresh walk
|
||||
assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
|
||||
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == rec.id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_bypass_seen") is True
|
||||
assert "_backfill_posts" not in co
|
||||
|
||||
# Stop clears the bypass flag too (shared lifecycle).
|
||||
stopped = await svc.stop_backfill(rec.id)
|
||||
|
||||
Reference in New Issue
Block a user