feat(subscriptions): live posts-processed progress on backfill/recovery — #704 step 2
The running badge only showed the chunk counter; now it shows posts walked — real walk progress. The ingester already reports posts_processed per chunk (step 1); the backfill lifecycle accumulates it into config_overrides._backfill_posts across chunks. SourceRecord exposes backfill_posts; start_backfill/start_recovery clear it (fresh walk); stop clears it too. SourceRow/SourceCard badge renders "Recovering · 45 posts" (falls back to "(N)" chunks before any posts are counted). Per-chunk accumulation (no mid-walk DB write) — simple and race-free; a small over-count from each chunk re-walking its resumed page is fine for a progress indicator. Tests: lifecycle accumulates posts_processed across chunks; start clears a prior _backfill_posts and the record exposes it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -515,6 +515,13 @@ class DownloadService:
|
||||
new_overrides = dict(src.config_overrides or {})
|
||||
chunks = int(new_overrides.get("_backfill_chunks", 0)) + 1
|
||||
new_overrides["_backfill_chunks"] = chunks
|
||||
# plan #704 (#5): accumulate posts-processed across chunks so the badge
|
||||
# shows live walk progress, not just the chunk count. Cleared on
|
||||
# (re)start. (Slight over-count: each chunk re-walks the resumed page —
|
||||
# fine for a progress indicator.)
|
||||
new_overrides["_backfill_posts"] = int(
|
||||
new_overrides.get("_backfill_posts", 0)
|
||||
) + int(getattr(dl_result, "posts_processed", 0) or 0)
|
||||
|
||||
completed = (
|
||||
dl_result.success
|
||||
|
||||
@@ -70,6 +70,9 @@ class SourceRecord:
|
||||
# plan #697: a running deep-walk that bypasses the Patreon seen-ledger
|
||||
# (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
|
||||
backfill_bypass_seen: bool
|
||||
# plan #704: cumulative posts processed across the walk's chunks — live
|
||||
# progress for the badge.
|
||||
backfill_posts: int
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
@@ -91,6 +94,7 @@ class SourceRecord:
|
||||
"backfill_state": self.backfill_state,
|
||||
"backfill_chunks": self.backfill_chunks,
|
||||
"backfill_bypass_seen": self.backfill_bypass_seen,
|
||||
"backfill_posts": self.backfill_posts,
|
||||
}
|
||||
|
||||
|
||||
@@ -167,6 +171,7 @@ class SourceService:
|
||||
backfill_state=co.get("_backfill_state"),
|
||||
backfill_chunks=int(co.get("_backfill_chunks", 0)),
|
||||
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
||||
backfill_posts=int(co.get("_backfill_posts", 0)),
|
||||
)
|
||||
|
||||
async def _row_to_record(self, source: Source) -> SourceRecord:
|
||||
@@ -317,7 +322,8 @@ class SourceService:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
co = dict(source.config_overrides or {})
|
||||
co["_backfill_state"] = "running"
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"):
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||
@@ -345,7 +351,8 @@ class SourceService:
|
||||
co = dict(source.config_overrides or {})
|
||||
co["_backfill_state"] = "running"
|
||||
co["_backfill_bypass_seen"] = True
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks"):
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||
@@ -362,7 +369,7 @@ class SourceService:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
co = dict(source.config_overrides or {})
|
||||
for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
|
||||
"_backfill_chunks", "_backfill_bypass_seen"):
|
||||
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = 0
|
||||
|
||||
Reference in New Issue
Block a user