feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 44s
CI / integration (push) Successful in 3m21s
CI / lint (push) Successful in 4s

A plain backfill gates post-body capture on the seen-ledger, so a post whose
media is already on disk AND whose post key is already seen never gets its body
recaptured (operator-flagged: Industrial Lust description missing). Recovery
recaptures unconditionally but re-downloads the whole source.

New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the
post-record gate so EVERY post's body + external links are re-captured
(detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and
surfaces already-present media via a separate non-deleting relink channel so the
importer backfills ImageRecord.source_filehash for inline-image localization.

- ingest_core: recapture mode + recapture_records gate bypass + relink collect
- patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path),
  never refetches seen-missing media, still downloads genuinely-new
- importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks
- download_service: mode derivation + phase-3 relink loop + lifecycle clear
- source_service/api: start_recapture + backfill_recapture field + action
- frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/
  Row/Card/SubscriptionsTab + sources store
- tests across ingester/downloader/importer/source_service/api/download_service

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-14 20:58:40 -04:00
parent 96c29c370b
commit 65ec29ba9b
18 changed files with 467 additions and 29 deletions
+81 -2
View File
@@ -88,16 +88,23 @@ class _FakeDownloader:
self.post_records = 0
def download_post(self, post, media_items, artist_slug, *, is_seen,
should_stop=lambda: False):
should_stop=lambda: False, recapture=False):
outcomes = []
for m in media_items:
if should_stop():
break
if is_seen(m):
seen = is_seen(m)
# Recapture mirrors the real downloader: a seen item isn't tier-1
# short-circuited — it falls through so an on-disk file surfaces as
# skipped_disk (for source_filehash backfill); a seen file NOT on
# disk is left alone (skipped_seen, never refetched).
if seen and not recapture:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.on_disk:
p = self.tmp_path / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
elif seen:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.error:
self.download_calls += 1
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
@@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
assert downloader.post_records == 1
def _seed_seen(sync_engine, source_id, key, post_id=None):
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id))
s.commit()
@pytest.mark.asyncio
async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
source_id, sync_engine, tmp_path,
):
"""#830: a plain backfill GATES post-record capture on the seen-ledger, so a
post whose `post:<id>` key is already seen is NOT recaptured (the operator's
'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and
re-captures the body/links even though the post key is seen — without
re-downloading the on-disk media."""
m1 = _media("p1", 1)
# Pre-seed BOTH the media key and the synthetic post key as already seen.
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
# 1) Plain backfill: post key is seen → gate skips body recapture.
client = _FakeClient([(None, [("p1", [m1])])])
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
res_backfill = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="backfill",
)
assert res_backfill.post_record_paths == [] # gated → not recaptured
assert downloader.post_records == 0
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
# the on-disk file is surfaced for source_filehash relink.
client2 = _FakeClient([(None, [("p1", [m1])])])
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
res_recap = ing2.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="recapture",
)
assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured
assert downloader2.post_records == 1
assert res_recap.files_downloaded == 0 # no media re-download
assert downloader2.download_calls == 0
# The on-disk media is surfaced as a (path, CDN url) relink pair.
assert len(res_recap.relink_source_paths) == 1
rel_path, rel_url = res_recap.relink_source_paths[0]
assert rel_url == m1.url
@pytest.mark.asyncio
async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
source_id, sync_engine, tmp_path,
):
"""Recapture must NOT re-download a seen item that's absent from disk (that's
recovery's job) — it returns skipped_seen and contributes no relink pair."""
m1 = _media("p1", 1)
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
client = _FakeClient([(None, [("p1", [m1])])])
# NOT in on_disk → seen + missing.
downloader = _FakeDownloader(tmp_path)
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="recapture",
)
assert result.files_downloaded == 0
assert downloader.download_calls == 0
assert result.relink_source_paths == []
# --- dead-letter ledger (plan #705 #7) ------------------------------------