feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -88,16 +88,23 @@ class _FakeDownloader:
|
||||
self.post_records = 0
|
||||
|
||||
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
||||
should_stop=lambda: False):
|
||||
should_stop=lambda: False, recapture=False):
|
||||
outcomes = []
|
||||
for m in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
if is_seen(m):
|
||||
seen = is_seen(m)
|
||||
# Recapture mirrors the real downloader: a seen item isn't tier-1
|
||||
# short-circuited — it falls through so an on-disk file surfaces as
|
||||
# skipped_disk (for source_filehash backfill); a seen file NOT on
|
||||
# disk is left alone (skipped_seen, never refetched).
|
||||
if seen and not recapture:
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||
elif _ledger_key(m) in self.on_disk:
|
||||
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
|
||||
elif seen:
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||
elif _ledger_key(m) in self.error:
|
||||
self.download_calls += 1
|
||||
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
|
||||
@@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
|
||||
assert downloader.post_records == 1
|
||||
|
||||
|
||||
def _seed_seen(sync_engine, source_id, key, post_id=None):
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id))
|
||||
s.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""#830: a plain backfill GATES post-record capture on the seen-ledger, so a
|
||||
post whose `post:<id>` key is already seen is NOT recaptured (the operator's
|
||||
'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and
|
||||
re-captures the body/links even though the post key is seen — without
|
||||
re-downloading the on-disk media."""
|
||||
m1 = _media("p1", 1)
|
||||
# Pre-seed BOTH the media key and the synthetic post key as already seen.
|
||||
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
|
||||
|
||||
# 1) Plain backfill: post key is seen → gate skips body recapture.
|
||||
client = _FakeClient([(None, [("p1", [m1])])])
|
||||
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
res_backfill = ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill",
|
||||
)
|
||||
assert res_backfill.post_record_paths == [] # gated → not recaptured
|
||||
assert downloader.post_records == 0
|
||||
|
||||
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
|
||||
# the on-disk file is surfaced for source_filehash relink.
|
||||
client2 = _FakeClient([(None, [("p1", [m1])])])
|
||||
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
|
||||
res_recap = ing2.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="recapture",
|
||||
)
|
||||
assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured
|
||||
assert downloader2.post_records == 1
|
||||
assert res_recap.files_downloaded == 0 # no media re-download
|
||||
assert downloader2.download_calls == 0
|
||||
# The on-disk media is surfaced as a (path, CDN url) relink pair.
|
||||
assert len(res_recap.relink_source_paths) == 1
|
||||
rel_path, rel_url = res_recap.relink_source_paths[0]
|
||||
assert rel_url == m1.url
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""Recapture must NOT re-download a seen item that's absent from disk (that's
|
||||
recovery's job) — it returns skipped_seen and contributes no relink pair."""
|
||||
m1 = _media("p1", 1)
|
||||
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||
client = _FakeClient([(None, [("p1", [m1])])])
|
||||
# NOT in on_disk → seen + missing.
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
result = ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="recapture",
|
||||
)
|
||||
assert result.files_downloaded == 0
|
||||
assert downloader.download_calls == 0
|
||||
assert result.relink_source_paths == []
|
||||
|
||||
|
||||
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user