feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -206,6 +206,54 @@ def test_skip_disk(tmp_path):
|
||||
assert existing.read_bytes() == b"already here" # untouched
|
||||
|
||||
|
||||
def test_recapture_surfaces_seen_on_disk_without_download(tmp_path):
|
||||
"""#830 recapture: a seen item that IS on disk is surfaced as skipped_disk
|
||||
(with its path, for source_filehash backfill) — NOT tier-1 short-circuited,
|
||||
and never re-downloaded."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
|
||||
post_dir.mkdir(parents=True)
|
||||
existing = post_dir / "01_media1.png"
|
||||
existing.write_bytes(b"already here")
|
||||
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "skipped_disk"
|
||||
assert outcomes[0].path == existing
|
||||
assert session.calls == [] # no re-download
|
||||
assert existing.read_bytes() == b"already here" # untouched
|
||||
|
||||
|
||||
def test_recapture_does_not_refetch_seen_missing(tmp_path):
|
||||
"""#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen,
|
||||
no download) — refetching is recovery's job, not recapture's."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "skipped_seen"
|
||||
assert outcomes[0].path is None
|
||||
assert session.calls == [] # not refetched
|
||||
|
||||
|
||||
def test_recapture_still_downloads_genuinely_new(tmp_path):
|
||||
"""#830 recapture: media that is neither seen nor on disk still downloads — a
|
||||
recapture walk is thorough, it just doesn't re-pull existing files."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: False, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert outcomes[0].path is not None
|
||||
|
||||
|
||||
def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
|
||||
Reference in New Issue
Block a user