feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -60,8 +60,10 @@ def _make_fake_dl_result(
|
||||
*, success=True, written_paths=None, quarantined_paths=None,
|
||||
files_downloaded=0, error_type=None, error_message=None,
|
||||
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
||||
relink_source_paths=None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
relink_source_paths=relink_source_paths or [],
|
||||
success=success,
|
||||
url="https://patreon.com/alice",
|
||||
artist_slug="alice",
|
||||
@@ -793,6 +795,31 @@ async def test_recovery_mode_selected_and_flag_cleared_on_complete(
|
||||
assert "_backfill_bypass_seen" not in co
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recapture_mode_selected_and_flag_cleared_on_complete(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""#830: `_backfill_recapture` alongside a running backfill selects recapture
|
||||
mode. A clean rc=0 walk completes the shared lifecycle AND clears the
|
||||
recapture flag so the next tick is a plain tick again."""
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_recapture": True}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
assert calls[0]["mode"] == "recapture"
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_state") == "complete"
|
||||
assert "_backfill_recapture" not in co
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_error_type_maps_to_ok_status(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
Reference in New Issue
Block a user