feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+15
-11
@@ -138,14 +138,16 @@ async def delete_source(source_id: int):
|
||||
|
||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||
async def set_backfill(source_id: int):
|
||||
"""Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
|
||||
Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
|
||||
walks the full post history in time-boxed chunks until it reaches the bottom
|
||||
(then the source shows 'complete'); 'recover' is the same walk but bypasses
|
||||
the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
|
||||
current pHash threshold; 'stop' cancels either back to tick mode. Returns the
|
||||
updated source dict (incl. backfill_state / backfill_chunks /
|
||||
backfill_bypass_seen)."""
|
||||
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
|
||||
recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
|
||||
(default "start"). 'start' walks the full post history in time-boxed chunks
|
||||
until it reaches the bottom (then the source shows 'complete'); 'recover' is
|
||||
the same walk but bypasses the Patreon seen-ledger to re-fetch
|
||||
dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
|
||||
re-grabs EVERY post's body + external links and localizes on-disk inline
|
||||
images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
|
||||
Returns the updated source dict (incl. backfill_state / backfill_chunks /
|
||||
backfill_bypass_seen / backfill_recapture)."""
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.credential_service import CredentialService
|
||||
@@ -157,10 +159,10 @@ async def set_backfill(source_id: int):
|
||||
|
||||
payload = await request.get_json(silent=True) or {}
|
||||
action = payload.get("action", "start")
|
||||
if action not in ("start", "stop", "recover"):
|
||||
if action not in ("start", "stop", "recover", "recapture"):
|
||||
return _bad(
|
||||
"invalid_action",
|
||||
detail="action must be 'start', 'stop', or 'recover'",
|
||||
detail="action must be 'start', 'stop', 'recover', or 'recapture'",
|
||||
)
|
||||
|
||||
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
|
||||
@@ -170,7 +172,7 @@ async def set_backfill(source_id: int):
|
||||
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
|
||||
# an arm action. The credential read happens in a session that's CLOSED
|
||||
# before the verify network call (don't hold a DB conn across the request).
|
||||
if action in ("start", "recover"):
|
||||
if action in ("start", "recover", "recapture"):
|
||||
async with get_session() as session:
|
||||
rec = await SourceService(session).get(source_id)
|
||||
if rec is None:
|
||||
@@ -200,6 +202,8 @@ async def set_backfill(source_id: int):
|
||||
record = await svc.start_backfill(source_id)
|
||||
elif action == "recover":
|
||||
record = await svc.start_recovery(source_id)
|
||||
elif action == "recapture":
|
||||
record = await svc.start_recapture(source_id)
|
||||
else:
|
||||
record = await svc.stop_backfill(source_id)
|
||||
except LookupError:
|
||||
|
||||
Reference in New Issue
Block a user