feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+15
-11
@@ -138,14 +138,16 @@ async def delete_source(source_id: int):
|
||||
|
||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||
async def set_backfill(source_id: int):
|
||||
"""Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
|
||||
Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
|
||||
walks the full post history in time-boxed chunks until it reaches the bottom
|
||||
(then the source shows 'complete'); 'recover' is the same walk but bypasses
|
||||
the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
|
||||
current pHash threshold; 'stop' cancels either back to tick mode. Returns the
|
||||
updated source dict (incl. backfill_state / backfill_chunks /
|
||||
backfill_bypass_seen)."""
|
||||
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
|
||||
recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
|
||||
(default "start"). 'start' walks the full post history in time-boxed chunks
|
||||
until it reaches the bottom (then the source shows 'complete'); 'recover' is
|
||||
the same walk but bypasses the Patreon seen-ledger to re-fetch
|
||||
dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
|
||||
re-grabs EVERY post's body + external links and localizes on-disk inline
|
||||
images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
|
||||
Returns the updated source dict (incl. backfill_state / backfill_chunks /
|
||||
backfill_bypass_seen / backfill_recapture)."""
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.credential_service import CredentialService
|
||||
@@ -157,10 +159,10 @@ async def set_backfill(source_id: int):
|
||||
|
||||
payload = await request.get_json(silent=True) or {}
|
||||
action = payload.get("action", "start")
|
||||
if action not in ("start", "stop", "recover"):
|
||||
if action not in ("start", "stop", "recover", "recapture"):
|
||||
return _bad(
|
||||
"invalid_action",
|
||||
detail="action must be 'start', 'stop', or 'recover'",
|
||||
detail="action must be 'start', 'stop', 'recover', or 'recapture'",
|
||||
)
|
||||
|
||||
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
|
||||
@@ -170,7 +172,7 @@ async def set_backfill(source_id: int):
|
||||
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
|
||||
# an arm action. The credential read happens in a session that's CLOSED
|
||||
# before the verify network call (don't hold a DB conn across the request).
|
||||
if action in ("start", "recover"):
|
||||
if action in ("start", "recover", "recapture"):
|
||||
async with get_session() as session:
|
||||
rec = await SourceService(session).get(source_id)
|
||||
if rec is None:
|
||||
@@ -200,6 +202,8 @@ async def set_backfill(source_id: int):
|
||||
record = await svc.start_backfill(source_id)
|
||||
elif action == "recover":
|
||||
record = await svc.start_recovery(source_id)
|
||||
elif action == "recapture":
|
||||
record = await svc.start_recapture(source_id)
|
||||
else:
|
||||
record = await svc.stop_backfill(source_id)
|
||||
except LookupError:
|
||||
|
||||
@@ -111,6 +111,10 @@ class DownloadService:
|
||||
# `_backfill_bypass_seen` flag rides alongside the running backfill state;
|
||||
# download mode is "recovery" when both are set.
|
||||
bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
|
||||
# #830 recapture: re-grab post bodies/links + localize on-disk inline
|
||||
# images, WITHOUT re-downloading media. Rides alongside the backfill
|
||||
# state like _backfill_bypass_seen; mutually exclusive with it.
|
||||
recapture = bool(overrides.get("_backfill_recapture"))
|
||||
if in_backfill:
|
||||
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
||||
@@ -130,6 +134,8 @@ class DownloadService:
|
||||
if uses_native_ingester(ctx["platform"]):
|
||||
if in_backfill and bypass_seen:
|
||||
mode = "recovery"
|
||||
elif in_backfill and recapture:
|
||||
mode = "recapture"
|
||||
elif in_backfill:
|
||||
mode = "backfill"
|
||||
else:
|
||||
@@ -396,6 +402,20 @@ class DownloadService:
|
||||
|
||||
await loop.run_in_executor(None, _upsert)
|
||||
|
||||
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
||||
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
||||
# the duplicate file). Empty outside recapture mode.
|
||||
for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []:
|
||||
rel_path = Path(rel_str)
|
||||
if not rel_path.exists(): # noqa: ASYNC240
|
||||
continue
|
||||
|
||||
def _relink(p=rel_path, u=rel_url):
|
||||
return self.importer.relink_source_filehash(p, u, artist=artist)
|
||||
|
||||
await loop.run_in_executor(None, _relink)
|
||||
|
||||
# Kick the off-platform file-host downloader for any links this run
|
||||
# recorded (mega/gdrive/…). Global + idempotent (only claims pending/
|
||||
# retryable rows); the beat sweep is the backstop. Lazy import dodges a
|
||||
@@ -508,6 +528,8 @@ class DownloadService:
|
||||
# plan #697: a recovery walk shares this lifecycle; clear its bypass
|
||||
# flag on completion so the next routine tick honors the seen-ledger.
|
||||
new_overrides.pop("_backfill_bypass_seen", None)
|
||||
# #830: same for the recapture flag.
|
||||
new_overrides.pop("_backfill_recapture", None)
|
||||
src.config_overrides = new_overrides
|
||||
src.backfill_runs_remaining = 0
|
||||
return
|
||||
|
||||
@@ -153,6 +153,11 @@ class DownloadResult:
|
||||
# Importer.upsert_post_record (keyed on external_post_id → updates, never
|
||||
# doubles, the same Post a media import would create).
|
||||
post_record_paths: list[str] = field(default_factory=list)
|
||||
# Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
|
||||
# pairs for already-present media whose ImageRecord.source_filehash should be
|
||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||
# on the gallery-dl path and outside recapture.
|
||||
relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
|
||||
@@ -1005,6 +1005,35 @@ class Importer:
|
||||
return sv.strip()
|
||||
return None
|
||||
|
||||
def relink_source_filehash(
|
||||
self, path: Path, source_url: str, *, artist: Artist | None = None,
|
||||
) -> bool:
|
||||
"""#830 recapture: backfill source_url + source_filehash on an EXISTING
|
||||
on-disk image's ImageRecord (matched by sha256) so its post-body inline
|
||||
`<img src=CDN>` remaps to the local copy at render time — WITHOUT
|
||||
re-downloading and WITHOUT unlinking the file (unlike the import path's
|
||||
duplicate-hash branch). NULL-only, mirroring _apply_sidecar: never
|
||||
clobbers an already-set filehash. Returns True only when a row was
|
||||
updated. No-op when the file is gone, the url has no filehash, no record
|
||||
matches the bytes, or the record already carries a filehash.
|
||||
"""
|
||||
fh = filehash_from_url(source_url)
|
||||
if not fh:
|
||||
return False
|
||||
try:
|
||||
sha = _sha256_of(path)
|
||||
except OSError:
|
||||
return False
|
||||
record = self.session.execute(
|
||||
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
||||
).scalar_one_or_none()
|
||||
if record is None or record.source_filehash is not None:
|
||||
return False
|
||||
record.source_url = source_url
|
||||
record.source_filehash = fh
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def _apply_sidecar(
|
||||
self,
|
||||
record: ImageRecord,
|
||||
|
||||
@@ -105,18 +105,28 @@ class Ingester:
|
||||
) -> DownloadResult:
|
||||
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
||||
|
||||
`mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
|
||||
seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
|
||||
files). The walk stops on:
|
||||
`mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery
|
||||
bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk
|
||||
still skips kept files). Recapture (#830) is the cheap "re-grab post
|
||||
text" walk: it bypasses the post-record gate (so EVERY post's body +
|
||||
external links are re-captured / detail-fetched) but KEEPS the media
|
||||
seen-ledger — on-disk media is NOT re-downloaded, only surfaced so its
|
||||
ImageRecord's source_filehash can be backfilled for inline-image
|
||||
localization. The walk stops on:
|
||||
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
|
||||
- tick early-out (seen_threshold contiguous seen) → success
|
||||
- reaching the bottom of the feed → success (rc 0)
|
||||
A client-level failure (drift / auth / network) fails the whole run loud.
|
||||
"""
|
||||
bypass_seen = mode == "recovery"
|
||||
recapture = mode == "recapture"
|
||||
# Both recovery and recapture re-capture EVERY post's body + links — they
|
||||
# bypass the post-record seen-gate (recovery via bypass_seen, recapture
|
||||
# explicitly). A plain backfill stays gated (capture once per post).
|
||||
recapture_records = bypass_seen or recapture
|
||||
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
|
||||
# tick has no resumable backfill state.
|
||||
checkpoint = mode in ("backfill", "recovery")
|
||||
checkpoint = mode in ("backfill", "recovery", "recapture")
|
||||
ledger_key = self._ledger_key
|
||||
# Optional seams (Patreon native ingester): capture pure-text posts that
|
||||
# have NO downloadable media so the artist archive is complete. Absent on
|
||||
@@ -129,6 +139,10 @@ class Ingester:
|
||||
written: list[str] = []
|
||||
post_records: list[str] = []
|
||||
quarantined_paths: list[str] = []
|
||||
# #830 recapture: (on-disk path, CDN source_url) pairs for already-present
|
||||
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
|
||||
# re-downloading or unlinking the file. Empty outside recapture mode.
|
||||
relink: list[tuple[str, str]] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -165,6 +179,7 @@ class Ingester:
|
||||
quarantined_paths=list(quarantined_paths),
|
||||
written_paths=written,
|
||||
post_record_paths=list(post_records),
|
||||
relink_source_paths=list(relink),
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
@@ -241,7 +256,7 @@ class Ingester:
|
||||
if rk is not None:
|
||||
pkey, ppid = rk
|
||||
already = (
|
||||
set() if bypass_seen
|
||||
set() if recapture_records
|
||||
else self._seen_keys(source_id, [pkey])
|
||||
)
|
||||
if pkey not in already:
|
||||
@@ -275,6 +290,7 @@ class Ingester:
|
||||
outcomes = self.downloader.download_post(
|
||||
post, media, artist_slug, is_seen=_is_skip,
|
||||
should_stop=lambda: time.monotonic() - start >= time_budget_seconds,
|
||||
recapture=recapture,
|
||||
)
|
||||
|
||||
to_mark: list[tuple[str, str]] = []
|
||||
@@ -300,6 +316,12 @@ class Ingester:
|
||||
to_clear.append(key)
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
# #830 recapture: surface (on-disk path, CDN url) so phase
|
||||
# 3 can backfill source_filehash for inline-image
|
||||
# localization — a SEPARATE non-deleting channel, never the
|
||||
# import list (which would unlink the file, per above).
|
||||
if recapture and outcome.path is not None:
|
||||
relink.append((str(outcome.path), media_item.url))
|
||||
elif outcome.status == "skipped_seen":
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
|
||||
@@ -198,6 +198,7 @@ class PatreonDownloader:
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one post; return per-item outcomes.
|
||||
|
||||
@@ -212,6 +213,11 @@ class PatreonDownloader:
|
||||
re-checks the budget between posts), so we honour the deadline mid-post
|
||||
and return the items done so far — the rest re-fetch next chunk (they
|
||||
were never marked seen). Bounds chunk overrun to one media download.
|
||||
|
||||
`recapture` (#830): don't re-download already-present media, but DO surface
|
||||
on-disk media as `skipped_disk` (with its path) even when the seen-ledger
|
||||
would tier-1 skip it — so the engine can backfill source_filehash for
|
||||
inline-image localization. Genuinely-missing seen media is NOT refetched.
|
||||
"""
|
||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
@@ -221,7 +227,10 @@ class PatreonDownloader:
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(post, media, post_dir, artist_slug, i, is_seen)
|
||||
self._download_one(
|
||||
post, media, post_dir, artist_slug, i, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
@@ -243,9 +252,15 @@ class PatreonDownloader:
|
||||
artist_slug: str,
|
||||
index: int,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
# tier-1: seen ledger (injected; no DB here).
|
||||
if is_seen(media):
|
||||
# tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
|
||||
# short-circuit here — we fall through to the disk check so an on-disk
|
||||
# seen file is surfaced as skipped_disk (with its path) for source_filehash
|
||||
# backfill; a seen file that's NOT on disk is left alone (not refetched).
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
nn = f"{index:02d}"
|
||||
@@ -265,6 +280,12 @@ class PatreonDownloader:
|
||||
media=media, status="skipped_disk", path=existing, error=None
|
||||
)
|
||||
|
||||
# recapture: a seen item that isn't on disk is NOT re-downloaded (that's
|
||||
# recovery's job) — recapture only re-grabs post text + localizes existing
|
||||
# files. Returns skipped_seen so the run-of-seen / counts stay consistent.
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Pace real downloads only (the skips above already returned). plan #703.
|
||||
|
||||
@@ -70,6 +70,10 @@ class SourceRecord:
|
||||
# plan #697: a running deep-walk that bypasses the Patreon seen-ledger
|
||||
# (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
|
||||
backfill_bypass_seen: bool
|
||||
# #830: a running deep-walk in RECAPTURE mode (re-grab post bodies/links +
|
||||
# localize on-disk inline images, no media re-download). Lets the badge label
|
||||
# it "Recapturing".
|
||||
backfill_recapture: bool
|
||||
# plan #704: cumulative posts processed across the walk's chunks — live
|
||||
# progress for the badge.
|
||||
backfill_posts: int
|
||||
@@ -94,6 +98,7 @@ class SourceRecord:
|
||||
"backfill_state": self.backfill_state,
|
||||
"backfill_chunks": self.backfill_chunks,
|
||||
"backfill_bypass_seen": self.backfill_bypass_seen,
|
||||
"backfill_recapture": self.backfill_recapture,
|
||||
"backfill_posts": self.backfill_posts,
|
||||
}
|
||||
|
||||
@@ -171,6 +176,7 @@ class SourceService:
|
||||
backfill_state=co.get("_backfill_state"),
|
||||
backfill_chunks=int(co.get("_backfill_chunks", 0)),
|
||||
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
||||
backfill_recapture=bool(co.get("_backfill_recapture")),
|
||||
backfill_posts=int(co.get("_backfill_posts", 0)),
|
||||
)
|
||||
|
||||
@@ -359,6 +365,34 @@ class SourceService:
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def start_recapture(self, source_id: int) -> SourceRecord:
|
||||
"""#830: arm a RECAPTURE walk — a backfill that re-grabs EVERY post's body
|
||||
+ external links (detail-fetching empty bodies) and localizes already-on-
|
||||
disk inline images, WITHOUT re-downloading media. Reuses the entire #693
|
||||
backfill state machine plus a `_backfill_recapture` flag that flips
|
||||
download mode to recapture. Distinct from recovery (which re-downloads the
|
||||
whole source); the two flags are mutually exclusive, so arming recapture
|
||||
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
||||
fresh from the top. The flag is cleared on completion (download_service)
|
||||
and on stop. Recapture is Patreon-only (the native ingester's post-record
|
||||
capture); inert elsewhere. The UI gates the action to Patreon sources."""
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
co = dict(source.config_overrides or {})
|
||||
co["_backfill_state"] = "running"
|
||||
co["_backfill_recapture"] = True
|
||||
co.pop("_backfill_bypass_seen", None) # mutually exclusive with recovery
|
||||
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def stop_backfill(self, source_id: int) -> SourceRecord:
|
||||
"""Plan #693: cancel an in-progress backfill — back to idle/tick mode.
|
||||
Clears the running state + cursor/chunk/stall bookkeeping."""
|
||||
@@ -369,7 +403,8 @@ class SourceService:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
co = dict(source.config_overrides or {})
|
||||
for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
|
||||
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"):
|
||||
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_recapture",
|
||||
"_backfill_posts"):
|
||||
co.pop(k, None)
|
||||
source.config_overrides = co
|
||||
source.backfill_runs_remaining = 0
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<v-icon>{{ running ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
|
||||
<v-tooltip activator="parent" location="top">
|
||||
{{ running
|
||||
? (recovering ? 'Stop recovery' : 'Stop backfill')
|
||||
? (recovering ? 'Stop recovery' : (recapturing ? 'Stop recapture' : 'Stop backfill'))
|
||||
: 'Backfill — one-time walk of the FULL post history until complete' }}
|
||||
</v-tooltip>
|
||||
</v-btn>
|
||||
@@ -38,6 +38,17 @@
|
||||
under the current similarity threshold
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="isPatreon && !running"
|
||||
prepend-icon="mdi-text-box-search-outline"
|
||||
@click="emit('recapture', source)"
|
||||
>
|
||||
<v-list-item-title>Recapture post text & links</v-list-item-title>
|
||||
<v-list-item-subtitle>
|
||||
Re-grab every post's body + external links and localize inline images
|
||||
already on disk — without re-downloading media
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider v-if="isPatreon && !running" />
|
||||
<v-list-item
|
||||
base-color="error"
|
||||
@@ -58,10 +69,11 @@ const props = defineProps({
|
||||
source: { type: Object, required: true },
|
||||
checking: { type: Boolean, default: false },
|
||||
})
|
||||
const emit = defineEmits(['check', 'backfill', 'recover', 'remove'])
|
||||
const emit = defineEmits(['check', 'backfill', 'recover', 'recapture', 'remove'])
|
||||
|
||||
const running = computed(() => props.source.backfill_state === 'running')
|
||||
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
||||
const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||
const isPatreon = computed(() => props.source.platform === 'patreon')
|
||||
</script>
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
@@ -60,6 +60,7 @@
|
||||
@check="$emit('check', $event)"
|
||||
@backfill="$emit('backfill', $event)"
|
||||
@recover="$emit('recover', $event)"
|
||||
@recapture="$emit('recapture', $event)"
|
||||
@remove="$emit('remove', $event)"
|
||||
/>
|
||||
</div>
|
||||
@@ -79,7 +80,7 @@ const props = defineProps({
|
||||
checking: { type: Boolean, default: false },
|
||||
warningThreshold: { type: Number, default: 5 },
|
||||
})
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
||||
|
||||
function onToggleEnabled(value) {
|
||||
emit('toggle', { source: props.source, enabled: value })
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
@@ -71,6 +71,7 @@
|
||||
@check="$emit('check', $event)"
|
||||
@backfill="$emit('backfill', $event)"
|
||||
@recover="$emit('recover', $event)"
|
||||
@recapture="$emit('recapture', $event)"
|
||||
@remove="$emit('remove', $event)"
|
||||
/>
|
||||
</td>
|
||||
@@ -87,7 +88,7 @@ const props = defineProps({
|
||||
checking: { type: Boolean, default: false },
|
||||
warningThreshold: { type: Number, default: 5 },
|
||||
})
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
||||
|
||||
function onToggleEnabled(value) {
|
||||
emit('toggle', { source: props.source, enabled: value })
|
||||
|
||||
@@ -138,7 +138,7 @@
|
||||
:source="item.singleSource"
|
||||
:checking="store.checkingIds.has(item.singleSource.id)"
|
||||
@check="onCheck" @backfill="onBackfill"
|
||||
@recover="onRecover" @remove="removeSource"
|
||||
@recover="onRecover" @recapture="onRecapture" @remove="removeSource"
|
||||
/>
|
||||
<v-btn icon size="small" variant="text" @click.stop="openEditSource(item.singleSource)">
|
||||
<v-icon>mdi-pencil</v-icon>
|
||||
@@ -214,6 +214,7 @@
|
||||
@check="onCheck"
|
||||
@backfill="onBackfill"
|
||||
@recover="onRecover"
|
||||
@recapture="onRecapture"
|
||||
/>
|
||||
<tr v-if="item.sources.length === 0">
|
||||
<td colspan="8" class="fc-subs__sources-empty">
|
||||
@@ -291,6 +292,7 @@
|
||||
@check="onCheck"
|
||||
@backfill="onBackfill"
|
||||
@recover="onRecover"
|
||||
@recapture="onRecapture"
|
||||
/>
|
||||
<div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
|
||||
No sources yet. Tap + to add one.
|
||||
@@ -646,6 +648,22 @@ async function onRecover(source) {
|
||||
}
|
||||
}
|
||||
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, without re-downloading
|
||||
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
||||
async function onRecapture(source) {
|
||||
try {
|
||||
await store.recaptureSource(source.id, source.artist_id)
|
||||
toast({ text: `Recapture started for ${source.artist_name}`, type: 'success' })
|
||||
// recaptureSource patches the source in place — no full reload.
|
||||
} catch (e) {
|
||||
toast({
|
||||
text: `Recapture start failed: ${e?.body?.detail || e?.message || e}`,
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAll(group) {
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
|
||||
@@ -131,6 +131,14 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
_patchSource(body)
|
||||
return body
|
||||
}
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, WITHOUT re-downloading
|
||||
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
||||
async function recaptureSource(id, artistIdHint = null) {
|
||||
const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recapture' } })
|
||||
_patchSource(body)
|
||||
return body
|
||||
}
|
||||
|
||||
// Plan #708 B4: dry-run preview — count what a backfill WOULD download
|
||||
// (native platforms), without downloading. Read-only, so no cache invalidate.
|
||||
@@ -167,6 +175,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
startBackfill,
|
||||
stopBackfill,
|
||||
recoverSource,
|
||||
recaptureSource,
|
||||
previewSource,
|
||||
findOrCreateArtist, autocompleteArtist,
|
||||
loadScheduleStatus,
|
||||
|
||||
@@ -324,6 +324,32 @@ async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
|
||||
assert stopped_body["backfill_bypass_seen"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_recapture_arms_flag(client, artist, db):
|
||||
"""#830: action='recapture' arms a backfill that re-grabs post bodies/links +
|
||||
localizes on-disk inline images; the response exposes backfill_recapture=True
|
||||
(badge 'Recapturing') and NOT backfill_bypass_seen. Stop clears it."""
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-recapture", enabled=True,
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
sid = src.id
|
||||
|
||||
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recapture"})
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["backfill_state"] == "running"
|
||||
assert body["backfill_recapture"] is True
|
||||
assert body["backfill_bypass_seen"] is False
|
||||
|
||||
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
|
||||
stopped_body = await stopped.get_json()
|
||||
assert stopped_body["backfill_state"] is None
|
||||
assert stopped_body["backfill_recapture"] is False
|
||||
|
||||
|
||||
# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
|
||||
|
||||
|
||||
|
||||
@@ -60,8 +60,10 @@ def _make_fake_dl_result(
|
||||
*, success=True, written_paths=None, quarantined_paths=None,
|
||||
files_downloaded=0, error_type=None, error_message=None,
|
||||
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
||||
relink_source_paths=None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
relink_source_paths=relink_source_paths or [],
|
||||
success=success,
|
||||
url="https://patreon.com/alice",
|
||||
artist_slug="alice",
|
||||
@@ -793,6 +795,31 @@ async def test_recovery_mode_selected_and_flag_cleared_on_complete(
|
||||
assert "_backfill_bypass_seen" not in co
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recapture_mode_selected_and_flag_cleared_on_complete(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""#830: `_backfill_recapture` alongside a running backfill selects recapture
|
||||
mode. A clean rc=0 walk completes the shared lifecycle AND clears the
|
||||
recapture flag so the next tick is a plain tick again."""
|
||||
_artist, source = seed_artist_and_source
|
||||
source.config_overrides = {"_backfill_state": "running", "_backfill_recapture": True}
|
||||
source.backfill_runs_remaining = 5
|
||||
await db.commit()
|
||||
|
||||
svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
assert calls[0]["mode"] == "recapture"
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_state") == "complete"
|
||||
assert "_backfill_recapture" not in co
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_error_type_maps_to_ok_status(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
@@ -206,6 +206,54 @@ def test_skip_disk(tmp_path):
|
||||
assert existing.read_bytes() == b"already here" # untouched
|
||||
|
||||
|
||||
def test_recapture_surfaces_seen_on_disk_without_download(tmp_path):
|
||||
"""#830 recapture: a seen item that IS on disk is surfaced as skipped_disk
|
||||
(with its path, for source_filehash backfill) — NOT tier-1 short-circuited,
|
||||
and never re-downloaded."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
|
||||
post_dir.mkdir(parents=True)
|
||||
existing = post_dir / "01_media1.png"
|
||||
existing.write_bytes(b"already here")
|
||||
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "skipped_disk"
|
||||
assert outcomes[0].path == existing
|
||||
assert session.calls == [] # no re-download
|
||||
assert existing.read_bytes() == b"already here" # untouched
|
||||
|
||||
|
||||
def test_recapture_does_not_refetch_seen_missing(tmp_path):
|
||||
"""#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen,
|
||||
no download) — refetching is recovery's job, not recapture's."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: True, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "skipped_seen"
|
||||
assert outcomes[0].path is None
|
||||
assert session.calls == [] # not refetched
|
||||
|
||||
|
||||
def test_recapture_still_downloads_genuinely_new(tmp_path):
|
||||
"""#830 recapture: media that is neither seen nor on disk still downloads — a
|
||||
recapture walk is thorough, it just doesn't re-pull existing files."""
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
outcomes = dl.download_post(
|
||||
_post(), [_img("media1.png")], "artist-x",
|
||||
is_seen=lambda m: False, recapture=True,
|
||||
)
|
||||
assert outcomes[0].status == "downloaded"
|
||||
assert outcomes[0].path is not None
|
||||
|
||||
|
||||
def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
|
||||
session = _FakeSession()
|
||||
dl = _downloader(tmp_path, session=session)
|
||||
|
||||
@@ -88,16 +88,23 @@ class _FakeDownloader:
|
||||
self.post_records = 0
|
||||
|
||||
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
||||
should_stop=lambda: False):
|
||||
should_stop=lambda: False, recapture=False):
|
||||
outcomes = []
|
||||
for m in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
if is_seen(m):
|
||||
seen = is_seen(m)
|
||||
# Recapture mirrors the real downloader: a seen item isn't tier-1
|
||||
# short-circuited — it falls through so an on-disk file surfaces as
|
||||
# skipped_disk (for source_filehash backfill); a seen file NOT on
|
||||
# disk is left alone (skipped_seen, never refetched).
|
||||
if seen and not recapture:
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||
elif _ledger_key(m) in self.on_disk:
|
||||
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
|
||||
elif seen:
|
||||
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||
elif _ledger_key(m) in self.error:
|
||||
self.download_calls += 1
|
||||
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
|
||||
@@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
|
||||
assert downloader.post_records == 1
|
||||
|
||||
|
||||
def _seed_seen(sync_engine, source_id, key, post_id=None):
|
||||
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||
with factory() as s:
|
||||
s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id))
|
||||
s.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""#830: a plain backfill GATES post-record capture on the seen-ledger, so a
|
||||
post whose `post:<id>` key is already seen is NOT recaptured (the operator's
|
||||
'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and
|
||||
re-captures the body/links even though the post key is seen — without
|
||||
re-downloading the on-disk media."""
|
||||
m1 = _media("p1", 1)
|
||||
# Pre-seed BOTH the media key and the synthetic post key as already seen.
|
||||
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
|
||||
|
||||
# 1) Plain backfill: post key is seen → gate skips body recapture.
|
||||
client = _FakeClient([(None, [("p1", [m1])])])
|
||||
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
res_backfill = ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="backfill",
|
||||
)
|
||||
assert res_backfill.post_record_paths == [] # gated → not recaptured
|
||||
assert downloader.post_records == 0
|
||||
|
||||
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
|
||||
# the on-disk file is surfaced for source_filehash relink.
|
||||
client2 = _FakeClient([(None, [("p1", [m1])])])
|
||||
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
|
||||
res_recap = ing2.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="recapture",
|
||||
)
|
||||
assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured
|
||||
assert downloader2.post_records == 1
|
||||
assert res_recap.files_downloaded == 0 # no media re-download
|
||||
assert downloader2.download_calls == 0
|
||||
# The on-disk media is surfaced as a (path, CDN url) relink pair.
|
||||
assert len(res_recap.relink_source_paths) == 1
|
||||
rel_path, rel_url = res_recap.relink_source_paths[0]
|
||||
assert rel_url == m1.url
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
|
||||
source_id, sync_engine, tmp_path,
|
||||
):
|
||||
"""Recapture must NOT re-download a seen item that's absent from disk (that's
|
||||
recovery's job) — it returns skipped_seen and contributes no relink pair."""
|
||||
m1 = _media("p1", 1)
|
||||
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||
client = _FakeClient([(None, [("p1", [m1])])])
|
||||
# NOT in on_disk → seen + missing.
|
||||
downloader = _FakeDownloader(tmp_path)
|
||||
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||
result = ing.run(
|
||||
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||
url="https://patreon.com/ingest", mode="recapture",
|
||||
)
|
||||
assert result.files_downloaded == 0
|
||||
assert downloader.download_calls == 0
|
||||
assert result.relink_source_paths == []
|
||||
|
||||
|
||||
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -120,6 +120,44 @@ def test_sidecar_source_url_persists_filehash(importer, import_layout):
|
||||
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||
|
||||
|
||||
def test_relink_source_filehash_backfills_existing(importer, import_layout):
|
||||
"""#830 recapture (Part B): relink_source_filehash backfills source_url +
|
||||
source_filehash on an existing on-disk image's ImageRecord (matched by
|
||||
sha256) WITHOUT re-importing or unlinking the file. NULL-only — never
|
||||
clobbers a filehash already set."""
|
||||
import_root, _ = import_layout
|
||||
m = import_root / "Alice" / "img.jpg"
|
||||
_split(m, "v")
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
rec = importer.session.get(ImageRecord, r.image_id)
|
||||
assert rec.source_filehash is None
|
||||
|
||||
url = "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/img.jpg"
|
||||
assert importer.relink_source_filehash(m, url) is True
|
||||
importer.session.refresh(rec)
|
||||
assert rec.source_url == url
|
||||
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||
assert m.exists() # never unlinked
|
||||
|
||||
# NULL-only: a second relink with a different url is a no-op.
|
||||
other = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/x.jpg"
|
||||
assert importer.relink_source_filehash(m, other) is False
|
||||
importer.session.refresh(rec)
|
||||
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||
|
||||
|
||||
def test_relink_source_filehash_no_match_is_noop(importer, import_layout):
|
||||
"""A path whose bytes match no ImageRecord (never imported) is a clean no-op,
|
||||
not an error."""
|
||||
import_root, _ = import_layout
|
||||
m = import_root / "Alice" / "ghost.jpg"
|
||||
_split(m, "h")
|
||||
assert importer.relink_source_filehash(
|
||||
m, "https://cdn.test/p/0123456789abcdef0123456789abcdef/ghost.jpg"
|
||||
) is False
|
||||
|
||||
|
||||
def test_reimport_same_post_idempotent(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
# Threshold 0: only an exact phash match collapses. Orthogonal splits
|
||||
|
||||
@@ -267,6 +267,47 @@ async def test_start_recovery_arms_bypass_flag(db):
|
||||
assert "_backfill_bypass_seen" not in (co2 or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_recapture_arms_recapture_flag(db):
|
||||
"""#830: start_recapture arms the backfill state machine PLUS the
|
||||
_backfill_recapture flag, surfaced on the record so the badge can label it
|
||||
'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop
|
||||
clears it."""
|
||||
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
artist = await _artist(db, "Alice")
|
||||
svc = SourceService(db)
|
||||
rec = await svc.create(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice",
|
||||
)
|
||||
# Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion).
|
||||
await svc.start_recovery(rec.id)
|
||||
|
||||
updated = await svc.start_recapture(rec.id)
|
||||
assert updated.backfill_state == "running"
|
||||
assert updated.backfill_recapture is True
|
||||
assert updated.backfill_bypass_seen is False
|
||||
assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
|
||||
|
||||
co = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == rec.id)
|
||||
)).scalar_one()
|
||||
assert co.get("_backfill_recapture") is True
|
||||
assert "_backfill_bypass_seen" not in co
|
||||
|
||||
# to_dict carries the new field for the API/UI.
|
||||
assert updated.to_dict()["backfill_recapture"] is True
|
||||
|
||||
# Stop clears the recapture flag too.
|
||||
stopped = await svc.stop_backfill(rec.id)
|
||||
assert stopped.backfill_recapture is False
|
||||
co2 = (await db.execute(
|
||||
select(Source.config_overrides).where(Source.id == rec.id)
|
||||
)).scalar_one()
|
||||
assert "_backfill_recapture" not in (co2 or {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_backfill_raises_when_source_missing(db):
|
||||
svc = SourceService(db)
|
||||
|
||||
Reference in New Issue
Block a user