feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
CI / lint (push) Successful in 4s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 44s
CI / integration (push) Successful in 3m21s

A plain backfill gates post-body capture on the seen-ledger, so a post whose
media is already on disk AND whose post key is already seen never gets its body
recaptured (operator-flagged: Industrial Lust description missing). Recovery
recaptures unconditionally but re-downloads the whole source.

New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the
post-record gate so EVERY post's body + external links are re-captured
(detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and
surfaces already-present media via a separate non-deleting relink channel so the
importer backfills ImageRecord.source_filehash for inline-image localization.

- ingest_core: recapture mode + recapture_records gate bypass + relink collect
- patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path),
  never refetches seen-missing media, still downloads genuinely-new
- importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks
- download_service: mode derivation + phase-3 relink loop + lifecycle clear
- source_service/api: start_recapture + backfill_recapture field + action
- frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/
  Row/Card/SubscriptionsTab + sources store
- tests across ingester/downloader/importer/source_service/api/download_service

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-14 20:58:40 -04:00
parent 96c29c370b
commit 65ec29ba9b
18 changed files with 467 additions and 29 deletions
+15 -11
View File
@@ -138,14 +138,16 @@ async def delete_source(source_id: int):
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
async def set_backfill(source_id: int):
"""Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
walks the full post history in time-boxed chunks until it reaches the bottom
(then the source shows 'complete'); 'recover' is the same walk but bypasses
the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
current pHash threshold; 'stop' cancels either back to tick mode. Returns the
updated source dict (incl. backfill_state / backfill_chunks /
backfill_bypass_seen)."""
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
(default "start"). 'start' walks the full post history in time-boxed chunks
until it reaches the bottom (then the source shows 'complete'); 'recover' is
the same walk but bypasses the Patreon seen-ledger to re-fetch
dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
re-grabs EVERY post's body + external links and localizes on-disk inline
images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
Returns the updated source dict (incl. backfill_state / backfill_chunks /
backfill_bypass_seen / backfill_recapture)."""
from pathlib import Path
from ..services.credential_service import CredentialService
@@ -157,10 +159,10 @@ async def set_backfill(source_id: int):
payload = await request.get_json(silent=True) or {}
action = payload.get("action", "start")
if action not in ("start", "stop", "recover"):
if action not in ("start", "stop", "recover", "recapture"):
return _bad(
"invalid_action",
detail="action must be 'start', 'stop', or 'recover'",
detail="action must be 'start', 'stop', 'recover', or 'recapture'",
)
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
@@ -170,7 +172,7 @@ async def set_backfill(source_id: int):
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
# an arm action. The credential read happens in a session that's CLOSED
# before the verify network call (don't hold a DB conn across the request).
if action in ("start", "recover"):
if action in ("start", "recover", "recapture"):
async with get_session() as session:
rec = await SourceService(session).get(source_id)
if rec is None:
@@ -200,6 +202,8 @@ async def set_backfill(source_id: int):
record = await svc.start_backfill(source_id)
elif action == "recover":
record = await svc.start_recovery(source_id)
elif action == "recapture":
record = await svc.start_recapture(source_id)
else:
record = await svc.stop_backfill(source_id)
except LookupError:
+22
View File
@@ -111,6 +111,10 @@ class DownloadService:
# `_backfill_bypass_seen` flag rides alongside the running backfill state;
# download mode is "recovery" when both are set.
bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
# #830 recapture: re-grab post bodies/links + localize on-disk inline
# images, WITHOUT re-downloading media. Rides alongside the backfill
# state like _backfill_bypass_seen; mutually exclusive with it.
recapture = bool(overrides.get("_backfill_recapture"))
if in_backfill:
skip_value: bool | str = BACKFILL_SKIP_VALUE
source_config.timeout = BACKFILL_CHUNK_SECONDS
@@ -130,6 +134,8 @@ class DownloadService:
if uses_native_ingester(ctx["platform"]):
if in_backfill and bypass_seen:
mode = "recovery"
elif in_backfill and recapture:
mode = "recapture"
elif in_backfill:
mode = "backfill"
else:
@@ -396,6 +402,20 @@ class DownloadService:
await loop.run_in_executor(None, _upsert)
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
# their post-body inline <img src=CDN> remaps to the local copy. A
# SEPARATE non-deleting channel (NOT the import list — that would unlink
# the duplicate file). Empty outside recapture mode.
for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []:
rel_path = Path(rel_str)
if not rel_path.exists(): # noqa: ASYNC240
continue
def _relink(p=rel_path, u=rel_url):
return self.importer.relink_source_filehash(p, u, artist=artist)
await loop.run_in_executor(None, _relink)
# Kick the off-platform file-host downloader for any links this run
# recorded (mega/gdrive/…). Global + idempotent (only claims pending/
# retryable rows); the beat sweep is the backstop. Lazy import dodges a
@@ -508,6 +528,8 @@ class DownloadService:
# plan #697: a recovery walk shares this lifecycle; clear its bypass
# flag on completion so the next routine tick honors the seen-ledger.
new_overrides.pop("_backfill_bypass_seen", None)
# #830: same for the recapture flag.
new_overrides.pop("_backfill_recapture", None)
src.config_overrides = new_overrides
src.backfill_runs_remaining = 0
return
+5
View File
@@ -153,6 +153,11 @@ class DownloadResult:
# Importer.upsert_post_record (keyed on external_post_id → updates, never
# doubles, the same Post a media import would create).
post_record_paths: list[str] = field(default_factory=list)
# Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
# pairs for already-present media whose ImageRecord.source_filehash should be
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
# on the gallery-dl path and outside recapture.
relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
stdout: str = ""
stderr: str = ""
return_code: int = 0
+29
View File
@@ -1005,6 +1005,35 @@ class Importer:
return sv.strip()
return None
def relink_source_filehash(
self, path: Path, source_url: str, *, artist: Artist | None = None,
) -> bool:
"""#830 recapture: backfill source_url + source_filehash on an EXISTING
on-disk image's ImageRecord (matched by sha256) so its post-body inline
`<img src=CDN>` remaps to the local copy at render time — WITHOUT
re-downloading and WITHOUT unlinking the file (unlike the import path's
duplicate-hash branch). NULL-only, mirroring _apply_sidecar: never
clobbers an already-set filehash. Returns True only when a row was
updated. No-op when the file is gone, the url has no filehash, no record
matches the bytes, or the record already carries a filehash.
"""
fh = filehash_from_url(source_url)
if not fh:
return False
try:
sha = _sha256_of(path)
except OSError:
return False
record = self.session.execute(
select(ImageRecord).where(ImageRecord.sha256 == sha)
).scalar_one_or_none()
if record is None or record.source_filehash is not None:
return False
record.source_url = source_url
record.source_filehash = fh
self.session.commit()
return True
def _apply_sidecar(
self,
record: ImageRecord,
+27 -5
View File
@@ -105,18 +105,28 @@ class Ingester:
) -> DownloadResult:
"""Walk + download for one source, returning a gallery-dl-shaped result.
`mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
files). The walk stops on:
`mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery
bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk
still skips kept files). Recapture (#830) is the cheap "re-grab post
text" walk: it bypasses the post-record gate (so EVERY post's body +
external links are re-captured / detail-fetched) but KEEPS the media
seen-ledger — on-disk media is NOT re-downloaded, only surfaced so its
ImageRecord's source_filehash can be backfilled for inline-image
localization. The walk stops on:
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
- tick early-out (seen_threshold contiguous seen) → success
- reaching the bottom of the feed → success (rc 0)
A client-level failure (drift / auth / network) fails the whole run loud.
"""
bypass_seen = mode == "recovery"
recapture = mode == "recapture"
# Both recovery and recapture re-capture EVERY post's body + links — they
# bypass the post-record seen-gate (recovery via bypass_seen, recapture
# explicitly). A plain backfill stays gated (capture once per post).
recapture_records = bypass_seen or recapture
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
# tick has no resumable backfill state.
checkpoint = mode in ("backfill", "recovery")
checkpoint = mode in ("backfill", "recovery", "recapture")
ledger_key = self._ledger_key
# Optional seams (Patreon native ingester): capture pure-text posts that
# have NO downloadable media so the artist archive is complete. Absent on
@@ -129,6 +139,10 @@ class Ingester:
written: list[str] = []
post_records: list[str] = []
quarantined_paths: list[str] = []
# #830 recapture: (on-disk path, CDN source_url) pairs for already-present
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
# re-downloading or unlinking the file. Empty outside recapture mode.
relink: list[tuple[str, str]] = []
downloaded = 0
errors = 0
quarantined = 0
@@ -165,6 +179,7 @@ class Ingester:
quarantined_paths=list(quarantined_paths),
written_paths=written,
post_record_paths=list(post_records),
relink_source_paths=list(relink),
stdout="\n".join(log_lines),
stderr="",
return_code=return_code,
@@ -241,7 +256,7 @@ class Ingester:
if rk is not None:
pkey, ppid = rk
already = (
set() if bypass_seen
set() if recapture_records
else self._seen_keys(source_id, [pkey])
)
if pkey not in already:
@@ -275,6 +290,7 @@ class Ingester:
outcomes = self.downloader.download_post(
post, media, artist_slug, is_seen=_is_skip,
should_stop=lambda: time.monotonic() - start >= time_budget_seconds,
recapture=recapture,
)
to_mark: list[tuple[str, str]] = []
@@ -300,6 +316,12 @@ class Ingester:
to_clear.append(key)
skipped_count += 1
consecutive_seen += 1
# #830 recapture: surface (on-disk path, CDN url) so phase
# 3 can backfill source_filehash for inline-image
# localization — a SEPARATE non-deleting channel, never the
# import list (which would unlink the file, per above).
if recapture and outcome.path is not None:
relink.append((str(outcome.path), media_item.url))
elif outcome.status == "skipped_seen":
skipped_count += 1
consecutive_seen += 1
+24 -3
View File
@@ -198,6 +198,7 @@ class PatreonDownloader:
*,
is_seen: Callable[[object], bool] = lambda m: False,
should_stop: Callable[[], bool] = lambda: False,
recapture: bool = False,
) -> list[MediaOutcome]:
"""Download every media item of one post; return per-item outcomes.
@@ -212,6 +213,11 @@ class PatreonDownloader:
re-checks the budget between posts), so we honour the deadline mid-post
and return the items done so far — the rest re-fetch next chunk (they
were never marked seen). Bounds chunk overrun to one media download.
`recapture` (#830): don't re-download already-present media, but DO surface
on-disk media as `skipped_disk` (with its path) even when the seen-ledger
would tier-1 skip it — so the engine can backfill source_filehash for
inline-image localization. Genuinely-missing seen media is NOT refetched.
"""
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
outcomes: list[MediaOutcome] = []
@@ -221,7 +227,10 @@ class PatreonDownloader:
break
try:
outcomes.append(
self._download_one(post, media, post_dir, artist_slug, i, is_seen)
self._download_one(
post, media, post_dir, artist_slug, i, is_seen,
recapture=recapture,
)
)
except Exception as exc: # resilient: isolate one item's failure
log.warning(
@@ -243,9 +252,15 @@ class PatreonDownloader:
artist_slug: str,
index: int,
is_seen: Callable[[object], bool],
*,
recapture: bool = False,
) -> MediaOutcome:
# tier-1: seen ledger (injected; no DB here).
if is_seen(media):
# tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
# short-circuit here — we fall through to the disk check so an on-disk
# seen file is surfaced as skipped_disk (with its path) for source_filehash
# backfill; a seen file that's NOT on disk is left alone (not refetched).
seen = is_seen(media)
if seen and not recapture:
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
nn = f"{index:02d}"
@@ -265,6 +280,12 @@ class PatreonDownloader:
media=media, status="skipped_disk", path=existing, error=None
)
# recapture: a seen item that isn't on disk is NOT re-downloaded (that's
# recovery's job) — recapture only re-grabs post text + localizes existing
# files. Returns skipped_seen so the run-of-seen / counts stay consistent.
if seen:
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
post_dir.mkdir(parents=True, exist_ok=True)
# Pace real downloads only (the skips above already returned). plan #703.
+36 -1
View File
@@ -70,6 +70,10 @@ class SourceRecord:
# plan #697: a running deep-walk that bypasses the Patreon seen-ledger
# (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
backfill_bypass_seen: bool
# #830: a running deep-walk in RECAPTURE mode (re-grab post bodies/links +
# localize on-disk inline images, no media re-download). Lets the badge label
# it "Recapturing".
backfill_recapture: bool
# plan #704: cumulative posts processed across the walk's chunks — live
# progress for the badge.
backfill_posts: int
@@ -94,6 +98,7 @@ class SourceRecord:
"backfill_state": self.backfill_state,
"backfill_chunks": self.backfill_chunks,
"backfill_bypass_seen": self.backfill_bypass_seen,
"backfill_recapture": self.backfill_recapture,
"backfill_posts": self.backfill_posts,
}
@@ -171,6 +176,7 @@ class SourceService:
backfill_state=co.get("_backfill_state"),
backfill_chunks=int(co.get("_backfill_chunks", 0)),
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
backfill_recapture=bool(co.get("_backfill_recapture")),
backfill_posts=int(co.get("_backfill_posts", 0)),
)
@@ -359,6 +365,34 @@ class SourceService:
await self.session.commit()
return await self._row_to_record(source)
async def start_recapture(self, source_id: int) -> SourceRecord:
"""#830: arm a RECAPTURE walk — a backfill that re-grabs EVERY post's body
+ external links (detail-fetching empty bodies) and localizes already-on-
disk inline images, WITHOUT re-downloading media. Reuses the entire #693
backfill state machine plus a `_backfill_recapture` flag that flips
download mode to recapture. Distinct from recovery (which re-downloads the
whole source); the two flags are mutually exclusive, so arming recapture
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
fresh from the top. The flag is cleared on completion (download_service)
and on stop. Recapture is Patreon-only (the native ingester's post-record
capture); inert elsewhere. The UI gates the action to Patreon sources."""
source = (await self.session.execute(
select(Source).where(Source.id == source_id)
)).scalar_one_or_none()
if source is None:
raise LookupError(f"source id={source_id} not found")
co = dict(source.config_overrides or {})
co["_backfill_state"] = "running"
co["_backfill_recapture"] = True
co.pop("_backfill_bypass_seen", None) # mutually exclusive with recovery
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
"_backfill_posts"):
co.pop(k, None)
source.config_overrides = co
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
await self.session.commit()
return await self._row_to_record(source)
async def stop_backfill(self, source_id: int) -> SourceRecord:
"""Plan #693: cancel an in-progress backfill — back to idle/tick mode.
Clears the running state + cursor/chunk/stall bookkeeping."""
@@ -369,7 +403,8 @@ class SourceService:
raise LookupError(f"source id={source_id} not found")
co = dict(source.config_overrides or {})
for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"):
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_recapture",
"_backfill_posts"):
co.pop(k, None)
source.config_overrides = co
source.backfill_runs_remaining = 0
@@ -21,7 +21,7 @@
<v-icon>{{ running ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
<v-tooltip activator="parent" location="top">
{{ running
? (recovering ? 'Stop recovery' : 'Stop backfill')
? (recovering ? 'Stop recovery' : (recapturing ? 'Stop recapture' : 'Stop backfill'))
: 'Backfill one-time walk of the FULL post history until complete' }}
</v-tooltip>
</v-btn>
@@ -38,6 +38,17 @@
under the current similarity threshold
</v-list-item-subtitle>
</v-list-item>
<v-list-item
v-if="isPatreon && !running"
prepend-icon="mdi-text-box-search-outline"
@click="emit('recapture', source)"
>
<v-list-item-title>Recapture post text &amp; links</v-list-item-title>
<v-list-item-subtitle>
Re-grab every post's body + external links and localize inline images
already on disk without re-downloading media
</v-list-item-subtitle>
</v-list-item>
<v-divider v-if="isPatreon && !running" />
<v-list-item
base-color="error"
@@ -58,10 +69,11 @@ const props = defineProps({
source: { type: Object, required: true },
checking: { type: Boolean, default: false },
})
const emit = defineEmits(['check', 'backfill', 'recover', 'remove'])
const emit = defineEmits(['check', 'backfill', 'recover', 'recapture', 'remove'])
const running = computed(() => props.source.backfill_state === 'running')
const recovering = computed(() => !!props.source.backfill_bypass_seen)
const recapturing = computed(() => !!props.source.backfill_recapture)
const isPatreon = computed(() => props.source.platform === 'patreon')
</script>
@@ -40,7 +40,7 @@
<v-chip
v-else-if="source.backfill_state === 'running'"
size="x-small" color="info" variant="tonal" label
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
}}{{ source.backfill_posts
? ` · ${source.backfill_posts} posts`
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
@@ -60,6 +60,7 @@
@check="$emit('check', $event)"
@backfill="$emit('backfill', $event)"
@recover="$emit('recover', $event)"
@recapture="$emit('recapture', $event)"
@remove="$emit('remove', $event)"
/>
</div>
@@ -79,7 +80,7 @@ const props = defineProps({
checking: { type: Boolean, default: false },
warningThreshold: { type: Number, default: 5 },
})
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
function onToggleEnabled(value) {
emit('toggle', { source: props.source, enabled: value })
@@ -51,7 +51,7 @@
<v-chip
v-else-if="source.backfill_state === 'running'"
size="x-small" color="info" variant="tonal" label
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
}}{{ source.backfill_posts
? ` · ${source.backfill_posts} posts`
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
@@ -71,6 +71,7 @@
@check="$emit('check', $event)"
@backfill="$emit('backfill', $event)"
@recover="$emit('recover', $event)"
@recapture="$emit('recapture', $event)"
@remove="$emit('remove', $event)"
/>
</td>
@@ -87,7 +88,7 @@ const props = defineProps({
checking: { type: Boolean, default: false },
warningThreshold: { type: Number, default: 5 },
})
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
function onToggleEnabled(value) {
emit('toggle', { source: props.source, enabled: value })
@@ -138,7 +138,7 @@
:source="item.singleSource"
:checking="store.checkingIds.has(item.singleSource.id)"
@check="onCheck" @backfill="onBackfill"
@recover="onRecover" @remove="removeSource"
@recover="onRecover" @recapture="onRecapture" @remove="removeSource"
/>
<v-btn icon size="small" variant="text" @click.stop="openEditSource(item.singleSource)">
<v-icon>mdi-pencil</v-icon>
@@ -214,6 +214,7 @@
@check="onCheck"
@backfill="onBackfill"
@recover="onRecover"
@recapture="onRecapture"
/>
<tr v-if="item.sources.length === 0">
<td colspan="8" class="fc-subs__sources-empty">
@@ -291,6 +292,7 @@
@check="onCheck"
@backfill="onBackfill"
@recover="onRecover"
@recapture="onRecapture"
/>
<div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
No sources yet. Tap + to add one.
@@ -646,6 +648,22 @@ async function onRecover(source) {
}
}
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
// external links and localize on-disk inline images, without re-downloading
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
async function onRecapture(source) {
try {
await store.recaptureSource(source.id, source.artist_id)
toast({ text: `Recapture started for ${source.artist_name}`, type: 'success' })
// recaptureSource patches the source in place — no full reload.
} catch (e) {
toast({
text: `Recapture start failed: ${e?.body?.detail || e?.message || e}`,
type: 'error',
})
}
}
async function checkAll(group) {
let ok = 0
let conflict = 0
+9
View File
@@ -131,6 +131,14 @@ export const useSourcesStore = defineStore('sources', () => {
_patchSource(body)
return body
}
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
// external links and localize on-disk inline images, WITHOUT re-downloading
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
async function recaptureSource(id, artistIdHint = null) {
const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recapture' } })
_patchSource(body)
return body
}
// Plan #708 B4: dry-run preview — count what a backfill WOULD download
// (native platforms), without downloading. Read-only, so no cache invalidate.
@@ -167,6 +175,7 @@ export const useSourcesStore = defineStore('sources', () => {
startBackfill,
stopBackfill,
recoverSource,
recaptureSource,
previewSource,
findOrCreateArtist, autocompleteArtist,
loadScheduleStatus,
+26
View File
@@ -324,6 +324,32 @@ async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
assert stopped_body["backfill_bypass_seen"] is False
@pytest.mark.asyncio
async def test_backfill_endpoint_recapture_arms_flag(client, artist, db):
"""#830: action='recapture' arms a backfill that re-grabs post bodies/links +
localizes on-disk inline images; the response exposes backfill_recapture=True
(badge 'Recapturing') and NOT backfill_bypass_seen. Stop clears it."""
src = Source(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice-recapture", enabled=True,
)
db.add(src)
await db.commit()
sid = src.id
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recapture"})
assert resp.status_code == 200
body = await resp.get_json()
assert body["backfill_state"] == "running"
assert body["backfill_recapture"] is True
assert body["backfill_bypass_seen"] is False
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
stopped_body = await stopped.get_json()
assert stopped_body["backfill_state"] is None
assert stopped_body["backfill_recapture"] is False
# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
+27
View File
@@ -60,8 +60,10 @@ def _make_fake_dl_result(
*, success=True, written_paths=None, quarantined_paths=None,
files_downloaded=0, error_type=None, error_message=None,
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
relink_source_paths=None,
):
return SimpleNamespace(
relink_source_paths=relink_source_paths or [],
success=success,
url="https://patreon.com/alice",
artist_slug="alice",
@@ -793,6 +795,31 @@ async def test_recovery_mode_selected_and_flag_cleared_on_complete(
assert "_backfill_bypass_seen" not in co
@pytest.mark.asyncio
async def test_recapture_mode_selected_and_flag_cleared_on_complete(
db, db_sync, tmp_path, seed_artist_and_source,
):
"""#830: `_backfill_recapture` alongside a running backfill selects recapture
mode. A clean rc=0 walk completes the shared lifecycle AND clears the
recapture flag so the next tick is a plain tick again."""
_artist, source = seed_artist_and_source
source.config_overrides = {"_backfill_state": "running", "_backfill_recapture": True}
source.backfill_runs_remaining = 5
await db.commit()
svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
success=True, written_paths=[], files_downloaded=0,
))
await svc.download_source(source.id)
assert calls[0]["mode"] == "recapture"
co = (await db.execute(
select(Source.config_overrides).where(Source.id == source.id)
)).scalar_one()
assert co.get("_backfill_state") == "complete"
assert "_backfill_recapture" not in co
@pytest.mark.asyncio
async def test_partial_error_type_maps_to_ok_status(
db, db_sync, tmp_path, seed_artist_and_source,
+48
View File
@@ -206,6 +206,54 @@ def test_skip_disk(tmp_path):
assert existing.read_bytes() == b"already here" # untouched
def test_recapture_surfaces_seen_on_disk_without_download(tmp_path):
"""#830 recapture: a seen item that IS on disk is surfaced as skipped_disk
(with its path, for source_filehash backfill) — NOT tier-1 short-circuited,
and never re-downloaded."""
session = _FakeSession()
dl = _downloader(tmp_path, session=session)
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
post_dir.mkdir(parents=True)
existing = post_dir / "01_media1.png"
existing.write_bytes(b"already here")
outcomes = dl.download_post(
_post(), [_img("media1.png")], "artist-x",
is_seen=lambda m: True, recapture=True,
)
assert outcomes[0].status == "skipped_disk"
assert outcomes[0].path == existing
assert session.calls == [] # no re-download
assert existing.read_bytes() == b"already here" # untouched
def test_recapture_does_not_refetch_seen_missing(tmp_path):
"""#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen,
no download) — refetching is recovery's job, not recapture's."""
session = _FakeSession()
dl = _downloader(tmp_path, session=session)
outcomes = dl.download_post(
_post(), [_img("media1.png")], "artist-x",
is_seen=lambda m: True, recapture=True,
)
assert outcomes[0].status == "skipped_seen"
assert outcomes[0].path is None
assert session.calls == [] # not refetched
def test_recapture_still_downloads_genuinely_new(tmp_path):
"""#830 recapture: media that is neither seen nor on disk still downloads — a
recapture walk is thorough, it just doesn't re-pull existing files."""
session = _FakeSession()
dl = _downloader(tmp_path, session=session)
outcomes = dl.download_post(
_post(), [_img("media1.png")], "artist-x",
is_seen=lambda m: False, recapture=True,
)
assert outcomes[0].status == "downloaded"
assert outcomes[0].path is not None
def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
session = _FakeSession()
dl = _downloader(tmp_path, session=session)
+81 -2
View File
@@ -88,16 +88,23 @@ class _FakeDownloader:
self.post_records = 0
def download_post(self, post, media_items, artist_slug, *, is_seen,
should_stop=lambda: False):
should_stop=lambda: False, recapture=False):
outcomes = []
for m in media_items:
if should_stop():
break
if is_seen(m):
seen = is_seen(m)
# Recapture mirrors the real downloader: a seen item isn't tier-1
# short-circuited — it falls through so an on-disk file surfaces as
# skipped_disk (for source_filehash backfill); a seen file NOT on
# disk is left alone (skipped_seen, never refetched).
if seen and not recapture:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.on_disk:
p = self.tmp_path / f"{m.post_id}_{m.filename}"
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
elif seen:
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
elif _ledger_key(m) in self.error:
self.download_calls += 1
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
@@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
assert downloader.post_records == 1
def _seed_seen(sync_engine, source_id, key, post_id=None):
factory = sessionmaker(sync_engine, expire_on_commit=False)
with factory() as s:
s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id))
s.commit()
@pytest.mark.asyncio
async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
source_id, sync_engine, tmp_path,
):
"""#830: a plain backfill GATES post-record capture on the seen-ledger, so a
post whose `post:<id>` key is already seen is NOT recaptured (the operator's
'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and
re-captures the body/links even though the post key is seen — without
re-downloading the on-disk media."""
m1 = _media("p1", 1)
# Pre-seed BOTH the media key and the synthetic post key as already seen.
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
# 1) Plain backfill: post key is seen → gate skips body recapture.
client = _FakeClient([(None, [("p1", [m1])])])
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
ing = _ingester(sync_engine, tmp_path, client, downloader)
res_backfill = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="backfill",
)
assert res_backfill.post_record_paths == [] # gated → not recaptured
assert downloader.post_records == 0
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
# the on-disk file is surfaced for source_filehash relink.
client2 = _FakeClient([(None, [("p1", [m1])])])
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
res_recap = ing2.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="recapture",
)
assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured
assert downloader2.post_records == 1
assert res_recap.files_downloaded == 0 # no media re-download
assert downloader2.download_calls == 0
# The on-disk media is surfaced as a (path, CDN url) relink pair.
assert len(res_recap.relink_source_paths) == 1
rel_path, rel_url = res_recap.relink_source_paths[0]
assert rel_url == m1.url
@pytest.mark.asyncio
async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
source_id, sync_engine, tmp_path,
):
"""Recapture must NOT re-download a seen item that's absent from disk (that's
recovery's job) — it returns skipped_seen and contributes no relink pair."""
m1 = _media("p1", 1)
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
client = _FakeClient([(None, [("p1", [m1])])])
# NOT in on_disk → seen + missing.
downloader = _FakeDownloader(tmp_path)
ing = _ingester(sync_engine, tmp_path, client, downloader)
result = ing.run(
source_id=source_id, campaign_id="c1", artist_slug="ingest",
url="https://patreon.com/ingest", mode="recapture",
)
assert result.files_downloaded == 0
assert downloader.download_calls == 0
assert result.relink_source_paths == []
# --- dead-letter ledger (plan #705 #7) ------------------------------------
+38
View File
@@ -120,6 +120,44 @@ def test_sidecar_source_url_persists_filehash(importer, import_layout):
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
def test_relink_source_filehash_backfills_existing(importer, import_layout):
"""#830 recapture (Part B): relink_source_filehash backfills source_url +
source_filehash on an existing on-disk image's ImageRecord (matched by
sha256) WITHOUT re-importing or unlinking the file. NULL-only — never
clobbers a filehash already set."""
import_root, _ = import_layout
m = import_root / "Alice" / "img.jpg"
_split(m, "v")
r = importer.import_one(m)
assert r.status == "imported"
rec = importer.session.get(ImageRecord, r.image_id)
assert rec.source_filehash is None
url = "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/img.jpg"
assert importer.relink_source_filehash(m, url) is True
importer.session.refresh(rec)
assert rec.source_url == url
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
assert m.exists() # never unlinked
# NULL-only: a second relink with a different url is a no-op.
other = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/x.jpg"
assert importer.relink_source_filehash(m, other) is False
importer.session.refresh(rec)
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
def test_relink_source_filehash_no_match_is_noop(importer, import_layout):
"""A path whose bytes match no ImageRecord (never imported) is a clean no-op,
not an error."""
import_root, _ = import_layout
m = import_root / "Alice" / "ghost.jpg"
_split(m, "h")
assert importer.relink_source_filehash(
m, "https://cdn.test/p/0123456789abcdef0123456789abcdef/ghost.jpg"
) is False
def test_reimport_same_post_idempotent(importer, import_layout):
import_root, _ = import_layout
# Threshold 0: only an exact phash match collapses. Orthogonal splits
+41
View File
@@ -267,6 +267,47 @@ async def test_start_recovery_arms_bypass_flag(db):
assert "_backfill_bypass_seen" not in (co2 or {})
@pytest.mark.asyncio
async def test_start_recapture_arms_recapture_flag(db):
"""#830: start_recapture arms the backfill state machine PLUS the
_backfill_recapture flag, surfaced on the record so the badge can label it
'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop
clears it."""
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
artist = await _artist(db, "Alice")
svc = SourceService(db)
rec = await svc.create(
artist_id=artist.id, platform="patreon",
url="https://patreon.com/alice",
)
# Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion).
await svc.start_recovery(rec.id)
updated = await svc.start_recapture(rec.id)
assert updated.backfill_state == "running"
assert updated.backfill_recapture is True
assert updated.backfill_bypass_seen is False
assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
co = (await db.execute(
select(Source.config_overrides).where(Source.id == rec.id)
)).scalar_one()
assert co.get("_backfill_recapture") is True
assert "_backfill_bypass_seen" not in co
# to_dict carries the new field for the API/UI.
assert updated.to_dict()["backfill_recapture"] is True
# Stop clears the recapture flag too.
stopped = await svc.stop_backfill(rec.id)
assert stopped.backfill_recapture is False
co2 = (await db.execute(
select(Source.config_overrides).where(Source.id == rec.id)
)).scalar_one()
assert "_backfill_recapture" not in (co2 or {})
@pytest.mark.asyncio
async def test_start_backfill_raises_when_source_missing(db):
svc = SourceService(db)