feat(ingest): Recapture mode — re-grab post bodies/links + localize on-disk inline images (#830)
A plain backfill gates post-body capture on the seen-ledger, so a post whose media is already on disk AND whose post key is already seen never gets its body recaptured (operator-flagged: Industrial Lust description missing). Recovery recaptures unconditionally but re-downloads the whole source. New 'recapture' walk mode (4th beside tick/backfill/recovery): bypasses the post-record gate so EVERY post's body + external links are re-captured (detail-fetching empty bodies) WITHOUT re-downloading on-disk media; and surfaces already-present media via a separate non-deleting relink channel so the importer backfills ImageRecord.source_filehash for inline-image localization. - ingest_core: recapture mode + recapture_records gate bypass + relink collect - patreon_downloader: recapture surfaces seen-on-disk as skipped_disk(path), never refetches seen-missing media, still downloads genuinely-new - importer.relink_source_filehash: NULL-only sha256 backfill, never unlinks - download_service: mode derivation + phase-3 relink loop + lifecycle clear - source_service/api: start_recapture + backfill_recapture field + action - frontend: Recapture kebab action + 'Recapturing' badge across SourceActions/ Row/Card/SubscriptionsTab + sources store - tests across ingester/downloader/importer/source_service/api/download_service Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+15
-11
@@ -138,14 +138,16 @@ async def delete_source(source_id: int):
|
|||||||
|
|
||||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||||
async def set_backfill(source_id: int):
|
async def set_backfill(source_id: int):
|
||||||
"""Plan #693/#697: start/stop a run-until-done backfill, or start a recovery.
|
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
|
||||||
Body: `{"action": "start" | "stop" | "recover"}` (default "start"). 'start'
|
recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
|
||||||
walks the full post history in time-boxed chunks until it reaches the bottom
|
(default "start"). 'start' walks the full post history in time-boxed chunks
|
||||||
(then the source shows 'complete'); 'recover' is the same walk but bypasses
|
until it reaches the bottom (then the source shows 'complete'); 'recover' is
|
||||||
the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the
|
the same walk but bypasses the Patreon seen-ledger to re-fetch
|
||||||
current pHash threshold; 'stop' cancels either back to tick mode. Returns the
|
dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
|
||||||
updated source dict (incl. backfill_state / backfill_chunks /
|
re-grabs EVERY post's body + external links and localizes on-disk inline
|
||||||
backfill_bypass_seen)."""
|
images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
|
||||||
|
Returns the updated source dict (incl. backfill_state / backfill_chunks /
|
||||||
|
backfill_bypass_seen / backfill_recapture)."""
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..services.credential_service import CredentialService
|
from ..services.credential_service import CredentialService
|
||||||
@@ -157,10 +159,10 @@ async def set_backfill(source_id: int):
|
|||||||
|
|
||||||
payload = await request.get_json(silent=True) or {}
|
payload = await request.get_json(silent=True) or {}
|
||||||
action = payload.get("action", "start")
|
action = payload.get("action", "start")
|
||||||
if action not in ("start", "stop", "recover"):
|
if action not in ("start", "stop", "recover", "recapture"):
|
||||||
return _bad(
|
return _bad(
|
||||||
"invalid_action",
|
"invalid_action",
|
||||||
detail="action must be 'start', 'stop', or 'recover'",
|
detail="action must be 'start', 'stop', 'recover', or 'recapture'",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
|
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
|
||||||
@@ -170,7 +172,7 @@ async def set_backfill(source_id: int):
|
|||||||
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
|
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
|
||||||
# an arm action. The credential read happens in a session that's CLOSED
|
# an arm action. The credential read happens in a session that's CLOSED
|
||||||
# before the verify network call (don't hold a DB conn across the request).
|
# before the verify network call (don't hold a DB conn across the request).
|
||||||
if action in ("start", "recover"):
|
if action in ("start", "recover", "recapture"):
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
rec = await SourceService(session).get(source_id)
|
rec = await SourceService(session).get(source_id)
|
||||||
if rec is None:
|
if rec is None:
|
||||||
@@ -200,6 +202,8 @@ async def set_backfill(source_id: int):
|
|||||||
record = await svc.start_backfill(source_id)
|
record = await svc.start_backfill(source_id)
|
||||||
elif action == "recover":
|
elif action == "recover":
|
||||||
record = await svc.start_recovery(source_id)
|
record = await svc.start_recovery(source_id)
|
||||||
|
elif action == "recapture":
|
||||||
|
record = await svc.start_recapture(source_id)
|
||||||
else:
|
else:
|
||||||
record = await svc.stop_backfill(source_id)
|
record = await svc.stop_backfill(source_id)
|
||||||
except LookupError:
|
except LookupError:
|
||||||
|
|||||||
@@ -111,6 +111,10 @@ class DownloadService:
|
|||||||
# `_backfill_bypass_seen` flag rides alongside the running backfill state;
|
# `_backfill_bypass_seen` flag rides alongside the running backfill state;
|
||||||
# download mode is "recovery" when both are set.
|
# download mode is "recovery" when both are set.
|
||||||
bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
|
bypass_seen = bool(overrides.get("_backfill_bypass_seen"))
|
||||||
|
# #830 recapture: re-grab post bodies/links + localize on-disk inline
|
||||||
|
# images, WITHOUT re-downloading media. Rides alongside the backfill
|
||||||
|
# state like _backfill_bypass_seen; mutually exclusive with it.
|
||||||
|
recapture = bool(overrides.get("_backfill_recapture"))
|
||||||
if in_backfill:
|
if in_backfill:
|
||||||
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||||
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
source_config.timeout = BACKFILL_CHUNK_SECONDS
|
||||||
@@ -130,6 +134,8 @@ class DownloadService:
|
|||||||
if uses_native_ingester(ctx["platform"]):
|
if uses_native_ingester(ctx["platform"]):
|
||||||
if in_backfill and bypass_seen:
|
if in_backfill and bypass_seen:
|
||||||
mode = "recovery"
|
mode = "recovery"
|
||||||
|
elif in_backfill and recapture:
|
||||||
|
mode = "recapture"
|
||||||
elif in_backfill:
|
elif in_backfill:
|
||||||
mode = "backfill"
|
mode = "backfill"
|
||||||
else:
|
else:
|
||||||
@@ -396,6 +402,20 @@ class DownloadService:
|
|||||||
|
|
||||||
await loop.run_in_executor(None, _upsert)
|
await loop.run_in_executor(None, _upsert)
|
||||||
|
|
||||||
|
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
||||||
|
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||||
|
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
||||||
|
# the duplicate file). Empty outside recapture mode.
|
||||||
|
for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []:
|
||||||
|
rel_path = Path(rel_str)
|
||||||
|
if not rel_path.exists(): # noqa: ASYNC240
|
||||||
|
continue
|
||||||
|
|
||||||
|
def _relink(p=rel_path, u=rel_url):
|
||||||
|
return self.importer.relink_source_filehash(p, u, artist=artist)
|
||||||
|
|
||||||
|
await loop.run_in_executor(None, _relink)
|
||||||
|
|
||||||
# Kick the off-platform file-host downloader for any links this run
|
# Kick the off-platform file-host downloader for any links this run
|
||||||
# recorded (mega/gdrive/…). Global + idempotent (only claims pending/
|
# recorded (mega/gdrive/…). Global + idempotent (only claims pending/
|
||||||
# retryable rows); the beat sweep is the backstop. Lazy import dodges a
|
# retryable rows); the beat sweep is the backstop. Lazy import dodges a
|
||||||
@@ -508,6 +528,8 @@ class DownloadService:
|
|||||||
# plan #697: a recovery walk shares this lifecycle; clear its bypass
|
# plan #697: a recovery walk shares this lifecycle; clear its bypass
|
||||||
# flag on completion so the next routine tick honors the seen-ledger.
|
# flag on completion so the next routine tick honors the seen-ledger.
|
||||||
new_overrides.pop("_backfill_bypass_seen", None)
|
new_overrides.pop("_backfill_bypass_seen", None)
|
||||||
|
# #830: same for the recapture flag.
|
||||||
|
new_overrides.pop("_backfill_recapture", None)
|
||||||
src.config_overrides = new_overrides
|
src.config_overrides = new_overrides
|
||||||
src.backfill_runs_remaining = 0
|
src.backfill_runs_remaining = 0
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -153,6 +153,11 @@ class DownloadResult:
|
|||||||
# Importer.upsert_post_record (keyed on external_post_id → updates, never
|
# Importer.upsert_post_record (keyed on external_post_id → updates, never
|
||||||
# doubles, the same Post a media import would create).
|
# doubles, the same Post a media import would create).
|
||||||
post_record_paths: list[str] = field(default_factory=list)
|
post_record_paths: list[str] = field(default_factory=list)
|
||||||
|
# Native ingester, recapture mode only (#830): (on-disk path, CDN source_url)
|
||||||
|
# pairs for already-present media whose ImageRecord.source_filehash should be
|
||||||
|
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||||
|
# on the gallery-dl path and outside recapture.
|
||||||
|
relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
|
||||||
stdout: str = ""
|
stdout: str = ""
|
||||||
stderr: str = ""
|
stderr: str = ""
|
||||||
return_code: int = 0
|
return_code: int = 0
|
||||||
|
|||||||
@@ -1005,6 +1005,35 @@ class Importer:
|
|||||||
return sv.strip()
|
return sv.strip()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def relink_source_filehash(
|
||||||
|
self, path: Path, source_url: str, *, artist: Artist | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""#830 recapture: backfill source_url + source_filehash on an EXISTING
|
||||||
|
on-disk image's ImageRecord (matched by sha256) so its post-body inline
|
||||||
|
`<img src=CDN>` remaps to the local copy at render time — WITHOUT
|
||||||
|
re-downloading and WITHOUT unlinking the file (unlike the import path's
|
||||||
|
duplicate-hash branch). NULL-only, mirroring _apply_sidecar: never
|
||||||
|
clobbers an already-set filehash. Returns True only when a row was
|
||||||
|
updated. No-op when the file is gone, the url has no filehash, no record
|
||||||
|
matches the bytes, or the record already carries a filehash.
|
||||||
|
"""
|
||||||
|
fh = filehash_from_url(source_url)
|
||||||
|
if not fh:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
sha = _sha256_of(path)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
record = self.session.execute(
|
||||||
|
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
if record is None or record.source_filehash is not None:
|
||||||
|
return False
|
||||||
|
record.source_url = source_url
|
||||||
|
record.source_filehash = fh
|
||||||
|
self.session.commit()
|
||||||
|
return True
|
||||||
|
|
||||||
def _apply_sidecar(
|
def _apply_sidecar(
|
||||||
self,
|
self,
|
||||||
record: ImageRecord,
|
record: ImageRecord,
|
||||||
|
|||||||
@@ -105,18 +105,28 @@ class Ingester:
|
|||||||
) -> DownloadResult:
|
) -> DownloadResult:
|
||||||
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
"""Walk + download for one source, returning a gallery-dl-shaped result.
|
||||||
|
|
||||||
`mode` is "tick" | "backfill" | "recovery". Recovery bypasses the tier-1
|
`mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery
|
||||||
seen-ledger AND the dead-letter ledger (tier-2 disk still skips kept
|
bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk
|
||||||
files). The walk stops on:
|
still skips kept files). Recapture (#830) is the cheap "re-grab post
|
||||||
|
text" walk: it bypasses the post-record gate (so EVERY post's body +
|
||||||
|
external links are re-captured / detail-fetched) but KEEPS the media
|
||||||
|
seen-ledger — on-disk media is NOT re-downloaded, only surfaced so its
|
||||||
|
ImageRecord's source_filehash can be backfilled for inline-image
|
||||||
|
localization. The walk stops on:
|
||||||
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
|
- budget exhaustion (time_budget_seconds) → TIMEOUT / PARTIAL
|
||||||
- tick early-out (seen_threshold contiguous seen) → success
|
- tick early-out (seen_threshold contiguous seen) → success
|
||||||
- reaching the bottom of the feed → success (rc 0)
|
- reaching the bottom of the feed → success (rc 0)
|
||||||
A client-level failure (drift / auth / network) fails the whole run loud.
|
A client-level failure (drift / auth / network) fails the whole run loud.
|
||||||
"""
|
"""
|
||||||
bypass_seen = mode == "recovery"
|
bypass_seen = mode == "recovery"
|
||||||
|
recapture = mode == "recapture"
|
||||||
|
# Both recovery and recapture re-capture EVERY post's body + links — they
|
||||||
|
# bypass the post-record seen-gate (recovery via bypass_seen, recapture
|
||||||
|
# explicitly). A plain backfill stays gated (capture once per post).
|
||||||
|
recapture_records = bypass_seen or recapture
|
||||||
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
|
# Only deep walks checkpoint their cursor mid-flight (plan #705 #6); a
|
||||||
# tick has no resumable backfill state.
|
# tick has no resumable backfill state.
|
||||||
checkpoint = mode in ("backfill", "recovery")
|
checkpoint = mode in ("backfill", "recovery", "recapture")
|
||||||
ledger_key = self._ledger_key
|
ledger_key = self._ledger_key
|
||||||
# Optional seams (Patreon native ingester): capture pure-text posts that
|
# Optional seams (Patreon native ingester): capture pure-text posts that
|
||||||
# have NO downloadable media so the artist archive is complete. Absent on
|
# have NO downloadable media so the artist archive is complete. Absent on
|
||||||
@@ -129,6 +139,10 @@ class Ingester:
|
|||||||
written: list[str] = []
|
written: list[str] = []
|
||||||
post_records: list[str] = []
|
post_records: list[str] = []
|
||||||
quarantined_paths: list[str] = []
|
quarantined_paths: list[str] = []
|
||||||
|
# #830 recapture: (on-disk path, CDN source_url) pairs for already-present
|
||||||
|
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
|
||||||
|
# re-downloading or unlinking the file. Empty outside recapture mode.
|
||||||
|
relink: list[tuple[str, str]] = []
|
||||||
downloaded = 0
|
downloaded = 0
|
||||||
errors = 0
|
errors = 0
|
||||||
quarantined = 0
|
quarantined = 0
|
||||||
@@ -165,6 +179,7 @@ class Ingester:
|
|||||||
quarantined_paths=list(quarantined_paths),
|
quarantined_paths=list(quarantined_paths),
|
||||||
written_paths=written,
|
written_paths=written,
|
||||||
post_record_paths=list(post_records),
|
post_record_paths=list(post_records),
|
||||||
|
relink_source_paths=list(relink),
|
||||||
stdout="\n".join(log_lines),
|
stdout="\n".join(log_lines),
|
||||||
stderr="",
|
stderr="",
|
||||||
return_code=return_code,
|
return_code=return_code,
|
||||||
@@ -241,7 +256,7 @@ class Ingester:
|
|||||||
if rk is not None:
|
if rk is not None:
|
||||||
pkey, ppid = rk
|
pkey, ppid = rk
|
||||||
already = (
|
already = (
|
||||||
set() if bypass_seen
|
set() if recapture_records
|
||||||
else self._seen_keys(source_id, [pkey])
|
else self._seen_keys(source_id, [pkey])
|
||||||
)
|
)
|
||||||
if pkey not in already:
|
if pkey not in already:
|
||||||
@@ -275,6 +290,7 @@ class Ingester:
|
|||||||
outcomes = self.downloader.download_post(
|
outcomes = self.downloader.download_post(
|
||||||
post, media, artist_slug, is_seen=_is_skip,
|
post, media, artist_slug, is_seen=_is_skip,
|
||||||
should_stop=lambda: time.monotonic() - start >= time_budget_seconds,
|
should_stop=lambda: time.monotonic() - start >= time_budget_seconds,
|
||||||
|
recapture=recapture,
|
||||||
)
|
)
|
||||||
|
|
||||||
to_mark: list[tuple[str, str]] = []
|
to_mark: list[tuple[str, str]] = []
|
||||||
@@ -300,6 +316,12 @@ class Ingester:
|
|||||||
to_clear.append(key)
|
to_clear.append(key)
|
||||||
skipped_count += 1
|
skipped_count += 1
|
||||||
consecutive_seen += 1
|
consecutive_seen += 1
|
||||||
|
# #830 recapture: surface (on-disk path, CDN url) so phase
|
||||||
|
# 3 can backfill source_filehash for inline-image
|
||||||
|
# localization — a SEPARATE non-deleting channel, never the
|
||||||
|
# import list (which would unlink the file, per above).
|
||||||
|
if recapture and outcome.path is not None:
|
||||||
|
relink.append((str(outcome.path), media_item.url))
|
||||||
elif outcome.status == "skipped_seen":
|
elif outcome.status == "skipped_seen":
|
||||||
skipped_count += 1
|
skipped_count += 1
|
||||||
consecutive_seen += 1
|
consecutive_seen += 1
|
||||||
|
|||||||
@@ -198,6 +198,7 @@ class PatreonDownloader:
|
|||||||
*,
|
*,
|
||||||
is_seen: Callable[[object], bool] = lambda m: False,
|
is_seen: Callable[[object], bool] = lambda m: False,
|
||||||
should_stop: Callable[[], bool] = lambda: False,
|
should_stop: Callable[[], bool] = lambda: False,
|
||||||
|
recapture: bool = False,
|
||||||
) -> list[MediaOutcome]:
|
) -> list[MediaOutcome]:
|
||||||
"""Download every media item of one post; return per-item outcomes.
|
"""Download every media item of one post; return per-item outcomes.
|
||||||
|
|
||||||
@@ -212,6 +213,11 @@ class PatreonDownloader:
|
|||||||
re-checks the budget between posts), so we honour the deadline mid-post
|
re-checks the budget between posts), so we honour the deadline mid-post
|
||||||
and return the items done so far — the rest re-fetch next chunk (they
|
and return the items done so far — the rest re-fetch next chunk (they
|
||||||
were never marked seen). Bounds chunk overrun to one media download.
|
were never marked seen). Bounds chunk overrun to one media download.
|
||||||
|
|
||||||
|
`recapture` (#830): don't re-download already-present media, but DO surface
|
||||||
|
on-disk media as `skipped_disk` (with its path) even when the seen-ledger
|
||||||
|
would tier-1 skip it — so the engine can backfill source_filehash for
|
||||||
|
inline-image localization. Genuinely-missing seen media is NOT refetched.
|
||||||
"""
|
"""
|
||||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||||
outcomes: list[MediaOutcome] = []
|
outcomes: list[MediaOutcome] = []
|
||||||
@@ -221,7 +227,10 @@ class PatreonDownloader:
|
|||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
outcomes.append(
|
outcomes.append(
|
||||||
self._download_one(post, media, post_dir, artist_slug, i, is_seen)
|
self._download_one(
|
||||||
|
post, media, post_dir, artist_slug, i, is_seen,
|
||||||
|
recapture=recapture,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
except Exception as exc: # resilient: isolate one item's failure
|
except Exception as exc: # resilient: isolate one item's failure
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -243,9 +252,15 @@ class PatreonDownloader:
|
|||||||
artist_slug: str,
|
artist_slug: str,
|
||||||
index: int,
|
index: int,
|
||||||
is_seen: Callable[[object], bool],
|
is_seen: Callable[[object], bool],
|
||||||
|
*,
|
||||||
|
recapture: bool = False,
|
||||||
) -> MediaOutcome:
|
) -> MediaOutcome:
|
||||||
# tier-1: seen ledger (injected; no DB here).
|
# tier-1: seen ledger (injected; no DB here). In recapture mode we DON'T
|
||||||
if is_seen(media):
|
# short-circuit here — we fall through to the disk check so an on-disk
|
||||||
|
# seen file is surfaced as skipped_disk (with its path) for source_filehash
|
||||||
|
# backfill; a seen file that's NOT on disk is left alone (not refetched).
|
||||||
|
seen = is_seen(media)
|
||||||
|
if seen and not recapture:
|
||||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
|
||||||
nn = f"{index:02d}"
|
nn = f"{index:02d}"
|
||||||
@@ -265,6 +280,12 @@ class PatreonDownloader:
|
|||||||
media=media, status="skipped_disk", path=existing, error=None
|
media=media, status="skipped_disk", path=existing, error=None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# recapture: a seen item that isn't on disk is NOT re-downloaded (that's
|
||||||
|
# recovery's job) — recapture only re-grabs post text + localizes existing
|
||||||
|
# files. Returns skipped_seen so the run-of-seen / counts stay consistent.
|
||||||
|
if seen:
|
||||||
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
|
||||||
post_dir.mkdir(parents=True, exist_ok=True)
|
post_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Pace real downloads only (the skips above already returned). plan #703.
|
# Pace real downloads only (the skips above already returned). plan #703.
|
||||||
|
|||||||
@@ -70,6 +70,10 @@ class SourceRecord:
|
|||||||
# plan #697: a running deep-walk that bypasses the Patreon seen-ledger
|
# plan #697: a running deep-walk that bypasses the Patreon seen-ledger
|
||||||
# (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
|
# (recovery) vs. a normal backfill. Lets the badge label it "Recovering".
|
||||||
backfill_bypass_seen: bool
|
backfill_bypass_seen: bool
|
||||||
|
# #830: a running deep-walk in RECAPTURE mode (re-grab post bodies/links +
|
||||||
|
# localize on-disk inline images, no media re-download). Lets the badge label
|
||||||
|
# it "Recapturing".
|
||||||
|
backfill_recapture: bool
|
||||||
# plan #704: cumulative posts processed across the walk's chunks — live
|
# plan #704: cumulative posts processed across the walk's chunks — live
|
||||||
# progress for the badge.
|
# progress for the badge.
|
||||||
backfill_posts: int
|
backfill_posts: int
|
||||||
@@ -94,6 +98,7 @@ class SourceRecord:
|
|||||||
"backfill_state": self.backfill_state,
|
"backfill_state": self.backfill_state,
|
||||||
"backfill_chunks": self.backfill_chunks,
|
"backfill_chunks": self.backfill_chunks,
|
||||||
"backfill_bypass_seen": self.backfill_bypass_seen,
|
"backfill_bypass_seen": self.backfill_bypass_seen,
|
||||||
|
"backfill_recapture": self.backfill_recapture,
|
||||||
"backfill_posts": self.backfill_posts,
|
"backfill_posts": self.backfill_posts,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,6 +176,7 @@ class SourceService:
|
|||||||
backfill_state=co.get("_backfill_state"),
|
backfill_state=co.get("_backfill_state"),
|
||||||
backfill_chunks=int(co.get("_backfill_chunks", 0)),
|
backfill_chunks=int(co.get("_backfill_chunks", 0)),
|
||||||
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
||||||
|
backfill_recapture=bool(co.get("_backfill_recapture")),
|
||||||
backfill_posts=int(co.get("_backfill_posts", 0)),
|
backfill_posts=int(co.get("_backfill_posts", 0)),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -359,6 +365,34 @@ class SourceService:
|
|||||||
await self.session.commit()
|
await self.session.commit()
|
||||||
return await self._row_to_record(source)
|
return await self._row_to_record(source)
|
||||||
|
|
||||||
|
async def start_recapture(self, source_id: int) -> SourceRecord:
|
||||||
|
"""#830: arm a RECAPTURE walk — a backfill that re-grabs EVERY post's body
|
||||||
|
+ external links (detail-fetching empty bodies) and localizes already-on-
|
||||||
|
disk inline images, WITHOUT re-downloading media. Reuses the entire #693
|
||||||
|
backfill state machine plus a `_backfill_recapture` flag that flips
|
||||||
|
download mode to recapture. Distinct from recovery (which re-downloads the
|
||||||
|
whole source); the two flags are mutually exclusive, so arming recapture
|
||||||
|
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
||||||
|
fresh from the top. The flag is cleared on completion (download_service)
|
||||||
|
and on stop. Recapture is Patreon-only (the native ingester's post-record
|
||||||
|
capture); inert elsewhere. The UI gates the action to Patreon sources."""
|
||||||
|
source = (await self.session.execute(
|
||||||
|
select(Source).where(Source.id == source_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if source is None:
|
||||||
|
raise LookupError(f"source id={source_id} not found")
|
||||||
|
co = dict(source.config_overrides or {})
|
||||||
|
co["_backfill_state"] = "running"
|
||||||
|
co["_backfill_recapture"] = True
|
||||||
|
co.pop("_backfill_bypass_seen", None) # mutually exclusive with recovery
|
||||||
|
for k in ("_backfill_cursor", "_backfill_cursor_stalls", "_backfill_chunks",
|
||||||
|
"_backfill_posts"):
|
||||||
|
co.pop(k, None)
|
||||||
|
source.config_overrides = co
|
||||||
|
source.backfill_runs_remaining = BACKFILL_MAX_CHUNKS
|
||||||
|
await self.session.commit()
|
||||||
|
return await self._row_to_record(source)
|
||||||
|
|
||||||
async def stop_backfill(self, source_id: int) -> SourceRecord:
|
async def stop_backfill(self, source_id: int) -> SourceRecord:
|
||||||
"""Plan #693: cancel an in-progress backfill — back to idle/tick mode.
|
"""Plan #693: cancel an in-progress backfill — back to idle/tick mode.
|
||||||
Clears the running state + cursor/chunk/stall bookkeeping."""
|
Clears the running state + cursor/chunk/stall bookkeeping."""
|
||||||
@@ -369,7 +403,8 @@ class SourceService:
|
|||||||
raise LookupError(f"source id={source_id} not found")
|
raise LookupError(f"source id={source_id} not found")
|
||||||
co = dict(source.config_overrides or {})
|
co = dict(source.config_overrides or {})
|
||||||
for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
|
for k in ("_backfill_state", "_backfill_cursor", "_backfill_cursor_stalls",
|
||||||
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_posts"):
|
"_backfill_chunks", "_backfill_bypass_seen", "_backfill_recapture",
|
||||||
|
"_backfill_posts"):
|
||||||
co.pop(k, None)
|
co.pop(k, None)
|
||||||
source.config_overrides = co
|
source.config_overrides = co
|
||||||
source.backfill_runs_remaining = 0
|
source.backfill_runs_remaining = 0
|
||||||
|
|||||||
@@ -21,7 +21,7 @@
|
|||||||
<v-icon>{{ running ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
|
<v-icon>{{ running ? 'mdi-stop' : 'mdi-magnify-scan' }}</v-icon>
|
||||||
<v-tooltip activator="parent" location="top">
|
<v-tooltip activator="parent" location="top">
|
||||||
{{ running
|
{{ running
|
||||||
? (recovering ? 'Stop recovery' : 'Stop backfill')
|
? (recovering ? 'Stop recovery' : (recapturing ? 'Stop recapture' : 'Stop backfill'))
|
||||||
: 'Backfill — one-time walk of the FULL post history until complete' }}
|
: 'Backfill — one-time walk of the FULL post history until complete' }}
|
||||||
</v-tooltip>
|
</v-tooltip>
|
||||||
</v-btn>
|
</v-btn>
|
||||||
@@ -38,6 +38,17 @@
|
|||||||
under the current similarity threshold
|
under the current similarity threshold
|
||||||
</v-list-item-subtitle>
|
</v-list-item-subtitle>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
|
<v-list-item
|
||||||
|
v-if="isPatreon && !running"
|
||||||
|
prepend-icon="mdi-text-box-search-outline"
|
||||||
|
@click="emit('recapture', source)"
|
||||||
|
>
|
||||||
|
<v-list-item-title>Recapture post text & links</v-list-item-title>
|
||||||
|
<v-list-item-subtitle>
|
||||||
|
Re-grab every post's body + external links and localize inline images
|
||||||
|
already on disk — without re-downloading media
|
||||||
|
</v-list-item-subtitle>
|
||||||
|
</v-list-item>
|
||||||
<v-divider v-if="isPatreon && !running" />
|
<v-divider v-if="isPatreon && !running" />
|
||||||
<v-list-item
|
<v-list-item
|
||||||
base-color="error"
|
base-color="error"
|
||||||
@@ -58,10 +69,11 @@ const props = defineProps({
|
|||||||
source: { type: Object, required: true },
|
source: { type: Object, required: true },
|
||||||
checking: { type: Boolean, default: false },
|
checking: { type: Boolean, default: false },
|
||||||
})
|
})
|
||||||
const emit = defineEmits(['check', 'backfill', 'recover', 'remove'])
|
const emit = defineEmits(['check', 'backfill', 'recover', 'recapture', 'remove'])
|
||||||
|
|
||||||
const running = computed(() => props.source.backfill_state === 'running')
|
const running = computed(() => props.source.backfill_state === 'running')
|
||||||
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
||||||
|
const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||||
const isPatreon = computed(() => props.source.platform === 'patreon')
|
const isPatreon = computed(() => props.source.platform === 'patreon')
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -40,7 +40,7 @@
|
|||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'running'"
|
v-else-if="source.backfill_state === 'running'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="tonal" label
|
||||||
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
|
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||||
}}{{ source.backfill_posts
|
}}{{ source.backfill_posts
|
||||||
? ` · ${source.backfill_posts} posts`
|
? ` · ${source.backfill_posts} posts`
|
||||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||||
@@ -60,6 +60,7 @@
|
|||||||
@check="$emit('check', $event)"
|
@check="$emit('check', $event)"
|
||||||
@backfill="$emit('backfill', $event)"
|
@backfill="$emit('backfill', $event)"
|
||||||
@recover="$emit('recover', $event)"
|
@recover="$emit('recover', $event)"
|
||||||
|
@recapture="$emit('recapture', $event)"
|
||||||
@remove="$emit('remove', $event)"
|
@remove="$emit('remove', $event)"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
@@ -79,7 +80,7 @@ const props = defineProps({
|
|||||||
checking: { type: Boolean, default: false },
|
checking: { type: Boolean, default: false },
|
||||||
warningThreshold: { type: Number, default: 5 },
|
warningThreshold: { type: Number, default: 5 },
|
||||||
})
|
})
|
||||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
|
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
||||||
|
|
||||||
function onToggleEnabled(value) {
|
function onToggleEnabled(value) {
|
||||||
emit('toggle', { source: props.source, enabled: value })
|
emit('toggle', { source: props.source, enabled: value })
|
||||||
|
|||||||
@@ -51,7 +51,7 @@
|
|||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'running'"
|
v-else-if="source.backfill_state === 'running'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="tonal" label
|
||||||
>{{ source.backfill_bypass_seen ? 'Recovering' : 'Backfilling'
|
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||||
}}{{ source.backfill_posts
|
}}{{ source.backfill_posts
|
||||||
? ` · ${source.backfill_posts} posts`
|
? ` · ${source.backfill_posts} posts`
|
||||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||||
@@ -71,6 +71,7 @@
|
|||||||
@check="$emit('check', $event)"
|
@check="$emit('check', $event)"
|
||||||
@backfill="$emit('backfill', $event)"
|
@backfill="$emit('backfill', $event)"
|
||||||
@recover="$emit('recover', $event)"
|
@recover="$emit('recover', $event)"
|
||||||
|
@recapture="$emit('recapture', $event)"
|
||||||
@remove="$emit('remove', $event)"
|
@remove="$emit('remove', $event)"
|
||||||
/>
|
/>
|
||||||
</td>
|
</td>
|
||||||
@@ -87,7 +88,7 @@ const props = defineProps({
|
|||||||
checking: { type: Boolean, default: false },
|
checking: { type: Boolean, default: false },
|
||||||
warningThreshold: { type: Number, default: 5 },
|
warningThreshold: { type: Number, default: 5 },
|
||||||
})
|
})
|
||||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover'])
|
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
||||||
|
|
||||||
function onToggleEnabled(value) {
|
function onToggleEnabled(value) {
|
||||||
emit('toggle', { source: props.source, enabled: value })
|
emit('toggle', { source: props.source, enabled: value })
|
||||||
|
|||||||
@@ -138,7 +138,7 @@
|
|||||||
:source="item.singleSource"
|
:source="item.singleSource"
|
||||||
:checking="store.checkingIds.has(item.singleSource.id)"
|
:checking="store.checkingIds.has(item.singleSource.id)"
|
||||||
@check="onCheck" @backfill="onBackfill"
|
@check="onCheck" @backfill="onBackfill"
|
||||||
@recover="onRecover" @remove="removeSource"
|
@recover="onRecover" @recapture="onRecapture" @remove="removeSource"
|
||||||
/>
|
/>
|
||||||
<v-btn icon size="small" variant="text" @click.stop="openEditSource(item.singleSource)">
|
<v-btn icon size="small" variant="text" @click.stop="openEditSource(item.singleSource)">
|
||||||
<v-icon>mdi-pencil</v-icon>
|
<v-icon>mdi-pencil</v-icon>
|
||||||
@@ -214,6 +214,7 @@
|
|||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
@backfill="onBackfill"
|
@backfill="onBackfill"
|
||||||
@recover="onRecover"
|
@recover="onRecover"
|
||||||
|
@recapture="onRecapture"
|
||||||
/>
|
/>
|
||||||
<tr v-if="item.sources.length === 0">
|
<tr v-if="item.sources.length === 0">
|
||||||
<td colspan="8" class="fc-subs__sources-empty">
|
<td colspan="8" class="fc-subs__sources-empty">
|
||||||
@@ -291,6 +292,7 @@
|
|||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
@backfill="onBackfill"
|
@backfill="onBackfill"
|
||||||
@recover="onRecover"
|
@recover="onRecover"
|
||||||
|
@recapture="onRecapture"
|
||||||
/>
|
/>
|
||||||
<div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
|
<div v-if="item.sources.length === 0" class="fc-subs__sources-empty">
|
||||||
No sources yet. Tap + to add one.
|
No sources yet. Tap + to add one.
|
||||||
@@ -646,6 +648,22 @@ async function onRecover(source) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||||
|
// external links and localize on-disk inline images, without re-downloading
|
||||||
|
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
||||||
|
async function onRecapture(source) {
|
||||||
|
try {
|
||||||
|
await store.recaptureSource(source.id, source.artist_id)
|
||||||
|
toast({ text: `Recapture started for ${source.artist_name}`, type: 'success' })
|
||||||
|
// recaptureSource patches the source in place — no full reload.
|
||||||
|
} catch (e) {
|
||||||
|
toast({
|
||||||
|
text: `Recapture start failed: ${e?.body?.detail || e?.message || e}`,
|
||||||
|
type: 'error',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function checkAll(group) {
|
async function checkAll(group) {
|
||||||
let ok = 0
|
let ok = 0
|
||||||
let conflict = 0
|
let conflict = 0
|
||||||
|
|||||||
@@ -131,6 +131,14 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
_patchSource(body)
|
_patchSource(body)
|
||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||||
|
// external links and localize on-disk inline images, WITHOUT re-downloading
|
||||||
|
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
||||||
|
async function recaptureSource(id, artistIdHint = null) {
|
||||||
|
const body = await api.post(`/api/sources/${id}/backfill`, { body: { action: 'recapture' } })
|
||||||
|
_patchSource(body)
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
// Plan #708 B4: dry-run preview — count what a backfill WOULD download
|
// Plan #708 B4: dry-run preview — count what a backfill WOULD download
|
||||||
// (native platforms), without downloading. Read-only, so no cache invalidate.
|
// (native platforms), without downloading. Read-only, so no cache invalidate.
|
||||||
@@ -167,6 +175,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
startBackfill,
|
startBackfill,
|
||||||
stopBackfill,
|
stopBackfill,
|
||||||
recoverSource,
|
recoverSource,
|
||||||
|
recaptureSource,
|
||||||
previewSource,
|
previewSource,
|
||||||
findOrCreateArtist, autocompleteArtist,
|
findOrCreateArtist, autocompleteArtist,
|
||||||
loadScheduleStatus,
|
loadScheduleStatus,
|
||||||
|
|||||||
@@ -324,6 +324,32 @@ async def test_backfill_endpoint_recover_arms_bypass(client, artist, db):
|
|||||||
assert stopped_body["backfill_bypass_seen"] is False
|
assert stopped_body["backfill_bypass_seen"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_endpoint_recapture_arms_flag(client, artist, db):
|
||||||
|
"""#830: action='recapture' arms a backfill that re-grabs post bodies/links +
|
||||||
|
localizes on-disk inline images; the response exposes backfill_recapture=True
|
||||||
|
(badge 'Recapturing') and NOT backfill_bypass_seen. Stop clears it."""
|
||||||
|
src = Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/alice-recapture", enabled=True,
|
||||||
|
)
|
||||||
|
db.add(src)
|
||||||
|
await db.commit()
|
||||||
|
sid = src.id
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/sources/{sid}/backfill", json={"action": "recapture"})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["backfill_state"] == "running"
|
||||||
|
assert body["backfill_recapture"] is True
|
||||||
|
assert body["backfill_bypass_seen"] is False
|
||||||
|
|
||||||
|
stopped = await client.post(f"/api/sources/{sid}/backfill", json={"action": "stop"})
|
||||||
|
stopped_body = await stopped.get_json()
|
||||||
|
assert stopped_body["backfill_state"] is None
|
||||||
|
assert stopped_body["backfill_recapture"] is False
|
||||||
|
|
||||||
|
|
||||||
# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
|
# --- Plan #703 #2: pre-flight credential verify on backfill/recover arm -----
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -60,8 +60,10 @@ def _make_fake_dl_result(
|
|||||||
*, success=True, written_paths=None, quarantined_paths=None,
|
*, success=True, written_paths=None, quarantined_paths=None,
|
||||||
files_downloaded=0, error_type=None, error_message=None,
|
files_downloaded=0, error_type=None, error_message=None,
|
||||||
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
||||||
|
relink_source_paths=None,
|
||||||
):
|
):
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
|
relink_source_paths=relink_source_paths or [],
|
||||||
success=success,
|
success=success,
|
||||||
url="https://patreon.com/alice",
|
url="https://patreon.com/alice",
|
||||||
artist_slug="alice",
|
artist_slug="alice",
|
||||||
@@ -793,6 +795,31 @@ async def test_recovery_mode_selected_and_flag_cleared_on_complete(
|
|||||||
assert "_backfill_bypass_seen" not in co
|
assert "_backfill_bypass_seen" not in co
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_recapture_mode_selected_and_flag_cleared_on_complete(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
):
|
||||||
|
"""#830: `_backfill_recapture` alongside a running backfill selects recapture
|
||||||
|
mode. A clean rc=0 walk completes the shared lifecycle AND clears the
|
||||||
|
recapture flag so the next tick is a plain tick again."""
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
source.config_overrides = {"_backfill_state": "running", "_backfill_recapture": True}
|
||||||
|
source.backfill_runs_remaining = 5
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
svc, calls = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||||
|
success=True, written_paths=[], files_downloaded=0,
|
||||||
|
))
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
assert calls[0]["mode"] == "recapture"
|
||||||
|
co = (await db.execute(
|
||||||
|
select(Source.config_overrides).where(Source.id == source.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert co.get("_backfill_state") == "complete"
|
||||||
|
assert "_backfill_recapture" not in co
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_partial_error_type_maps_to_ok_status(
|
async def test_partial_error_type_maps_to_ok_status(
|
||||||
db, db_sync, tmp_path, seed_artist_and_source,
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
|||||||
@@ -206,6 +206,54 @@ def test_skip_disk(tmp_path):
|
|||||||
assert existing.read_bytes() == b"already here" # untouched
|
assert existing.read_bytes() == b"already here" # untouched
|
||||||
|
|
||||||
|
|
||||||
|
def test_recapture_surfaces_seen_on_disk_without_download(tmp_path):
|
||||||
|
"""#830 recapture: a seen item that IS on disk is surfaced as skipped_disk
|
||||||
|
(with its path, for source_filehash backfill) — NOT tier-1 short-circuited,
|
||||||
|
and never re-downloaded."""
|
||||||
|
session = _FakeSession()
|
||||||
|
dl = _downloader(tmp_path, session=session)
|
||||||
|
post_dir = tmp_path / "artist-x" / "patreon" / "2026-05-01_1001_My Post Title"
|
||||||
|
post_dir.mkdir(parents=True)
|
||||||
|
existing = post_dir / "01_media1.png"
|
||||||
|
existing.write_bytes(b"already here")
|
||||||
|
|
||||||
|
outcomes = dl.download_post(
|
||||||
|
_post(), [_img("media1.png")], "artist-x",
|
||||||
|
is_seen=lambda m: True, recapture=True,
|
||||||
|
)
|
||||||
|
assert outcomes[0].status == "skipped_disk"
|
||||||
|
assert outcomes[0].path == existing
|
||||||
|
assert session.calls == [] # no re-download
|
||||||
|
assert existing.read_bytes() == b"already here" # untouched
|
||||||
|
|
||||||
|
|
||||||
|
def test_recapture_does_not_refetch_seen_missing(tmp_path):
|
||||||
|
"""#830 recapture: a seen item that's NOT on disk is left alone (skipped_seen,
|
||||||
|
no download) — refetching is recovery's job, not recapture's."""
|
||||||
|
session = _FakeSession()
|
||||||
|
dl = _downloader(tmp_path, session=session)
|
||||||
|
outcomes = dl.download_post(
|
||||||
|
_post(), [_img("media1.png")], "artist-x",
|
||||||
|
is_seen=lambda m: True, recapture=True,
|
||||||
|
)
|
||||||
|
assert outcomes[0].status == "skipped_seen"
|
||||||
|
assert outcomes[0].path is None
|
||||||
|
assert session.calls == [] # not refetched
|
||||||
|
|
||||||
|
|
||||||
|
def test_recapture_still_downloads_genuinely_new(tmp_path):
|
||||||
|
"""#830 recapture: media that is neither seen nor on disk still downloads — a
|
||||||
|
recapture walk is thorough, it just doesn't re-pull existing files."""
|
||||||
|
session = _FakeSession()
|
||||||
|
dl = _downloader(tmp_path, session=session)
|
||||||
|
outcomes = dl.download_post(
|
||||||
|
_post(), [_img("media1.png")], "artist-x",
|
||||||
|
is_seen=lambda m: False, recapture=True,
|
||||||
|
)
|
||||||
|
assert outcomes[0].status == "downloaded"
|
||||||
|
assert outcomes[0].path is not None
|
||||||
|
|
||||||
|
|
||||||
def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
|
def test_video_routes_to_ytdlp(tmp_path, monkeypatch):
|
||||||
session = _FakeSession()
|
session = _FakeSession()
|
||||||
dl = _downloader(tmp_path, session=session)
|
dl = _downloader(tmp_path, session=session)
|
||||||
|
|||||||
@@ -88,16 +88,23 @@ class _FakeDownloader:
|
|||||||
self.post_records = 0
|
self.post_records = 0
|
||||||
|
|
||||||
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
def download_post(self, post, media_items, artist_slug, *, is_seen,
|
||||||
should_stop=lambda: False):
|
should_stop=lambda: False, recapture=False):
|
||||||
outcomes = []
|
outcomes = []
|
||||||
for m in media_items:
|
for m in media_items:
|
||||||
if should_stop():
|
if should_stop():
|
||||||
break
|
break
|
||||||
if is_seen(m):
|
seen = is_seen(m)
|
||||||
|
# Recapture mirrors the real downloader: a seen item isn't tier-1
|
||||||
|
# short-circuited — it falls through so an on-disk file surfaces as
|
||||||
|
# skipped_disk (for source_filehash backfill); a seen file NOT on
|
||||||
|
# disk is left alone (skipped_seen, never refetched).
|
||||||
|
if seen and not recapture:
|
||||||
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||||
elif _ledger_key(m) in self.on_disk:
|
elif _ledger_key(m) in self.on_disk:
|
||||||
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
p = self.tmp_path / f"{m.post_id}_{m.filename}"
|
||||||
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
|
outcomes.append(MediaOutcome(media=m, status="skipped_disk", path=p, error=None))
|
||||||
|
elif seen:
|
||||||
|
outcomes.append(MediaOutcome(media=m, status="skipped_seen", path=None, error=None))
|
||||||
elif _ledger_key(m) in self.error:
|
elif _ledger_key(m) in self.error:
|
||||||
self.download_calls += 1
|
self.download_calls += 1
|
||||||
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
|
outcomes.append(MediaOutcome(media=m, status="error", path=None, error="boom"))
|
||||||
@@ -581,6 +588,78 @@ async def test_backfill_recaptures_body_for_already_downloaded_post(
|
|||||||
assert downloader.post_records == 1
|
assert downloader.post_records == 1
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_seen(sync_engine, source_id, key, post_id=None):
|
||||||
|
factory = sessionmaker(sync_engine, expire_on_commit=False)
|
||||||
|
with factory() as s:
|
||||||
|
s.add(PatreonSeenMedia(source_id=source_id, filehash=key, post_id=post_id))
|
||||||
|
s.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
|
||||||
|
source_id, sync_engine, tmp_path,
|
||||||
|
):
|
||||||
|
"""#830: a plain backfill GATES post-record capture on the seen-ledger, so a
|
||||||
|
post whose `post:<id>` key is already seen is NOT recaptured (the operator's
|
||||||
|
'Industrial Lust body missing' bug). RECAPTURE mode bypasses that gate and
|
||||||
|
re-captures the body/links even though the post key is seen — without
|
||||||
|
re-downloading the on-disk media."""
|
||||||
|
m1 = _media("p1", 1)
|
||||||
|
# Pre-seed BOTH the media key and the synthetic post key as already seen.
|
||||||
|
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||||
|
_seed_seen(sync_engine, source_id, "post:p1", post_id="p1")
|
||||||
|
|
||||||
|
# 1) Plain backfill: post key is seen → gate skips body recapture.
|
||||||
|
client = _FakeClient([(None, [("p1", [m1])])])
|
||||||
|
downloader = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||||
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||||
|
res_backfill = ing.run(
|
||||||
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||||
|
url="https://patreon.com/ingest", mode="backfill",
|
||||||
|
)
|
||||||
|
assert res_backfill.post_record_paths == [] # gated → not recaptured
|
||||||
|
assert downloader.post_records == 0
|
||||||
|
|
||||||
|
# 2) Recapture: gate bypassed → body recaptured; media NOT re-downloaded;
|
||||||
|
# the on-disk file is surfaced for source_filehash relink.
|
||||||
|
client2 = _FakeClient([(None, [("p1", [m1])])])
|
||||||
|
downloader2 = _FakeDownloader(tmp_path, on_disk={_ledger_key(m1)})
|
||||||
|
ing2 = _ingester(sync_engine, tmp_path, client2, downloader2)
|
||||||
|
res_recap = ing2.run(
|
||||||
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||||
|
url="https://patreon.com/ingest", mode="recapture",
|
||||||
|
)
|
||||||
|
assert len(res_recap.post_record_paths) == 1 # gate bypassed → recaptured
|
||||||
|
assert downloader2.post_records == 1
|
||||||
|
assert res_recap.files_downloaded == 0 # no media re-download
|
||||||
|
assert downloader2.download_calls == 0
|
||||||
|
# The on-disk media is surfaced as a (path, CDN url) relink pair.
|
||||||
|
assert len(res_recap.relink_source_paths) == 1
|
||||||
|
rel_path, rel_url = res_recap.relink_source_paths[0]
|
||||||
|
assert rel_url == m1.url
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_recapture_does_not_refetch_seen_media_missing_from_disk(
|
||||||
|
source_id, sync_engine, tmp_path,
|
||||||
|
):
|
||||||
|
"""Recapture must NOT re-download a seen item that's absent from disk (that's
|
||||||
|
recovery's job) — it returns skipped_seen and contributes no relink pair."""
|
||||||
|
m1 = _media("p1", 1)
|
||||||
|
_seed_seen(sync_engine, source_id, _ledger_key(m1), post_id="p1")
|
||||||
|
client = _FakeClient([(None, [("p1", [m1])])])
|
||||||
|
# NOT in on_disk → seen + missing.
|
||||||
|
downloader = _FakeDownloader(tmp_path)
|
||||||
|
ing = _ingester(sync_engine, tmp_path, client, downloader)
|
||||||
|
result = ing.run(
|
||||||
|
source_id=source_id, campaign_id="c1", artist_slug="ingest",
|
||||||
|
url="https://patreon.com/ingest", mode="recapture",
|
||||||
|
)
|
||||||
|
assert result.files_downloaded == 0
|
||||||
|
assert downloader.download_calls == 0
|
||||||
|
assert result.relink_source_paths == []
|
||||||
|
|
||||||
|
|
||||||
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
# --- dead-letter ledger (plan #705 #7) ------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -120,6 +120,44 @@ def test_sidecar_source_url_persists_filehash(importer, import_layout):
|
|||||||
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
|
def test_relink_source_filehash_backfills_existing(importer, import_layout):
|
||||||
|
"""#830 recapture (Part B): relink_source_filehash backfills source_url +
|
||||||
|
source_filehash on an existing on-disk image's ImageRecord (matched by
|
||||||
|
sha256) WITHOUT re-importing or unlinking the file. NULL-only — never
|
||||||
|
clobbers a filehash already set."""
|
||||||
|
import_root, _ = import_layout
|
||||||
|
m = import_root / "Alice" / "img.jpg"
|
||||||
|
_split(m, "v")
|
||||||
|
r = importer.import_one(m)
|
||||||
|
assert r.status == "imported"
|
||||||
|
rec = importer.session.get(ImageRecord, r.image_id)
|
||||||
|
assert rec.source_filehash is None
|
||||||
|
|
||||||
|
url = "https://cdn.test/p/0123456789ABCDEF0123456789abcdef/img.jpg"
|
||||||
|
assert importer.relink_source_filehash(m, url) is True
|
||||||
|
importer.session.refresh(rec)
|
||||||
|
assert rec.source_url == url
|
||||||
|
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||||
|
assert m.exists() # never unlinked
|
||||||
|
|
||||||
|
# NULL-only: a second relink with a different url is a no-op.
|
||||||
|
other = "https://cdn.test/p/ffffffffffffffffffffffffffffffff/x.jpg"
|
||||||
|
assert importer.relink_source_filehash(m, other) is False
|
||||||
|
importer.session.refresh(rec)
|
||||||
|
assert rec.source_filehash == "0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
|
def test_relink_source_filehash_no_match_is_noop(importer, import_layout):
|
||||||
|
"""A path whose bytes match no ImageRecord (never imported) is a clean no-op,
|
||||||
|
not an error."""
|
||||||
|
import_root, _ = import_layout
|
||||||
|
m = import_root / "Alice" / "ghost.jpg"
|
||||||
|
_split(m, "h")
|
||||||
|
assert importer.relink_source_filehash(
|
||||||
|
m, "https://cdn.test/p/0123456789abcdef0123456789abcdef/ghost.jpg"
|
||||||
|
) is False
|
||||||
|
|
||||||
|
|
||||||
def test_reimport_same_post_idempotent(importer, import_layout):
|
def test_reimport_same_post_idempotent(importer, import_layout):
|
||||||
import_root, _ = import_layout
|
import_root, _ = import_layout
|
||||||
# Threshold 0: only an exact phash match collapses. Orthogonal splits
|
# Threshold 0: only an exact phash match collapses. Orthogonal splits
|
||||||
|
|||||||
@@ -267,6 +267,47 @@ async def test_start_recovery_arms_bypass_flag(db):
|
|||||||
assert "_backfill_bypass_seen" not in (co2 or {})
|
assert "_backfill_bypass_seen" not in (co2 or {})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_start_recapture_arms_recapture_flag(db):
|
||||||
|
"""#830: start_recapture arms the backfill state machine PLUS the
|
||||||
|
_backfill_recapture flag, surfaced on the record so the badge can label it
|
||||||
|
'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop
|
||||||
|
clears it."""
|
||||||
|
from backend.app.services.source_service import BACKFILL_MAX_CHUNKS
|
||||||
|
|
||||||
|
artist = await _artist(db, "Alice")
|
||||||
|
svc = SourceService(db)
|
||||||
|
rec = await svc.create(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/alice",
|
||||||
|
)
|
||||||
|
# Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion).
|
||||||
|
await svc.start_recovery(rec.id)
|
||||||
|
|
||||||
|
updated = await svc.start_recapture(rec.id)
|
||||||
|
assert updated.backfill_state == "running"
|
||||||
|
assert updated.backfill_recapture is True
|
||||||
|
assert updated.backfill_bypass_seen is False
|
||||||
|
assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS
|
||||||
|
|
||||||
|
co = (await db.execute(
|
||||||
|
select(Source.config_overrides).where(Source.id == rec.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert co.get("_backfill_recapture") is True
|
||||||
|
assert "_backfill_bypass_seen" not in co
|
||||||
|
|
||||||
|
# to_dict carries the new field for the API/UI.
|
||||||
|
assert updated.to_dict()["backfill_recapture"] is True
|
||||||
|
|
||||||
|
# Stop clears the recapture flag too.
|
||||||
|
stopped = await svc.stop_backfill(rec.id)
|
||||||
|
assert stopped.backfill_recapture is False
|
||||||
|
co2 = (await db.execute(
|
||||||
|
select(Source.config_overrides).where(Source.id == rec.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert "_backfill_recapture" not in (co2 or {})
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_start_backfill_raises_when_source_missing(db):
|
async def test_start_backfill_raises_when_source_missing(db):
|
||||||
svc = SourceService(db)
|
svc = SourceService(db)
|
||||||
|
|||||||
Reference in New Issue
Block a user