feat(ingest): per-post body-capture + recapture diagnostics logging
Operator-flagged: a recapture 'caught nothing' for a post and there were no logs explaining why. Three silent spots now log, so a recapture's per-post outcome is diagnosable (retention bounds the volume): - patreon_client.fetch_post_detail_content: the 200-OK-but-null-content branch was silent — now logs 'fetched N chars' on success AND 'empty/null content (tier-gated or no text)' on the empty case (the most common silent miss). - patreon_downloader.write_post_record: logs each post's FINAL body outcome (captured N chars / NO body) read off the memoized attrs after detail-fetch. - ingest_core summary: appends post-record + relinked counts to the run summary (surfaces on the event stdout the operator already reads). - download_service phase3: logs how many on-disk images got source_filehash relinked (N/total) per recapture. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -406,7 +406,9 @@ class DownloadService:
|
||||
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
||||
# the duplicate file). Empty outside recapture mode.
|
||||
for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []:
|
||||
relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
|
||||
relinked = 0
|
||||
for rel_str, rel_url in relink_pairs:
|
||||
rel_path = Path(rel_str)
|
||||
if not rel_path.exists(): # noqa: ASYNC240
|
||||
continue
|
||||
@@ -414,7 +416,16 @@ class DownloadService:
|
||||
def _relink(p=rel_path, u=rel_url):
|
||||
return self.importer.relink_source_filehash(p, u, artist=artist)
|
||||
|
||||
await loop.run_in_executor(None, _relink)
|
||||
if await loop.run_in_executor(None, _relink):
|
||||
relinked += 1
|
||||
if relink_pairs:
|
||||
# recapture diagnostic: how many on-disk images got their
|
||||
# source_filehash backfilled (inline-image localization). < total is
|
||||
# normal — files already carrying a filehash are skipped (NULL-only).
|
||||
log.info(
|
||||
"recap: relinked source_filehash on %d/%d on-disk image(s)",
|
||||
relinked, len(relink_pairs),
|
||||
)
|
||||
|
||||
# Kick the off-platform file-host downloader for any links this run
|
||||
# recorded (mega/gdrive/…). Global + idempotent (only claims pending/
|
||||
|
||||
@@ -402,7 +402,8 @@ class Ingester:
|
||||
f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
|
||||
f"{skipped_count} skipped, {quarantined} quarantined, "
|
||||
f"{dead_lettered} dead-lettered, {errors} error(s), "
|
||||
f"{posts_processed} post(s)"
|
||||
f"{posts_processed} post(s), {len(post_records)} post-record(s), "
|
||||
f"{len(relink)} relinked"
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
|
||||
@@ -609,7 +609,17 @@ class PatreonClient:
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
attrs = data.get("attributes") if isinstance(data, dict) else None
|
||||
content = attrs.get("content") if isinstance(attrs, dict) else None
|
||||
return content if isinstance(content, str) and content.strip() else None
|
||||
if isinstance(content, str) and content.strip():
|
||||
log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
|
||||
return content
|
||||
# 200 OK but content is null/empty — the common silent case (a tier-gated
|
||||
# post serves content:null, or the post genuinely has no text body). Logged
|
||||
# so a recapture that "caught nothing" is diagnosable, not invisible.
|
||||
log.info(
|
||||
"post-detail: empty/null content (post %s) — body unavailable "
|
||||
"(tier-gated, or post has no text)", post_id,
|
||||
)
|
||||
return None
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -574,8 +574,20 @@ class PatreonDownloader:
|
||||
leading underscore keeps it from colliding with a media sidecar
|
||||
(`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
|
||||
by find_sidecar. Returns the path, or None when the post has no id."""
|
||||
if not str(post.get("id") or ""):
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
return None
|
||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
path = self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
# Per-post body-capture diagnostic: _write_sidecar_data has, by now,
|
||||
# memoized any detail-fetched body onto post["attributes"]["content"], so
|
||||
# this reflects the FINAL body (feed or detail). Logs the success (N chars)
|
||||
# AND the skip (empty) so a recapture's per-post outcome is visible.
|
||||
body = (post.get("attributes") or {}).get("content")
|
||||
n = len(body) if isinstance(body, str) else 0
|
||||
if n:
|
||||
log.info("post-record: post %s body captured (%d chars)", pid, n)
|
||||
else:
|
||||
log.info("post-record: post %s has NO body (feed + detail both empty)", pid)
|
||||
return path
|
||||
|
||||
@@ -637,6 +637,9 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
|
||||
assert len(res_recap.relink_source_paths) == 1
|
||||
rel_path, rel_url = res_recap.relink_source_paths[0]
|
||||
assert rel_url == m1.url
|
||||
# Diagnostic summary surfaces the post-record + relink counts (operator reads
|
||||
# this off the event stdout to see what a recapture actually did).
|
||||
assert "1 post-record(s), 1 relinked" in res_recap.stdout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user