From b999480db5407f2ad78438edf29dad603faf7b2e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 14 Jun 2026 22:10:59 -0400 Subject: [PATCH] feat(ingest): per-post body-capture + recapture diagnostics logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged: a recapture 'caught nothing' for a post and there were no logs explaining why. Three silent spots now log, so a recapture's per-post outcome is diagnosable (retention bounds the volume): - patreon_client.fetch_post_detail_content: the 200-OK-but-null-content branch was silent — now logs 'fetched N chars' on success AND 'empty/null content (tier-gated or no text)' on the empty case (the most common silent miss). - patreon_downloader.write_post_record: logs each post's FINAL body outcome (captured N chars / NO body) read off the memoized attrs after detail-fetch. - ingest_core summary: appends post-record + relinked counts to the run summary (surfaces on the event stdout the operator already reads). - download_service phase3: logs how many on-disk images got source_filehash relinked (N/total) per recapture. Co-Authored-By: Claude Opus 4.8 --- backend/app/services/download_service.py | 15 +++++++++++++-- backend/app/services/ingest_core.py | 3 ++- backend/app/services/patreon_client.py | 12 +++++++++++- backend/app/services/patreon_downloader.py | 16 ++++++++++++++-- tests/test_patreon_ingester.py | 3 +++ 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index bbf48fb..bb099a1 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -406,7 +406,9 @@ class DownloadService: # their post-body inline remaps to the local copy. A # SEPARATE non-deleting channel (NOT the import list — that would unlink # the duplicate file). Empty outside recapture mode. - for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []: + relink_pairs = getattr(dl_result, "relink_source_paths", None) or [] + relinked = 0 + for rel_str, rel_url in relink_pairs: rel_path = Path(rel_str) if not rel_path.exists(): # noqa: ASYNC240 continue @@ -414,7 +416,16 @@ class DownloadService: def _relink(p=rel_path, u=rel_url): return self.importer.relink_source_filehash(p, u, artist=artist) - await loop.run_in_executor(None, _relink) + if await loop.run_in_executor(None, _relink): + relinked += 1 + if relink_pairs: + # recapture diagnostic: how many on-disk images got their + # source_filehash backfilled (inline-image localization). < total is + # normal — files already carrying a filehash are skipped (NULL-only). + log.info( + "recap: relinked source_filehash on %d/%d on-disk image(s)", + relinked, len(relink_pairs), + ) # Kick the off-platform file-host downloader for any links this run # recorded (mega/gdrive/…). Global + idempotent (only claims pending/ diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index 8cd293b..933af56 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -402,7 +402,8 @@ class Ingester: f"{self._platform} ingest ({mode}): {downloaded} downloaded, " f"{skipped_count} skipped, {quarantined} quarantined, " f"{dead_lettered} dead-lettered, {errors} error(s), " - f"{posts_processed} post(s)" + f"{posts_processed} post(s), {len(post_records)} post-record(s), " + f"{len(relink)} relinked" + (", reached end" if reached_bottom else "") + (", time-boxed" if budget_hit else "") ) diff --git a/backend/app/services/patreon_client.py b/backend/app/services/patreon_client.py index f79f627..f90f92c 100644 --- a/backend/app/services/patreon_client.py +++ b/backend/app/services/patreon_client.py @@ -609,7 +609,17 @@ class PatreonClient: data = payload.get("data") if isinstance(payload, dict) else None attrs = data.get("attributes") if isinstance(data, dict) else None content = attrs.get("content") if isinstance(attrs, dict) else None - return content if isinstance(content, str) and content.strip() else None + if isinstance(content, str) and content.strip(): + log.info("post-detail: fetched %d chars (post %s)", len(content), post_id) + return content + # 200 OK but content is null/empty — the common silent case (a tier-gated + # post serves content:null, or the post genuinely has no text body). Logged + # so a recapture that "caught nothing" is diagnosable, not invisible. + log.info( + "post-detail: empty/null content (post %s) — body unavailable " + "(tier-gated, or post has no text)", post_id, + ) + return None # -- verify ------------------------------------------------------------ diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index c142150..bd11fa9 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -574,8 +574,20 @@ class PatreonDownloader: leading underscore keeps it from colliding with a media sidecar (`_.json`) and from being resolved as some media file's sidecar by find_sidecar. Returns the path, or None when the post has no id.""" - if not str(post.get("id") or ""): + pid = str(post.get("id") or "") + if not pid: return None post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) post_dir.mkdir(parents=True, exist_ok=True) - return self._write_sidecar_data(post, post_dir / "_post.json") + path = self._write_sidecar_data(post, post_dir / "_post.json") + # Per-post body-capture diagnostic: _write_sidecar_data has, by now, + # memoized any detail-fetched body onto post["attributes"]["content"], so + # this reflects the FINAL body (feed or detail). Logs the success (N chars) + # AND the skip (empty) so a recapture's per-post outcome is visible. + body = (post.get("attributes") or {}).get("content") + n = len(body) if isinstance(body, str) else 0 + if n: + log.info("post-record: post %s body captured (%d chars)", pid, n) + else: + log.info("post-record: post %s has NO body (feed + detail both empty)", pid) + return path diff --git a/tests/test_patreon_ingester.py b/tests/test_patreon_ingester.py index 358a7cc..529acc7 100644 --- a/tests/test_patreon_ingester.py +++ b/tests/test_patreon_ingester.py @@ -637,6 +637,9 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it( assert len(res_recap.relink_source_paths) == 1 rel_path, rel_url = res_recap.relink_source_paths[0] assert rel_url == m1.url + # Diagnostic summary surfaces the post-record + relink counts (operator reads + # this off the event stdout to see what a recapture actually did). + assert "1 post-record(s), 1 relinked" in res_recap.stdout @pytest.mark.asyncio