Merge pull request 'Merge dev → main: recapture body-fetch fix + diagnostics (#842)' (#104) from dev into main

backend/app/services/download_service.py

@@ -406,7 +406,9 @@ class DownloadService:
         # their post-body inline <img src=CDN> remaps to the local copy. A
         # SEPARATE non-deleting channel (NOT the import list — that would unlink
         # the duplicate file). Empty outside recapture mode.
-        for rel_str, rel_url in getattr(dl_result, "relink_source_paths", None) or []:
+        relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
+        relinked = 0
+        for rel_str, rel_url in relink_pairs:
             rel_path = Path(rel_str)
             if not rel_path.exists():  # noqa: ASYNC240
                 continue
@@ -414,7 +416,16 @@ class DownloadService:
             def _relink(p=rel_path, u=rel_url):
                 return self.importer.relink_source_filehash(p, u, artist=artist)
 
-            await loop.run_in_executor(None, _relink)
+            if await loop.run_in_executor(None, _relink):
+                relinked += 1
+        if relink_pairs:
+            # recapture diagnostic: how many on-disk images got their
+            # source_filehash backfilled (inline-image localization). < total is
+            # normal — files already carrying a filehash are skipped (NULL-only).
+            log.info(
+                "recap: relinked source_filehash on %d/%d on-disk image(s)",
+                relinked, len(relink_pairs),
+            )
 
         # Kick the off-platform file-host downloader for any links this run
         # recorded (mega/gdrive/…). Global + idempotent (only claims pending/

backend/app/services/ingest_core.py

@@ -402,7 +402,8 @@ class Ingester:
             f"{self._platform} ingest ({mode}): {downloaded} downloaded, "
             f"{skipped_count} skipped, {quarantined} quarantined, "
             f"{dead_lettered} dead-lettered, {errors} error(s), "
-            f"{posts_processed} post(s)"
+            f"{posts_processed} post(s), {len(post_records)} post-record(s), "
+            f"{len(relink)} relinked"
             + (", reached end" if reached_bottom else "")
             + (", time-boxed" if budget_hit else "")
         )

backend/app/services/patreon_client.py

@@ -609,7 +609,44 @@ class PatreonClient:
         data = payload.get("data") if isinstance(payload, dict) else None
         attrs = data.get("attributes") if isinstance(data, dict) else None
         content = attrs.get("content") if isinstance(attrs, dict) else None
-        return content if isinstance(content, str) and content.strip() else None
+        if isinstance(content, str) and content.strip():
+            log.info("post-detail: fetched %d chars (post %s)", len(content), post_id)
+            return content
+        # Sparse `fields[post]=content` came back null/empty. Patreon serves null
+        # under the sparse fieldset for some post types (polls, embeds/films,
+        # body-only announcements) even when the body plainly exists. Re-fetch the
+        # FULL post resource ONCE — only the empty cases pay this extra GET — which
+        # both DIAGNOSES (logs post_type) and RECOVERS the body when the sparse
+        # fieldset was the cause (#842).
+        try:
+            full = self._session.get(
+                url, params={"json-api-version": "1.0"}, timeout=_TIMEOUT_SECONDS,
+            )
+        except requests.RequestException as exc:
+            log.warning("post-detail full re-fetch failed (post %s): %s", post_id, exc)
+            return None
+        fa = None
+        if full.status_code == 200:
+            try:
+                fp = full.json()
+            except ValueError:
+                fp = None
+            fdata = fp.get("data") if isinstance(fp, dict) else None
+            fa = fdata.get("attributes") if isinstance(fdata, dict) else None
+        fcontent = fa.get("content") if isinstance(fa, dict) else None
+        ptype = fa.get("post_type") if isinstance(fa, dict) else None
+        if isinstance(fcontent, str) and fcontent.strip():
+            log.warning(
+                "post-detail: sparse fieldset gave null but FULL fetch has %d chars "
+                "(post %s, post_type=%s) — using full body",
+                len(fcontent), post_id, ptype,
+            )
+            return fcontent
+        log.info(
+            "post-detail: empty/null content (post %s, post_type=%s) even on full "
+            "fetch — body not in the post resource", post_id, ptype,
+        )
+        return None
 
     # -- verify ------------------------------------------------------------
 

backend/app/services/patreon_downloader.py

@@ -574,8 +574,20 @@ class PatreonDownloader:
         leading underscore keeps it from colliding with a media sidecar
         (`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
         by find_sidecar. Returns the path, or None when the post has no id."""
-        if not str(post.get("id") or ""):
+        pid = str(post.get("id") or "")
+        if not pid:
             return None
         post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
         post_dir.mkdir(parents=True, exist_ok=True)
-        return self._write_sidecar_data(post, post_dir / "_post.json")
+        path = self._write_sidecar_data(post, post_dir / "_post.json")
+        # Per-post body-capture diagnostic: _write_sidecar_data has, by now,
+        # memoized any detail-fetched body onto post["attributes"]["content"], so
+        # this reflects the FINAL body (feed or detail). Logs the success (N chars)
+        # AND the skip (empty) so a recapture's per-post outcome is visible.
+        body = (post.get("attributes") or {}).get("content")
+        n = len(body) if isinstance(body, str) else 0
+        if n:
+            log.info("post-record: post %s body captured (%d chars)", pid, n)
+        else:
+            log.info("post-record: post %s has NO body (feed + detail both empty)", pid)
+        return path

tests/test_patreon_client.py

@@ -380,6 +380,28 @@ def test_fetch_post_detail_content_empty_body_is_none(monkeypatch):
     assert client.fetch_post_detail_content("42") is None
 
 
+def test_fetch_post_detail_content_falls_back_to_full_fetch(monkeypatch):
+    """#842: when the sparse fields[post]=content request returns null (Patreon
+    does this for polls / embeds / body-only posts), a one-time FULL re-fetch
+    recovers the body — so inline-only / no-media posts still get their text."""
+    client = PatreonClient(cookies_path=None)
+    seen_params: list = []
+
+    def _get(url, params=None, timeout=None):
+        seen_params.append(params or {})
+        if "fields[post]" in (params or {}):
+            return _FakeResp(200, json_data={"data": {"attributes": {"content": None}}})
+        return _FakeResp(200, json_data={
+            "data": {"attributes": {"content": "<p>body via full</p>", "post_type": "poll"}}
+        })
+
+    monkeypatch.setattr(client._session, "get", _get)
+    assert client.fetch_post_detail_content("42") == "<p>body via full</p>"
+    # Sparse request first, then a full re-fetch without the sparse fieldset.
+    assert "fields[post]" in seen_params[0]
+    assert "fields[post]" not in seen_params[1]
+
+
 def test_fetch_post_detail_content_blank_id_skips_request(monkeypatch):
     calls: list = []
     client = PatreonClient(cookies_path=None)

tests/test_patreon_ingester.py

@@ -637,6 +637,9 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     assert len(res_recap.relink_source_paths) == 1
     rel_path, rel_url = res_recap.relink_source_paths[0]
     assert rel_url == m1.url
+    # Diagnostic summary surfaces the post-record + relink counts (operator reads
+    # this off the event stdout to see what a recapture actually did).
+    assert "1 post-record(s), 1 relinked" in res_recap.stdout
 
 
 @pytest.mark.asyncio