Merge pull request 'Merge dev → main: per-post body-capture diagnostics in the event UI (#842)' (#105) from dev into main

backend/app/services/download_service.py

@@ -477,6 +477,10 @@ class DownloadService:
             # recurring "post shows a zip but no images" report. Each entry is
             # {file, reason}; None when every archive extracted cleanly.
             "unextracted_archives": unextracted_archives or None,
+            # #842: per-post body-capture outcomes ({post_id, title, post_type,
+            # body_chars}) so the UI shows how each post was handled — a 0-char
+            # body next to its post_type explains an empty post at a glance.
+            "post_diagnostics": getattr(dl_result, "post_diagnostics", None) or None,
         }
         await self._update_source_health(
             source_id=ctx["source_id"], status=status, error_message=ev.error,

backend/app/services/gallery_dl.py

@@ -158,6 +158,11 @@ class DownloadResult:
     # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
     # on the gallery-dl path and outside recapture.
     relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
+    # Native ingester (#842): per-post body-capture diagnostics
+    # ({post_id, title, post_type, body_chars}) surfaced on the DownloadEvent so
+    # the operator can see in the UI how each post was handled. Empty on the
+    # gallery-dl path and for posts whose body wasn't (re)captured this run.
+    post_diagnostics: list[dict] = field(default_factory=list)
     stdout: str = ""
     stderr: str = ""
     return_code: int = 0

backend/app/services/ingest_core.py

@@ -143,6 +143,11 @@ class Ingester:
         # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
         # re-downloading or unlinking the file. Empty outside recapture mode.
         relink: list[tuple[str, str]] = []
+        # #842: per-post body-capture diagnostics surfaced on the DownloadEvent so
+        # the operator can see IN THE UI how each post was handled — crucially the
+        # post_type + final body length, which explains an empty body (e.g. a poll
+        # whose body the API never returns) without digging through worker logs.
+        post_diag: list[dict] = []
         downloaded = 0
         errors = 0
         quarantined = 0
@@ -180,6 +185,7 @@ class Ingester:
                 written_paths=written,
                 post_record_paths=list(post_records),
                 relink_source_paths=list(relink),
+                post_diagnostics=list(post_diag),
                 stdout="\n".join(log_lines),
                 stderr="",
                 return_code=return_code,
@@ -264,6 +270,19 @@ class Ingester:
                             if rec_path is not None:
                                 post_records.append(str(rec_path))
                                 self._mark_seen(source_id, [(pkey, ppid)])
+                            # Read the FINAL body (write_post_record memoized any
+                            # detail-fetched content onto the post) + post_type
+                            # (already in the feed fieldset) for the per-post UI
+                            # diagnostic. A body_chars of 0 with the post_type is
+                            # the operator's "why is this one empty" answer.
+                            pattrs = post.get("attributes") or {}
+                            pbody = pattrs.get("content")
+                            post_diag.append({
+                                "post_id": ppid,
+                                "title": pattrs.get("title") or None,
+                                "post_type": pattrs.get("post_type") or None,
+                                "body_chars": len(pbody) if isinstance(pbody, str) else 0,
+                            })
 
                 media = self.client.extract_media(post, included)
                 if not media:

frontend/src/components/downloads/DownloadDetailModal.vue

@@ -28,6 +28,54 @@
           <div><span class="fc-dl-grid__label">Errors</span><span>{{ summary.errors ?? 0 }}</span></div>
         </div>
 
+        <template v-if="postDiag.length">
+          <h3 class="text-subtitle-2 mt-4">Post capture</h3>
+          <div class="fc-dl-grid">
+            <div><span class="fc-dl-grid__label">Posts</span><span>{{ postDiag.length }}</span></div>
+            <div><span class="fc-dl-grid__label">With body</span><span>{{ postWithBody }}</span></div>
+            <div><span class="fc-dl-grid__label">Empty body</span><span>{{ postEmpty.length }}</span></div>
+          </div>
+          <v-expansion-panels class="mt-2">
+            <v-expansion-panel v-if="postEmpty.length">
+              <v-expansion-panel-title>
+                <span>Empty-body posts ({{ postEmpty.length }})</span>
+              </v-expansion-panel-title>
+              <v-expansion-panel-text>
+                <table class="fc-dl-table">
+                  <thead><tr><th>Post</th><th>Type</th><th>Chars</th></tr></thead>
+                  <tbody>
+                    <tr v-for="p in postEmpty" :key="p.post_id" class="fc-dl-empty">
+                      <td>{{ p.title || `#${p.post_id}` }}</td>
+                      <td><code>{{ p.post_type || '—' }}</code></td>
+                      <td>{{ p.body_chars }}</td>
+                    </tr>
+                  </tbody>
+                </table>
+              </v-expansion-panel-text>
+            </v-expansion-panel>
+            <v-expansion-panel>
+              <v-expansion-panel-title>
+                <span>All posts ({{ postDiag.length }})</span>
+              </v-expansion-panel-title>
+              <v-expansion-panel-text>
+                <table class="fc-dl-table">
+                  <thead><tr><th>Post</th><th>Type</th><th>Chars</th></tr></thead>
+                  <tbody>
+                    <tr
+                      v-for="p in postDiag" :key="p.post_id"
+                      :class="{ 'fc-dl-empty': !p.body_chars }"
+                    >
+                      <td>{{ p.title || `#${p.post_id}` }}</td>
+                      <td><code>{{ p.post_type || '—' }}</code></td>
+                      <td>{{ p.body_chars }}</td>
+                    </tr>
+                  </tbody>
+                </table>
+              </v-expansion-panel-text>
+            </v-expansion-panel>
+          </v-expansion-panels>
+        </template>
+
         <template v-if="quarantinedPaths.length">
           <h3 class="text-subtitle-2 mt-4">Quarantined paths</h3>
           <ul class="fc-dl-quar">
@@ -115,18 +163,31 @@ const summary = computed(() => props.event?.metadata?.import_summary || {})
 const quarantinedPaths = computed(() => props.event?.metadata?.quarantined_paths || [])
 const errorsWarnings = computed(() => props.event?.metadata?.stderr_errors_warnings || '')
 
+// #842: per-post body-capture diagnostics — how each post was handled, with
+// post_type so an empty body is self-explanatory in the UI.
+const postDiag = computed(() => props.event?.metadata?.post_diagnostics || [])
+const postEmpty = computed(() => postDiag.value.filter((p) => !p.body_chars))
+const postWithBody = computed(() => postDiag.value.length - postEmpty.value.length)
+
 // One combined block for "research the issue elsewhere" — header line +
 // error + full stdout/stderr. Built lazily from the current event.
 const allDiagnostics = computed(() => {
   const e = props.event
   if (!e) return ''
   const md = e.metadata || {}
+  const emptyPosts = postEmpty.value
+    .map((p) => `  • ${p.title || '#' + p.post_id} [${p.post_type || '—'}] ${p.body_chars} chars`)
+    .join('\n')
   return [
     `event #${e.id} · ${e.platform || '—'} · ${e.artist_name || '—'}`,
     `status: ${e.status}`,
     `started: ${e.started_at}  finished: ${e.finished_at || '(running)'}`,
     e.error ? `\n--- error ---\n${e.error}` : '',
     errorsWarnings.value ? `\n--- errors & warnings ---\n${errorsWarnings.value}` : '',
+    postDiag.value.length
+      ? `\n--- post capture (${postWithBody.value}/${postDiag.value.length} with body) ---`
+      + (emptyPosts ? `\nempty-body posts:\n${emptyPosts}` : '')
+      : '',
     `\n--- stdout ---\n${md.stdout || '(empty)'}`,
     `\n--- stderr ---\n${md.stderr || '(empty)'}`,
   ].filter(Boolean).join('\n')
@@ -181,6 +242,25 @@ function onClose() {
   word-break: break-all;
 }
 .fc-dl-quar { padding-left: 1.5rem; font-size: 0.85rem; }
+.fc-dl-table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 0.8rem;
+}
+.fc-dl-table th,
+.fc-dl-table td {
+  text-align: left;
+  padding: 0.2rem 0.5rem;
+  border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant) / 0.14);
+}
+.fc-dl-table th:last-child,
+.fc-dl-table td:last-child {
+  text-align: right;
+  font-variant-numeric: tabular-nums;
+}
+.fc-dl-table .fc-dl-empty {
+  color: rgb(var(--v-theme-warning));
+}
 .fc-dl-blockhead {
   display: flex; align-items: center; justify-content: space-between;
   gap: 0.5rem;

tests/test_download_service.py

@@ -60,10 +60,11 @@ def _make_fake_dl_result(
     *, success=True, written_paths=None, quarantined_paths=None,
     files_downloaded=0, error_type=None, error_message=None,
     stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
-    relink_source_paths=None,
+    relink_source_paths=None, post_diagnostics=None,
 ):
     return SimpleNamespace(
         relink_source_paths=relink_source_paths or [],
+        post_diagnostics=post_diagnostics or [],
         success=success,
         url="https://patreon.com/alice",
         artist_slug="alice",
@@ -820,6 +821,28 @@ async def test_recapture_mode_selected_and_flag_cleared_on_complete(
     assert "_backfill_recapture" not in co
 
 
+@pytest.mark.asyncio
+async def test_post_diagnostics_written_to_event_metadata(
+    db, db_sync, tmp_path, seed_artist_and_source,
+):
+    """#842: per-post body diagnostics land on the DownloadEvent.metadata_ so the
+    UI can show how each post was handled (post_type + body length)."""
+    _artist, source = seed_artist_and_source
+    diag = [
+        {"post_id": "p1", "title": "Has body", "post_type": "image_file", "body_chars": 42},
+        {"post_id": "p2", "title": "Empty poll", "post_type": "poll", "body_chars": 0},
+    ]
+    svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
+        success=True, post_diagnostics=diag,
+    ))
+    await svc.download_source(source.id)
+
+    ev = (await db.execute(
+        select(DownloadEvent).order_by(DownloadEvent.id.desc()).limit(1)
+    )).scalar_one()
+    assert ev.metadata_["post_diagnostics"] == diag
+
+
 @pytest.mark.asyncio
 async def test_partial_error_type_maps_to_ok_status(
     db, db_sync, tmp_path, seed_artist_and_source,

tests/test_patreon_ingester.py

@@ -60,7 +60,21 @@ class _FakeClient:
         for page_cursor, posts in self._pages:
             for post_id, media in posts:
                 self.consumed_posts += 1
-                yield {"id": post_id, "_media": media}, {}, page_cursor
+                # Carry feed attributes (title/post_type/content) like the real
+                # client — ingest_core reads these for the per-post diagnostic.
+                yield (
+                    {
+                        "id": post_id,
+                        "_media": media,
+                        "attributes": {
+                            "title": f"Post {post_id}",
+                            "post_type": "image_file",
+                            "content": f"<p>body {post_id}</p>",
+                        },
+                    },
+                    {},
+                    page_cursor,
+                )
 
     def extract_media(self, post, included_index):
         return post["_media"]
@@ -640,6 +654,13 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
     # Diagnostic summary surfaces the post-record + relink counts (operator reads
     # this off the event stdout to see what a recapture actually did).
     assert "1 post-record(s), 1 relinked" in res_recap.stdout
+    # #842: per-post body diagnostic carries post_type + body length so the UI can
+    # show how each post was handled (and explain empties by post_type).
+    assert len(res_recap.post_diagnostics) == 1
+    diag = res_recap.post_diagnostics[0]
+    assert diag["post_id"] == "p1"
+    assert diag["post_type"] == "image_file"
+    assert diag["body_chars"] == len("<p>body p1</p>")
 
 
 @pytest.mark.asyncio