feat(downloads): per-post body-capture diagnostics in the event UI (#842)
Operator can't (and shouldn't have to) hunt worker logs to see why a recapture
left a post body empty. Surface per-post handling ON THE EVENT, in the UI.
The feed already requests post_type (in _FIELDS_POST), so ingest_core builds a
per-post diagnostic {post_id, title, post_type, body_chars} with zero extra
fetching — a 0-char body next to its post_type explains an empty post at a
glance (e.g. polls/embeds whose body the API never returns).
- ingest_core: accumulate post_diagnostics; thread via DownloadResult
- download_service: write to DownloadEvent.metadata_['post_diagnostics']
- DownloadDetailModal: 'Post capture' section — totals + empty-body table
(post_type + chars, flagged) + all-posts table; included in Copy-all
- tests: ingester diag (post_type + body_chars), download_service metadata
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -477,6 +477,10 @@ class DownloadService:
|
||||
# recurring "post shows a zip but no images" report. Each entry is
|
||||
# {file, reason}; None when every archive extracted cleanly.
|
||||
"unextracted_archives": unextracted_archives or None,
|
||||
# #842: per-post body-capture outcomes ({post_id, title, post_type,
|
||||
# body_chars}) so the UI shows how each post was handled — a 0-char
|
||||
# body next to its post_type explains an empty post at a glance.
|
||||
"post_diagnostics": getattr(dl_result, "post_diagnostics", None) or None,
|
||||
}
|
||||
await self._update_source_health(
|
||||
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
||||
|
||||
@@ -158,6 +158,11 @@ class DownloadResult:
|
||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||
# on the gallery-dl path and outside recapture.
|
||||
relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
|
||||
# Native ingester (#842): per-post body-capture diagnostics
|
||||
# ({post_id, title, post_type, body_chars}) surfaced on the DownloadEvent so
|
||||
# the operator can see in the UI how each post was handled. Empty on the
|
||||
# gallery-dl path and for posts whose body wasn't (re)captured this run.
|
||||
post_diagnostics: list[dict] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
|
||||
@@ -143,6 +143,11 @@ class Ingester:
|
||||
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
|
||||
# re-downloading or unlinking the file. Empty outside recapture mode.
|
||||
relink: list[tuple[str, str]] = []
|
||||
# #842: per-post body-capture diagnostics surfaced on the DownloadEvent so
|
||||
# the operator can see IN THE UI how each post was handled — crucially the
|
||||
# post_type + final body length, which explains an empty body (e.g. a poll
|
||||
# whose body the API never returns) without digging through worker logs.
|
||||
post_diag: list[dict] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -180,6 +185,7 @@ class Ingester:
|
||||
written_paths=written,
|
||||
post_record_paths=list(post_records),
|
||||
relink_source_paths=list(relink),
|
||||
post_diagnostics=list(post_diag),
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
@@ -264,6 +270,19 @@ class Ingester:
|
||||
if rec_path is not None:
|
||||
post_records.append(str(rec_path))
|
||||
self._mark_seen(source_id, [(pkey, ppid)])
|
||||
# Read the FINAL body (write_post_record memoized any
|
||||
# detail-fetched content onto the post) + post_type
|
||||
# (already in the feed fieldset) for the per-post UI
|
||||
# diagnostic. A body_chars of 0 with the post_type is
|
||||
# the operator's "why is this one empty" answer.
|
||||
pattrs = post.get("attributes") or {}
|
||||
pbody = pattrs.get("content")
|
||||
post_diag.append({
|
||||
"post_id": ppid,
|
||||
"title": pattrs.get("title") or None,
|
||||
"post_type": pattrs.get("post_type") or None,
|
||||
"body_chars": len(pbody) if isinstance(pbody, str) else 0,
|
||||
})
|
||||
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
|
||||
Reference in New Issue
Block a user