diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py index a59c6c1..bb099a1 100644 --- a/backend/app/services/download_service.py +++ b/backend/app/services/download_service.py @@ -477,10 +477,6 @@ class DownloadService: # recurring "post shows a zip but no images" report. Each entry is # {file, reason}; None when every archive extracted cleanly. "unextracted_archives": unextracted_archives or None, - # #842: per-post body-capture outcomes ({post_id, title, post_type, - # body_chars}) so the UI shows how each post was handled — a 0-char - # body next to its post_type explains an empty post at a glance. - "post_diagnostics": getattr(dl_result, "post_diagnostics", None) or None, } await self._update_source_health( source_id=ctx["source_id"], status=status, error_message=ev.error, diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py index 0c25eee..6797b89 100644 --- a/backend/app/services/gallery_dl.py +++ b/backend/app/services/gallery_dl.py @@ -158,11 +158,6 @@ class DownloadResult: # backfilled (inline-image localization) WITHOUT re-download or unlink. Empty # on the gallery-dl path and outside recapture. relink_source_paths: list[tuple[str, str]] = field(default_factory=list) - # Native ingester (#842): per-post body-capture diagnostics - # ({post_id, title, post_type, body_chars}) surfaced on the DownloadEvent so - # the operator can see in the UI how each post was handled. Empty on the - # gallery-dl path and for posts whose body wasn't (re)captured this run. - post_diagnostics: list[dict] = field(default_factory=list) stdout: str = "" stderr: str = "" return_code: int = 0 diff --git a/backend/app/services/ingest_core.py b/backend/app/services/ingest_core.py index c2b1665..365dda8 100644 --- a/backend/app/services/ingest_core.py +++ b/backend/app/services/ingest_core.py @@ -143,11 +143,6 @@ class Ingester: # media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT # re-downloading or unlinking the file. Empty outside recapture mode. relink: list[tuple[str, str]] = [] - # #842: per-post body-capture diagnostics surfaced on the DownloadEvent so - # the operator can see IN THE UI how each post was handled — crucially the - # post_type + final body length, which explains an empty body (e.g. a poll - # whose body the API never returns) without digging through worker logs. - post_diag: list[dict] = [] downloaded = 0 errors = 0 quarantined = 0 @@ -185,7 +180,6 @@ class Ingester: written_paths=written, post_record_paths=list(post_records), relink_source_paths=list(relink), - post_diagnostics=list(post_diag), stdout="\n".join(log_lines), stderr="", return_code=return_code, @@ -266,23 +260,20 @@ class Ingester: else self._seen_keys(source_id, [pkey]) ) if pkey not in already: - rec_path = write_post_record(post, artist_slug) - if rec_path is not None: - post_records.append(str(rec_path)) + rec = write_post_record(post, artist_slug) + if rec.path is not None: + post_records.append(str(rec.path)) self._mark_seen(source_id, [(pkey, ppid)]) - # Read the FINAL body (write_post_record memoized any - # detail-fetched content onto the post) + post_type - # (already in the feed fieldset) for the per-post UI - # diagnostic. A body_chars of 0 with the post_type is - # the operator's "why is this one empty" answer. - pattrs = post.get("attributes") or {} - pbody = pattrs.get("content") - post_diag.append({ - "post_id": ppid, - "title": pattrs.get("title") or None, - "post_type": pattrs.get("post_type") or None, - "body_chars": len(pbody) if isinstance(pbody, str) else 0, - }) + # Per-post handling line in the run stdout (the existing + # "Raw stdout" panel) — the downloader already read the + # post; we only format its outcome here. post_type beside + # a 0-char body is the "why is this one empty" answer. + log_lines.append( + f" post {ppid} [{rec.post_type or '?'}] " + f"body: {rec.body_chars} chars" + + ("" if rec.body_chars else " — EMPTY") + + (f" — {rec.title}" if rec.title else "") + ) media = self.client.extract_media(post, included) if not media: diff --git a/backend/app/services/patreon_downloader.py b/backend/app/services/patreon_downloader.py index bd11fa9..176b546 100644 --- a/backend/app/services/patreon_downloader.py +++ b/backend/app/services/patreon_downloader.py @@ -151,6 +151,20 @@ class MediaOutcome: error: str | None +@dataclass +class PostRecordOutcome: + """Result of write_post_record — mirrors the download_post → MediaOutcome + contract so the engine reports per-post handling without re-reading the post. + `path` is the _post.json sidecar (None when the post had no id); the rest is + the captured body's shape (post_type + final char count) for the run log. + """ + + path: Path | None + post_type: str | None + title: str | None + body_chars: int + + class PatreonDownloader: """Download resolved Patreon media to gallery-dl's on-disk layout. @@ -567,27 +581,33 @@ class PatreonDownloader: sidecar_path.write_text(json.dumps(data, indent=2)) return sidecar_path - def write_post_record(self, post: dict, artist_slug: str) -> Path | None: + def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome: """Write a post-ONLY sidecar (no media file) for a media-less post, so the importer can still upsert the Post + its body — text posts often hold the only copy of an external link. Named `_post.json`: the leading underscore keeps it from colliding with a media sidecar (`_.json`) and from being resolved as some media file's sidecar - by find_sidecar. Returns the path, or None when the post has no id.""" + by find_sidecar. + + Returns a PostRecordOutcome (path None when the post has no id) carrying + the captured body's shape — post_type + final char count — so the engine + can log per-post handling without re-reading the post itself. + """ + attrs = post.get("attributes") or {} + title = attrs.get("title") if isinstance(attrs.get("title"), str) else None + post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None pid = str(post.get("id") or "") if not pid: - return None + return PostRecordOutcome( + path=None, post_type=post_type, title=title, body_chars=0, + ) post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post) post_dir.mkdir(parents=True, exist_ok=True) path = self._write_sidecar_data(post, post_dir / "_post.json") - # Per-post body-capture diagnostic: _write_sidecar_data has, by now, - # memoized any detail-fetched body onto post["attributes"]["content"], so - # this reflects the FINAL body (feed or detail). Logs the success (N chars) - # AND the skip (empty) so a recapture's per-post outcome is visible. + # _write_sidecar_data has by now memoized any detail-fetched body onto + # post["attributes"]["content"], so re-read it for the FINAL char count. body = (post.get("attributes") or {}).get("content") - n = len(body) if isinstance(body, str) else 0 - if n: - log.info("post-record: post %s body captured (%d chars)", pid, n) - else: - log.info("post-record: post %s has NO body (feed + detail both empty)", pid) - return path + body_chars = len(body) if isinstance(body, str) else 0 + return PostRecordOutcome( + path=path, post_type=post_type, title=title, body_chars=body_chars, + ) diff --git a/frontend/src/components/downloads/DownloadDetailModal.vue b/frontend/src/components/downloads/DownloadDetailModal.vue index 4026a05..260bf5e 100644 --- a/frontend/src/components/downloads/DownloadDetailModal.vue +++ b/frontend/src/components/downloads/DownloadDetailModal.vue @@ -28,54 +28,6 @@
Errors{{ summary.errors ?? 0 }}
- -