refactor(ingest): per-post handling into run stdout via a downloader outcome (#842)
Two corrections from operator review: 1. Reuse the existing 'Raw stdout' panel instead of a bespoke structured UI section — the native ingester now writes a per-post line into the run stdout (parity with gallery-dl's per-file stdout), so the per-post handling shows in the panel the operator already uses. 2. DRY: stop re-reading post['attributes'] inline in ingest_core. write_post_record now returns a PostRecordOutcome (path, post_type, title, body_chars) — mirroring the download_post -> MediaOutcome contract — and the downloader owns the read; ingest_core only formats the outcome into the log line. Reverts the post_diagnostics metadata field + DownloadDetailModal 'Post capture' section added earlier. Per-post line: 'post <id> [<post_type>] body: N chars' (+ ' — EMPTY' when 0), so an empty body is self-explanatory by post_type. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -151,6 +151,20 @@ class MediaOutcome:
|
||||
error: str | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PostRecordOutcome:
|
||||
"""Result of write_post_record — mirrors the download_post → MediaOutcome
|
||||
contract so the engine reports per-post handling without re-reading the post.
|
||||
`path` is the _post.json sidecar (None when the post had no id); the rest is
|
||||
the captured body's shape (post_type + final char count) for the run log.
|
||||
"""
|
||||
|
||||
path: Path | None
|
||||
post_type: str | None
|
||||
title: str | None
|
||||
body_chars: int
|
||||
|
||||
|
||||
class PatreonDownloader:
|
||||
"""Download resolved Patreon media to gallery-dl's on-disk layout.
|
||||
|
||||
@@ -567,27 +581,33 @@ class PatreonDownloader:
|
||||
sidecar_path.write_text(json.dumps(data, indent=2))
|
||||
return sidecar_path
|
||||
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> Path | None:
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
|
||||
"""Write a post-ONLY sidecar (no media file) for a media-less post, so
|
||||
the importer can still upsert the Post + its body — text posts often hold
|
||||
the only copy of an external <a href> link. Named `_post.json`: the
|
||||
leading underscore keeps it from colliding with a media sidecar
|
||||
(`<NN>_<stem>.json`) and from being resolved as some media file's sidecar
|
||||
by find_sidecar. Returns the path, or None when the post has no id."""
|
||||
by find_sidecar.
|
||||
|
||||
Returns a PostRecordOutcome (path None when the post has no id) carrying
|
||||
the captured body's shape — post_type + final char count — so the engine
|
||||
can log per-post handling without re-reading the post itself.
|
||||
"""
|
||||
attrs = post.get("attributes") or {}
|
||||
title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
|
||||
post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
return None
|
||||
return PostRecordOutcome(
|
||||
path=None, post_type=post_type, title=title, body_chars=0,
|
||||
)
|
||||
post_dir = self.images_root / artist_slug / "patreon" / _post_dir_name(post)
|
||||
post_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = self._write_sidecar_data(post, post_dir / "_post.json")
|
||||
# Per-post body-capture diagnostic: _write_sidecar_data has, by now,
|
||||
# memoized any detail-fetched body onto post["attributes"]["content"], so
|
||||
# this reflects the FINAL body (feed or detail). Logs the success (N chars)
|
||||
# AND the skip (empty) so a recapture's per-post outcome is visible.
|
||||
# _write_sidecar_data has by now memoized any detail-fetched body onto
|
||||
# post["attributes"]["content"], so re-read it for the FINAL char count.
|
||||
body = (post.get("attributes") or {}).get("content")
|
||||
n = len(body) if isinstance(body, str) else 0
|
||||
if n:
|
||||
log.info("post-record: post %s body captured (%d chars)", pid, n)
|
||||
else:
|
||||
log.info("post-record: post %s has NO body (feed + detail both empty)", pid)
|
||||
return path
|
||||
body_chars = len(body) if isinstance(body, str) else 0
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=post_type, title=title, body_chars=body_chars,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user