feat(downloads): per-post body-capture diagnostics in the event UI (#842)
Operator can't (and shouldn't have to) hunt worker logs to see why a recapture
left a post body empty. Surface per-post handling ON THE EVENT, in the UI.
The feed already requests post_type (in _FIELDS_POST), so ingest_core builds a
per-post diagnostic {post_id, title, post_type, body_chars} with zero extra
fetching — a 0-char body next to its post_type explains an empty post at a
glance (e.g. polls/embeds whose body the API never returns).
- ingest_core: accumulate post_diagnostics; thread via DownloadResult
- download_service: write to DownloadEvent.metadata_['post_diagnostics']
- DownloadDetailModal: 'Post capture' section — totals + empty-body table
(post_type + chars, flagged) + all-posts table; included in Copy-all
- tests: ingester diag (post_type + body_chars), download_service metadata
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -60,10 +60,11 @@ def _make_fake_dl_result(
|
||||
*, success=True, written_paths=None, quarantined_paths=None,
|
||||
files_downloaded=0, error_type=None, error_message=None,
|
||||
stdout="", stderr="", cursor=None, run_stats=None, posts_processed=0,
|
||||
relink_source_paths=None,
|
||||
relink_source_paths=None, post_diagnostics=None,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
relink_source_paths=relink_source_paths or [],
|
||||
post_diagnostics=post_diagnostics or [],
|
||||
success=success,
|
||||
url="https://patreon.com/alice",
|
||||
artist_slug="alice",
|
||||
@@ -820,6 +821,28 @@ async def test_recapture_mode_selected_and_flag_cleared_on_complete(
|
||||
assert "_backfill_recapture" not in co
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_diagnostics_written_to_event_metadata(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""#842: per-post body diagnostics land on the DownloadEvent.metadata_ so the
|
||||
UI can show how each post was handled (post_type + body length)."""
|
||||
_artist, source = seed_artist_and_source
|
||||
diag = [
|
||||
{"post_id": "p1", "title": "Has body", "post_type": "image_file", "body_chars": 42},
|
||||
{"post_id": "p2", "title": "Empty poll", "post_type": "poll", "body_chars": 0},
|
||||
]
|
||||
svc, _ = _backfill_svc(db, db_sync, tmp_path, _make_fake_dl_result(
|
||||
success=True, post_diagnostics=diag,
|
||||
))
|
||||
await svc.download_source(source.id)
|
||||
|
||||
ev = (await db.execute(
|
||||
select(DownloadEvent).order_by(DownloadEvent.id.desc()).limit(1)
|
||||
)).scalar_one()
|
||||
assert ev.metadata_["post_diagnostics"] == diag
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_error_type_maps_to_ok_status(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
|
||||
@@ -60,7 +60,21 @@ class _FakeClient:
|
||||
for page_cursor, posts in self._pages:
|
||||
for post_id, media in posts:
|
||||
self.consumed_posts += 1
|
||||
yield {"id": post_id, "_media": media}, {}, page_cursor
|
||||
# Carry feed attributes (title/post_type/content) like the real
|
||||
# client — ingest_core reads these for the per-post diagnostic.
|
||||
yield (
|
||||
{
|
||||
"id": post_id,
|
||||
"_media": media,
|
||||
"attributes": {
|
||||
"title": f"Post {post_id}",
|
||||
"post_type": "image_file",
|
||||
"content": f"<p>body {post_id}</p>",
|
||||
},
|
||||
},
|
||||
{},
|
||||
page_cursor,
|
||||
)
|
||||
|
||||
def extract_media(self, post, included_index):
|
||||
return post["_media"]
|
||||
@@ -640,6 +654,13 @@ async def test_backfill_skips_already_captured_post_but_recapture_forces_it(
|
||||
# Diagnostic summary surfaces the post-record + relink counts (operator reads
|
||||
# this off the event stdout to see what a recapture actually did).
|
||||
assert "1 post-record(s), 1 relinked" in res_recap.stdout
|
||||
# #842: per-post body diagnostic carries post_type + body length so the UI can
|
||||
# show how each post was handled (and explain empties by post_type).
|
||||
assert len(res_recap.post_diagnostics) == 1
|
||||
diag = res_recap.post_diagnostics[0]
|
||||
assert diag["post_id"] == "p1"
|
||||
assert diag["post_type"] == "image_file"
|
||||
assert diag["body_chars"] == len("<p>body p1</p>")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user