fix(subscribestar): port gallery-dl date extraction (wrapped dates) + parse canary
Image posts wrap the post date in an <a> permalink (<div class="post-date"><a href="/posts/ID">DATE</a></div>); text-only posts don't. Our hand-written <div class="post-date">([^<]+)</div> regex matched ONLY the unwrapped case, so every image post got a null published_at and sorted to the top of the feed looking broken (cheunart 2026-06-17). Port gallery-dl's _data_from_post method: text up to the first </, then after the last > — handles both. Verified against the live raw feed (all 6 dates now parse). Robust logging (operator request): _parse_posts now logs per-page parse stats (posts / dated / with-body) and a WARNING canary when posts parse but NONE get a date or body while the raw markers are present — i.e. our extraction diverged from the live markup. Makes this failure class diagnosable from the worker log alone, no authed re-fetch needed. Test: a permalink-wrapped date parses to ISO. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -86,7 +86,13 @@ _GDL_HEADERS = {
|
|||||||
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
|
# hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
|
||||||
_POST_OPEN = '<div class="post '
|
_POST_OPEN = '<div class="post '
|
||||||
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
|
_POST_ID_RE = re.compile(r'data-id="(\d+)"')
|
||||||
_POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
|
# The date may be plain text OR wrapped in an <a> permalink — image posts wrap it
|
||||||
|
# (`<div class="post-date"><a href="/posts/ID">DATE</a></div>`), text-only posts
|
||||||
|
# don't. gallery-dl's `_data_from_post` handles both: take text up to the first
|
||||||
|
# `</`, then whatever follows the last `>`. A naive
|
||||||
|
# `<div class="post-date">([^<]+)</div>` regex matched ONLY the unwrapped case, so
|
||||||
|
# every image post got a null date and sorted to the top (cheunart 2026-06-17).
|
||||||
|
_DATE_OPEN = 'class="post-date">'
|
||||||
# The body lives between these two LITERAL markers — exactly gallery-dl's
|
# The body lives between these two LITERAL markers — exactly gallery-dl's
|
||||||
# `_data_from_post` extraction: from the post_content-text wrapper open to the
|
# `_data_from_post` extraction: from the post_content-text wrapper open to the
|
||||||
# youtube-uploads div that always follows post-content. A balanced-</div> regex
|
# youtube-uploads div that always follows post-content. A balanced-</div> regex
|
||||||
@@ -369,8 +375,10 @@ class SubscribeStarClient:
|
|||||||
if not m:
|
if not m:
|
||||||
return None
|
return None
|
||||||
post_id = m.group(1)
|
post_id = m.group(1)
|
||||||
date_m = _POST_DATE_RE.search(chunk)
|
# gallery-dl date method: text up to first '</', then after the last '>'
|
||||||
published = _parse_ss_datetime(date_m.group(1)) if date_m else None
|
# — handles both plain and <a>-wrapped dates (see _DATE_OPEN comment).
|
||||||
|
raw_date = _extr(chunk, _DATE_OPEN, "</").rpartition(">")[2]
|
||||||
|
published = _parse_ss_datetime(raw_date) if raw_date else None
|
||||||
content = _extract_content(chunk)
|
content = _extract_content(chunk)
|
||||||
return {
|
return {
|
||||||
"id": post_id,
|
"id": post_id,
|
||||||
@@ -393,6 +401,28 @@ class SubscribeStarClient:
|
|||||||
post = self._parse_post(chunk)
|
post = self._parse_post(chunk)
|
||||||
if post is not None:
|
if post is not None:
|
||||||
posts.append(post)
|
posts.append(post)
|
||||||
|
if posts:
|
||||||
|
dated = sum(1 for p in posts if p["attributes"].get("published_at"))
|
||||||
|
bodied = sum(1 for p in posts if p["attributes"].get("content"))
|
||||||
|
log.info(
|
||||||
|
"SubscribeStar parsed %d posts (%d dated, %d with body)",
|
||||||
|
len(posts), dated, bodied,
|
||||||
|
)
|
||||||
|
# Canary for this exact failure class: posts parsed but NONE got a
|
||||||
|
# date or a body, while the raw markers ARE present → our extraction
|
||||||
|
# diverged from the live markup (e.g. the <a>-wrapped date bug). Log
|
||||||
|
# the marker counts so the cause is diagnosable from the worker log
|
||||||
|
# alone, without re-fetching the authed page.
|
||||||
|
if dated == 0 or bodied == 0:
|
||||||
|
log.warning(
|
||||||
|
"SubscribeStar parse canary: %d posts but dated=%d bodied=%d; "
|
||||||
|
"raw markers post-date=%d post_content-text=%d data-gallery=%d "
|
||||||
|
"— extraction likely diverged from the live markup",
|
||||||
|
len(posts), dated, bodied,
|
||||||
|
html.count('class="post-date"'),
|
||||||
|
html.count("post_content-text"),
|
||||||
|
html.count("data-gallery"),
|
||||||
|
)
|
||||||
return posts
|
return posts
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -82,6 +82,21 @@ def test_parse_ss_datetime_variants():
|
|||||||
assert _parse_ss_datetime("nonsense") is None
|
assert _parse_ss_datetime("nonsense") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_date_parses_when_wrapped_in_permalink_anchor():
|
||||||
|
# Image posts wrap the date in an <a> permalink; a plain regex missed them
|
||||||
|
# → null dates (cheunart 2026-06-17). gallery-dl's method handles both.
|
||||||
|
client = SubscribeStarClient(None)
|
||||||
|
wrapped = (
|
||||||
|
'<div class="post is-shown" data-id="900">'
|
||||||
|
'<div class="post-date"><a href="/posts/900">May 01, 2026 12:00 pm</a></div>'
|
||||||
|
'<div class="post-content" data-role="post_content-text">'
|
||||||
|
'<div class="trix-content"><p>x</p></div></div>'
|
||||||
|
'<div class="post-uploads for-youtube"></div></div>'
|
||||||
|
)
|
||||||
|
[p] = client._parse_posts(wrapped)
|
||||||
|
assert p["attributes"]["published_at"] == "2026-05-01T12:00:00"
|
||||||
|
|
||||||
|
|
||||||
# -- client: request headers (regression: live-fetch drift) -----------------
|
# -- client: request headers (regression: live-fetch drift) -----------------
|
||||||
|
|
||||||
def test_request_profile_matches_gallery_dl(monkeypatch):
|
def test_request_profile_matches_gallery_dl(monkeypatch):
|
||||||
|
|||||||
Reference in New Issue
Block a user