Merge pull request 'fix(subscribestar): port gallery-dl date extraction (wrapped dates) + parse canary' (#123) from dev into main

backend/app/services/subscribestar_client.py

@@ -86,7 +86,13 @@ _GDL_HEADERS = {
 # hyphenated siblings (`post-content`/`post-date`/`post-body`/`post-uploads`).
 _POST_OPEN = '<div class="post '
 _POST_ID_RE = re.compile(r'data-id="(\d+)"')
-_POST_DATE_RE = re.compile(r'<div class="post-date">([^<]+)</div>')
+# The date may be plain text OR wrapped in an <a> permalink — image posts wrap it
+# (`<div class="post-date"><a href="/posts/ID">DATE</a></div>`), text-only posts
+# don't. gallery-dl's `_data_from_post` handles both: take text up to the first
+# `</`, then whatever follows the last `>`. A naive
+# `<div class="post-date">([^<]+)</div>` regex matched ONLY the unwrapped case, so
+# every image post got a null date and sorted to the top (cheunart 2026-06-17).
+_DATE_OPEN = 'class="post-date">'
 # The body lives between these two LITERAL markers — exactly gallery-dl's
 # `_data_from_post` extraction: from the post_content-text wrapper open to the
 # youtube-uploads div that always follows post-content. A balanced-</div> regex
@@ -369,8 +375,10 @@ class SubscribeStarClient:
         if not m:
             return None
         post_id = m.group(1)
-        date_m = _POST_DATE_RE.search(chunk)
-        published = _parse_ss_datetime(date_m.group(1)) if date_m else None
+        # gallery-dl date method: text up to first '</', then after the last '>'
+        # — handles both plain and <a>-wrapped dates (see _DATE_OPEN comment).
+        raw_date = _extr(chunk, _DATE_OPEN, "</").rpartition(">")[2]
+        published = _parse_ss_datetime(raw_date) if raw_date else None
         content = _extract_content(chunk)
         return {
             "id": post_id,
@@ -393,6 +401,28 @@ class SubscribeStarClient:
             post = self._parse_post(chunk)
             if post is not None:
                 posts.append(post)
+        if posts:
+            dated = sum(1 for p in posts if p["attributes"].get("published_at"))
+            bodied = sum(1 for p in posts if p["attributes"].get("content"))
+            log.info(
+                "SubscribeStar parsed %d posts (%d dated, %d with body)",
+                len(posts), dated, bodied,
+            )
+            # Canary for this exact failure class: posts parsed but NONE got a
+            # date or a body, while the raw markers ARE present → our extraction
+            # diverged from the live markup (e.g. the <a>-wrapped date bug). Log
+            # the marker counts so the cause is diagnosable from the worker log
+            # alone, without re-fetching the authed page.
+            if dated == 0 or bodied == 0:
+                log.warning(
+                    "SubscribeStar parse canary: %d posts but dated=%d bodied=%d; "
+                    "raw markers post-date=%d post_content-text=%d data-gallery=%d "
+                    "— extraction likely diverged from the live markup",
+                    len(posts), dated, bodied,
+                    html.count('class="post-date"'),
+                    html.count("post_content-text"),
+                    html.count("data-gallery"),
+                )
         return posts
 
     @staticmethod

tests/test_subscribestar_native.py

@@ -82,6 +82,21 @@ def test_parse_ss_datetime_variants():
     assert _parse_ss_datetime("nonsense") is None
 
 
+def test_date_parses_when_wrapped_in_permalink_anchor():
+    # Image posts wrap the date in an <a> permalink; a plain regex missed them
+    # → null dates (cheunart 2026-06-17). gallery-dl's method handles both.
+    client = SubscribeStarClient(None)
+    wrapped = (
+        '<div class="post is-shown" data-id="900">'
+        '<div class="post-date"><a href="/posts/900">May 01, 2026 12:00 pm</a></div>'
+        '<div class="post-content" data-role="post_content-text">'
+        '<div class="trix-content"><p>x</p></div></div>'
+        '<div class="post-uploads for-youtube"></div></div>'
+    )
+    [p] = client._parse_posts(wrapped)
+    assert p["attributes"]["published_at"] == "2026-05-01T12:00:00"
+
+
 # -- client: request headers (regression: live-fetch drift) -----------------
 
 def test_request_profile_matches_gallery_dl(monkeypatch):