feat(artist): resolve patreon + subscribestar display names at add-time (#130 step 5)
Parity with pixiv (operator ask): the extension add now resolves the real display name for our other native platforms too, not just the URL handle. patreon_resolver.resolve_display_name reads the campaigns API's attributes.name; SubscribeStarClient.resolve_display_name pulls the creator name off the profile page (og:title, else the <title> stripped of the SubscribeStar suffix). extension_service._resolve_artist_name dispatches per platform (pixiv=token, patreon/subscribestar=cookies via get_cookies_path), best-effort in an executor, falling back to the readable URL handle on any failure. Still all curator core — the extension is unchanged (sends only the URL). gallery-dl platforms keep the handle (readable, no native client). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -8,6 +8,7 @@ and returns a JSON-shaped dict for the API layer.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -18,6 +19,8 @@ from ..utils.slug import slugify
|
||||
from .db_helpers import get_or_create
|
||||
from .source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
@@ -82,9 +85,9 @@ class ExtensionService:
|
||||
)).scalar_one()
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
# New source → name the artist properly. Pixiv's URL yields only the
|
||||
# numeric id, so resolve the real display name via the app API.
|
||||
name = await self._resolve_artist_name(platform, raw_slug)
|
||||
# New source → name the artist properly by resolving the real display
|
||||
# name from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
@@ -110,25 +113,48 @@ class ExtensionService:
|
||||
"created_artist": created_artist,
|
||||
}
|
||||
|
||||
async def _resolve_artist_name(self, platform: str, raw_slug: str) -> str:
|
||||
"""A human display name for a new artist. Pixiv's URL carries only the
|
||||
numeric user id, so resolve the real name via the app API (curator holds
|
||||
the OAuth token); every other platform's URL handle is already readable.
|
||||
Falls back to raw_slug on any failure (no token, API error)."""
|
||||
if platform != "pixiv" or self._crypto is None:
|
||||
async def _resolve_artist_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str:
|
||||
"""The real display name for a new artist, resolved from the platform at
|
||||
add-time (#130). Our native platforms each have a name source — pixiv the
|
||||
app API (token), patreon the campaigns API, subscribestar the profile
|
||||
page (both cookies). Other platforms (and any failure — no credential,
|
||||
network error) fall back to the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .pixiv_client import PixivClient
|
||||
|
||||
token = await CredentialService(self.session, self._crypto).get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
cred = CredentialService(self.session, self._crypto)
|
||||
loop = asyncio.get_running_loop()
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
try:
|
||||
if platform == "pixiv":
|
||||
token = await cred.get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
from .pixiv_client import PixivClient
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
elif platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
else: # subscribestar
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
|
||||
@@ -139,6 +139,32 @@ def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
||||
return campaign_id
|
||||
|
||||
|
||||
def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
|
||||
"""The Patreon campaign's display name for `vanity` via the campaigns API
|
||||
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
||||
on any failure — the caller falls back to the vanity handle. Sync: call from
|
||||
an executor."""
|
||||
jar = _load_cookie_jar(cookies_path)
|
||||
try:
|
||||
resp = requests.get(
|
||||
_CAMPAIGNS_URL,
|
||||
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
||||
cookies=jar,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json().get("data")
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
|
||||
return None
|
||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||
return None
|
||||
name = (data[0].get("attributes") or {}).get("name")
|
||||
return name.strip() if isinstance(name, str) and name.strip() else None
|
||||
|
||||
|
||||
def _scrape_campaign_id(html: str) -> str | None:
|
||||
"""First campaign id found in creator-page HTML via the known embeddings."""
|
||||
if not isinstance(html, str):
|
||||
|
||||
@@ -273,6 +273,30 @@ def _parse_ss_datetime(text: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
_OG_TITLE_RE = re.compile(
|
||||
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
# Trailing " | SubscribeStar" / " on SubscribeStar" the profile <title> carries.
|
||||
_SS_TITLE_SUFFIX_RE = re.compile(
|
||||
r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _extract_creator_name(html: str) -> str | None:
|
||||
"""The creator's display name from a SubscribeStar profile page: prefer the
|
||||
og:title meta (it's the bare creator name), else the <title> with the
|
||||
SubscribeStar suffix stripped. None when neither yields anything (#130)."""
|
||||
m = _OG_TITLE_RE.search(html)
|
||||
name = unescape(m.group(1)).strip() if m else ""
|
||||
if not name:
|
||||
t = _TITLE_RE.search(html)
|
||||
raw = unescape(t.group(1)).strip() if t else ""
|
||||
name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
|
||||
return name or None
|
||||
|
||||
|
||||
class SubscribeStarClient:
|
||||
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
||||
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
||||
@@ -604,6 +628,23 @@ class SubscribeStarClient:
|
||||
current = next_href
|
||||
first_page = False
|
||||
|
||||
# -- display name -------------------------------------------------------
|
||||
|
||||
def resolve_display_name(self, campaign_id: str) -> str | None:
|
||||
"""The creator's display name from their profile page, used to name the
|
||||
Artist at add-time (#130). `campaign_id` is the creator URL. None on any
|
||||
failure — the caller falls back to the URL handle. Sync: run in an
|
||||
executor."""
|
||||
base, slug = _split_creator_url(campaign_id)
|
||||
if not slug:
|
||||
return None
|
||||
self._session.headers["Referer"] = f"{base}/"
|
||||
try:
|
||||
html = self._feed_html(f"{base}/{slug}")
|
||||
except SubscribeStarAPIError:
|
||||
return None
|
||||
return _extract_creator_name(html)
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||
|
||||
+30
-13
@@ -85,25 +85,42 @@ async def test_quick_add_reuses_source_artist_after_rename(client, ext_key):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_artist_name_pixiv_uses_api(db, monkeypatch):
|
||||
# #130: a pixiv add resolves the display name via the app API (with a token),
|
||||
# not the numeric id; falls back to the id without crypto/token.
|
||||
async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch):
|
||||
# #130: each native platform resolves its real display name at add-time
|
||||
# (pixiv=token API, patreon=campaigns API, subscribestar=profile page);
|
||||
# gallery-dl platforms and any failure fall back to the URL handle.
|
||||
from backend.app.services import patreon_resolver
|
||||
from backend.app.services.credential_service import CredentialService
|
||||
from backend.app.services.extension_service import ExtensionService
|
||||
from backend.app.services.pixiv_client import PixivClient
|
||||
from backend.app.services.subscribestar_client import SubscribeStarClient
|
||||
|
||||
async def _tok(self, platform):
|
||||
return "tok" if platform == "pixiv" else None
|
||||
monkeypatch.setattr(CredentialService, "get_token", _tok)
|
||||
monkeypatch.setattr(
|
||||
PixivClient, "resolve_display_name", lambda self, uid: "Kurotsuchi Machi"
|
||||
)
|
||||
return "tok"
|
||||
|
||||
svc = ExtensionService(db, crypto=object()) # crypto unused (token stubbed)
|
||||
assert await svc._resolve_artist_name("pixiv", "555") == "Kurotsuchi Machi"
|
||||
assert await svc._resolve_artist_name("patreon", "alice") == "alice" # passthrough
|
||||
# No crypto → no resolution attempt → fall back to the raw handle.
|
||||
assert await ExtensionService(db)._resolve_artist_name("pixiv", "555") == "555"
|
||||
async def _cookies(self, platform):
|
||||
return "/tmp/cookies.txt"
|
||||
|
||||
monkeypatch.setattr(CredentialService, "get_token", _tok)
|
||||
monkeypatch.setattr(CredentialService, "get_cookies_path", _cookies)
|
||||
monkeypatch.setattr(PixivClient, "resolve_display_name", lambda self, uid: "Pixiv Name")
|
||||
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: "Patreon Name")
|
||||
monkeypatch.setattr(SubscribeStarClient, "resolve_display_name", lambda self, u: "SS Name")
|
||||
|
||||
svc = ExtensionService(db, crypto=object()) # crypto seam only (calls stubbed)
|
||||
assert await svc._resolve_artist_name(
|
||||
"pixiv", "555", "https://www.pixiv.net/users/555") == "Pixiv Name"
|
||||
assert await svc._resolve_artist_name(
|
||||
"patreon", "maewix", "https://patreon.com/maewix") == "Patreon Name"
|
||||
assert await svc._resolve_artist_name(
|
||||
"subscribestar", "sabu", "https://subscribestar.adult/sabu") == "SS Name"
|
||||
# gallery-dl platform → readable handle passthrough (no resolver).
|
||||
assert await svc._resolve_artist_name("hentaifoundry", "Foo", "u") == "Foo"
|
||||
# No crypto → no resolution attempt → the raw handle.
|
||||
assert await ExtensionService(db)._resolve_artist_name("pixiv", "555", "u") == "555"
|
||||
# Resolver returns None → fall back to the handle.
|
||||
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: None)
|
||||
assert await svc._resolve_artist_name("patreon", "maewix", "u") == "maewix"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("url,platform,slug", [
|
||||
|
||||
@@ -5,9 +5,34 @@ import pytest
|
||||
from backend.app.services.patreon_resolver import (
|
||||
resolve_campaign_id,
|
||||
resolve_campaign_id_for_source,
|
||||
resolve_display_name,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_display_name_from_campaign():
|
||||
fake = MagicMock()
|
||||
fake.status_code = 200
|
||||
fake.json.return_value = {"data": [
|
||||
{"id": "1", "type": "campaign", "attributes": {"name": "Maewix Studios"}},
|
||||
]}
|
||||
with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
|
||||
assert resolve_display_name("maewix", None) == "Maewix Studios"
|
||||
|
||||
|
||||
def test_resolve_display_name_none_on_empty_or_error():
|
||||
empty = MagicMock()
|
||||
empty.status_code = 200
|
||||
empty.json.return_value = {"data": []}
|
||||
with patch("backend.app.services.patreon_resolver.requests.get", return_value=empty):
|
||||
assert resolve_display_name("maewix", None) is None
|
||||
import requests as _rq
|
||||
with patch(
|
||||
"backend.app.services.patreon_resolver.requests.get",
|
||||
side_effect=_rq.ConnectionError("x"),
|
||||
):
|
||||
assert resolve_display_name("maewix", None) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolves_on_happy_path():
|
||||
fake_response = MagicMock()
|
||||
|
||||
@@ -21,6 +21,7 @@ from backend.app.services.subscribestar_client import (
|
||||
SubscribeStarAuthError,
|
||||
SubscribeStarClient,
|
||||
SubscribeStarDriftError,
|
||||
_extract_creator_name,
|
||||
_parse_ss_datetime,
|
||||
_split_creator_url,
|
||||
)
|
||||
@@ -102,6 +103,27 @@ def test_split_creator_url_leaves_com_and_adult_untouched():
|
||||
"https://www.subscribestar.com"
|
||||
|
||||
|
||||
def test_extract_creator_name_prefers_og_title():
|
||||
html = (
|
||||
'<head><meta property="og:title" content="Sabu & Friends">'
|
||||
'<title>Sabu | SubscribeStar</title></head>'
|
||||
)
|
||||
assert _extract_creator_name(html) == "Sabu & Friends"
|
||||
|
||||
|
||||
def test_extract_creator_name_title_fallback_strips_suffix():
|
||||
assert _extract_creator_name(
|
||||
"<head><title>Elasid on SubscribeStar</title></head>"
|
||||
) == "Elasid"
|
||||
assert _extract_creator_name(
|
||||
"<head><title>Cheun | SubscribeStar — adult</title></head>"
|
||||
) == "Cheun"
|
||||
|
||||
|
||||
def test_extract_creator_name_none_when_absent():
|
||||
assert _extract_creator_name("<html><body>no title</body></html>") is None
|
||||
|
||||
|
||||
def test_date_parses_when_wrapped_in_permalink_anchor():
|
||||
# Image posts wrap the date in an <a> permalink; a plain regex missed them
|
||||
# → null dates (cheunart 2026-06-17). gallery-dl's method handles both.
|
||||
|
||||
Reference in New Issue
Block a user