feat(artist): resolve patreon + subscribestar display names at add-time (#130 step 5)
Parity with pixiv (operator ask): the extension add now resolves the real display name for our other native platforms too, not just the URL handle. patreon_resolver.resolve_display_name reads the campaigns API's attributes.name; SubscribeStarClient.resolve_display_name pulls the creator name off the profile page (og:title, else the <title> stripped of the SubscribeStar suffix). extension_service._resolve_artist_name dispatches per platform (pixiv=token, patreon/subscribestar=cookies via get_cookies_path), best-effort in an executor, falling back to the readable URL handle on any failure. Still all curator core — the extension is unchanged (sends only the URL). gallery-dl platforms keep the handle (readable, no native client). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -8,6 +8,7 @@ and returns a JSON-shaped dict for the API layer.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -18,6 +19,8 @@ from ..utils.slug import slugify
|
||||
from .db_helpers import get_or_create
|
||||
from .source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
@@ -82,9 +85,9 @@ class ExtensionService:
|
||||
)).scalar_one()
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
# New source → name the artist properly. Pixiv's URL yields only the
|
||||
# numeric id, so resolve the real display name via the app API.
|
||||
name = await self._resolve_artist_name(platform, raw_slug)
|
||||
# New source → name the artist properly by resolving the real display
|
||||
# name from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
@@ -110,25 +113,48 @@ class ExtensionService:
|
||||
"created_artist": created_artist,
|
||||
}
|
||||
|
||||
async def _resolve_artist_name(self, platform: str, raw_slug: str) -> str:
|
||||
"""A human display name for a new artist. Pixiv's URL carries only the
|
||||
numeric user id, so resolve the real name via the app API (curator holds
|
||||
the OAuth token); every other platform's URL handle is already readable.
|
||||
Falls back to raw_slug on any failure (no token, API error)."""
|
||||
if platform != "pixiv" or self._crypto is None:
|
||||
async def _resolve_artist_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str:
|
||||
"""The real display name for a new artist, resolved from the platform at
|
||||
add-time (#130). Our native platforms each have a name source — pixiv the
|
||||
app API (token), patreon the campaigns API, subscribestar the profile
|
||||
page (both cookies). Other platforms (and any failure — no credential,
|
||||
network error) fall back to the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .pixiv_client import PixivClient
|
||||
|
||||
token = await CredentialService(self.session, self._crypto).get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
cred = CredentialService(self.session, self._crypto)
|
||||
loop = asyncio.get_running_loop()
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
try:
|
||||
if platform == "pixiv":
|
||||
token = await cred.get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
from .pixiv_client import PixivClient
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
elif platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
else: # subscribestar
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
|
||||
@@ -139,6 +139,32 @@ def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
||||
return campaign_id
|
||||
|
||||
|
||||
def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
|
||||
"""The Patreon campaign's display name for `vanity` via the campaigns API
|
||||
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
||||
on any failure — the caller falls back to the vanity handle. Sync: call from
|
||||
an executor."""
|
||||
jar = _load_cookie_jar(cookies_path)
|
||||
try:
|
||||
resp = requests.get(
|
||||
_CAMPAIGNS_URL,
|
||||
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
||||
cookies=jar,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json().get("data")
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
|
||||
return None
|
||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||
return None
|
||||
name = (data[0].get("attributes") or {}).get("name")
|
||||
return name.strip() if isinstance(name, str) and name.strip() else None
|
||||
|
||||
|
||||
def _scrape_campaign_id(html: str) -> str | None:
|
||||
"""First campaign id found in creator-page HTML via the known embeddings."""
|
||||
if not isinstance(html, str):
|
||||
|
||||
@@ -273,6 +273,30 @@ def _parse_ss_datetime(text: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
_OG_TITLE_RE = re.compile(
|
||||
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
# Trailing " | SubscribeStar" / " on SubscribeStar" the profile <title> carries.
|
||||
_SS_TITLE_SUFFIX_RE = re.compile(
|
||||
r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _extract_creator_name(html: str) -> str | None:
|
||||
"""The creator's display name from a SubscribeStar profile page: prefer the
|
||||
og:title meta (it's the bare creator name), else the <title> with the
|
||||
SubscribeStar suffix stripped. None when neither yields anything (#130)."""
|
||||
m = _OG_TITLE_RE.search(html)
|
||||
name = unescape(m.group(1)).strip() if m else ""
|
||||
if not name:
|
||||
t = _TITLE_RE.search(html)
|
||||
raw = unescape(t.group(1)).strip() if t else ""
|
||||
name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
|
||||
return name or None
|
||||
|
||||
|
||||
class SubscribeStarClient:
|
||||
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
||||
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
||||
@@ -604,6 +628,23 @@ class SubscribeStarClient:
|
||||
current = next_href
|
||||
first_page = False
|
||||
|
||||
# -- display name -------------------------------------------------------
|
||||
|
||||
def resolve_display_name(self, campaign_id: str) -> str | None:
|
||||
"""The creator's display name from their profile page, used to name the
|
||||
Artist at add-time (#130). `campaign_id` is the creator URL. None on any
|
||||
failure — the caller falls back to the URL handle. Sync: run in an
|
||||
executor."""
|
||||
base, slug = _split_creator_url(campaign_id)
|
||||
if not slug:
|
||||
return None
|
||||
self._session.headers["Referer"] = f"{base}/"
|
||||
try:
|
||||
html = self._feed_html(f"{base}/{slug}")
|
||||
except SubscribeStarAPIError:
|
||||
return None
|
||||
return _extract_creator_name(html)
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||
|
||||
Reference in New Issue
Block a user