feat(artist): resolve patreon + subscribestar display names at add-time (#130 step 5)
Parity with pixiv (operator ask): the extension add now resolves the real display name for our other native platforms too, not just the URL handle. patreon_resolver.resolve_display_name reads the campaigns API's attributes.name; SubscribeStarClient.resolve_display_name pulls the creator name off the profile page (og:title, else the <title> stripped of the SubscribeStar suffix). extension_service._resolve_artist_name dispatches per platform (pixiv=token, patreon/subscribestar=cookies via get_cookies_path), best-effort in an executor, falling back to the readable URL handle on any failure. Still all curator core — the extension is unchanged (sends only the URL). gallery-dl platforms keep the handle (readable, no native client). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CDgx8bQS5YrGRK76v8HUnM
This commit is contained in:
@@ -8,6 +8,7 @@ and returns a JSON-shaped dict for the API layer.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
@@ -18,6 +19,8 @@ from ..utils.slug import slugify
|
|||||||
from .db_helpers import get_or_create
|
from .db_helpers import get_or_create
|
||||||
from .source_service import BACKFILL_MAX_CHUNKS
|
from .source_service import BACKFILL_MAX_CHUNKS
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class UnknownPlatformError(Exception):
|
class UnknownPlatformError(Exception):
|
||||||
"""URL didn't match any platform pattern."""
|
"""URL didn't match any platform pattern."""
|
||||||
@@ -82,9 +85,9 @@ class ExtensionService:
|
|||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||||
|
|
||||||
# New source → name the artist properly. Pixiv's URL yields only the
|
# New source → name the artist properly by resolving the real display
|
||||||
# numeric id, so resolve the real display name via the app API.
|
# name from the platform (falls back to the URL handle).
|
||||||
name = await self._resolve_artist_name(platform, raw_slug)
|
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||||
artist, created_artist = await self._find_or_create_artist(name)
|
artist, created_artist = await self._find_or_create_artist(name)
|
||||||
source, created_source = await self._find_or_create_source(
|
source, created_source = await self._find_or_create_source(
|
||||||
artist_id=artist.id, platform=platform, url=url,
|
artist_id=artist.id, platform=platform, url=url,
|
||||||
@@ -110,25 +113,48 @@ class ExtensionService:
|
|||||||
"created_artist": created_artist,
|
"created_artist": created_artist,
|
||||||
}
|
}
|
||||||
|
|
||||||
async def _resolve_artist_name(self, platform: str, raw_slug: str) -> str:
|
async def _resolve_artist_name(
|
||||||
"""A human display name for a new artist. Pixiv's URL carries only the
|
self, platform: str, raw_slug: str, url: str
|
||||||
numeric user id, so resolve the real name via the app API (curator holds
|
) -> str:
|
||||||
the OAuth token); every other platform's URL handle is already readable.
|
"""The real display name for a new artist, resolved from the platform at
|
||||||
Falls back to raw_slug on any failure (no token, API error)."""
|
add-time (#130). Our native platforms each have a name source — pixiv the
|
||||||
if platform != "pixiv" or self._crypto is None:
|
app API (token), patreon the campaigns API, subscribestar the profile
|
||||||
|
page (both cookies). Other platforms (and any failure — no credential,
|
||||||
|
network error) fall back to the URL handle, which is already readable.
|
||||||
|
The resolvers are sync, so they run in an executor."""
|
||||||
|
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
||||||
return raw_slug
|
return raw_slug
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from .credential_service import CredentialService
|
from .credential_service import CredentialService
|
||||||
from .pixiv_client import PixivClient
|
cred = CredentialService(self.session, self._crypto)
|
||||||
|
|
||||||
token = await CredentialService(self.session, self._crypto).get_token("pixiv")
|
|
||||||
if not token:
|
|
||||||
return raw_slug
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
name = await loop.run_in_executor(
|
try:
|
||||||
None, PixivClient(token).resolve_display_name, raw_slug
|
if platform == "pixiv":
|
||||||
)
|
token = await cred.get_token("pixiv")
|
||||||
|
if not token:
|
||||||
|
return raw_slug
|
||||||
|
from .pixiv_client import PixivClient
|
||||||
|
name = await loop.run_in_executor(
|
||||||
|
None, PixivClient(token).resolve_display_name, raw_slug
|
||||||
|
)
|
||||||
|
elif platform == "patreon":
|
||||||
|
cookies = await cred.get_cookies_path("patreon")
|
||||||
|
from .patreon_resolver import resolve_display_name
|
||||||
|
name = await loop.run_in_executor(
|
||||||
|
None, resolve_display_name, raw_slug,
|
||||||
|
str(cookies) if cookies else None,
|
||||||
|
)
|
||||||
|
else: # subscribestar
|
||||||
|
cookies = await cred.get_cookies_path("subscribestar")
|
||||||
|
from .subscribestar_client import SubscribeStarClient
|
||||||
|
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||||
|
name = await loop.run_in_executor(
|
||||||
|
None, client.resolve_display_name, url
|
||||||
|
)
|
||||||
|
except Exception as exc: # resolution is best-effort — never block the add
|
||||||
|
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||||
|
return raw_slug
|
||||||
return name or raw_slug
|
return name or raw_slug
|
||||||
|
|
||||||
async def probe(self, url: str) -> dict:
|
async def probe(self, url: str) -> dict:
|
||||||
|
|||||||
@@ -139,6 +139,32 @@ def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
|||||||
return campaign_id
|
return campaign_id
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
|
||||||
|
"""The Patreon campaign's display name for `vanity` via the campaigns API
|
||||||
|
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
||||||
|
on any failure — the caller falls back to the vanity handle. Sync: call from
|
||||||
|
an executor."""
|
||||||
|
jar = _load_cookie_jar(cookies_path)
|
||||||
|
try:
|
||||||
|
resp = requests.get(
|
||||||
|
_CAMPAIGNS_URL,
|
||||||
|
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
||||||
|
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
||||||
|
cookies=jar,
|
||||||
|
timeout=_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
data = resp.json().get("data")
|
||||||
|
except (requests.RequestException, ValueError) as exc:
|
||||||
|
log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
|
||||||
|
return None
|
||||||
|
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||||
|
return None
|
||||||
|
name = (data[0].get("attributes") or {}).get("name")
|
||||||
|
return name.strip() if isinstance(name, str) and name.strip() else None
|
||||||
|
|
||||||
|
|
||||||
def _scrape_campaign_id(html: str) -> str | None:
|
def _scrape_campaign_id(html: str) -> str | None:
|
||||||
"""First campaign id found in creator-page HTML via the known embeddings."""
|
"""First campaign id found in creator-page HTML via the known embeddings."""
|
||||||
if not isinstance(html, str):
|
if not isinstance(html, str):
|
||||||
|
|||||||
@@ -273,6 +273,30 @@ def _parse_ss_datetime(text: str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
_OG_TITLE_RE = re.compile(
|
||||||
|
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||||
|
# Trailing " | SubscribeStar" / " on SubscribeStar" the profile <title> carries.
|
||||||
|
_SS_TITLE_SUFFIX_RE = re.compile(
|
||||||
|
r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_creator_name(html: str) -> str | None:
|
||||||
|
"""The creator's display name from a SubscribeStar profile page: prefer the
|
||||||
|
og:title meta (it's the bare creator name), else the <title> with the
|
||||||
|
SubscribeStar suffix stripped. None when neither yields anything (#130)."""
|
||||||
|
m = _OG_TITLE_RE.search(html)
|
||||||
|
name = unescape(m.group(1)).strip() if m else ""
|
||||||
|
if not name:
|
||||||
|
t = _TITLE_RE.search(html)
|
||||||
|
raw = unescape(t.group(1)).strip() if t else ""
|
||||||
|
name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
|
||||||
|
return name or None
|
||||||
|
|
||||||
|
|
||||||
class SubscribeStarClient:
|
class SubscribeStarClient:
|
||||||
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
||||||
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
||||||
@@ -604,6 +628,23 @@ class SubscribeStarClient:
|
|||||||
current = next_href
|
current = next_href
|
||||||
first_page = False
|
first_page = False
|
||||||
|
|
||||||
|
# -- display name -------------------------------------------------------
|
||||||
|
|
||||||
|
def resolve_display_name(self, campaign_id: str) -> str | None:
|
||||||
|
"""The creator's display name from their profile page, used to name the
|
||||||
|
Artist at add-time (#130). `campaign_id` is the creator URL. None on any
|
||||||
|
failure — the caller falls back to the URL handle. Sync: run in an
|
||||||
|
executor."""
|
||||||
|
base, slug = _split_creator_url(campaign_id)
|
||||||
|
if not slug:
|
||||||
|
return None
|
||||||
|
self._session.headers["Referer"] = f"{base}/"
|
||||||
|
try:
|
||||||
|
html = self._feed_html(f"{base}/{slug}")
|
||||||
|
except SubscribeStarAPIError:
|
||||||
|
return None
|
||||||
|
return _extract_creator_name(html)
|
||||||
|
|
||||||
# -- verify ------------------------------------------------------------
|
# -- verify ------------------------------------------------------------
|
||||||
|
|
||||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||||
|
|||||||
+30
-13
@@ -85,25 +85,42 @@ async def test_quick_add_reuses_source_artist_after_rename(client, ext_key):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_resolve_artist_name_pixiv_uses_api(db, monkeypatch):
|
async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch):
|
||||||
# #130: a pixiv add resolves the display name via the app API (with a token),
|
# #130: each native platform resolves its real display name at add-time
|
||||||
# not the numeric id; falls back to the id without crypto/token.
|
# (pixiv=token API, patreon=campaigns API, subscribestar=profile page);
|
||||||
|
# gallery-dl platforms and any failure fall back to the URL handle.
|
||||||
|
from backend.app.services import patreon_resolver
|
||||||
from backend.app.services.credential_service import CredentialService
|
from backend.app.services.credential_service import CredentialService
|
||||||
from backend.app.services.extension_service import ExtensionService
|
from backend.app.services.extension_service import ExtensionService
|
||||||
from backend.app.services.pixiv_client import PixivClient
|
from backend.app.services.pixiv_client import PixivClient
|
||||||
|
from backend.app.services.subscribestar_client import SubscribeStarClient
|
||||||
|
|
||||||
async def _tok(self, platform):
|
async def _tok(self, platform):
|
||||||
return "tok" if platform == "pixiv" else None
|
return "tok"
|
||||||
monkeypatch.setattr(CredentialService, "get_token", _tok)
|
|
||||||
monkeypatch.setattr(
|
|
||||||
PixivClient, "resolve_display_name", lambda self, uid: "Kurotsuchi Machi"
|
|
||||||
)
|
|
||||||
|
|
||||||
svc = ExtensionService(db, crypto=object()) # crypto unused (token stubbed)
|
async def _cookies(self, platform):
|
||||||
assert await svc._resolve_artist_name("pixiv", "555") == "Kurotsuchi Machi"
|
return "/tmp/cookies.txt"
|
||||||
assert await svc._resolve_artist_name("patreon", "alice") == "alice" # passthrough
|
|
||||||
# No crypto → no resolution attempt → fall back to the raw handle.
|
monkeypatch.setattr(CredentialService, "get_token", _tok)
|
||||||
assert await ExtensionService(db)._resolve_artist_name("pixiv", "555") == "555"
|
monkeypatch.setattr(CredentialService, "get_cookies_path", _cookies)
|
||||||
|
monkeypatch.setattr(PixivClient, "resolve_display_name", lambda self, uid: "Pixiv Name")
|
||||||
|
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: "Patreon Name")
|
||||||
|
monkeypatch.setattr(SubscribeStarClient, "resolve_display_name", lambda self, u: "SS Name")
|
||||||
|
|
||||||
|
svc = ExtensionService(db, crypto=object()) # crypto seam only (calls stubbed)
|
||||||
|
assert await svc._resolve_artist_name(
|
||||||
|
"pixiv", "555", "https://www.pixiv.net/users/555") == "Pixiv Name"
|
||||||
|
assert await svc._resolve_artist_name(
|
||||||
|
"patreon", "maewix", "https://patreon.com/maewix") == "Patreon Name"
|
||||||
|
assert await svc._resolve_artist_name(
|
||||||
|
"subscribestar", "sabu", "https://subscribestar.adult/sabu") == "SS Name"
|
||||||
|
# gallery-dl platform → readable handle passthrough (no resolver).
|
||||||
|
assert await svc._resolve_artist_name("hentaifoundry", "Foo", "u") == "Foo"
|
||||||
|
# No crypto → no resolution attempt → the raw handle.
|
||||||
|
assert await ExtensionService(db)._resolve_artist_name("pixiv", "555", "u") == "555"
|
||||||
|
# Resolver returns None → fall back to the handle.
|
||||||
|
monkeypatch.setattr(patreon_resolver, "resolve_display_name", lambda v, c: None)
|
||||||
|
assert await svc._resolve_artist_name("patreon", "maewix", "u") == "maewix"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("url,platform,slug", [
|
@pytest.mark.parametrize("url,platform,slug", [
|
||||||
|
|||||||
@@ -5,9 +5,34 @@ import pytest
|
|||||||
from backend.app.services.patreon_resolver import (
|
from backend.app.services.patreon_resolver import (
|
||||||
resolve_campaign_id,
|
resolve_campaign_id,
|
||||||
resolve_campaign_id_for_source,
|
resolve_campaign_id_for_source,
|
||||||
|
resolve_display_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_display_name_from_campaign():
|
||||||
|
fake = MagicMock()
|
||||||
|
fake.status_code = 200
|
||||||
|
fake.json.return_value = {"data": [
|
||||||
|
{"id": "1", "type": "campaign", "attributes": {"name": "Maewix Studios"}},
|
||||||
|
]}
|
||||||
|
with patch("backend.app.services.patreon_resolver.requests.get", return_value=fake):
|
||||||
|
assert resolve_display_name("maewix", None) == "Maewix Studios"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_display_name_none_on_empty_or_error():
|
||||||
|
empty = MagicMock()
|
||||||
|
empty.status_code = 200
|
||||||
|
empty.json.return_value = {"data": []}
|
||||||
|
with patch("backend.app.services.patreon_resolver.requests.get", return_value=empty):
|
||||||
|
assert resolve_display_name("maewix", None) is None
|
||||||
|
import requests as _rq
|
||||||
|
with patch(
|
||||||
|
"backend.app.services.patreon_resolver.requests.get",
|
||||||
|
side_effect=_rq.ConnectionError("x"),
|
||||||
|
):
|
||||||
|
assert resolve_display_name("maewix", None) is None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_resolves_on_happy_path():
|
async def test_resolves_on_happy_path():
|
||||||
fake_response = MagicMock()
|
fake_response = MagicMock()
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from backend.app.services.subscribestar_client import (
|
|||||||
SubscribeStarAuthError,
|
SubscribeStarAuthError,
|
||||||
SubscribeStarClient,
|
SubscribeStarClient,
|
||||||
SubscribeStarDriftError,
|
SubscribeStarDriftError,
|
||||||
|
_extract_creator_name,
|
||||||
_parse_ss_datetime,
|
_parse_ss_datetime,
|
||||||
_split_creator_url,
|
_split_creator_url,
|
||||||
)
|
)
|
||||||
@@ -102,6 +103,27 @@ def test_split_creator_url_leaves_com_and_adult_untouched():
|
|||||||
"https://www.subscribestar.com"
|
"https://www.subscribestar.com"
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_creator_name_prefers_og_title():
|
||||||
|
html = (
|
||||||
|
'<head><meta property="og:title" content="Sabu & Friends">'
|
||||||
|
'<title>Sabu | SubscribeStar</title></head>'
|
||||||
|
)
|
||||||
|
assert _extract_creator_name(html) == "Sabu & Friends"
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_creator_name_title_fallback_strips_suffix():
|
||||||
|
assert _extract_creator_name(
|
||||||
|
"<head><title>Elasid on SubscribeStar</title></head>"
|
||||||
|
) == "Elasid"
|
||||||
|
assert _extract_creator_name(
|
||||||
|
"<head><title>Cheun | SubscribeStar — adult</title></head>"
|
||||||
|
) == "Cheun"
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_creator_name_none_when_absent():
|
||||||
|
assert _extract_creator_name("<html><body>no title</body></html>") is None
|
||||||
|
|
||||||
|
|
||||||
def test_date_parses_when_wrapped_in_permalink_anchor():
|
def test_date_parses_when_wrapped_in_permalink_anchor():
|
||||||
# Image posts wrap the date in an <a> permalink; a plain regex missed them
|
# Image posts wrap the date in an <a> permalink; a plain regex missed them
|
||||||
# → null dates (cheunart 2026-06-17). gallery-dl's method handles both.
|
# → null dates (cheunart 2026-06-17). gallery-dl's method handles both.
|
||||||
|
|||||||
Reference in New Issue
Block a user