"""FC-3g: backend support for the Firefox extension. `ExtensionService.quick_add_source(url)` derives platform + artist slug from a URL using regex patterns mirrored from extension/lib/platforms.js, then find-or-creates Artist + Source rows and returns a JSON-shaped dict for the API layer. """ from __future__ import annotations import logging import re from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from ..models import Artist, Source from ..utils.slug import slugify from .db_helpers import get_or_create from .source_service import BACKFILL_MAX_CHUNKS log = logging.getLogger(__name__) # The probe runs while the chip is drawing; names that take longer are skipped. _NAME_LOOKUP_SECONDS = 6.0 class UnknownPlatformError(Exception): """URL didn't match any platform pattern.""" class InvalidUrlError(Exception): """URL was empty or missing a scheme.""" class UnknownArtistError(Exception): """quick-add named an `artist_id` that does not exist.""" # Mirrored byte-for-byte from extension/lib/platforms.js # PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand — # reviewers catch drift. _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ ("patreon", re.compile( # Three creator URL shapes — bare (patreon.com/Atole), `c/`, and `cw/` # (the "creator workspace" URL served once subscribed, see # patreon_resolver._VANITY_RE). A trailing sub-path is allowed so a # creator's inner page still derives the slug. Nav pages stay excluded. r"^https?://(?:www\.)?patreon\.com/" r"(?:cw/|c/)?" r"(?!(?:home|search|messages|notifications|library|settings|posts)(?:[/?#]|$))" r"(?P[^/?#]+)", re.IGNORECASE, )), ("subscribestar", re.compile( r"^https?://(?:www\.)?subscribestar\.(?:com|adult)/" r"(?!feed$|messages$|library$)" r"(?P[^/?#]+)/?$", re.IGNORECASE, )), ("hentaifoundry", re.compile( r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P[^/?#]+)", re.IGNORECASE, )), # A Discord URL names a server or a channel, never a creator, so the slug is # `` or `/` and the artist is chosen, not derived. # A trailing message id (a jump link) still names its channel. DMs (`@me`) # are not sources; thread links (`/threads/`) are left to the manual form. ("discord", re.compile( r"^https?://(?:www\.|ptb\.|canary\.)?discord\.com/channels/" r"(?P\d+(?:/\d+)?)(?:/\d+)?/?(?:[?#].*)?$", re.IGNORECASE, )), ] DISCORD = "discord" def canonical_source_url(platform: str, url: str, slug: str) -> str: """The URL a new source is stored under. Discord's is rebuilt from the ids — the form the manual Add form and the ingester use — so a jump link, a ptb/canary host or a trailing slash never makes a second source for the same channel. Every other platform keeps the URL as given.""" if platform == DISCORD: return f"https://discord.com/channels/{slug}" return url def _discord_ids(url: str) -> tuple[str | None, str | None] | None: """`(server_id, channel_id)` of a stored Discord source URL, None if it does not parse (a DM or thread link, or an old malformed row).""" from .discord_client import DiscordAPIError, parse_source_url try: return parse_source_url(url) except DiscordAPIError: return None class ExtensionService: def __init__(self, session: AsyncSession, crypto=None) -> None: self.session = session # Optional decryptor for resolving a platform's display name at # add-time. None → skip resolution, fall back to the handle. self._crypto = crypto async def quick_add_source( self, url: str, *, artist_id: int | None = None, artist_name: str | None = None, use_platform_name: bool = False, ) -> dict: """Add `url` as a source. `artist_id` connects it to an existing artist, `artist_name` to that artist (created if new); with neither, the artist is resolved from the platform as before. `use_platform_name` applies the operator's convention that the Patreon name is canon: a Patreon source added to an existing artist renames that artist to the creator's Patreon display name. Name only — the slug, and every path keyed off it, never moves (#130). Ignored on every other platform, and when the name can't be read.""" platform, raw_slug = self._derive(url) url = canonical_source_url(platform, url, raw_slug) renamed_from = None # Identity by SOURCE handle (#130): an existing (platform, url) source # keeps its artist on re-add — even if that artist was since renamed (its # frozen slug no longer matches the current name), and even when the # add named a different artist. Only a genuinely new source # resolves/creates an artist. existing = await self._existing_source(platform, url) if existing is not None: artist = (await self.session.execute( select(Artist).where(Artist.id == existing.artist_id) )).scalar_one() return self._shape(existing, artist, created_source=False, created_artist=False) if artist_id is not None: artist = (await self.session.execute( select(Artist).where(Artist.id == artist_id) )).scalar_one_or_none() if artist is None: raise UnknownArtistError(f"no artist with id {artist_id}") created_artist = False if use_platform_name and platform == "patreon": renamed_from = await self._adopt_patreon_name(artist, raw_slug, url) else: name = (artist_name or "").strip() if not name: # Name the artist properly by resolving the real display name # from the platform (falls back to the URL handle). name = await self._resolve_artist_name(platform, raw_slug, url) artist, created_artist = await self._find_or_create_artist(name) source, created_source = await self._find_or_create_source( artist_id=artist.id, platform=platform, url=url, ) shaped = self._shape(source, artist, created_source, created_artist) if renamed_from is not None: shaped["renamed_from"] = renamed_from return shaped async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None: """Rename `artist` to the Patreon display name; the old name when it changed, else None. Unreadable name → no rename, never the handle.""" name = await self._platform_display_name("patreon", raw_slug, url) if not name or name == artist.name: return None old = artist.name artist.name = name await self.session.commit() return old async def _existing_source(self, platform: str, url: str) -> Source | None: """The source this URL already is, whichever artist owns it. Discord compares ids, not strings, so a row stored before canonicalisation (a ptb host, a trailing slash) is still found.""" if platform != DISCORD: return (await self.session.execute( select(Source).where(Source.platform == platform, Source.url == url) )).scalars().first() want = _discord_ids(url) rows = (await self.session.execute( select(Source).where(Source.platform == DISCORD).order_by(Source.id) )).scalars().all() return next((s for s in rows if _discord_ids(s.url) == want), None) @staticmethod def _shape(source, artist, created_source: bool, created_artist: bool) -> dict: return { "source": { "id": source.id, "artist_id": source.artist_id, "platform": source.platform, "url": source.url, "enabled": source.enabled, }, "artist": { "id": artist.id, "name": artist.name, "slug": artist.slug, }, "created_source": created_source, "created_artist": created_artist, } async def _resolve_artist_name( self, platform: str, raw_slug: str, url: str ) -> str: """The real display name for a new artist, resolved from the platform at add-time (#130). Our native platforms each have a name source — patreon the campaigns API, subscribestar the profile page (both cookies). Other platforms (and any failure — no credential, network error) fall back to the URL handle, which is already readable. The resolvers are sync, so they run in an executor.""" if platform == DISCORD: # The server's name: what the operator knows the community as. server_id = raw_slug.split("/", 1)[0] names = await self._discord_names(server_id, None) return names.get("server") or f"Discord {server_id}" return await self._platform_display_name(platform, raw_slug, url) or raw_slug async def _platform_display_name( self, platform: str, raw_slug: str, url: str ) -> str | None: """The creator's display name as Patreon or SubscribeStar shows it, read with the stored cookies; None when it can't be read (no credential, a network error, a slow answer, any other platform). None, not the handle, so a caller can tell a real name from a fallback — a rename to the Patreon name must never rename to a URL handle instead.""" if self._crypto is None or platform not in ("patreon", "subscribestar"): return None import asyncio from .credential_service import CredentialService cred = CredentialService(self.session, self._crypto) loop = asyncio.get_running_loop() try: if platform == "patreon": cookies = await cred.get_cookies_path("patreon") from .patreon_resolver import resolve_display_name call = loop.run_in_executor( None, resolve_display_name, raw_slug, str(cookies) if cookies else None, ) else: # subscribestar cookies = await cred.get_cookies_path("subscribestar") from .subscribestar_client import SubscribeStarClient client = SubscribeStarClient(str(cookies) if cookies else None) call = loop.run_in_executor(None, client.resolve_display_name, url) name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS) except Exception as exc: # resolution is best-effort — never block the add log.warning("artist display-name resolution failed (%s): %s", platform, exc) return None return (name or "").strip() or None async def probe(self, url: str, *, names: bool = False) -> dict: """Read-only resolution of a creator-page URL against the FC DB. Returns one of: - {state: 'unknown_platform'} — URL didn't match any platform's strict artist-page pattern - {state: 'new', platform, slug} — would create both artist and source on quick-add - {state: 'artist_match', platform, slug, artist} — artist exists, this exact URL isn't a Source yet (collapses the sidecar-synthetic case too — the synthetic anchor counts as an existing artist row but not as a pollable Source for this URL) - {state: 'source_match', platform, slug, artist, source} — exact (artist, platform, url) Source already exists `names` (the Add panel asks, the chip does not) adds `display_name`: the creator's name as Patreon/SubscribeStar shows it, or None. It costs a request to the platform, so a plain page view never pays it. Side-effect-free: two SELECTs at most, plus that one lookup. """ try: platform, raw_slug = self._derive(url) except (UnknownPlatformError, InvalidUrlError): return {"state": "unknown_platform"} if platform == DISCORD: return await self._probe_discord(raw_slug) slug = slugify(raw_slug) result: dict = {"platform": platform, "slug": slug} if names: result["display_name"] = await self._platform_display_name( platform, raw_slug, url, ) artist = (await self.session.execute( select(Artist).where(Artist.slug == slug) )).scalar_one_or_none() if artist is None: return {"state": "new", **result} source = (await self.session.execute( select(Source).where( Source.artist_id == artist.id, Source.platform == platform, Source.url == url, ) )).scalar_one_or_none() if source is None: return {"state": "artist_match", **result, "artist": self._artist_payload(artist)} return { "state": "source_match", **result, "artist": self._artist_payload(artist), "source": self._source_payload(source), } async def _probe_discord(self, raw_slug: str) -> dict: """probe for a Discord server or channel. The states mean what they mean elsewhere, but the artist is never read off the URL: - source_match: this channel is a source — or the whole server is (`covered_by_server`), which already walks every channel; - artist_match: another source on this server belongs to an artist, the one this channel most likely belongs to too (a suggestion the Add panel preselects, not a decision); - new: nothing on this server yet. `discord` carries the ids, both canonical URLs and the display names, read with the stored token; a name that can't be read is None.""" server_id, _, channel_id = raw_slug.partition("/") channel_id = channel_id or None rows = (await self.session.execute( select(Source, Artist) .join(Artist, Artist.id == Source.artist_id) .where(Source.platform == DISCORD) .order_by(Source.id) )).all() exact = server_whole = on_server = None for source, artist in rows: ids = _discord_ids(source.url) if ids is None or ids[0] != server_id: continue if ids[1] == channel_id and exact is None: exact = (source, artist) elif ids[1] is None and server_whole is None: server_whole = (source, artist) if on_server is None: on_server = (source, artist) names = await self._discord_names(server_id, channel_id) base = f"https://discord.com/channels/{server_id}" result: dict = { "platform": DISCORD, "slug": raw_slug, "discord": { "server_id": server_id, "channel_id": channel_id, "server_name": names.get("server"), "channel_name": names.get("channel"), "server_url": base, "channel_url": f"{base}/{channel_id}" if channel_id else None, }, } hit = exact or server_whole if hit is not None: source, artist = hit result.update( state="source_match", artist=self._artist_payload(artist), source=self._source_payload(source), covered_by_server=exact is None, ) elif on_server is not None: result.update(state="artist_match", artist=self._artist_payload(on_server[1])) else: result["state"] = "new" return result async def _discord_names(self, server_id: str | None, channel_id: str | None) -> dict: """Server/channel display names via the stored Discord token. Never raises and never waits out a rate limit: it runs while the operator looks at a page, so a slow or missing answer just means no names.""" if self._crypto is None: return {} import asyncio from .credential_service import CredentialService from .discord_client import DiscordClient try: token = await CredentialService(self.session, self._crypto).get_token(DISCORD) if not token: return {} client = DiscordClient(token, max_retries=0) loop = asyncio.get_running_loop() return await asyncio.wait_for( loop.run_in_executor(None, client.describe, server_id, channel_id), timeout=_NAME_LOOKUP_SECONDS, ) except Exception as exc: # names are decoration — never fail the call log.info("Discord name lookup failed: %s", exc) return {} @staticmethod def _artist_payload(artist) -> dict: return {"id": artist.id, "name": artist.name, "slug": artist.slug} @staticmethod def _source_payload(source) -> dict: return { "id": source.id, "artist_id": source.artist_id, "platform": source.platform, "url": source.url, "enabled": source.enabled, } def _derive(self, url: str) -> tuple[str, str]: if not isinstance(url, str) or not url.strip(): raise InvalidUrlError("url is empty") if not url.startswith(("http://", "https://")): raise InvalidUrlError(f"url must include http:// or https:// scheme: {url!r}") for platform, pattern in _PLATFORM_PATTERNS: m = pattern.match(url) if m: return platform, m.group("slug") raise UnknownPlatformError(f"no platform pattern matched {url!r}") async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]: """Race-safe find-or-create on Artist by slug. Mirrors the savepoint + IntegrityError recovery pattern used in Importer._find_or_create_source/post (see reference_scalar_one_or_none_duplicates memory). Without this, two concurrent quick-add-source calls hitting the same artist would both miss the existence check and the second INSERT would 500 against uq_artist_slug. """ slug = slugify(raw_name) async def _create() -> Artist: artist = Artist(name=raw_name, slug=slug, is_subscription=True) self.session.add(artist) await self.session.flush() return artist return await get_or_create( self.session, select(Artist).where(Artist.slug == slug), _create ) async def _find_or_create_source( self, *, artist_id: int, platform: str, url: str, ) -> tuple[Source, bool]: """Race-safe — same pattern as _find_or_create_artist above. The uq_source_artist_platform_url constraint catches the duplicate insert; we roll the savepoint back and re-select.""" select_existing = select(Source).where( Source.artist_id == artist_id, Source.platform == platform, Source.url == url, ) async def _create() -> Source: # New subscription sources arm run-until-done backfill (plan #693) # so the first ticks walk the full history (otherwise gallery-dl's # exit:20 short-circuits before the archive is built). Mirrors # SourceService.create — without it, Firefox quick-add on a creator # with >20 unsynced posts would surface as "check failed" with no # diagnosis. Audit 2026-06-02. src = Source( artist_id=artist_id, platform=platform, url=url, enabled=True, config_overrides={"_backfill_state": "running"}, backfill_runs_remaining=BACKFILL_MAX_CHUNKS, ) self.session.add(src) await self.session.flush() return src src, created = await get_or_create( self.session, select_existing, _create ) if created: await self.session.commit() return src, created