CI / lint (push) Successful in 3s
CI / extension-version (push) Successful in 4s
CI / frontend-build (push) Successful in 23s
extension / lint (push) Successful in 26s
CI / backend-lint-and-test (push) Successful in 28s
CI / integration (push) Successful in 3m43s
Executes the 2026-07-05 product decision (FC downloaders = art-dedicated
services only), which removed Twitter/X and Bluesky but left deviantart
fully wired for seven weeks — the half-retired state rule 22 exists to
prevent.
Removed: the PlatformInfo module and its registry entry, the gallery-dl
extractor block, extension_service's artist-page pattern, the extension's
PLATFORMS + PLATFORM_ARTIST_PATTERNS entries, its manifest host permission
and content-script match, the frontend icon/colour/label, and the operator-
facing "supported platforms" list that still advertised it.
Two judgment calls, both recorded in migration 0088:
* existing `source` rows are DISABLED, not deleted. The row is the only
record of the artist's DeviantArt URL. Disabling is also required for
correctness rather than tidiness: with the platform unregistered the
download path falls through to gallery-dl, which carries its OWN
deviantart extractor, so an enabled row would have kept downloading
from a dropped platform.
* the `credential` row IS deleted — a live session cookie for a site FC
will never call again.
Adds the invariant whose absence is why manifest.json drifted in the first
place: nothing tied its domain lists back to the platform table. The
extension suite now asserts both directions, plus that no host permission
belongs to an unclaimed domain (`*://*/*` exempted — FC is self-hosted at
an operator-chosen URL the extension cannot enumerate).
Extension version 1.0.10 -> 1.0.11: ci.yml's guard hard-fails a packaged
extension change without a bump. No release is cut — build.yml's
sign-extension job only runs on main.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
287 lines
12 KiB
Python
287 lines
12 KiB
Python
"""FC-3g: backend support for the Firefox extension.
|
|
|
|
`ExtensionService.quick_add_source(url)` derives platform + artist
|
|
slug from a URL using regex patterns mirrored from
|
|
extension/lib/platforms.js, then find-or-creates Artist + Source rows
|
|
and returns a JSON-shaped dict for the API layer.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ..models import Artist, Source
|
|
from ..utils.slug import slugify
|
|
from .db_helpers import get_or_create
|
|
from .source_service import BACKFILL_MAX_CHUNKS
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class UnknownPlatformError(Exception):
|
|
"""URL didn't match any platform pattern."""
|
|
|
|
|
|
class InvalidUrlError(Exception):
|
|
"""URL was empty or missing a scheme."""
|
|
|
|
|
|
# Mirrored byte-for-byte from extension/lib/platforms.js
|
|
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
|
# reviewers catch drift.
|
|
_PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|
("patreon", re.compile(
|
|
# Three creator URL shapes — bare (patreon.com/Atole), `c/`, and `cw/`
|
|
# (the "creator workspace" URL served once subscribed, see
|
|
# patreon_resolver._VANITY_RE). A trailing sub-path is allowed so a
|
|
# creator's inner page still derives the slug. Nav pages stay excluded.
|
|
r"^https?://(?:www\.)?patreon\.com/"
|
|
r"(?:cw/|c/)?"
|
|
r"(?!(?:home|search|messages|notifications|library|settings|posts)(?:[/?#]|$))"
|
|
r"(?P<slug>[^/?#]+)",
|
|
re.IGNORECASE,
|
|
)),
|
|
("subscribestar", re.compile(
|
|
r"^https?://(?:www\.)?subscribestar\.(?:com|adult)/"
|
|
r"(?!feed$|messages$|library$)"
|
|
r"(?P<slug>[^/?#]+)/?$",
|
|
re.IGNORECASE,
|
|
)),
|
|
("hentaifoundry", re.compile(
|
|
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
|
re.IGNORECASE,
|
|
)),
|
|
("pixiv", re.compile(
|
|
r"^https?://(?:www\.)?pixiv\.net/(?:en/)?users/(?P<slug>\d+)",
|
|
re.IGNORECASE,
|
|
)),
|
|
]
|
|
|
|
|
|
class ExtensionService:
|
|
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
|
self.session = session
|
|
# Optional decryptor for resolving a token-auth platform's display name
|
|
# (pixiv) at add-time. None → skip resolution, fall back to the handle.
|
|
self._crypto = crypto
|
|
|
|
async def quick_add_source(self, url: str) -> dict:
|
|
platform, raw_slug = self._derive(url)
|
|
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
|
# keeps its artist on re-add — even if that artist was since renamed (its
|
|
# frozen slug no longer matches the current name). Only a genuinely new
|
|
# source resolves/creates an artist.
|
|
existing = (await self.session.execute(
|
|
select(Source).where(Source.platform == platform, Source.url == url)
|
|
)).scalar_one_or_none()
|
|
if existing is not None:
|
|
artist = (await self.session.execute(
|
|
select(Artist).where(Artist.id == existing.artist_id)
|
|
)).scalar_one()
|
|
return self._shape(existing, artist, created_source=False, created_artist=False)
|
|
|
|
# New source → name the artist properly by resolving the real display
|
|
# name from the platform (falls back to the URL handle).
|
|
name = await self._resolve_artist_name(platform, raw_slug, url)
|
|
artist, created_artist = await self._find_or_create_artist(name)
|
|
source, created_source = await self._find_or_create_source(
|
|
artist_id=artist.id, platform=platform, url=url,
|
|
)
|
|
return self._shape(source, artist, created_source, created_artist)
|
|
|
|
@staticmethod
|
|
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
|
return {
|
|
"source": {
|
|
"id": source.id,
|
|
"artist_id": source.artist_id,
|
|
"platform": source.platform,
|
|
"url": source.url,
|
|
"enabled": source.enabled,
|
|
},
|
|
"artist": {
|
|
"id": artist.id,
|
|
"name": artist.name,
|
|
"slug": artist.slug,
|
|
},
|
|
"created_source": created_source,
|
|
"created_artist": created_artist,
|
|
}
|
|
|
|
async def _resolve_artist_name(
|
|
self, platform: str, raw_slug: str, url: str
|
|
) -> str:
|
|
"""The real display name for a new artist, resolved from the platform at
|
|
add-time (#130). Our native platforms each have a name source — pixiv the
|
|
app API (token), patreon the campaigns API, subscribestar the profile
|
|
page (both cookies). Other platforms (and any failure — no credential,
|
|
network error) fall back to the URL handle, which is already readable.
|
|
The resolvers are sync, so they run in an executor."""
|
|
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
|
return raw_slug
|
|
import asyncio
|
|
|
|
from .credential_service import CredentialService
|
|
cred = CredentialService(self.session, self._crypto)
|
|
loop = asyncio.get_running_loop()
|
|
try:
|
|
if platform == "pixiv":
|
|
token = await cred.get_token("pixiv")
|
|
if not token:
|
|
return raw_slug
|
|
from .pixiv_client import PixivClient
|
|
name = await loop.run_in_executor(
|
|
None, PixivClient(token).resolve_display_name, raw_slug
|
|
)
|
|
elif platform == "patreon":
|
|
cookies = await cred.get_cookies_path("patreon")
|
|
from .patreon_resolver import resolve_display_name
|
|
name = await loop.run_in_executor(
|
|
None, resolve_display_name, raw_slug,
|
|
str(cookies) if cookies else None,
|
|
)
|
|
else: # subscribestar
|
|
cookies = await cred.get_cookies_path("subscribestar")
|
|
from .subscribestar_client import SubscribeStarClient
|
|
client = SubscribeStarClient(str(cookies) if cookies else None)
|
|
name = await loop.run_in_executor(
|
|
None, client.resolve_display_name, url
|
|
)
|
|
except Exception as exc: # resolution is best-effort — never block the add
|
|
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
|
return raw_slug
|
|
return name or raw_slug
|
|
|
|
async def probe(self, url: str) -> dict:
|
|
"""Read-only resolution of a creator-page URL against the FC DB.
|
|
Returns one of:
|
|
- {state: 'unknown_platform'} — URL didn't match any
|
|
platform's strict artist-page pattern
|
|
- {state: 'new', platform, slug} — would create both
|
|
artist and source on quick-add
|
|
- {state: 'artist_match', platform, slug, artist}
|
|
— artist exists, this
|
|
exact URL isn't a Source yet (collapses the sidecar-synthetic
|
|
case too — the synthetic anchor counts as an existing artist
|
|
row but not as a pollable Source for this URL)
|
|
- {state: 'source_match', platform, slug, artist, source}
|
|
— exact (artist, platform,
|
|
url) Source already exists
|
|
|
|
Side-effect-free: two SELECTs at most.
|
|
"""
|
|
try:
|
|
platform, raw_slug = self._derive(url)
|
|
except (UnknownPlatformError, InvalidUrlError):
|
|
return {"state": "unknown_platform"}
|
|
|
|
slug = slugify(raw_slug)
|
|
artist = (await self.session.execute(
|
|
select(Artist).where(Artist.slug == slug)
|
|
)).scalar_one_or_none()
|
|
if artist is None:
|
|
return {"state": "new", "platform": platform, "slug": slug}
|
|
|
|
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
|
|
|
source = (await self.session.execute(
|
|
select(Source).where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == platform,
|
|
Source.url == url,
|
|
)
|
|
)).scalar_one_or_none()
|
|
if source is None:
|
|
return {
|
|
"state": "artist_match",
|
|
"platform": platform,
|
|
"slug": slug,
|
|
"artist": artist_payload,
|
|
}
|
|
|
|
return {
|
|
"state": "source_match",
|
|
"platform": platform,
|
|
"slug": slug,
|
|
"artist": artist_payload,
|
|
"source": {
|
|
"id": source.id,
|
|
"artist_id": source.artist_id,
|
|
"platform": source.platform,
|
|
"url": source.url,
|
|
"enabled": source.enabled,
|
|
},
|
|
}
|
|
|
|
def _derive(self, url: str) -> tuple[str, str]:
|
|
if not isinstance(url, str) or not url.strip():
|
|
raise InvalidUrlError("url is empty")
|
|
if not url.startswith(("http://", "https://")):
|
|
raise InvalidUrlError(f"url must include http:// or https:// scheme: {url!r}")
|
|
for platform, pattern in _PLATFORM_PATTERNS:
|
|
m = pattern.match(url)
|
|
if m:
|
|
return platform, m.group("slug")
|
|
raise UnknownPlatformError(f"no platform pattern matched {url!r}")
|
|
|
|
async def _find_or_create_artist(self, raw_name: str) -> tuple[Artist, bool]:
|
|
"""Race-safe find-or-create on Artist by slug. Mirrors the
|
|
savepoint + IntegrityError recovery pattern used in
|
|
Importer._find_or_create_source/post (see
|
|
reference_scalar_one_or_none_duplicates memory). Without this,
|
|
two concurrent quick-add-source calls hitting the same artist
|
|
would both miss the existence check and the second INSERT would
|
|
500 against uq_artist_slug.
|
|
"""
|
|
slug = slugify(raw_name)
|
|
|
|
async def _create() -> Artist:
|
|
artist = Artist(name=raw_name, slug=slug, is_subscription=True)
|
|
self.session.add(artist)
|
|
await self.session.flush()
|
|
return artist
|
|
|
|
return await get_or_create(
|
|
self.session, select(Artist).where(Artist.slug == slug), _create
|
|
)
|
|
|
|
async def _find_or_create_source(
|
|
self, *, artist_id: int, platform: str, url: str,
|
|
) -> tuple[Source, bool]:
|
|
"""Race-safe — same pattern as _find_or_create_artist above. The
|
|
uq_source_artist_platform_url constraint catches the duplicate
|
|
insert; we roll the savepoint back and re-select."""
|
|
select_existing = select(Source).where(
|
|
Source.artist_id == artist_id,
|
|
Source.platform == platform,
|
|
Source.url == url,
|
|
)
|
|
|
|
async def _create() -> Source:
|
|
# New subscription sources arm run-until-done backfill (plan #693)
|
|
# so the first ticks walk the full history (otherwise gallery-dl's
|
|
# exit:20 short-circuits before the archive is built). Mirrors
|
|
# SourceService.create — without it, Firefox quick-add on a creator
|
|
# with >20 unsynced posts would surface as "check failed" with no
|
|
# diagnosis. Audit 2026-06-02.
|
|
src = Source(
|
|
artist_id=artist_id, platform=platform,
|
|
url=url, enabled=True,
|
|
config_overrides={"_backfill_state": "running"},
|
|
backfill_runs_remaining=BACKFILL_MAX_CHUNKS,
|
|
)
|
|
self.session.add(src)
|
|
await self.session.flush()
|
|
return src
|
|
|
|
src, created = await get_or_create(
|
|
self.session, select_existing, _create
|
|
)
|
|
if created:
|
|
await self.session.commit()
|
|
return src, created
|