diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py index adc6d6a..f467eb0 100644 --- a/backend/app/services/credential_service.py +++ b/backend/app/services/credential_service.py @@ -165,99 +165,16 @@ class CredentialService: def _augment_cookies(platform: str, netscape: str) -> str: - """Inject platform-specific synthetic cookies needed to bypass server - gates or extractor quirks. - - subscribestar.adult: the server gates artist pages behind the - `_personalization_id` age-confirmation cookie. The site's frontend JS - uses localStorage to suppress the age popup once dismissed, so after - the cookie's annual expiry the user can't easily get a fresh one — - visiting the site in a logged-in session doesn't re-show the popup - and doesn't re-issue the cookie. gallery-dl's own login flow (which - FC doesn't use; we capture cookies via the extension instead) - sidesteps this by manually setting `18_plus_agreement_generic=true` - on `.subscribestar.adult` — the server accepts that as the - age-confirmation marker. - - hentaifoundry: gallery-dl's extractor uses - `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` to - decide whether the user is logged in. `requests` does EXACT domain - matching on .get(); the extension rewrites every captured cookie to - a leading-dot subdomain-wide form (`.hentai-foundry.com`), which - fails that exact match. The fallback path hits a HEAD - `?enterAgree=1` that 401s. Emit host-only duplicates of PHPSESSID + - YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup succeeds. - (The original `.hentai-foundry.com` entries stay — the actual HTTP - requests use RFC 6265 subdomain matching, which works either way.) - - All injections are idempotent (no-op if the target cookie is already - present) and platform-scoped. - - Operator-flagged 2026-05-27: subscribestar age-confirmation, then - hentaifoundry 401 on /?enterAgree=1. - """ - if platform == "subscribestar": - return _augment_subscribestar(netscape) - if platform == "hentaifoundry": - return _augment_hentaifoundry(netscape) - return netscape - - -def _augment_subscribestar(netscape: str) -> str: - if "18_plus_agreement_generic" in netscape: + """Delegate to the platform's `augment_cookies` hook if one is + registered (subscribestar, hentaifoundry, etc. — see + `services/platforms/.py`). No-op when the platform doesn't + register a hook (Patreon, DeviantArt). Centralizing the + quirks-per-platform in the platforms package means adding a new + platform's cookie quirks doesn't require touching this file.""" + info = PLATFORMS.get(platform) + if info is None or info.augment_cookies is None: return netscape - # Far-future expiry — gallery-dl's own login flow sets this with no - # explicit expiry; the server only checks presence/value. - expiry = 4102444800 # 2100-01-01 UTC, opaque "far future" - line = "\t".join([ - ".subscribestar.adult", "TRUE", "/", "TRUE", - str(expiry), "18_plus_agreement_generic", "true", - ]) - body = netscape.rstrip("\n") - if not body: - body = "# Netscape HTTP Cookie File" - return body + "\n" + line + "\n" - - -_HF_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") - - -def _augment_hentaifoundry(netscape: str) -> str: - body = netscape.rstrip("\n") - if not body: - return netscape - lines = body.split("\n") - existing_host_only = set() - by_name: dict[str, list[str]] = {} - for raw in lines: - if not raw or raw.startswith("#"): - continue - parts = raw.split("\t") - if len(parts) < 7: - continue - domain, _flag, _path, _secure, _exp, name, _value = parts[:7] - if name not in _HF_HOST_ONLY_NAMES: - continue - if domain == "www.hentai-foundry.com": - existing_host_only.add(name) - elif domain in (".hentai-foundry.com", "hentai-foundry.com"): - by_name.setdefault(name, []).append(raw) - - appended = [] - for name in _HF_HOST_ONLY_NAMES: - if name in existing_host_only or name not in by_name: - continue - # Duplicate the FIRST subdomain-wide line as host-only on - # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks - # it host-only in netscape format. - parts = by_name[name][0].split("\t") - parts[0] = "www.hentai-foundry.com" - parts[1] = "FALSE" - appended.append("\t".join(parts[:7])) - - if not appended: - return netscape - return body + "\n" + "\n".join(appended) + "\n" + return info.augment_cookies(netscape) def _to_netscape(plaintext: str) -> str: diff --git a/backend/app/services/platforms.py b/backend/app/services/platforms.py deleted file mode 100644 index bd3723a..0000000 --- a/backend/app/services/platforms.py +++ /dev/null @@ -1,140 +0,0 @@ -"""FC-3b platforms registry — the single source of truth for what -FabledCurator supports. - -Lifted from GallerySubscriber's -~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py -and ~/.../extension/lib/platforms.js. Six platforms; auth_type and -URL patterns match GS exactly so the existing browser extension -hits FC unmodified. -""" - -from dataclasses import dataclass -from typing import Literal - - -@dataclass(frozen=True) -class PlatformInfo: - key: str - name: str - description: str - auth_type: Literal["cookies", "token"] - requires_auth: bool - url_pattern: str - url_examples: list[str] - default_config: dict - notes: str | None = None - - -# Common defaults used across most platforms; embedded per-platform -# below so per-platform overrides remain explicit. -_DEFAULTS = { - "sleep": 3.0, - "sleep_request": 1.5, - "skip_existing": True, - "save_metadata": True, - "timeout": 3600, -} - - -PLATFORMS: dict[str, PlatformInfo] = { - "patreon": PlatformInfo( - key="patreon", - name="Patreon", - description="Download posts from Patreon creators", - auth_type="cookies", - requires_auth=True, - url_pattern=r"^https?://(www\.)?patreon\.com/", - url_examples=[ - "https://www.patreon.com/example_artist", - "https://www.patreon.com/user?u=12345678", - ], - default_config={**_DEFAULTS, "content_types": ["images", "attachments"]}, - ), - "subscribestar": PlatformInfo( - key="subscribestar", - name="SubscribeStar", - description="Download posts from SubscribeStar creators", - auth_type="cookies", - requires_auth=True, - url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", - url_examples=[ - "https://subscribestar.adult/example_artist", - "https://www.subscribestar.com/example_artist", - ], - default_config={**_DEFAULTS, "content_types": ["all"]}, - ), - "hentaifoundry": PlatformInfo( - key="hentaifoundry", - name="Hentai Foundry", - description="Download artwork from Hentai Foundry artists", - auth_type="cookies", - requires_auth=False, - url_pattern=r"^https?://(www\.)?hentai-foundry\.com/", - url_examples=[ - "https://www.hentai-foundry.com/user/example_artist", - "https://www.hentai-foundry.com/pictures/user/example_artist", - ], - default_config={**_DEFAULTS, "content_types": ["pictures"]}, - ), - "discord": PlatformInfo( - key="discord", - name="Discord", - description="Download attachments from Discord channels", - auth_type="token", - requires_auth=True, - url_pattern=r"^https?://(www\.)?discord\.com/channels/", - url_examples=["https://discord.com/channels/123456789/987654321"], - default_config={**_DEFAULTS, "content_types": ["all"]}, - notes="Requires Discord user token (not bot token).", - ), - "pixiv": PlatformInfo( - key="pixiv", - name="Pixiv", - description="Download artwork from Pixiv artists", - auth_type="token", - requires_auth=True, - url_pattern=r"^https?://(www\.)?pixiv\.net/", - url_examples=[ - "https://www.pixiv.net/users/12345678", - "https://www.pixiv.net/en/users/12345678", - ], - default_config={**_DEFAULTS, "content_types": ["all"]}, - notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.", - ), - "deviantart": PlatformInfo( - key="deviantart", - name="DeviantArt", - description="Download artwork from DeviantArt artists", - auth_type="cookies", - requires_auth=False, - url_pattern=r"^https?://(www\.)?deviantart\.com/", - url_examples=[ - "https://www.deviantart.com/example-artist", - "https://www.deviantart.com/example-artist/gallery", - ], - default_config={**_DEFAULTS, "content_types": ["gallery"]}, - ), -} - - -def known_platform_keys() -> frozenset[str]: - return frozenset(PLATFORMS.keys()) - - -def auth_type_for(platform: str) -> str | None: - info = PLATFORMS.get(platform) - return info.auth_type if info else None - - -def to_dict(info: PlatformInfo) -> dict: - return { - "key": info.key, - "name": info.name, - "description": info.description, - "auth_type": info.auth_type, - "requires_auth": info.requires_auth, - "url_pattern": info.url_pattern, - "url_examples": info.url_examples, - "default_config": info.default_config, - "notes": info.notes, - } diff --git a/backend/app/services/platforms/__init__.py b/backend/app/services/platforms/__init__.py new file mode 100644 index 0000000..4bd85be --- /dev/null +++ b/backend/app/services/platforms/__init__.py @@ -0,0 +1,98 @@ +"""FC-3b platforms registry — single source of truth for what +FabledCurator supports + where each platform's quirks live. + +Adding a new platform: drop a new module `.py` next to this +one, declare an `INFO = PlatformInfo(...)`, add the import + entry in +PLATFORMS below. Sidecar parsing, cookie materialization, and +`/api/platforms` pick it up automatically. + +Lifted from GallerySubscriber's +~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py +and ~/.../extension/lib/platforms.js. Six platforms; auth_type and +URL patterns match GS exactly so the existing browser extension +hits FC unmodified. +""" + +from . import ( + deviantart, + discord, + hentaifoundry, + patreon, + pixiv, + subscribestar, +) +from .base import ( + DEFAULT_DESCRIPTION_KEYS, + DEFAULT_EXTERNAL_POST_ID_KEYS, + PlatformInfo, +) + + +PLATFORMS: dict[str, PlatformInfo] = { + info.key: info + for info in ( + patreon.INFO, + subscribestar.INFO, + hentaifoundry.INFO, + discord.INFO, + pixiv.INFO, + deviantart.INFO, + ) +} + + +def known_platform_keys() -> frozenset[str]: + return frozenset(PLATFORMS.keys()) + + +def auth_type_for(platform: str) -> str | None: + info = PLATFORMS.get(platform) + return info.auth_type if info else None + + +def to_dict(info: PlatformInfo) -> dict: + """Serialize a PlatformInfo to a JSON-safe dict for /api/platforms. + + Behavioral fields (callables, sidecar-chain overrides) are + intentionally omitted — they aren't useful to API consumers. + """ + return { + "key": info.key, + "name": info.name, + "description": info.description, + "auth_type": info.auth_type, + "requires_auth": info.requires_auth, + "url_pattern": info.url_pattern, + "url_examples": info.url_examples, + "default_config": info.default_config, + "notes": info.notes, + } + + +def external_post_id_keys_for(platform: str | None) -> tuple[str, ...]: + """Resolve the external_post_id lookup chain for a given platform, + falling back to the module default when the platform isn't + registered or hasn't overridden the chain.""" + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.external_post_id_keys is not None: + return info.external_post_id_keys + return DEFAULT_EXTERNAL_POST_ID_KEYS + + +def description_keys_for(platform: str | None) -> tuple[str, ...]: + """Resolve the description body lookup chain for a given platform.""" + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.description_keys is not None: + return info.description_keys + return DEFAULT_DESCRIPTION_KEYS + + +__all__ = [ + "PLATFORMS", + "PlatformInfo", + "auth_type_for", + "description_keys_for", + "external_post_id_keys_for", + "known_platform_keys", + "to_dict", +] diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py new file mode 100644 index 0000000..8c496d5 --- /dev/null +++ b/backend/app/services/platforms/base.py @@ -0,0 +1,108 @@ +"""PlatformInfo dataclass + shared defaults + small helpers. + +Per-platform modules import from here, register their PlatformInfo via +INFO, optionally attaching `derive_post_url` and/or `augment_cookies` +callables for behavior that diverges from gallery-dl's mainline shape +(Patreon). + +Adding a new platform: drop a new module under `services/platforms/`, +declare an INFO, and add it to the import list in +`services/platforms/__init__.py`. Sidecar parsing, cookie +materialization, and the /api/platforms response pick it up +automatically. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Literal + + +# Sidecar parsing defaults. Per-platform PlatformInfo entries can +# override these by setting `external_post_id_keys=` / +# `description_keys=`. Most don't need to — the defaults already cover +# every platform FC supports. +# +# external_post_id chain: `post_id` MUST come before `id` because +# SubscribeStar gallery-dl puts the per-attachment id in `id` and the +# actual post id in `post_id`; picking `id` first fragments +# multi-image SubscribeStar posts into N Post rows. Patreon/Pixiv have +# no `post_id` so `id` still wins for them; HF uses `index`, Discord +# uses `message_id` — all reached via the remaining chain entries. +# (Banked 2026-05-27 during the sidecar audit.) +DEFAULT_EXTERNAL_POST_ID_KEYS: tuple[str, ...] = ( + "post_id", "id", "index", "message_id", +) + +# Description body chain: Discord's gallery-dl extractor uses `message` +# (no `content`); appended to the chain so Discord posts surface body +# text. +DEFAULT_DESCRIPTION_KEYS: tuple[str, ...] = ( + "content", "description", "caption", "message", +) + + +@dataclass(frozen=True) +class PlatformInfo: + # --- Identity / metadata --- + key: str + name: str + description: str + auth_type: Literal["cookies", "token"] + requires_auth: bool + url_pattern: str + url_examples: list[str] + default_config: dict + notes: str | None = None + + # --- Sidecar parsing overrides --- + # Each is None to mean "use the module default above"; a platform + # only sets one of these when its sidecar shape genuinely differs. + external_post_id_keys: tuple[str, ...] | None = None + description_keys: tuple[str, ...] | None = None + + # --- Behavioral hooks --- + # Synthesize a post permalink from sidecar data. Required when + # gallery-dl's `url` field is the file/CDN URL rather than the post + # permalink (subscribestar/pixiv/hf/discord). None = trust the bare + # `url` field (patreon, deviantart). + derive_post_url: Callable[[dict], str | None] | None = None + + # Post-process the materialized cookies.txt for gallery-dl. Used by + # platforms whose server gates or extractor quirks need synthetic + # cookies the extension can't capture (subscribestar age cookie, HF + # host-only PHPSESSID duplicate). None = no-op. + augment_cookies: Callable[[str], str] | None = None + + +def str_id_value(v) -> str | None: + """Coerce a JSON scalar id into a non-empty string, rejecting bool + (Python's bool is an int subclass so `isinstance(True, int)` is + True; without this guard a sidecar with `"id": true` would produce + external_post_id="True").""" + if isinstance(v, bool): + return None + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + +def str_field(v) -> str | None: + """Same idea as str_id_value but for plain string fields (no int + coercion).""" + if isinstance(v, str) and v.strip(): + return v.strip() + return None + + +# Shared gallery-dl invocation defaults. Embedded in each platform's +# default_config (with platform-specific overrides) so per-platform +# choices stay explicit. +GD_DEFAULTS = { + "sleep": 3.0, + "sleep_request": 1.5, + "skip_existing": True, + "save_metadata": True, + "timeout": 3600, +} diff --git a/backend/app/services/platforms/deviantart.py b/backend/app/services/platforms/deviantart.py new file mode 100644 index 0000000..e41fc3b --- /dev/null +++ b/backend/app/services/platforms/deviantart.py @@ -0,0 +1,23 @@ +"""DeviantArt — no exercised quirks yet. + +No operator-owned DeviantArt archive existed at the 2026-05-27 sidecar +audit, so we don't know yet whether DA's gallery-dl sidecars are +well-behaved or have their own quirks. When DA gets exercised for the +first time, add `derive_post_url` / `augment_cookies` here as needed. +""" + +from .base import GD_DEFAULTS, PlatformInfo + +INFO = PlatformInfo( + key="deviantart", + name="DeviantArt", + description="Download artwork from DeviantArt artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?deviantart\.com/", + url_examples=[ + "https://www.deviantart.com/example-artist", + "https://www.deviantart.com/example-artist/gallery", + ], + default_config={**GD_DEFAULTS, "content_types": ["gallery"]}, +) diff --git a/backend/app/services/platforms/discord.py b/backend/app/services/platforms/discord.py new file mode 100644 index 0000000..e6afbf1 --- /dev/null +++ b/backend/app/services/platforms/discord.py @@ -0,0 +1,38 @@ +"""Discord — one quirk + one already-default. + +post_url: gallery-dl's `url` is the CDN attachment URL. The "permalink" +for a Discord message uses the (server, channel, message) triple via +`discord.com/channels///`. Note that +permalinks are only resolvable for users in the same server — public +access doesn't work — but the URL is still useful to the operator +in-app. + +Description body is in `message` not `content`. That's already covered +by the default description chain in base.py (DEFAULT_DESCRIPTION_KEYS +ends with `message`). No description_keys override needed. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + sid = str_id_value(data.get("server_id")) + cid = str_id_value(data.get("channel_id")) + mid = str_id_value(data.get("message_id")) + if sid and cid and mid: + return f"https://discord.com/channels/{sid}/{cid}/{mid}" + return None + + +INFO = PlatformInfo( + key="discord", + name="Discord", + description="Download attachments from Discord channels", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?discord\.com/channels/", + url_examples=["https://discord.com/channels/123456789/987654321"], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + notes="Requires Discord user token (not bot token).", + derive_post_url=derive_post_url, +) diff --git a/backend/app/services/platforms/hentaifoundry.py b/backend/app/services/platforms/hentaifoundry.py new file mode 100644 index 0000000..d84ebef --- /dev/null +++ b/backend/app/services/platforms/hentaifoundry.py @@ -0,0 +1,83 @@ +"""HentaiFoundry — two quirks colocated. + +1. post_url: HF sidecars omit `url` entirely; `src` is the image URL. + Synthesize the permalink from `user` + `index` + (/pictures/user//). + +2. augment_cookies: gallery-dl's HF extractor checks + `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with + `requests`' EXACT domain matching. The extension's pre-v1.0.5 + `cookies.js` aggressively rewrote every captured cookie to the + leading-dot subdomain-wide form (`.hentai-foundry.com`), which fails + the exact lookup even though the cookie IS sent on actual HTTP + requests (RFC 6265 subdomain matching). The extractor falls into + an unauthenticated `?enterAgree=1` HEAD that 401s. Inject host-only + duplicates of PHPSESSID + YII_CSRF_TOKEN so the lookup succeeds. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_field, str_id_value + +_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") + + +def derive_post_url(data: dict) -> str | None: + user = str_field(data.get("user")) or str_field(data.get("artist")) + idx = str_id_value(data.get("index")) + if user and idx: + return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + return None + + +def augment_cookies(netscape: str) -> str: + body = netscape.rstrip("\n") + if not body: + return netscape + lines = body.split("\n") + existing_host_only: set[str] = set() + by_name: dict[str, list[str]] = {} + for raw in lines: + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + if len(parts) < 7: + continue + domain, _flag, _path, _secure, _exp, name, _value = parts[:7] + if name not in _HOST_ONLY_NAMES: + continue + if domain == "www.hentai-foundry.com": + existing_host_only.add(name) + elif domain in (".hentai-foundry.com", "hentai-foundry.com"): + by_name.setdefault(name, []).append(raw) + + appended: list[str] = [] + for name in _HOST_ONLY_NAMES: + if name in existing_host_only or name not in by_name: + continue + # Duplicate the first subdomain-wide line as host-only on + # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks + # the entry host-only in netscape format. + parts = by_name[name][0].split("\t") + parts[0] = "www.hentai-foundry.com" + parts[1] = "FALSE" + appended.append("\t".join(parts[:7])) + + if not appended: + return netscape + return body + "\n" + "\n".join(appended) + "\n" + + +INFO = PlatformInfo( + key="hentaifoundry", + name="Hentai Foundry", + description="Download artwork from Hentai Foundry artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?hentai-foundry\.com/", + url_examples=[ + "https://www.hentai-foundry.com/user/example_artist", + "https://www.hentai-foundry.com/pictures/user/example_artist", + ], + default_config={**GD_DEFAULTS, "content_types": ["pictures"]}, + derive_post_url=derive_post_url, + augment_cookies=augment_cookies, +) diff --git a/backend/app/services/platforms/patreon.py b/backend/app/services/platforms/patreon.py new file mode 100644 index 0000000..fa05bf2 --- /dev/null +++ b/backend/app/services/platforms/patreon.py @@ -0,0 +1,23 @@ +"""Patreon — no quirks. The reference platform. + +Patreon's gallery-dl sidecars are the well-behaved baseline: `url` is a +real permalink, `id` is the post id, `title` and `content` are +populated. No cookie quirks (session cookies are domain-wide). No +derivation overrides. +""" + +from .base import GD_DEFAULTS, PlatformInfo + +INFO = PlatformInfo( + key="patreon", + name="Patreon", + description="Download posts from Patreon creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?patreon\.com/", + url_examples=[ + "https://www.patreon.com/example_artist", + "https://www.patreon.com/user?u=12345678", + ], + default_config={**GD_DEFAULTS, "content_types": ["images", "attachments"]}, +) diff --git a/backend/app/services/platforms/pixiv.py b/backend/app/services/platforms/pixiv.py new file mode 100644 index 0000000..8664a1e --- /dev/null +++ b/backend/app/services/platforms/pixiv.py @@ -0,0 +1,32 @@ +"""Pixiv — one quirk. + +post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The +post permalink follows /artworks/. external_post_id (= `id`) was +already correct, so no override there. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + pid = str_id_value(data.get("id")) + if pid: + return f"https://www.pixiv.net/artworks/{pid}" + return None + + +INFO = PlatformInfo( + key="pixiv", + name="Pixiv", + description="Download artwork from Pixiv artists", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?pixiv\.net/", + url_examples=[ + "https://www.pixiv.net/users/12345678", + "https://www.pixiv.net/en/users/12345678", + ], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.", + derive_post_url=derive_post_url, +) diff --git a/backend/app/services/platforms/subscribestar.py b/backend/app/services/platforms/subscribestar.py new file mode 100644 index 0000000..554c26c --- /dev/null +++ b/backend/app/services/platforms/subscribestar.py @@ -0,0 +1,62 @@ +"""SubscribeStar — three quirks colocated. + +1. external_post_id: gallery-dl puts the per-attachment id in `id` + (e.g. 711509) and the actual post id in `post_id` (e.g. 360360). + The default chain in base.py already prefers `post_id`; this module + doesn't need to override it but the comment lives here too so a + future reader knows the chain's order was driven by this platform. + +2. post_url: gallery-dl's `url` is the file CDN URL + (`/post_uploads?payload=...`). Synthesize the post permalink from + `post_id`. + +3. augment_cookies: the server gates artist pages behind a + `_personalization_id` age-confirmation cookie that the user can't + easily refresh — SubscribeStar's frontend JS uses localStorage to + suppress the age popup once dismissed. gallery-dl's own login flow + sidesteps this by setting `18_plus_agreement_generic=true` on + `.subscribestar.adult`; we mirror that for cookies captured via the + extension. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + pid = str_id_value(data.get("post_id")) + if pid: + return f"https://www.subscribestar.com/posts/{pid}" + return None + + +def augment_cookies(netscape: str) -> str: + if "18_plus_agreement_generic" in netscape: + return netscape + # Far-future expiry — gallery-dl's own login flow sets this with no + # explicit expiry; the server only checks presence/value. + expiry = 4102444800 # 2100-01-01 UTC + line = "\t".join([ + ".subscribestar.adult", "TRUE", "/", "TRUE", + str(expiry), "18_plus_agreement_generic", "true", + ]) + body = netscape.rstrip("\n") + if not body: + body = "# Netscape HTTP Cookie File" + return body + "\n" + line + "\n" + + +INFO = PlatformInfo( + key="subscribestar", + name="SubscribeStar", + description="Download posts from SubscribeStar creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", + url_examples=[ + "https://subscribestar.adult/example_artist", + "https://www.subscribestar.com/example_artist", + ], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + derive_post_url=derive_post_url, + augment_cookies=augment_cookies, +) diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 13ced7c..0d81930 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -1,7 +1,9 @@ """Minimal gallery-dl sidecar parsing (one-time filesystem-import aid). -No per-platform branching: a small common key set with fallbacks; the -full JSON is kept in raw so anything unmapped is recoverable later. +Per-platform quirks (post_url synthesis, key-chain overrides) live in +the platforms registry — `backend/app/services/platforms/`. This module +is platform-agnostic: it looks up `category` in the sidecar and asks +the registry for the right behavior. """ import re @@ -9,6 +11,12 @@ from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path +from ..services.platforms import ( + PLATFORMS, + description_keys_for, + external_post_id_keys_for, +) + @dataclass(frozen=True) class SidecarData: @@ -55,6 +63,19 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None: return None +def _first_id(data: dict, keys: tuple[str, ...]) -> str | None: + """Like `_first_str` but accepts ints and rejects bool (Python's + bool subclasses int, so a literal `"id": true` would otherwise + yield external_post_id="True").""" + for k in keys: + v = data.get(k) + if isinstance(v, bool): + continue + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + # Strip HTML tags + collapse whitespace + take the first non-empty line. # Used to derive a display title from a body when the platform doesn't # expose a separate title field (subscribestar posts always write @@ -111,22 +132,7 @@ def parse_sidecar(data: dict) -> SidecarData: cat = data.get("category") platform = cat if isinstance(cat, str) and cat.strip() else None - # external_post_id lookup order: post_id MUST come before id. - # SubscribeStar gallery-dl writes the per-attachment id in `id` - # (e.g. 711509) and the actual post id in `post_id` (e.g. 360360); - # picking `id` first fragments every multi-image subscribestar post - # into N distinct Post rows in FC. Patreon/Pixiv have no `post_id` - # so `id` still wins for them; HF uses `index`, Discord uses - # `message_id` — all reached via the remaining chain entries. - # Operator-flagged 2026-05-27 during the sidecar audit. - external_post_id = None - for k in ("post_id", "id", "index", "message_id"): - v = data.get(k) - if isinstance(v, bool): - continue - if isinstance(v, (str, int)) and str(v).strip(): - external_post_id = str(v) - break + external_post_id = _first_id(data, external_post_id_keys_for(platform)) pc = data.get("page_count") if isinstance(pc, bool): @@ -146,30 +152,23 @@ def parse_sidecar(data: dict) -> SidecarData: if post_date is not None: break - # `message` is Discord gallery-dl's body field (no `content`); added - # 2026-05-27 to the description fallback chain. - description = _first_str( - data, ("content", "description", "caption", "message"), - ) + description = _first_str(data, description_keys_for(platform)) - # SubscribeStar posts always write `title: ""` and put the leading - # sentence inside `content` (confirmed against the operator's - # /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When - # no explicit title is present, synthesize one from the description - # body's first non-empty line. Patreon retains its explicit titles - # because they're non-empty and short-circuit the fallback. + # When `title` is empty (subscribestar always; sometimes elsewhere), + # synthesize from the description body's first non-empty text line. + # Patreon's explicit titles short-circuit the fallback. post_title = _first_str(data, ("title",)) if post_title is None and description: post_title = _first_line_text(description) - # post_url derivation: SubscribeStar/Pixiv/HF/Discord put the FILE - # download URL in `url`, not a post permalink. Synthesize the - # permalink from per-platform fields when possible. Patreon's `url` - # IS a permalink and is used as-is. For the four file-URL platforms, - # the bare `url` is NEVER trusted — derive or return None rather - # than persist a CDN URL in post.post_url. - if platform in _DERIVED_URL_PLATFORMS: - post_url = _derive_post_url(platform, data) + # post_url: ask the platform module to synthesize a permalink. + # When the platform registers a `derive_post_url`, it owns the + # field (the bare `url`/`post_url` value is a file CDN URL and + # must NEVER be persisted). When it doesn't register one, trust + # the sidecar's `url` (Patreon's case — real permalink). + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.derive_post_url is not None: + post_url = info.derive_post_url(data) else: post_url = _first_str(data, ("url", "post_url")) @@ -183,39 +182,3 @@ def parse_sidecar(data: dict) -> SidecarData: post_date=post_date, raw=data, ) - - -_DERIVED_URL_PLATFORMS = frozenset({ - "subscribestar", "pixiv", "hentaifoundry", "discord", -}) - - -def _derive_post_url(platform: str, data: dict) -> str | None: - """Synthesize the post-permalink URL from per-platform metadata. - - gallery-dl writes the file-download URL in `url` for these four - platforms; we need a real permalink for the PostCard "open original" - button. Returns None if the platform-specific fields are missing - (rare in well-formed sidecars but defensive). - """ - if platform == "subscribestar": - pid = data.get("post_id") - if isinstance(pid, (str, int)) and str(pid).strip(): - return f"https://www.subscribestar.com/posts/{pid}" - elif platform == "pixiv": - pid = data.get("id") - if isinstance(pid, (str, int)) and not isinstance(pid, bool) and str(pid).strip(): - return f"https://www.pixiv.net/artworks/{pid}" - elif platform == "hentaifoundry": - user = _first_str(data, ("user", "artist")) - idx = data.get("index") - if user and isinstance(idx, (str, int)) and not isinstance(idx, bool) and str(idx).strip(): - return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" - elif platform == "discord": - sid = data.get("server_id") - cid = data.get("channel_id") - mid = data.get("message_id") - if all(isinstance(v, (str, int)) and not isinstance(v, bool) and str(v).strip() - for v in (sid, cid, mid)): - return f"https://discord.com/channels/{sid}/{cid}/{mid}" - return None