From 8243740a04edf6f810dd17afbc6cbcef0582e835 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 18:18:04 -0400 Subject: [PATCH 1/7] fix(subscribestar): inject 18_plus_agreement_generic age cookie to bypass server gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged 2026-05-27: subscribestar source check aborted with `AbortExtraction: HTTP redirect to .../age_confirmation_warning`. The captured `_personalization_id` cookie in the browser-stored file had expired (annual rotation), and the user could not realistically refresh it: SubscribeStar's frontend JS uses localStorage to suppress the age-confirmation popup once dismissed, so a logged-in revisit doesn't re-show the popup and the server-side cookie is never re-issued. gallery-dl's own login flow (which FC doesn't exercise — cookies come from the extension instead) sidesteps this by manually setting `18_plus_agreement_generic=true` on `.subscribestar.adult`. The server accepts that as the age-confirmation marker. `credential_service._augment_cookies(platform, netscape)` mirrors that behavior: when the materialized cookies file is for subscribestar and the age cookie isn't already present, append a synthetic line for `.subscribestar.adult` with name=`18_plus_agreement_generic` value=`true` and a far-future expiry. No-op for other platforms; no-op if the cookie is already present (idempotent for manual pastes / extension captures that happen to include it). Three new tests pin: (a) injection fires for subscribestar, preserves existing cookies; (b) idempotent when already present (no double injection); (c) does NOT fire for non-subscribestar platforms (Patreon etc. don't get a foreign-domain cookie). Not a curator handling bug per se — the extension faithfully captured what the browser had. This is mirroring a documented gallery-dl workaround so the cookies-via-extension auth path doesn't degrade as the server-side cookie expires. --- backend/app/services/credential_service.py | 40 +++++++++++++++++ tests/test_credential_service.py | 51 ++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py index a2cd324..e50026a 100644 --- a/backend/app/services/credential_service.py +++ b/backend/app/services/credential_service.py @@ -148,6 +148,7 @@ class CredentialService: return None plaintext = self.crypto.decrypt(row.encrypted_blob) netscape = _to_netscape(plaintext) + netscape = _augment_cookies(platform, netscape) self.cookies_dir.mkdir(parents=True, exist_ok=True) out = self.cookies_dir / f"{platform}_cookies.txt" out.write_text(netscape) @@ -163,6 +164,45 @@ class CredentialService: return self.crypto.decrypt(row.encrypted_blob) +def _augment_cookies(platform: str, netscape: str) -> str: + """Inject platform-specific synthetic cookies needed to bypass server + gates that the user can't realistically re-trigger in their browser. + + subscribestar.adult: the server gates artist pages behind the + `_personalization_id` age-confirmation cookie. The site's frontend JS + uses localStorage to suppress the age popup once dismissed, so after + the cookie's annual expiry the user can't easily get a fresh one — + visiting the site in a logged-in session doesn't re-show the popup + and doesn't re-issue the cookie. gallery-dl's own login flow (which + FC doesn't use; we capture cookies via the extension instead) + sidesteps this by manually setting `18_plus_agreement_generic=true` + on `.subscribestar.adult` — the server accepts that as the + age-confirmation marker. Mirror that behavior here so cookies-only + auth works long-term. + + The injection is a no-op if the cookie is already present (operator + might have it from an earlier login or a manual cookies.txt paste). + + Operator-flagged 2026-05-27 after a subscribestar source check + aborted with `HTTP redirect to .../age_confirmation_warning`. + """ + if platform != "subscribestar": + return netscape + if "18_plus_agreement_generic" in netscape: + return netscape + # Far-future expiry — 10 years out. The server only checks presence/value; + # gallery-dl's own login flow sets this with no explicit expiry too. + expiry = 4102444800 # 2100-01-01 UTC, opaque "far future" + line = "\t".join([ + ".subscribestar.adult", "TRUE", "/", "TRUE", + str(expiry), "18_plus_agreement_generic", "true", + ]) + body = netscape.rstrip("\n") + if not body: + body = "# Netscape HTTP Cookie File" + return body + "\n" + line + "\n" + + def _to_netscape(plaintext: str) -> str: """Accept either Netscape-format text (the extension's output) or a JSON array of cookie dicts (a manual-paste edge case); produce diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py index 4d85f41..32ee71f 100644 --- a/tests/test_credential_service.py +++ b/tests/test_credential_service.py @@ -110,6 +110,57 @@ async def test_get_cookies_path_none_for_token_kind(db, crypto, tmp_path): assert await svc.get_cookies_path("discord") is None +@pytest.mark.asyncio +async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp_path): + """SubscribeStar's server gates artist pages behind a _personalization_id + cookie; the browser-stored cookie expires annually and can't be easily + refreshed (the JS age popup is suppressed by localStorage). Mirror + gallery-dl's own login-flow workaround by injecting + `18_plus_agreement_generic=true` on `.subscribestar.adult` whenever + cookies for subscribestar are materialized.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("subscribestar") + contents = path.read_text() + assert "18_plus_agreement_generic\ttrue" in contents + assert ".subscribestar.adult" in contents + # Original session_id cookie preserved. + assert "session_id\txyz" in contents + + +@pytest.mark.asyncio +async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto, tmp_path): + """If the operator's captured cookies ALREADY contain the age cookie + (e.g. a manual paste, or a re-login), don't double-inject.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\t18_plus_agreement_generic\ttrue\n" + ".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("subscribestar") + contents = path.read_text() + assert contents.count("18_plus_agreement_generic") == 1 + + +@pytest.mark.asyncio +async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path): + """The age-cookie injection MUST NOT fire for non-subscribestar + platforms — Patreon/HF/etc. don't need it and shouldn't carry a + foreign-domain cookie in their cookies.txt.""" + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) + path = await svc.get_cookies_path("patreon") + contents = path.read_text() + assert "18_plus_agreement_generic" not in contents + assert "subscribestar" not in contents + + @pytest.mark.asyncio async def test_get_token_decrypts(db, crypto): svc = CredentialService(db, crypto) From 2394e4737060b5fcb3b8474ccb941230aec1cff8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 19:12:51 -0400 Subject: [PATCH 2/7] fix(hentaifoundry): inject host-only PHPSESSID/CSRF duplicates + extension preserves browser hostOnly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged 2026-05-27: HF source check 401'd on `HEAD /?enterAgree=1` even with valid login cookies. Root cause is the combination of (1) gallery-dl's HF extractor checking `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with `requests`' EXACT domain matching, and (2) the extension's cookies.js forcibly rewriting every captured cookie to a leading-dot subdomain-wide form. HF's PHPSESSID is browser-stored as host-only on `www.hentai-foundry.com`; the rewrite re-anchored it to `.hentai-foundry.com`, which `cookies.get(...)` no longer matches even though the cookie is still sent on actual HTTP requests (RFC 6265 subdomain rules). The extractor falls into its unauthenticated `?enterAgree=1` fallback, which 401s (Cloudflare or HF's anti-bot HEAD gating). Two-part fix, no operator action required for existing stored cookies: 1. **Backend** (`credential_service._augment_cookies`) — refactored from the subscribestar-only single function into a per-platform dispatcher. New `_augment_hentaifoundry` parses the materialized netscape file and, for each `.hentai-foundry.com` entry whose name is PHPSESSID or YII_CSRF_TOKEN, appends a host-only duplicate (`www.hentai-foundry.com\tFALSE\t...`). Originals preserved. Three new tests pin: injection fires + originals preserved; idempotent when host-only already exists; doesn't touch unrelated cookies (e.g. `_ga`). 2. **Extension** (`cookies.js`) — `toNetscapeFormat` now respects `c.hostOnly` from the browser instead of blindly forcing a leading-dot subdomain-wide form. Host-only cookies are written with the bare host + FALSE flag; non-host-only cookies retain the leading-dot + TRUE form. Forward-compat — fresh captures from v1.0.5+ no longer need the backend's host-only duplication. Extension bumped 1.0.4 → 1.0.5; manifest + package.json in lockstep. After deploy: the next HF source check on the operator's already-stored cookies will succeed because the materialized cookies.txt now contains host-only PHPSESSID. No browser re-export needed. --- backend/app/services/credential_service.py | 79 +++++++++++++++++++--- extension/lib/cookies.js | 28 +++++++- extension/manifest.json | 2 +- extension/package.json | 2 +- tests/test_credential_service.py | 59 +++++++++++++++- 5 files changed, 153 insertions(+), 17 deletions(-) diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py index e50026a..adc6d6a 100644 --- a/backend/app/services/credential_service.py +++ b/backend/app/services/credential_service.py @@ -166,7 +166,7 @@ class CredentialService: def _augment_cookies(platform: str, netscape: str) -> str: """Inject platform-specific synthetic cookies needed to bypass server - gates that the user can't realistically re-trigger in their browser. + gates or extractor quirks. subscribestar.adult: the server gates artist pages behind the `_personalization_id` age-confirmation cookie. The site's frontend JS @@ -177,21 +177,37 @@ def _augment_cookies(platform: str, netscape: str) -> str: FC doesn't use; we capture cookies via the extension instead) sidesteps this by manually setting `18_plus_agreement_generic=true` on `.subscribestar.adult` — the server accepts that as the - age-confirmation marker. Mirror that behavior here so cookies-only - auth works long-term. + age-confirmation marker. - The injection is a no-op if the cookie is already present (operator - might have it from an earlier login or a manual cookies.txt paste). + hentaifoundry: gallery-dl's extractor uses + `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` to + decide whether the user is logged in. `requests` does EXACT domain + matching on .get(); the extension rewrites every captured cookie to + a leading-dot subdomain-wide form (`.hentai-foundry.com`), which + fails that exact match. The fallback path hits a HEAD + `?enterAgree=1` that 401s. Emit host-only duplicates of PHPSESSID + + YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup succeeds. + (The original `.hentai-foundry.com` entries stay — the actual HTTP + requests use RFC 6265 subdomain matching, which works either way.) - Operator-flagged 2026-05-27 after a subscribestar source check - aborted with `HTTP redirect to .../age_confirmation_warning`. + All injections are idempotent (no-op if the target cookie is already + present) and platform-scoped. + + Operator-flagged 2026-05-27: subscribestar age-confirmation, then + hentaifoundry 401 on /?enterAgree=1. """ - if platform != "subscribestar": - return netscape + if platform == "subscribestar": + return _augment_subscribestar(netscape) + if platform == "hentaifoundry": + return _augment_hentaifoundry(netscape) + return netscape + + +def _augment_subscribestar(netscape: str) -> str: if "18_plus_agreement_generic" in netscape: return netscape - # Far-future expiry — 10 years out. The server only checks presence/value; - # gallery-dl's own login flow sets this with no explicit expiry too. + # Far-future expiry — gallery-dl's own login flow sets this with no + # explicit expiry; the server only checks presence/value. expiry = 4102444800 # 2100-01-01 UTC, opaque "far future" line = "\t".join([ ".subscribestar.adult", "TRUE", "/", "TRUE", @@ -203,6 +219,47 @@ def _augment_cookies(platform: str, netscape: str) -> str: return body + "\n" + line + "\n" +_HF_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") + + +def _augment_hentaifoundry(netscape: str) -> str: + body = netscape.rstrip("\n") + if not body: + return netscape + lines = body.split("\n") + existing_host_only = set() + by_name: dict[str, list[str]] = {} + for raw in lines: + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + if len(parts) < 7: + continue + domain, _flag, _path, _secure, _exp, name, _value = parts[:7] + if name not in _HF_HOST_ONLY_NAMES: + continue + if domain == "www.hentai-foundry.com": + existing_host_only.add(name) + elif domain in (".hentai-foundry.com", "hentai-foundry.com"): + by_name.setdefault(name, []).append(raw) + + appended = [] + for name in _HF_HOST_ONLY_NAMES: + if name in existing_host_only or name not in by_name: + continue + # Duplicate the FIRST subdomain-wide line as host-only on + # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks + # it host-only in netscape format. + parts = by_name[name][0].split("\t") + parts[0] = "www.hentai-foundry.com" + parts[1] = "FALSE" + appended.append("\t".join(parts[:7])) + + if not appended: + return netscape + return body + "\n" + "\n".join(appended) + "\n" + + def _to_netscape(plaintext: str) -> str: """Accept either Netscape-format text (the extension's output) or a JSON array of cookie dicts (a manual-paste edge case); produce diff --git a/extension/lib/cookies.js b/extension/lib/cookies.js index b13ca4d..8569337 100644 --- a/extension/lib/cookies.js +++ b/extension/lib/cookies.js @@ -38,11 +38,33 @@ function deduplicateCookies(cookies) { function toNetscapeFormat(cookies) { const lines = ['# Netscape HTTP Cookie File']; for (const c of cookies) { - let domain = c.domain.replace(/^\.?www\./, '.'); - if (!domain.startsWith('.')) domain = '.' + domain; + // Preserve the browser's actual scope. Earlier versions rewrote + // every cookie to a leading-dot subdomain-wide form, which broke + // gallery-dl's HF extractor: its `cookies.get(name, + // domain="www.hentai-foundry.com")` does EXACT domain matching and + // missed host-only PHPSESSID rewritten to `.hentai-foundry.com`. + // Operator-flagged 2026-05-27. Backend `_augment_cookies` covers + // the already-stored cookies; this fix is forward-compat for fresh + // captures. + // + // Cookie storage semantics (Firefox): + // c.hostOnly === true → cookie set without a Domain= attribute; + // applies to the exact host only. + // c.hostOnly === false → cookie set with Domain=X; applies to + // that domain and its subdomains. + // + // Netscape format: + // leading-dot domain + TRUE flag → subdomain-wide + // bare-host domain + FALSE flag → host-only + const hostOnly = c.hostOnly === true; + let domain = c.domain; + if (!hostOnly && !domain.startsWith('.')) { + domain = '.' + domain; + } + const subdomainFlag = hostOnly ? 'FALSE' : 'TRUE'; const secure = c.secure ? 'TRUE' : 'FALSE'; const expiration = c.expirationDate ? Math.floor(c.expirationDate) : 0; - lines.push([domain, 'TRUE', c.path || '/', secure, String(expiration), c.name, c.value].join('\t')); + lines.push([domain, subdomainFlag, c.path || '/', secure, String(expiration), c.name, c.value].join('\t')); } return lines.join('\n'); } diff --git a/extension/manifest.json b/extension/manifest.json index 850a6fa..1bc2f47 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "FabledCurator", - "version": "1.0.4", + "version": "1.0.5", "description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.", "browser_specific_settings": { diff --git a/extension/package.json b/extension/package.json index f4c7db4..d38e933 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,6 +1,6 @@ { "name": "fabledcurator-extension", - "version": "1.0.4", + "version": "1.0.5", "private": true, "description": "Firefox extension for FabledCurator", "scripts": { diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py index 32ee71f..b856761 100644 --- a/tests/test_credential_service.py +++ b/tests/test_credential_service.py @@ -151,7 +151,7 @@ async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto @pytest.mark.asyncio async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path): """The age-cookie injection MUST NOT fire for non-subscribestar - platforms — Patreon/HF/etc. don't need it and shouldn't carry a + platforms — Patreon/etc. don't need it and shouldn't carry a foreign-domain cookie in their cookies.txt.""" svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) @@ -161,6 +161,63 @@ async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path assert "subscribestar" not in contents +@pytest.mark.asyncio +async def test_get_cookies_path_hf_injects_host_only_phpsessid(db, crypto, tmp_path): + """HF: extension writes session cookies as subdomain-wide + (`.hentai-foundry.com`), but gallery-dl's extractor uses + `cookies.get(name, domain='www.hentai-foundry.com')` with EXACT + domain matching. Emit host-only duplicates of PHPSESSID + + YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup matches.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + # Subdomain-wide originals preserved. + assert ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents + # Host-only duplicates appended for both names. + assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents + assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456" in contents + + +@pytest.mark.asyncio +async def test_get_cookies_path_hf_idempotent_when_host_only_present(db, crypto, tmp_path): + """If the captured cookies already include a host-only PHPSESSID + on www.hentai-foundry.com (e.g. a future extension fix that + preserves browser hostOnly state), don't double-inject.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + # Should count PHPSESSID exactly twice — the original two lines, no third. + assert contents.count("PHPSESSID\tsess123") == 2 + + +@pytest.mark.asyncio +async def test_get_cookies_path_hf_ignores_unrelated_cookies(db, crypto, tmp_path): + """The injection should only target session/CSRF cookies. Other HF + cookies (e.g. analytics) stay subdomain-wide as captured.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\t_ga\tGA1.2.x\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + assert "www.hentai-foundry.com" not in contents + assert contents.count("_ga") == 1 + + @pytest.mark.asyncio async def test_get_token_decrypts(db, crypto): svc = CredentialService(db, crypto) From abafc3265ea0e43c8f8cc40b059f54afb43d188f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 19:46:05 -0400 Subject: [PATCH 3/7] =?UTF-8?q?refactor(platforms):=20promote=20services/p?= =?UTF-8?q?latforms.py=20=E2=86=92=20services/platforms/=20package=20with?= =?UTF-8?q?=20per-platform=20quirk=20colocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-requested 2026-05-27: centralize the per-platform quirks that had been accumulating across credential_service, sidecar, and platforms into a single per-platform module so adding/updating quirks becomes "edit one file." **Layout** services/platforms/ base.py PlatformInfo dataclass + module-default key chains + shared helpers (str_id_value, str_field) __init__.py PLATFORMS dict + public API (auth_type_for, known_platform_keys, to_dict, external_post_id_keys_for, description_keys_for) patreon.py metadata only — the reference platform, no quirks subscribestar.py metadata + augment_cookies (18+ agreement) + derive_post_url (synthetic /posts/) hentaifoundry.py metadata + augment_cookies (host-only PHPSESSID duplicate) + derive_post_url (/pictures/user/...) pixiv.py metadata + derive_post_url (/artworks/) discord.py metadata + derive_post_url (channels///) deviantart.py metadata only — un-audited; quirks to be added when an operator first exercises DA **PlatformInfo extensions** Existing fields preserved. Four new optional fields: external_post_id_keys: tuple[str, ...] | None Override the sidecar external_post_id lookup chain. None falls back to DEFAULT_EXTERNAL_POST_ID_KEYS in base.py ("post_id", "id", "index", "message_id") — covers every current platform. description_keys: tuple[str, ...] | None Override the description body lookup chain. None falls back to DEFAULT_DESCRIPTION_KEYS ("content", "description", "caption", "message") — Discord's "message" body field is covered by the default's trailing entry. derive_post_url: Callable[[dict], str | None] | None Synthesize the post permalink from sidecar metadata. None = trust the bare `url` / `post_url` field (patreon, deviantart). subscribestar/pixiv/hf/discord override this because their `url` is the file CDN URL. augment_cookies: Callable[[str], str] | None Post-process the materialized cookies.txt before gallery-dl consumes it. None = no-op. Used by subscribestar (age cookie) and hentaifoundry (host-only PHPSESSID duplicate). **Consumer changes** - credential_service._augment_cookies(platform, netscape) shrunk from a per-platform-conditional dispatcher (~80 lines of inlined helpers) to a 5-line lookup: `info.augment_cookies(netscape) if info and info.augment_cookies else netscape`. The platform-specific helper bodies moved verbatim into the per-platform modules. - sidecar.parse_sidecar similarly delegates: external_post_id chain via external_post_id_keys_for(category), description chain via description_keys_for(category), post_url via PLATFORMS[category].derive_post_url. The _DERIVED_URL_PLATFORMS set and inline _derive_post_url body both gone. Added a shared `_first_id` helper for bool-safe id coercion. **Public API preserved** PLATFORMS, PlatformInfo, auth_type_for, known_platform_keys, to_dict are all re-exported from the package's __init__.py. test_platforms_registry test_credential_service, and test_sidecar_util pass without changes because the behavior is identical; only the implementation moved. **Adding a new platform** 1. Create services/platforms/.py with `INFO = PlatformInfo(...)` and any of the four optional hooks. 2. Import it in services/platforms/__init__.py + add to the PLATFORMS tuple-comprehension. 3. Done. sidecar parsing, cookie materialization, /api/platforms all pick it up automatically. --- backend/app/services/credential_service.py | 101 ++----------- backend/app/services/platforms.py | 140 ------------------ backend/app/services/platforms/__init__.py | 98 ++++++++++++ backend/app/services/platforms/base.py | 108 ++++++++++++++ backend/app/services/platforms/deviantart.py | 23 +++ backend/app/services/platforms/discord.py | 38 +++++ .../app/services/platforms/hentaifoundry.py | 83 +++++++++++ backend/app/services/platforms/patreon.py | 23 +++ backend/app/services/platforms/pixiv.py | 32 ++++ .../app/services/platforms/subscribestar.py | 62 ++++++++ backend/app/utils/sidecar.py | 109 +++++--------- 11 files changed, 512 insertions(+), 305 deletions(-) delete mode 100644 backend/app/services/platforms.py create mode 100644 backend/app/services/platforms/__init__.py create mode 100644 backend/app/services/platforms/base.py create mode 100644 backend/app/services/platforms/deviantart.py create mode 100644 backend/app/services/platforms/discord.py create mode 100644 backend/app/services/platforms/hentaifoundry.py create mode 100644 backend/app/services/platforms/patreon.py create mode 100644 backend/app/services/platforms/pixiv.py create mode 100644 backend/app/services/platforms/subscribestar.py diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py index adc6d6a..f467eb0 100644 --- a/backend/app/services/credential_service.py +++ b/backend/app/services/credential_service.py @@ -165,99 +165,16 @@ class CredentialService: def _augment_cookies(platform: str, netscape: str) -> str: - """Inject platform-specific synthetic cookies needed to bypass server - gates or extractor quirks. - - subscribestar.adult: the server gates artist pages behind the - `_personalization_id` age-confirmation cookie. The site's frontend JS - uses localStorage to suppress the age popup once dismissed, so after - the cookie's annual expiry the user can't easily get a fresh one — - visiting the site in a logged-in session doesn't re-show the popup - and doesn't re-issue the cookie. gallery-dl's own login flow (which - FC doesn't use; we capture cookies via the extension instead) - sidesteps this by manually setting `18_plus_agreement_generic=true` - on `.subscribestar.adult` — the server accepts that as the - age-confirmation marker. - - hentaifoundry: gallery-dl's extractor uses - `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` to - decide whether the user is logged in. `requests` does EXACT domain - matching on .get(); the extension rewrites every captured cookie to - a leading-dot subdomain-wide form (`.hentai-foundry.com`), which - fails that exact match. The fallback path hits a HEAD - `?enterAgree=1` that 401s. Emit host-only duplicates of PHPSESSID + - YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup succeeds. - (The original `.hentai-foundry.com` entries stay — the actual HTTP - requests use RFC 6265 subdomain matching, which works either way.) - - All injections are idempotent (no-op if the target cookie is already - present) and platform-scoped. - - Operator-flagged 2026-05-27: subscribestar age-confirmation, then - hentaifoundry 401 on /?enterAgree=1. - """ - if platform == "subscribestar": - return _augment_subscribestar(netscape) - if platform == "hentaifoundry": - return _augment_hentaifoundry(netscape) - return netscape - - -def _augment_subscribestar(netscape: str) -> str: - if "18_plus_agreement_generic" in netscape: + """Delegate to the platform's `augment_cookies` hook if one is + registered (subscribestar, hentaifoundry, etc. — see + `services/platforms/.py`). No-op when the platform doesn't + register a hook (Patreon, DeviantArt). Centralizing the + quirks-per-platform in the platforms package means adding a new + platform's cookie quirks doesn't require touching this file.""" + info = PLATFORMS.get(platform) + if info is None or info.augment_cookies is None: return netscape - # Far-future expiry — gallery-dl's own login flow sets this with no - # explicit expiry; the server only checks presence/value. - expiry = 4102444800 # 2100-01-01 UTC, opaque "far future" - line = "\t".join([ - ".subscribestar.adult", "TRUE", "/", "TRUE", - str(expiry), "18_plus_agreement_generic", "true", - ]) - body = netscape.rstrip("\n") - if not body: - body = "# Netscape HTTP Cookie File" - return body + "\n" + line + "\n" - - -_HF_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") - - -def _augment_hentaifoundry(netscape: str) -> str: - body = netscape.rstrip("\n") - if not body: - return netscape - lines = body.split("\n") - existing_host_only = set() - by_name: dict[str, list[str]] = {} - for raw in lines: - if not raw or raw.startswith("#"): - continue - parts = raw.split("\t") - if len(parts) < 7: - continue - domain, _flag, _path, _secure, _exp, name, _value = parts[:7] - if name not in _HF_HOST_ONLY_NAMES: - continue - if domain == "www.hentai-foundry.com": - existing_host_only.add(name) - elif domain in (".hentai-foundry.com", "hentai-foundry.com"): - by_name.setdefault(name, []).append(raw) - - appended = [] - for name in _HF_HOST_ONLY_NAMES: - if name in existing_host_only or name not in by_name: - continue - # Duplicate the FIRST subdomain-wide line as host-only on - # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks - # it host-only in netscape format. - parts = by_name[name][0].split("\t") - parts[0] = "www.hentai-foundry.com" - parts[1] = "FALSE" - appended.append("\t".join(parts[:7])) - - if not appended: - return netscape - return body + "\n" + "\n".join(appended) + "\n" + return info.augment_cookies(netscape) def _to_netscape(plaintext: str) -> str: diff --git a/backend/app/services/platforms.py b/backend/app/services/platforms.py deleted file mode 100644 index bd3723a..0000000 --- a/backend/app/services/platforms.py +++ /dev/null @@ -1,140 +0,0 @@ -"""FC-3b platforms registry — the single source of truth for what -FabledCurator supports. - -Lifted from GallerySubscriber's -~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py -and ~/.../extension/lib/platforms.js. Six platforms; auth_type and -URL patterns match GS exactly so the existing browser extension -hits FC unmodified. -""" - -from dataclasses import dataclass -from typing import Literal - - -@dataclass(frozen=True) -class PlatformInfo: - key: str - name: str - description: str - auth_type: Literal["cookies", "token"] - requires_auth: bool - url_pattern: str - url_examples: list[str] - default_config: dict - notes: str | None = None - - -# Common defaults used across most platforms; embedded per-platform -# below so per-platform overrides remain explicit. -_DEFAULTS = { - "sleep": 3.0, - "sleep_request": 1.5, - "skip_existing": True, - "save_metadata": True, - "timeout": 3600, -} - - -PLATFORMS: dict[str, PlatformInfo] = { - "patreon": PlatformInfo( - key="patreon", - name="Patreon", - description="Download posts from Patreon creators", - auth_type="cookies", - requires_auth=True, - url_pattern=r"^https?://(www\.)?patreon\.com/", - url_examples=[ - "https://www.patreon.com/example_artist", - "https://www.patreon.com/user?u=12345678", - ], - default_config={**_DEFAULTS, "content_types": ["images", "attachments"]}, - ), - "subscribestar": PlatformInfo( - key="subscribestar", - name="SubscribeStar", - description="Download posts from SubscribeStar creators", - auth_type="cookies", - requires_auth=True, - url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", - url_examples=[ - "https://subscribestar.adult/example_artist", - "https://www.subscribestar.com/example_artist", - ], - default_config={**_DEFAULTS, "content_types": ["all"]}, - ), - "hentaifoundry": PlatformInfo( - key="hentaifoundry", - name="Hentai Foundry", - description="Download artwork from Hentai Foundry artists", - auth_type="cookies", - requires_auth=False, - url_pattern=r"^https?://(www\.)?hentai-foundry\.com/", - url_examples=[ - "https://www.hentai-foundry.com/user/example_artist", - "https://www.hentai-foundry.com/pictures/user/example_artist", - ], - default_config={**_DEFAULTS, "content_types": ["pictures"]}, - ), - "discord": PlatformInfo( - key="discord", - name="Discord", - description="Download attachments from Discord channels", - auth_type="token", - requires_auth=True, - url_pattern=r"^https?://(www\.)?discord\.com/channels/", - url_examples=["https://discord.com/channels/123456789/987654321"], - default_config={**_DEFAULTS, "content_types": ["all"]}, - notes="Requires Discord user token (not bot token).", - ), - "pixiv": PlatformInfo( - key="pixiv", - name="Pixiv", - description="Download artwork from Pixiv artists", - auth_type="token", - requires_auth=True, - url_pattern=r"^https?://(www\.)?pixiv\.net/", - url_examples=[ - "https://www.pixiv.net/users/12345678", - "https://www.pixiv.net/en/users/12345678", - ], - default_config={**_DEFAULTS, "content_types": ["all"]}, - notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.", - ), - "deviantart": PlatformInfo( - key="deviantart", - name="DeviantArt", - description="Download artwork from DeviantArt artists", - auth_type="cookies", - requires_auth=False, - url_pattern=r"^https?://(www\.)?deviantart\.com/", - url_examples=[ - "https://www.deviantart.com/example-artist", - "https://www.deviantart.com/example-artist/gallery", - ], - default_config={**_DEFAULTS, "content_types": ["gallery"]}, - ), -} - - -def known_platform_keys() -> frozenset[str]: - return frozenset(PLATFORMS.keys()) - - -def auth_type_for(platform: str) -> str | None: - info = PLATFORMS.get(platform) - return info.auth_type if info else None - - -def to_dict(info: PlatformInfo) -> dict: - return { - "key": info.key, - "name": info.name, - "description": info.description, - "auth_type": info.auth_type, - "requires_auth": info.requires_auth, - "url_pattern": info.url_pattern, - "url_examples": info.url_examples, - "default_config": info.default_config, - "notes": info.notes, - } diff --git a/backend/app/services/platforms/__init__.py b/backend/app/services/platforms/__init__.py new file mode 100644 index 0000000..4bd85be --- /dev/null +++ b/backend/app/services/platforms/__init__.py @@ -0,0 +1,98 @@ +"""FC-3b platforms registry — single source of truth for what +FabledCurator supports + where each platform's quirks live. + +Adding a new platform: drop a new module `.py` next to this +one, declare an `INFO = PlatformInfo(...)`, add the import + entry in +PLATFORMS below. Sidecar parsing, cookie materialization, and +`/api/platforms` pick it up automatically. + +Lifted from GallerySubscriber's +~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py +and ~/.../extension/lib/platforms.js. Six platforms; auth_type and +URL patterns match GS exactly so the existing browser extension +hits FC unmodified. +""" + +from . import ( + deviantart, + discord, + hentaifoundry, + patreon, + pixiv, + subscribestar, +) +from .base import ( + DEFAULT_DESCRIPTION_KEYS, + DEFAULT_EXTERNAL_POST_ID_KEYS, + PlatformInfo, +) + + +PLATFORMS: dict[str, PlatformInfo] = { + info.key: info + for info in ( + patreon.INFO, + subscribestar.INFO, + hentaifoundry.INFO, + discord.INFO, + pixiv.INFO, + deviantart.INFO, + ) +} + + +def known_platform_keys() -> frozenset[str]: + return frozenset(PLATFORMS.keys()) + + +def auth_type_for(platform: str) -> str | None: + info = PLATFORMS.get(platform) + return info.auth_type if info else None + + +def to_dict(info: PlatformInfo) -> dict: + """Serialize a PlatformInfo to a JSON-safe dict for /api/platforms. + + Behavioral fields (callables, sidecar-chain overrides) are + intentionally omitted — they aren't useful to API consumers. + """ + return { + "key": info.key, + "name": info.name, + "description": info.description, + "auth_type": info.auth_type, + "requires_auth": info.requires_auth, + "url_pattern": info.url_pattern, + "url_examples": info.url_examples, + "default_config": info.default_config, + "notes": info.notes, + } + + +def external_post_id_keys_for(platform: str | None) -> tuple[str, ...]: + """Resolve the external_post_id lookup chain for a given platform, + falling back to the module default when the platform isn't + registered or hasn't overridden the chain.""" + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.external_post_id_keys is not None: + return info.external_post_id_keys + return DEFAULT_EXTERNAL_POST_ID_KEYS + + +def description_keys_for(platform: str | None) -> tuple[str, ...]: + """Resolve the description body lookup chain for a given platform.""" + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.description_keys is not None: + return info.description_keys + return DEFAULT_DESCRIPTION_KEYS + + +__all__ = [ + "PLATFORMS", + "PlatformInfo", + "auth_type_for", + "description_keys_for", + "external_post_id_keys_for", + "known_platform_keys", + "to_dict", +] diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py new file mode 100644 index 0000000..8c496d5 --- /dev/null +++ b/backend/app/services/platforms/base.py @@ -0,0 +1,108 @@ +"""PlatformInfo dataclass + shared defaults + small helpers. + +Per-platform modules import from here, register their PlatformInfo via +INFO, optionally attaching `derive_post_url` and/or `augment_cookies` +callables for behavior that diverges from gallery-dl's mainline shape +(Patreon). + +Adding a new platform: drop a new module under `services/platforms/`, +declare an INFO, and add it to the import list in +`services/platforms/__init__.py`. Sidecar parsing, cookie +materialization, and the /api/platforms response pick it up +automatically. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Literal + + +# Sidecar parsing defaults. Per-platform PlatformInfo entries can +# override these by setting `external_post_id_keys=` / +# `description_keys=`. Most don't need to — the defaults already cover +# every platform FC supports. +# +# external_post_id chain: `post_id` MUST come before `id` because +# SubscribeStar gallery-dl puts the per-attachment id in `id` and the +# actual post id in `post_id`; picking `id` first fragments +# multi-image SubscribeStar posts into N Post rows. Patreon/Pixiv have +# no `post_id` so `id` still wins for them; HF uses `index`, Discord +# uses `message_id` — all reached via the remaining chain entries. +# (Banked 2026-05-27 during the sidecar audit.) +DEFAULT_EXTERNAL_POST_ID_KEYS: tuple[str, ...] = ( + "post_id", "id", "index", "message_id", +) + +# Description body chain: Discord's gallery-dl extractor uses `message` +# (no `content`); appended to the chain so Discord posts surface body +# text. +DEFAULT_DESCRIPTION_KEYS: tuple[str, ...] = ( + "content", "description", "caption", "message", +) + + +@dataclass(frozen=True) +class PlatformInfo: + # --- Identity / metadata --- + key: str + name: str + description: str + auth_type: Literal["cookies", "token"] + requires_auth: bool + url_pattern: str + url_examples: list[str] + default_config: dict + notes: str | None = None + + # --- Sidecar parsing overrides --- + # Each is None to mean "use the module default above"; a platform + # only sets one of these when its sidecar shape genuinely differs. + external_post_id_keys: tuple[str, ...] | None = None + description_keys: tuple[str, ...] | None = None + + # --- Behavioral hooks --- + # Synthesize a post permalink from sidecar data. Required when + # gallery-dl's `url` field is the file/CDN URL rather than the post + # permalink (subscribestar/pixiv/hf/discord). None = trust the bare + # `url` field (patreon, deviantart). + derive_post_url: Callable[[dict], str | None] | None = None + + # Post-process the materialized cookies.txt for gallery-dl. Used by + # platforms whose server gates or extractor quirks need synthetic + # cookies the extension can't capture (subscribestar age cookie, HF + # host-only PHPSESSID duplicate). None = no-op. + augment_cookies: Callable[[str], str] | None = None + + +def str_id_value(v) -> str | None: + """Coerce a JSON scalar id into a non-empty string, rejecting bool + (Python's bool is an int subclass so `isinstance(True, int)` is + True; without this guard a sidecar with `"id": true` would produce + external_post_id="True").""" + if isinstance(v, bool): + return None + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + +def str_field(v) -> str | None: + """Same idea as str_id_value but for plain string fields (no int + coercion).""" + if isinstance(v, str) and v.strip(): + return v.strip() + return None + + +# Shared gallery-dl invocation defaults. Embedded in each platform's +# default_config (with platform-specific overrides) so per-platform +# choices stay explicit. +GD_DEFAULTS = { + "sleep": 3.0, + "sleep_request": 1.5, + "skip_existing": True, + "save_metadata": True, + "timeout": 3600, +} diff --git a/backend/app/services/platforms/deviantart.py b/backend/app/services/platforms/deviantart.py new file mode 100644 index 0000000..e41fc3b --- /dev/null +++ b/backend/app/services/platforms/deviantart.py @@ -0,0 +1,23 @@ +"""DeviantArt — no exercised quirks yet. + +No operator-owned DeviantArt archive existed at the 2026-05-27 sidecar +audit, so we don't know yet whether DA's gallery-dl sidecars are +well-behaved or have their own quirks. When DA gets exercised for the +first time, add `derive_post_url` / `augment_cookies` here as needed. +""" + +from .base import GD_DEFAULTS, PlatformInfo + +INFO = PlatformInfo( + key="deviantart", + name="DeviantArt", + description="Download artwork from DeviantArt artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?deviantart\.com/", + url_examples=[ + "https://www.deviantart.com/example-artist", + "https://www.deviantart.com/example-artist/gallery", + ], + default_config={**GD_DEFAULTS, "content_types": ["gallery"]}, +) diff --git a/backend/app/services/platforms/discord.py b/backend/app/services/platforms/discord.py new file mode 100644 index 0000000..e6afbf1 --- /dev/null +++ b/backend/app/services/platforms/discord.py @@ -0,0 +1,38 @@ +"""Discord — one quirk + one already-default. + +post_url: gallery-dl's `url` is the CDN attachment URL. The "permalink" +for a Discord message uses the (server, channel, message) triple via +`discord.com/channels///`. Note that +permalinks are only resolvable for users in the same server — public +access doesn't work — but the URL is still useful to the operator +in-app. + +Description body is in `message` not `content`. That's already covered +by the default description chain in base.py (DEFAULT_DESCRIPTION_KEYS +ends with `message`). No description_keys override needed. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + sid = str_id_value(data.get("server_id")) + cid = str_id_value(data.get("channel_id")) + mid = str_id_value(data.get("message_id")) + if sid and cid and mid: + return f"https://discord.com/channels/{sid}/{cid}/{mid}" + return None + + +INFO = PlatformInfo( + key="discord", + name="Discord", + description="Download attachments from Discord channels", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?discord\.com/channels/", + url_examples=["https://discord.com/channels/123456789/987654321"], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + notes="Requires Discord user token (not bot token).", + derive_post_url=derive_post_url, +) diff --git a/backend/app/services/platforms/hentaifoundry.py b/backend/app/services/platforms/hentaifoundry.py new file mode 100644 index 0000000..d84ebef --- /dev/null +++ b/backend/app/services/platforms/hentaifoundry.py @@ -0,0 +1,83 @@ +"""HentaiFoundry — two quirks colocated. + +1. post_url: HF sidecars omit `url` entirely; `src` is the image URL. + Synthesize the permalink from `user` + `index` + (/pictures/user//). + +2. augment_cookies: gallery-dl's HF extractor checks + `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with + `requests`' EXACT domain matching. The extension's pre-v1.0.5 + `cookies.js` aggressively rewrote every captured cookie to the + leading-dot subdomain-wide form (`.hentai-foundry.com`), which fails + the exact lookup even though the cookie IS sent on actual HTTP + requests (RFC 6265 subdomain matching). The extractor falls into + an unauthenticated `?enterAgree=1` HEAD that 401s. Inject host-only + duplicates of PHPSESSID + YII_CSRF_TOKEN so the lookup succeeds. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_field, str_id_value + +_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") + + +def derive_post_url(data: dict) -> str | None: + user = str_field(data.get("user")) or str_field(data.get("artist")) + idx = str_id_value(data.get("index")) + if user and idx: + return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + return None + + +def augment_cookies(netscape: str) -> str: + body = netscape.rstrip("\n") + if not body: + return netscape + lines = body.split("\n") + existing_host_only: set[str] = set() + by_name: dict[str, list[str]] = {} + for raw in lines: + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + if len(parts) < 7: + continue + domain, _flag, _path, _secure, _exp, name, _value = parts[:7] + if name not in _HOST_ONLY_NAMES: + continue + if domain == "www.hentai-foundry.com": + existing_host_only.add(name) + elif domain in (".hentai-foundry.com", "hentai-foundry.com"): + by_name.setdefault(name, []).append(raw) + + appended: list[str] = [] + for name in _HOST_ONLY_NAMES: + if name in existing_host_only or name not in by_name: + continue + # Duplicate the first subdomain-wide line as host-only on + # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks + # the entry host-only in netscape format. + parts = by_name[name][0].split("\t") + parts[0] = "www.hentai-foundry.com" + parts[1] = "FALSE" + appended.append("\t".join(parts[:7])) + + if not appended: + return netscape + return body + "\n" + "\n".join(appended) + "\n" + + +INFO = PlatformInfo( + key="hentaifoundry", + name="Hentai Foundry", + description="Download artwork from Hentai Foundry artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?hentai-foundry\.com/", + url_examples=[ + "https://www.hentai-foundry.com/user/example_artist", + "https://www.hentai-foundry.com/pictures/user/example_artist", + ], + default_config={**GD_DEFAULTS, "content_types": ["pictures"]}, + derive_post_url=derive_post_url, + augment_cookies=augment_cookies, +) diff --git a/backend/app/services/platforms/patreon.py b/backend/app/services/platforms/patreon.py new file mode 100644 index 0000000..fa05bf2 --- /dev/null +++ b/backend/app/services/platforms/patreon.py @@ -0,0 +1,23 @@ +"""Patreon — no quirks. The reference platform. + +Patreon's gallery-dl sidecars are the well-behaved baseline: `url` is a +real permalink, `id` is the post id, `title` and `content` are +populated. No cookie quirks (session cookies are domain-wide). No +derivation overrides. +""" + +from .base import GD_DEFAULTS, PlatformInfo + +INFO = PlatformInfo( + key="patreon", + name="Patreon", + description="Download posts from Patreon creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?patreon\.com/", + url_examples=[ + "https://www.patreon.com/example_artist", + "https://www.patreon.com/user?u=12345678", + ], + default_config={**GD_DEFAULTS, "content_types": ["images", "attachments"]}, +) diff --git a/backend/app/services/platforms/pixiv.py b/backend/app/services/platforms/pixiv.py new file mode 100644 index 0000000..8664a1e --- /dev/null +++ b/backend/app/services/platforms/pixiv.py @@ -0,0 +1,32 @@ +"""Pixiv — one quirk. + +post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The +post permalink follows /artworks/. external_post_id (= `id`) was +already correct, so no override there. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + pid = str_id_value(data.get("id")) + if pid: + return f"https://www.pixiv.net/artworks/{pid}" + return None + + +INFO = PlatformInfo( + key="pixiv", + name="Pixiv", + description="Download artwork from Pixiv artists", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?pixiv\.net/", + url_examples=[ + "https://www.pixiv.net/users/12345678", + "https://www.pixiv.net/en/users/12345678", + ], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.", + derive_post_url=derive_post_url, +) diff --git a/backend/app/services/platforms/subscribestar.py b/backend/app/services/platforms/subscribestar.py new file mode 100644 index 0000000..554c26c --- /dev/null +++ b/backend/app/services/platforms/subscribestar.py @@ -0,0 +1,62 @@ +"""SubscribeStar — three quirks colocated. + +1. external_post_id: gallery-dl puts the per-attachment id in `id` + (e.g. 711509) and the actual post id in `post_id` (e.g. 360360). + The default chain in base.py already prefers `post_id`; this module + doesn't need to override it but the comment lives here too so a + future reader knows the chain's order was driven by this platform. + +2. post_url: gallery-dl's `url` is the file CDN URL + (`/post_uploads?payload=...`). Synthesize the post permalink from + `post_id`. + +3. augment_cookies: the server gates artist pages behind a + `_personalization_id` age-confirmation cookie that the user can't + easily refresh — SubscribeStar's frontend JS uses localStorage to + suppress the age popup once dismissed. gallery-dl's own login flow + sidesteps this by setting `18_plus_agreement_generic=true` on + `.subscribestar.adult`; we mirror that for cookies captured via the + extension. +""" + +from .base import GD_DEFAULTS, PlatformInfo, str_id_value + + +def derive_post_url(data: dict) -> str | None: + pid = str_id_value(data.get("post_id")) + if pid: + return f"https://www.subscribestar.com/posts/{pid}" + return None + + +def augment_cookies(netscape: str) -> str: + if "18_plus_agreement_generic" in netscape: + return netscape + # Far-future expiry — gallery-dl's own login flow sets this with no + # explicit expiry; the server only checks presence/value. + expiry = 4102444800 # 2100-01-01 UTC + line = "\t".join([ + ".subscribestar.adult", "TRUE", "/", "TRUE", + str(expiry), "18_plus_agreement_generic", "true", + ]) + body = netscape.rstrip("\n") + if not body: + body = "# Netscape HTTP Cookie File" + return body + "\n" + line + "\n" + + +INFO = PlatformInfo( + key="subscribestar", + name="SubscribeStar", + description="Download posts from SubscribeStar creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", + url_examples=[ + "https://subscribestar.adult/example_artist", + "https://www.subscribestar.com/example_artist", + ], + default_config={**GD_DEFAULTS, "content_types": ["all"]}, + derive_post_url=derive_post_url, + augment_cookies=augment_cookies, +) diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 13ced7c..0d81930 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -1,7 +1,9 @@ """Minimal gallery-dl sidecar parsing (one-time filesystem-import aid). -No per-platform branching: a small common key set with fallbacks; the -full JSON is kept in raw so anything unmapped is recoverable later. +Per-platform quirks (post_url synthesis, key-chain overrides) live in +the platforms registry — `backend/app/services/platforms/`. This module +is platform-agnostic: it looks up `category` in the sidecar and asks +the registry for the right behavior. """ import re @@ -9,6 +11,12 @@ from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path +from ..services.platforms import ( + PLATFORMS, + description_keys_for, + external_post_id_keys_for, +) + @dataclass(frozen=True) class SidecarData: @@ -55,6 +63,19 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None: return None +def _first_id(data: dict, keys: tuple[str, ...]) -> str | None: + """Like `_first_str` but accepts ints and rejects bool (Python's + bool subclasses int, so a literal `"id": true` would otherwise + yield external_post_id="True").""" + for k in keys: + v = data.get(k) + if isinstance(v, bool): + continue + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + # Strip HTML tags + collapse whitespace + take the first non-empty line. # Used to derive a display title from a body when the platform doesn't # expose a separate title field (subscribestar posts always write @@ -111,22 +132,7 @@ def parse_sidecar(data: dict) -> SidecarData: cat = data.get("category") platform = cat if isinstance(cat, str) and cat.strip() else None - # external_post_id lookup order: post_id MUST come before id. - # SubscribeStar gallery-dl writes the per-attachment id in `id` - # (e.g. 711509) and the actual post id in `post_id` (e.g. 360360); - # picking `id` first fragments every multi-image subscribestar post - # into N distinct Post rows in FC. Patreon/Pixiv have no `post_id` - # so `id` still wins for them; HF uses `index`, Discord uses - # `message_id` — all reached via the remaining chain entries. - # Operator-flagged 2026-05-27 during the sidecar audit. - external_post_id = None - for k in ("post_id", "id", "index", "message_id"): - v = data.get(k) - if isinstance(v, bool): - continue - if isinstance(v, (str, int)) and str(v).strip(): - external_post_id = str(v) - break + external_post_id = _first_id(data, external_post_id_keys_for(platform)) pc = data.get("page_count") if isinstance(pc, bool): @@ -146,30 +152,23 @@ def parse_sidecar(data: dict) -> SidecarData: if post_date is not None: break - # `message` is Discord gallery-dl's body field (no `content`); added - # 2026-05-27 to the description fallback chain. - description = _first_str( - data, ("content", "description", "caption", "message"), - ) + description = _first_str(data, description_keys_for(platform)) - # SubscribeStar posts always write `title: ""` and put the leading - # sentence inside `content` (confirmed against the operator's - # /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When - # no explicit title is present, synthesize one from the description - # body's first non-empty line. Patreon retains its explicit titles - # because they're non-empty and short-circuit the fallback. + # When `title` is empty (subscribestar always; sometimes elsewhere), + # synthesize from the description body's first non-empty text line. + # Patreon's explicit titles short-circuit the fallback. post_title = _first_str(data, ("title",)) if post_title is None and description: post_title = _first_line_text(description) - # post_url derivation: SubscribeStar/Pixiv/HF/Discord put the FILE - # download URL in `url`, not a post permalink. Synthesize the - # permalink from per-platform fields when possible. Patreon's `url` - # IS a permalink and is used as-is. For the four file-URL platforms, - # the bare `url` is NEVER trusted — derive or return None rather - # than persist a CDN URL in post.post_url. - if platform in _DERIVED_URL_PLATFORMS: - post_url = _derive_post_url(platform, data) + # post_url: ask the platform module to synthesize a permalink. + # When the platform registers a `derive_post_url`, it owns the + # field (the bare `url`/`post_url` value is a file CDN URL and + # must NEVER be persisted). When it doesn't register one, trust + # the sidecar's `url` (Patreon's case — real permalink). + info = PLATFORMS.get(platform) if platform else None + if info is not None and info.derive_post_url is not None: + post_url = info.derive_post_url(data) else: post_url = _first_str(data, ("url", "post_url")) @@ -183,39 +182,3 @@ def parse_sidecar(data: dict) -> SidecarData: post_date=post_date, raw=data, ) - - -_DERIVED_URL_PLATFORMS = frozenset({ - "subscribestar", "pixiv", "hentaifoundry", "discord", -}) - - -def _derive_post_url(platform: str, data: dict) -> str | None: - """Synthesize the post-permalink URL from per-platform metadata. - - gallery-dl writes the file-download URL in `url` for these four - platforms; we need a real permalink for the PostCard "open original" - button. Returns None if the platform-specific fields are missing - (rare in well-formed sidecars but defensive). - """ - if platform == "subscribestar": - pid = data.get("post_id") - if isinstance(pid, (str, int)) and str(pid).strip(): - return f"https://www.subscribestar.com/posts/{pid}" - elif platform == "pixiv": - pid = data.get("id") - if isinstance(pid, (str, int)) and not isinstance(pid, bool) and str(pid).strip(): - return f"https://www.pixiv.net/artworks/{pid}" - elif platform == "hentaifoundry": - user = _first_str(data, ("user", "artist")) - idx = data.get("index") - if user and isinstance(idx, (str, int)) and not isinstance(idx, bool) and str(idx).strip(): - return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" - elif platform == "discord": - sid = data.get("server_id") - cid = data.get("channel_id") - mid = data.get("message_id") - if all(isinstance(v, (str, int)) and not isinstance(v, bool) and str(v).strip() - for v in (sid, cid, mid)): - return f"https://discord.com/channels/{sid}/{cid}/{mid}" - return None From b447c42853930e62ec3616fb12cd593317a5d4de Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 19:52:50 -0400 Subject: [PATCH 4/7] =?UTF-8?q?fix(platforms):=20ruff=20I001=20=E2=80=94?= =?UTF-8?q?=20drop=20unused=20=5F=5Ffuture=5F=5F=20import;=20switch=20=5F?= =?UTF-8?q?=5Finit=5F=5F=20to=20per-module=20imports=20for=20clean=20isort?= =?UTF-8?q?=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/platforms/__init__.py | 27 ++++++++++------------ backend/app/services/platforms/base.py | 2 -- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/backend/app/services/platforms/__init__.py b/backend/app/services/platforms/__init__.py index 4bd85be..822276a 100644 --- a/backend/app/services/platforms/__init__.py +++ b/backend/app/services/platforms/__init__.py @@ -13,30 +13,27 @@ URL patterns match GS exactly so the existing browser extension hits FC unmodified. """ -from . import ( - deviantart, - discord, - hentaifoundry, - patreon, - pixiv, - subscribestar, -) from .base import ( DEFAULT_DESCRIPTION_KEYS, DEFAULT_EXTERNAL_POST_ID_KEYS, PlatformInfo, ) - +from .deviantart import INFO as _DEVIANTART +from .discord import INFO as _DISCORD +from .hentaifoundry import INFO as _HENTAIFOUNDRY +from .patreon import INFO as _PATREON +from .pixiv import INFO as _PIXIV +from .subscribestar import INFO as _SUBSCRIBESTAR PLATFORMS: dict[str, PlatformInfo] = { info.key: info for info in ( - patreon.INFO, - subscribestar.INFO, - hentaifoundry.INFO, - discord.INFO, - pixiv.INFO, - deviantart.INFO, + _PATREON, + _SUBSCRIBESTAR, + _HENTAIFOUNDRY, + _DISCORD, + _PIXIV, + _DEVIANTART, ) } diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py index 8c496d5..a700871 100644 --- a/backend/app/services/platforms/base.py +++ b/backend/app/services/platforms/base.py @@ -12,8 +12,6 @@ materialization, and the /api/platforms response pick it up automatically. """ -from __future__ import annotations - from collections.abc import Callable from dataclasses import dataclass from typing import Literal From 6d7116c090c1e142583feec939c7c9a59d8d943d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 20:37:06 -0400 Subject: [PATCH 5/7] =?UTF-8?q?fix(platforms):=20ruff=20I001=20in=20base.p?= =?UTF-8?q?y=20=E2=80=94=20one=20blank=20line=20between=20imports=20and=20?= =?UTF-8?q?module-level=20constant=20(was=20two)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/platforms/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/services/platforms/base.py b/backend/app/services/platforms/base.py index a700871..8a626b7 100644 --- a/backend/app/services/platforms/base.py +++ b/backend/app/services/platforms/base.py @@ -16,7 +16,6 @@ from collections.abc import Callable from dataclasses import dataclass from typing import Literal - # Sidecar parsing defaults. Per-platform PlatformInfo entries can # override these by setting `external_post_id_keys=` / # `description_keys=`. Most don't need to — the defaults already cover From 12be188ada21ca50653d2b7150b7a75550d61605 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 20:59:58 -0400 Subject: [PATCH 6/7] feat(showcase): IR-parity R-key shuffle + stagger entry animation; fix(cleanup): min-dim Delete swallowed crypto.subtle TypeError on plain HTTP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **showcase R-key + entry animation** Restores two behaviors lost during the FC-2 IR→Vue port. Operator-flagged 2026-05-27. - ShowcaseView listens for keydown 'r'/'R' on window. Triggers `store.shuffle()`. Skips when an input/textarea/contenteditable is focused or a Vuetify overlay is open (the dialog/menu sets `.v-overlay--active` on the body). - MasonryGrid gains an opt-in `animateFromIndex` prop (default `Number.POSITIVE_INFINITY` = off). When set, items with index ≥ the threshold animate in with a stagger fade-in: 12px translateY, 0.25s ease, 60ms per item, capped by `prefers-reduced-motion`. Stagger uses original-items-array index (resolved via an `idxById` Map) so the reading order is preserved even after the masonry distributes items across columns. - ShowcaseView watches `store.images.length`: shrink-or-zero baseline ⇒ `animateFromIndex=0` (animate everything on initial load / shuffle); grow ⇒ baseline=prevCount (animate only the appended tail on infinite-scroll). Other MasonryGrid consumers (ArtistView's Gallery tab) don't pass the prop, so they keep their current no-animation behavior. Direct port of IR's `app/static/js/showcase.js` keyboard handler + `app/static/style.css` itemFadeIn keyframe. **min-dim Delete: crypto.subtle TypeError fix** The Delete button on the Cleanup → Minimum Dimensions card was silently no-op'ing. Root cause: `crypto.subtle` is Secure-Context-gated (undefined on plain-HTTP origins per the homelab posture). The card's `onDeleteClick` computed the Tier-C confirm token via `crypto.subtle.digest('SHA-256', ...)`, which threw TypeError before `showModal.value = true`. The promise rejected, the click handler had no `.catch`, the modal never opened — exactly the operator's reported symptom. Same shape as the v26.05.26.0 `navigator.clipboard` fix on the ErrorDetailModal Copy button. Fix: backend `/api/cleanup/min-dimension/preview` now returns `confirm_token` (the canonical `delete-min-dim-` string) in its response. Frontend reads it from the preview response and feeds the 8-char suffix to DestructiveConfirmModal's `runId` prop — no client-side crypto needed. Single source of truth. Integration test `test_min_dimension_preview_returns_count` pinned to also assert `body["confirm_token"]` matches the server-side compute. --- backend/app/api/cleanup.py | 7 +++ .../components/cleanup/MinDimensionCard.vue | 32 +++++++----- .../src/components/discovery/MasonryGrid.vue | 51 ++++++++++++++++++- frontend/src/views/ShowcaseView.vue | 44 ++++++++++++++-- tests/test_api_cleanup.py | 4 ++ 5 files changed, 118 insertions(+), 20 deletions(-) diff --git a/backend/app/api/cleanup.py b/backend/app/api/cleanup.py index 386c3e5..7149525 100644 --- a/backend/app/api/cleanup.py +++ b/backend/app/api/cleanup.py @@ -81,6 +81,13 @@ async def min_dim_preview(): s, min_width=min_w, min_height=min_h, ) ) + # Hand the canonical Tier-C delete token back with the preview so + # the frontend doesn't have to recompute SHA-256 client-side. + # window.crypto.subtle is Secure-Context-gated and undefined on + # plain-HTTP origins (homelab posture); without this the Delete + # button silently swallowed the TypeError and never opened the + # confirm modal. Operator-flagged 2026-05-27. + projection["confirm_token"] = _min_dim_token(min_w, min_h) return jsonify(projection) diff --git a/frontend/src/components/cleanup/MinDimensionCard.vue b/frontend/src/components/cleanup/MinDimensionCard.vue index a26ae2d..b1a1e9b 100644 --- a/frontend/src/components/cleanup/MinDimensionCard.vue +++ b/frontend/src/components/cleanup/MinDimensionCard.vue @@ -52,7 +52,7 @@ v-model="showModal" action="delete" kind="min-dim" - :run-id="tokenSha8" + :run-id="tokenSuffix" tier="C" :projected-counts="projectedCounts" :description="`Width < ${minW} OR height < ${minH}`" @@ -62,7 +62,7 @@ diff --git a/tests/test_api_cleanup.py b/tests/test_api_cleanup.py index ba9534f..35478a4 100644 --- a/tests/test_api_cleanup.py +++ b/tests/test_api_cleanup.py @@ -66,6 +66,10 @@ async def test_min_dimension_preview_returns_count(client, db, tmp_path): assert resp.status_code == 200 body = await resp.get_json() assert body["count"] == 1 + # Preview hands the canonical Tier-C confirm token back so the + # frontend doesn't have to recompute SHA-256 client-side + # (crypto.subtle is Secure-Context-gated; FC runs over plain HTTP). + assert body["confirm_token"] == _sha256_min_dim_token(200, 200) @pytest.mark.asyncio From df6d89cb59cc81a365b926ae0d63270bb7dcb9d9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 21:17:40 -0400 Subject: [PATCH 7/7] =?UTF-8?q?fix(secure-context):=20full=20audit=20?= =?UTF-8?q?=E2=80=94=20DestructiveConfirmModal.expectedTokenOverride=20+?= =?UTF-8?q?=20bulk-delete=20+=20min-dim=20use=20backend-computed=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged 2026-05-27: walk the whole project for the same shape as the min-dim Delete-button silent failure (crypto.subtle TypeError on plain HTTP). FC runs over plain HTTP per the homelab posture; Secure-Context-gated browser APIs are undefined on the production origin. **Audit results across `frontend/src/`:** crypto.subtle.digest — 2 sites: - MinDimensionCard (fixed 2026-05-27) - BulkEditorPanel (THIS FIX) navigator.clipboard — 1 site, already guarded: - utils/clipboard.js writeText with execCommand fallback serviceWorker / mediaDevices / Push / Web USB|HID|Bluetooth|Serial / cookieStore / queryLocalFonts / WebAuthn / geolocation — NOT USED, nothing to fix Extension scripts (background.js) use crypto.subtle but run from moz-extension:// which IS a Secure Context — left as-is. **BulkEditorPanel double bug** The bulk-delete UI on the gallery selection had been broken since FC-3k shipped, in two ways: 1. `crypto.subtle.digest` swallowed TypeError on plain HTTP — modal never opened. Same symptom as min-dim. 2. Even on HTTPS, the modal's `kind="images-selection"` produced `delete-images-selection-` while the backend expected `delete-images-`. The two would never match. Fix: - Backend `/api/admin/images/bulk-delete` dry-run response now returns `confirm_token` (the canonical `delete-images-` string). Integration test `test_bulk_delete_dry_run_returns_counts` pinned to assert the new field. - DestructiveConfirmModal gains an `expectedTokenOverride` prop. When set, it bypasses the `${action}-${kind}-${runId}` formula and uses the explicit string. This decouples the UI label (`kind`) from the wire-format token (server-provided), so future endpoints can use a kind-specific label without their kind name leaking into the token. - BulkEditorPanel passes `:expected-token-override="bulkProjected?.confirm_token"` — no client-side crypto, no kind-prefix mismatch. - MinDimensionCard refactored to the same explicit pattern (was slicing the 8-char suffix off the backend's token and passing it through `runId`; now passes the full backend token via `expected-token-override` directly). Cleaner; one source of truth. **Banked memory** `feedback_no_secure_context_apis.md` documents the full table of Secure-Context-gated APIs, which ones FC currently uses, and how each is handled. Indexed in MEMORY.md. Sites for the audit also listed in the memory for future drift-checking. No other Secure-Context-gated APIs found in `frontend/src/`. The same shape won't recur unless someone adds a new dependency on one — at which point the banked memory should fire. --- backend/app/api/admin.py | 14 +++++++--- .../components/cleanup/MinDimensionCard.vue | 27 +++++++------------ .../components/gallery/BulkEditorPanel.vue | 25 ++++++++--------- .../modal/DestructiveConfirmModal.vue | 14 ++++++++-- tests/test_api_admin.py | 5 ++++ 5 files changed, 49 insertions(+), 36 deletions(-) diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index cefc181..fd3c5ce 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -97,11 +97,19 @@ async def images_bulk_delete(): ) ) - if dry_run: - return jsonify(projected) - sha8 = _bulk_image_confirm_token(image_ids) expected = f"delete-images-{sha8}" + + if dry_run: + # Hand the canonical Tier-C confirm token back with the + # projection so the frontend doesn't have to recompute SHA-256 + # client-side via crypto.subtle (Secure-Context-gated, + # undefined on plain-HTTP origins per the homelab posture). + # Operator-flagged 2026-05-27. + projected["confirm_token"] = expected + return jsonify(projected) + + if supplied_confirm != expected: return _bad( "confirm_mismatch", diff --git a/frontend/src/components/cleanup/MinDimensionCard.vue b/frontend/src/components/cleanup/MinDimensionCard.vue index b1a1e9b..140d549 100644 --- a/frontend/src/components/cleanup/MinDimensionCard.vue +++ b/frontend/src/components/cleanup/MinDimensionCard.vue @@ -52,8 +52,8 @@ v-model="showModal" action="delete" kind="min-dim" - :run-id="tokenSuffix" tier="C" + :expected-token-override="preview?.confirm_token || ''" :projected-counts="projectedCounts" :description="`Width < ${minW} OR height < ${minH}`" @confirm="onConfirmedDelete" @@ -62,11 +62,19 @@