From 2394e4737060b5fcb3b8474ccb941230aec1cff8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 19:12:51 -0400 Subject: [PATCH] fix(hentaifoundry): inject host-only PHPSESSID/CSRF duplicates + extension preserves browser hostOnly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged 2026-05-27: HF source check 401'd on `HEAD /?enterAgree=1` even with valid login cookies. Root cause is the combination of (1) gallery-dl's HF extractor checking `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with `requests`' EXACT domain matching, and (2) the extension's cookies.js forcibly rewriting every captured cookie to a leading-dot subdomain-wide form. HF's PHPSESSID is browser-stored as host-only on `www.hentai-foundry.com`; the rewrite re-anchored it to `.hentai-foundry.com`, which `cookies.get(...)` no longer matches even though the cookie is still sent on actual HTTP requests (RFC 6265 subdomain rules). The extractor falls into its unauthenticated `?enterAgree=1` fallback, which 401s (Cloudflare or HF's anti-bot HEAD gating). Two-part fix, no operator action required for existing stored cookies: 1. **Backend** (`credential_service._augment_cookies`) — refactored from the subscribestar-only single function into a per-platform dispatcher. New `_augment_hentaifoundry` parses the materialized netscape file and, for each `.hentai-foundry.com` entry whose name is PHPSESSID or YII_CSRF_TOKEN, appends a host-only duplicate (`www.hentai-foundry.com\tFALSE\t...`). Originals preserved. Three new tests pin: injection fires + originals preserved; idempotent when host-only already exists; doesn't touch unrelated cookies (e.g. `_ga`). 2. **Extension** (`cookies.js`) — `toNetscapeFormat` now respects `c.hostOnly` from the browser instead of blindly forcing a leading-dot subdomain-wide form. Host-only cookies are written with the bare host + FALSE flag; non-host-only cookies retain the leading-dot + TRUE form. Forward-compat — fresh captures from v1.0.5+ no longer need the backend's host-only duplication. Extension bumped 1.0.4 → 1.0.5; manifest + package.json in lockstep. After deploy: the next HF source check on the operator's already-stored cookies will succeed because the materialized cookies.txt now contains host-only PHPSESSID. No browser re-export needed. --- backend/app/services/credential_service.py | 79 +++++++++++++++++++--- extension/lib/cookies.js | 28 +++++++- extension/manifest.json | 2 +- extension/package.json | 2 +- tests/test_credential_service.py | 59 +++++++++++++++- 5 files changed, 153 insertions(+), 17 deletions(-) diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py index e50026a..adc6d6a 100644 --- a/backend/app/services/credential_service.py +++ b/backend/app/services/credential_service.py @@ -166,7 +166,7 @@ class CredentialService: def _augment_cookies(platform: str, netscape: str) -> str: """Inject platform-specific synthetic cookies needed to bypass server - gates that the user can't realistically re-trigger in their browser. + gates or extractor quirks. subscribestar.adult: the server gates artist pages behind the `_personalization_id` age-confirmation cookie. The site's frontend JS @@ -177,21 +177,37 @@ def _augment_cookies(platform: str, netscape: str) -> str: FC doesn't use; we capture cookies via the extension instead) sidesteps this by manually setting `18_plus_agreement_generic=true` on `.subscribestar.adult` — the server accepts that as the - age-confirmation marker. Mirror that behavior here so cookies-only - auth works long-term. + age-confirmation marker. - The injection is a no-op if the cookie is already present (operator - might have it from an earlier login or a manual cookies.txt paste). + hentaifoundry: gallery-dl's extractor uses + `self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` to + decide whether the user is logged in. `requests` does EXACT domain + matching on .get(); the extension rewrites every captured cookie to + a leading-dot subdomain-wide form (`.hentai-foundry.com`), which + fails that exact match. The fallback path hits a HEAD + `?enterAgree=1` that 401s. Emit host-only duplicates of PHPSESSID + + YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup succeeds. + (The original `.hentai-foundry.com` entries stay — the actual HTTP + requests use RFC 6265 subdomain matching, which works either way.) - Operator-flagged 2026-05-27 after a subscribestar source check - aborted with `HTTP redirect to .../age_confirmation_warning`. + All injections are idempotent (no-op if the target cookie is already + present) and platform-scoped. + + Operator-flagged 2026-05-27: subscribestar age-confirmation, then + hentaifoundry 401 on /?enterAgree=1. """ - if platform != "subscribestar": - return netscape + if platform == "subscribestar": + return _augment_subscribestar(netscape) + if platform == "hentaifoundry": + return _augment_hentaifoundry(netscape) + return netscape + + +def _augment_subscribestar(netscape: str) -> str: if "18_plus_agreement_generic" in netscape: return netscape - # Far-future expiry — 10 years out. The server only checks presence/value; - # gallery-dl's own login flow sets this with no explicit expiry too. + # Far-future expiry — gallery-dl's own login flow sets this with no + # explicit expiry; the server only checks presence/value. expiry = 4102444800 # 2100-01-01 UTC, opaque "far future" line = "\t".join([ ".subscribestar.adult", "TRUE", "/", "TRUE", @@ -203,6 +219,47 @@ def _augment_cookies(platform: str, netscape: str) -> str: return body + "\n" + line + "\n" +_HF_HOST_ONLY_NAMES = ("PHPSESSID", "YII_CSRF_TOKEN") + + +def _augment_hentaifoundry(netscape: str) -> str: + body = netscape.rstrip("\n") + if not body: + return netscape + lines = body.split("\n") + existing_host_only = set() + by_name: dict[str, list[str]] = {} + for raw in lines: + if not raw or raw.startswith("#"): + continue + parts = raw.split("\t") + if len(parts) < 7: + continue + domain, _flag, _path, _secure, _exp, name, _value = parts[:7] + if name not in _HF_HOST_ONLY_NAMES: + continue + if domain == "www.hentai-foundry.com": + existing_host_only.add(name) + elif domain in (".hentai-foundry.com", "hentai-foundry.com"): + by_name.setdefault(name, []).append(raw) + + appended = [] + for name in _HF_HOST_ONLY_NAMES: + if name in existing_host_only or name not in by_name: + continue + # Duplicate the FIRST subdomain-wide line as host-only on + # www.hentai-foundry.com. Same value + expiry; flag=FALSE marks + # it host-only in netscape format. + parts = by_name[name][0].split("\t") + parts[0] = "www.hentai-foundry.com" + parts[1] = "FALSE" + appended.append("\t".join(parts[:7])) + + if not appended: + return netscape + return body + "\n" + "\n".join(appended) + "\n" + + def _to_netscape(plaintext: str) -> str: """Accept either Netscape-format text (the extension's output) or a JSON array of cookie dicts (a manual-paste edge case); produce diff --git a/extension/lib/cookies.js b/extension/lib/cookies.js index b13ca4d..8569337 100644 --- a/extension/lib/cookies.js +++ b/extension/lib/cookies.js @@ -38,11 +38,33 @@ function deduplicateCookies(cookies) { function toNetscapeFormat(cookies) { const lines = ['# Netscape HTTP Cookie File']; for (const c of cookies) { - let domain = c.domain.replace(/^\.?www\./, '.'); - if (!domain.startsWith('.')) domain = '.' + domain; + // Preserve the browser's actual scope. Earlier versions rewrote + // every cookie to a leading-dot subdomain-wide form, which broke + // gallery-dl's HF extractor: its `cookies.get(name, + // domain="www.hentai-foundry.com")` does EXACT domain matching and + // missed host-only PHPSESSID rewritten to `.hentai-foundry.com`. + // Operator-flagged 2026-05-27. Backend `_augment_cookies` covers + // the already-stored cookies; this fix is forward-compat for fresh + // captures. + // + // Cookie storage semantics (Firefox): + // c.hostOnly === true → cookie set without a Domain= attribute; + // applies to the exact host only. + // c.hostOnly === false → cookie set with Domain=X; applies to + // that domain and its subdomains. + // + // Netscape format: + // leading-dot domain + TRUE flag → subdomain-wide + // bare-host domain + FALSE flag → host-only + const hostOnly = c.hostOnly === true; + let domain = c.domain; + if (!hostOnly && !domain.startsWith('.')) { + domain = '.' + domain; + } + const subdomainFlag = hostOnly ? 'FALSE' : 'TRUE'; const secure = c.secure ? 'TRUE' : 'FALSE'; const expiration = c.expirationDate ? Math.floor(c.expirationDate) : 0; - lines.push([domain, 'TRUE', c.path || '/', secure, String(expiration), c.name, c.value].join('\t')); + lines.push([domain, subdomainFlag, c.path || '/', secure, String(expiration), c.name, c.value].join('\t')); } return lines.join('\n'); } diff --git a/extension/manifest.json b/extension/manifest.json index 850a6fa..1bc2f47 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "FabledCurator", - "version": "1.0.4", + "version": "1.0.5", "description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.", "browser_specific_settings": { diff --git a/extension/package.json b/extension/package.json index f4c7db4..d38e933 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,6 +1,6 @@ { "name": "fabledcurator-extension", - "version": "1.0.4", + "version": "1.0.5", "private": true, "description": "Firefox extension for FabledCurator", "scripts": { diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py index 32ee71f..b856761 100644 --- a/tests/test_credential_service.py +++ b/tests/test_credential_service.py @@ -151,7 +151,7 @@ async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto @pytest.mark.asyncio async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path): """The age-cookie injection MUST NOT fire for non-subscribestar - platforms — Patreon/HF/etc. don't need it and shouldn't carry a + platforms — Patreon/etc. don't need it and shouldn't carry a foreign-domain cookie in their cookies.txt.""" svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) @@ -161,6 +161,63 @@ async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path assert "subscribestar" not in contents +@pytest.mark.asyncio +async def test_get_cookies_path_hf_injects_host_only_phpsessid(db, crypto, tmp_path): + """HF: extension writes session cookies as subdomain-wide + (`.hentai-foundry.com`), but gallery-dl's extractor uses + `cookies.get(name, domain='www.hentai-foundry.com')` with EXACT + domain matching. Emit host-only duplicates of PHPSESSID + + YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup matches.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + # Subdomain-wide originals preserved. + assert ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents + # Host-only duplicates appended for both names. + assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents + assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456" in contents + + +@pytest.mark.asyncio +async def test_get_cookies_path_hf_idempotent_when_host_only_present(db, crypto, tmp_path): + """If the captured cookies already include a host-only PHPSESSID + on www.hentai-foundry.com (e.g. a future extension fix that + preserves browser hostOnly state), don't double-inject.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + # Should count PHPSESSID exactly twice — the original two lines, no third. + assert contents.count("PHPSESSID\tsess123") == 2 + + +@pytest.mark.asyncio +async def test_get_cookies_path_hf_ignores_unrelated_cookies(db, crypto, tmp_path): + """The injection should only target session/CSRF cookies. Other HF + cookies (e.g. analytics) stay subdomain-wide as captured.""" + netscape_in = ( + "# Netscape HTTP Cookie File\n" + ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\t_ga\tGA1.2.x\n" + ) + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in) + path = await svc.get_cookies_path("hentaifoundry") + contents = path.read_text() + assert "www.hentai-foundry.com" not in contents + assert contents.count("_ga") == 1 + + @pytest.mark.asyncio async def test_get_token_decrypts(db, crypto): svc = CredentialService(db, crypto)