bvandeusen
2394e47370
Operator-flagged 2026-05-27: HF source check 401'd on
`HEAD /?enterAgree=1` even with valid login cookies. Root cause is the
combination of (1) gallery-dl's HF extractor checking
`self.cookies.get("PHPSESSID", domain="www.hentai-foundry.com")` with
`requests`' EXACT domain matching, and (2) the extension's cookies.js
forcibly rewriting every captured cookie to a leading-dot subdomain-wide
form. HF's PHPSESSID is browser-stored as host-only on
`www.hentai-foundry.com`; the rewrite re-anchored it to
`.hentai-foundry.com`, which `cookies.get(...)` no longer matches even
though the cookie is still sent on actual HTTP requests (RFC 6265
subdomain rules). The extractor falls into its unauthenticated
`?enterAgree=1` fallback, which 401s (Cloudflare or HF's anti-bot HEAD
gating).
Two-part fix, no operator action required for existing stored cookies:
1. **Backend** (`credential_service._augment_cookies`) — refactored from
the subscribestar-only single function into a per-platform dispatcher.
New `_augment_hentaifoundry` parses the materialized netscape file
and, for each `.hentai-foundry.com` entry whose name is PHPSESSID or
YII_CSRF_TOKEN, appends a host-only duplicate
(`www.hentai-foundry.com\tFALSE\t...`). Originals preserved. Three
new tests pin: injection fires + originals preserved; idempotent
when host-only already exists; doesn't touch unrelated cookies
(e.g. `_ga`).
2. **Extension** (`cookies.js`) — `toNetscapeFormat` now respects
`c.hostOnly` from the browser instead of blindly forcing a
leading-dot subdomain-wide form. Host-only cookies are written with
the bare host + FALSE flag; non-host-only cookies retain the
leading-dot + TRUE form. Forward-compat — fresh captures from
v1.0.5+ no longer need the backend's host-only duplication.
Extension bumped 1.0.4 → 1.0.5; manifest + package.json in lockstep.
After deploy: the next HF source check on the operator's already-stored
cookies will succeed because the materialized cookies.txt now contains
host-only PHPSESSID. No browser re-export needed.
233 lines
9.4 KiB
Python
233 lines
9.4 KiB
Python
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import Credential
|
|
from backend.app.services.credential_crypto import CredentialCrypto
|
|
from backend.app.services.credential_service import (
|
|
CredentialService,
|
|
EmptyDataError,
|
|
UnknownPlatformError,
|
|
WrongAuthTypeError,
|
|
)
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
_NETSCAPE = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".patreon.com\tTRUE\t/\tTRUE\t1700000000\tsession_id\tabc\n"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def crypto(tmp_path):
|
|
return CredentialCrypto(tmp_path / "k")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_creates_then_updates(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
rec = await svc.upsert(
|
|
platform="patreon", credential_type="cookies", data=_NETSCAPE,
|
|
)
|
|
assert rec.platform == "patreon"
|
|
assert rec.credential_type == "cookies"
|
|
again = await svc.upsert(
|
|
platform="patreon", credential_type="cookies", data=_NETSCAPE + "x",
|
|
)
|
|
assert again.platform == "patreon"
|
|
rows = (await db.execute(
|
|
select(Credential).where(Credential.platform == "patreon")
|
|
)).scalars().all()
|
|
assert len(rows) == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_rejects_unknown_platform(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
with pytest.raises(UnknownPlatformError):
|
|
await svc.upsert(platform="fanbox", credential_type="cookies", data="x")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_rejects_wrong_auth_type(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
# discord is token-only
|
|
with pytest.raises(WrongAuthTypeError):
|
|
await svc.upsert(platform="discord", credential_type="cookies", data="x")
|
|
# patreon is cookies-only
|
|
with pytest.raises(WrongAuthTypeError):
|
|
await svc.upsert(platform="patreon", credential_type="token", data="x")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upsert_rejects_empty_data(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
with pytest.raises(EmptyDataError):
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=" ")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_excludes_blob(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE)
|
|
records = await svc.list()
|
|
assert len(records) == 1
|
|
d = records[0].to_dict()
|
|
assert "data" not in d
|
|
assert "encrypted_blob" not in d
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE)
|
|
await svc.delete("patreon")
|
|
assert await svc.get("patreon") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_writes_usable_file(db, crypto, tmp_path):
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE)
|
|
path = await svc.get_cookies_path("patreon")
|
|
assert path is not None and path.exists()
|
|
contents = path.read_text()
|
|
assert ".patreon.com" in contents
|
|
assert "session_id" in contents
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_none_for_missing(db, crypto, tmp_path):
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path)
|
|
assert await svc.get_cookies_path("patreon") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_none_for_token_kind(db, crypto, tmp_path):
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path)
|
|
await svc.upsert(platform="discord", credential_type="token", data="tok")
|
|
assert await svc.get_cookies_path("discord") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_subscribestar_injects_age_cookie(db, crypto, tmp_path):
|
|
"""SubscribeStar's server gates artist pages behind a _personalization_id
|
|
cookie; the browser-stored cookie expires annually and can't be easily
|
|
refreshed (the JS age popup is suppressed by localStorage). Mirror
|
|
gallery-dl's own login-flow workaround by injecting
|
|
`18_plus_agreement_generic=true` on `.subscribestar.adult` whenever
|
|
cookies for subscribestar are materialized."""
|
|
netscape_in = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n"
|
|
)
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in)
|
|
path = await svc.get_cookies_path("subscribestar")
|
|
contents = path.read_text()
|
|
assert "18_plus_agreement_generic\ttrue" in contents
|
|
assert ".subscribestar.adult" in contents
|
|
# Original session_id cookie preserved.
|
|
assert "session_id\txyz" in contents
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_subscribestar_idempotent_when_present(db, crypto, tmp_path):
|
|
"""If the operator's captured cookies ALREADY contain the age cookie
|
|
(e.g. a manual paste, or a re-login), don't double-inject."""
|
|
netscape_in = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\t18_plus_agreement_generic\ttrue\n"
|
|
".subscribestar.adult\tTRUE\t/\tTRUE\t1700000000\tsession_id\txyz\n"
|
|
)
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="subscribestar", credential_type="cookies", data=netscape_in)
|
|
path = await svc.get_cookies_path("subscribestar")
|
|
contents = path.read_text()
|
|
assert contents.count("18_plus_agreement_generic") == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_non_subscribestar_unchanged(db, crypto, tmp_path):
|
|
"""The age-cookie injection MUST NOT fire for non-subscribestar
|
|
platforms — Patreon/etc. don't need it and shouldn't carry a
|
|
foreign-domain cookie in their cookies.txt."""
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE)
|
|
path = await svc.get_cookies_path("patreon")
|
|
contents = path.read_text()
|
|
assert "18_plus_agreement_generic" not in contents
|
|
assert "subscribestar" not in contents
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_hf_injects_host_only_phpsessid(db, crypto, tmp_path):
|
|
"""HF: extension writes session cookies as subdomain-wide
|
|
(`.hentai-foundry.com`), but gallery-dl's extractor uses
|
|
`cookies.get(name, domain='www.hentai-foundry.com')` with EXACT
|
|
domain matching. Emit host-only duplicates of PHPSESSID +
|
|
YII_CSRF_TOKEN on `www.hentai-foundry.com` so the lookup matches."""
|
|
netscape_in = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n"
|
|
".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456\n"
|
|
)
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in)
|
|
path = await svc.get_cookies_path("hentaifoundry")
|
|
contents = path.read_text()
|
|
# Subdomain-wide originals preserved.
|
|
assert ".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents
|
|
# Host-only duplicates appended for both names.
|
|
assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123" in contents
|
|
assert "www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tYII_CSRF_TOKEN\ttoken456" in contents
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_hf_idempotent_when_host_only_present(db, crypto, tmp_path):
|
|
"""If the captured cookies already include a host-only PHPSESSID
|
|
on www.hentai-foundry.com (e.g. a future extension fix that
|
|
preserves browser hostOnly state), don't double-inject."""
|
|
netscape_in = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n"
|
|
"www.hentai-foundry.com\tFALSE\t/\tTRUE\t1900000000\tPHPSESSID\tsess123\n"
|
|
)
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in)
|
|
path = await svc.get_cookies_path("hentaifoundry")
|
|
contents = path.read_text()
|
|
# Should count PHPSESSID exactly twice — the original two lines, no third.
|
|
assert contents.count("PHPSESSID\tsess123") == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cookies_path_hf_ignores_unrelated_cookies(db, crypto, tmp_path):
|
|
"""The injection should only target session/CSRF cookies. Other HF
|
|
cookies (e.g. analytics) stay subdomain-wide as captured."""
|
|
netscape_in = (
|
|
"# Netscape HTTP Cookie File\n"
|
|
".hentai-foundry.com\tTRUE\t/\tTRUE\t1900000000\t_ga\tGA1.2.x\n"
|
|
)
|
|
svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies")
|
|
await svc.upsert(platform="hentaifoundry", credential_type="cookies", data=netscape_in)
|
|
path = await svc.get_cookies_path("hentaifoundry")
|
|
contents = path.read_text()
|
|
assert "www.hentai-foundry.com" not in contents
|
|
assert contents.count("_ga") == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_token_decrypts(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
await svc.upsert(platform="discord", credential_type="token", data="my-token")
|
|
assert await svc.get_token("discord") == "my-token"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_token_none_for_cookies_kind(db, crypto):
|
|
svc = CredentialService(db, crypto)
|
|
await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE)
|
|
assert await svc.get_token("patreon") is None
|