What you pay for, what Discord drops, and pixiv switched off #251

Merged
bvandeusen merged 37 commits from dev into main 2026-09-13 12:09:28 -04:00
17 changed files with 262 additions and 180 deletions
Showing only changes of commit e3fd8c67d4 - Show all commits
@@ -0,0 +1,68 @@
"""Disable sources on retired platforms, so the scheduler stops selecting them.
Milestone #406, phase 1 (switch pixiv off). Rule #171 records the scope decision.
## Why this is a migration and not a button
The live instance had one pixiv source still ENABLED when pixiv was retired
(read 2026-09-13, step 1) even though the operator believed it gone. Unregistering
a platform removes it from code; it does not touch the `source` rows that name it.
Left enabled, that row keeps being picked by the scheduler every interval, and
`download_backends` now refuses it with `unsupported_url` — forever, as a
climbing failure count on a source the operator has already given up.
A migration reaches the live instance on deploy without depending on anyone
finding the row and clicking it. The `run_download` guard is what makes a stale
enabled row SAFE; this is what makes it QUIET.
## Deliberately NOT done here
- **No rows are deleted.** Deleting a source sets its posts' `source_id` to NULL
(FK `ON DELETE SET NULL`), and `uq_post_artist_external_id_null_source` can
reject that if a source-less copy of one of those posts already exists. That
needs checking against real data first, which is phase 2's job (step 6). A
disable cannot collide with anything.
- **No posts or images are touched.** The art stays.
- **deviantart is included** because #3069 retired it and nothing disabled its
rows either. The read found none, so for it this is a no-op — written anyway,
so the statement names every retired platform rather than just the latest one.
## Hardcoded platform names
A migration is a record of one event, frozen in time, so it names the platforms
it acted on rather than importing today's registry — the registry will keep
changing and this revision must not.
Revision ID: 0097
Revises: 0096
Create Date: 2026-09-13
"""
from typing import Sequence, Union
from alembic import op
revision: str = "0097"
down_revision: Union[str, None] = "0096"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Clears the failure state the same way `SourceService.update` does when a
# source is disabled through the app (issue #1285), so a retired source
# does not keep showing as failing after it stops being polled. A disable
# done here and one done by clicking must leave identical rows.
op.execute(
"UPDATE source SET enabled = false, last_error = NULL, "
"error_type = NULL, consecutive_failures = 0 "
"WHERE enabled AND platform IN ('pixiv', 'deviantart')"
)
def downgrade() -> None:
# Irreversible by design: which of these rows were enabled before is not
# recorded, and re-enabling every retired-platform source would resume
# polling services the product no longer supports. Rule #22 owes no
# migration story back to a dropped platform.
pass
+34 -2
View File
@@ -28,12 +28,32 @@ from .patreon_ingester import PatreonIngester
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
from .pixiv_client import user_id_from_url from .pixiv_client import user_id_from_url
from .pixiv_ingester import PixivIngester from .pixiv_ingester import PixivIngester
from .platforms import known_platform_keys
from .subscribestar_ingester import SubscribeStarIngester from .subscribestar_ingester import SubscribeStarIngester
# Platforms whose download + verify go through the native ingester rather than # Platforms whose download + verify go through the native ingester rather than
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until # gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
# they migrate too. # they migrate too. pixiv left this set when it was retired (milestone #406).
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"}) NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
def _unsupported_platform_message(platform: str) -> str | None:
"""Why `platform` may not be downloaded or verified, or None if it may.
A source can outlive its platform. Retiring one (DeviantArt #3069, pixiv
#406) unregisters it, but its `Source` rows — and the `enabled` flag on
them — are data, and data survives a deploy. So this refuses at the two
functions every download and every credential probe pass through, instead
of trusting the scheduler's `enabled` filter and every future caller to
agree.
Without it a retired platform does not fail: it falls through to the
gallery-dl branch, which is precisely where a platform lands once it is no
longer native — and gallery-dl still has an extractor for it.
"""
if platform in known_platform_keys():
return None
return f"{platform!r} is not a supported platform (retired or unknown)"
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
# messages so the operator sees the exact lookup endpoint that was hit. # messages so the operator sees the exact lookup endpoint that was hit.
@@ -80,6 +100,13 @@ async def run_download(
backfill state machine and owns phase 3. backfill state machine and owns phase 3.
""" """
platform = ctx["platform"] platform = ctx["platform"]
refusal = _unsupported_platform_message(platform)
if refusal is not None:
return DownloadResult(
success=False, url=ctx["url"], artist_slug=ctx["artist_slug"],
platform=platform,
error_type=ErrorType.UNSUPPORTED_URL, error_message=refusal,
), None
if uses_native_ingester(platform): if uses_native_ingester(platform):
return await _run_native_ingester( return await _run_native_ingester(
ctx, source_config, mode, gdl, sync_session_factory ctx, source_config, mode, gdl, sync_session_factory
@@ -217,6 +244,11 @@ async def verify_source_credential(
network / nothing to test). Callers don't branch on platform — they call network / nothing to test). Callers don't branch on platform — they call
this and render the result. this and render the result.
""" """
refusal = _unsupported_platform_message(platform)
if refusal is not None:
# Inconclusive rather than False: nothing was probed, so nothing was
# rejected. False would tell the operator their credential is bad.
return None, refusal
if uses_native_ingester(platform): if uses_native_ingester(platform):
# Native ingester platforms verify via their own lightweight auth probe. # Native ingester platforms verify via their own lightweight auth probe.
# SubscribeStar's probe takes the creator URL directly; Patreon's # SubscribeStar's probe takes the creator URL directly; Patreon's
@@ -55,10 +55,6 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)", r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
re.IGNORECASE, re.IGNORECASE,
)), )),
("pixiv", re.compile(
r"^https?://(?:www\.)?pixiv\.net/(?:en/)?users/(?P<slug>\d+)",
re.IGNORECASE,
)),
] ]
+5 -3
View File
@@ -11,7 +11,11 @@ Lifted from GallerySubscriber's
and ~/.../extension/lib/platforms.js. Five platforms; auth_type and and ~/.../extension/lib/platforms.js. Five platforms; auth_type and
URL patterns match GS exactly so the existing browser extension URL patterns match GS exactly so the existing browser extension
hits FC unmodified. deviantart was dropped at #3069 (2026-08-27) — hits FC unmodified. deviantart was dropped at #3069 (2026-08-27) —
FC downloaders are art-dedicated services only. FC downloaders are art-dedicated services only. pixiv was retired at
milestone #406 (2026-09-13, rule #171): unregistered here first, which
switches it off everywhere this registry is consulted; `pixiv.py` and the
pixiv client/downloader/ingester stay in the tree, uncalled, until the
milestone's phase 2 deletes them.
""" """
from .base import ( from .base import (
@@ -22,7 +26,6 @@ from .base import (
from .discord import INFO as _DISCORD from .discord import INFO as _DISCORD
from .hentaifoundry import INFO as _HENTAIFOUNDRY from .hentaifoundry import INFO as _HENTAIFOUNDRY
from .patreon import INFO as _PATREON from .patreon import INFO as _PATREON
from .pixiv import INFO as _PIXIV
from .subscribestar import INFO as _SUBSCRIBESTAR from .subscribestar import INFO as _SUBSCRIBESTAR
PLATFORMS: dict[str, PlatformInfo] = { PLATFORMS: dict[str, PlatformInfo] = {
@@ -32,7 +35,6 @@ PLATFORMS: dict[str, PlatformInfo] = {
_SUBSCRIBESTAR, _SUBSCRIBESTAR,
_HENTAIFOUNDRY, _HENTAIFOUNDRY,
_DISCORD, _DISCORD,
_PIXIV,
) )
} }
+17 -115
View File
@@ -1,32 +1,35 @@
/** /**
* Background script — message router + Discord token capture * Background script — message router + Discord token capture (webRequest).
* (webRequest) + Pixiv PKCE OAuth. Direct port of GS background.js; * Direct port of GS background.js; api.js client points at FC instead of GS.
* api.js client points at FC instead of GS. *
* pixiv's PKCE OAuth flow lived here until FC retired pixiv (milestone #406).
* It was removed together with pixiv's host permissions rather than left
* behind: a webRequest listener on a host the manifest no longer grants is at
* best dead and at worst a startup failure for the whole background script.
*/ */
let discordToken = null; let discordToken = null;
let discordTokenCapturedAt = null; let discordTokenCapturedAt = null;
let pixivRefreshToken = null;
let pixivTokenCapturedAt = null;
let pixivOAuthPending = null;
const PIXIV_CLIENT_ID = 'MOBrBDS8blbauoSck0ZfDbtuzpyT';
const PIXIV_CLIENT_SECRET = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj';
const PIXIV_OAUTH_URL = 'https://app-api.pixiv.net/web/v1/login';
const PIXIV_TOKEN_URL = 'https://oauth.secure.pixiv.net/auth/token';
const PIXIV_REDIRECT_URI = 'https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback';
let initialized = false; let initialized = false;
async function ensureInitialized() { async function ensureInitialized() {
if (initialized) return; if (initialized) return;
await api.init(); await api.init();
await loadDiscordToken(); await loadDiscordToken();
await loadPixivToken(); await forgetRetiredPixivToken();
initialized = true; initialized = true;
} }
// A browser that authenticated pixiv before the retirement still holds a live
// OAuth refresh token in extension storage. Nothing reads it any more, and a
// credential for a service FC no longer uses is a liability with no benefit
// (the same reasoning as the server-side cleanup, issue #3980). Removing keys
// that are absent is a no-op, so this is safe on every startup.
async function forgetRetiredPixivToken() {
await browser.storage.local.remove(['pixivRefreshToken', 'pixivTokenCapturedAt']);
}
browser.runtime.onInstalled.addListener(() => ensureInitialized()); browser.runtime.onInstalled.addListener(() => ensureInitialized());
browser.runtime.onStartup.addListener(() => ensureInitialized()); browser.runtime.onStartup.addListener(() => ensureInitialized());
ensureInitialized().catch(e => console.error('init failed:', e)); ensureInitialized().catch(e => console.error('init failed:', e));
@@ -141,98 +144,6 @@ async function saveDiscordToken(token) {
await browser.storage.local.set({ discordToken: token, discordTokenCapturedAt }); await browser.storage.local.set({ discordToken: token, discordTokenCapturedAt });
} }
// ---- Pixiv PKCE OAuth ----
async function loadPixivToken() {
const s = await browser.storage.local.get(['pixivRefreshToken', 'pixivTokenCapturedAt']);
pixivRefreshToken = s.pixivRefreshToken || null;
pixivTokenCapturedAt = s.pixivTokenCapturedAt || null;
}
async function savePixivToken(token) {
pixivRefreshToken = token;
pixivTokenCapturedAt = new Date().toISOString();
await browser.storage.local.set({ pixivRefreshToken: token, pixivTokenCapturedAt });
}
function generateCodeVerifier() {
const a = new Uint8Array(32);
crypto.getRandomValues(a);
return base64UrlEncode(a);
}
async function generateCodeChallenge(verifier) {
const data = new TextEncoder().encode(verifier);
const hash = await crypto.subtle.digest('SHA-256', data);
return base64UrlEncode(new Uint8Array(hash));
}
function base64UrlEncode(buf) {
return btoa(String.fromCharCode(...buf)).replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
}
async function initiatePixivOAuth() {
const codeVerifier = generateCodeVerifier();
const codeChallenge = await generateCodeChallenge(codeVerifier);
const params = new URLSearchParams({
code_challenge: codeChallenge,
code_challenge_method: 'S256',
client: 'pixiv-android',
});
const tab = await browser.tabs.create({ url: `${PIXIV_OAUTH_URL}?${params}` });
return new Promise((resolve, reject) => {
pixivOAuthPending = { codeVerifier, tabId: tab.id, resolve, reject };
setTimeout(() => {
if (pixivOAuthPending && pixivOAuthPending.tabId === tab.id) {
pixivOAuthPending = null;
reject(new Error('Pixiv OAuth timed out (5 min)'));
}
}, 5 * 60 * 1000);
});
}
browser.webRequest.onBeforeRedirect.addListener(
async (details) => {
if (!pixivOAuthPending) return;
if (details.tabId !== pixivOAuthPending.tabId) return;
const url = new URL(details.redirectUrl);
const code = url.searchParams.get('code');
if (!code) return;
const verifier = pixivOAuthPending.codeVerifier;
const resolve = pixivOAuthPending.resolve;
const reject = pixivOAuthPending.reject;
pixivOAuthPending = null;
try {
const tokenResp = await fetch(PIXIV_TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
client_id: PIXIV_CLIENT_ID,
client_secret: PIXIV_CLIENT_SECRET,
code,
code_verifier: verifier,
grant_type: 'authorization_code',
include_policy: 'true',
redirect_uri: PIXIV_REDIRECT_URI,
}),
});
const body = await tokenResp.json();
if (!body.refresh_token) {
reject(new Error(`Pixiv token exchange failed: ${JSON.stringify(body)}`));
return;
}
await savePixivToken(body.refresh_token);
try { await browser.tabs.remove(details.tabId); } catch {}
resolve(body.refresh_token);
} catch (e) {
reject(e);
}
},
{ urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] },
);
// Extract → verify → upload one cookie-auth platform. Returns a structured // Extract → verify → upload one cookie-auth platform. Returns a structured
// outcome so the two callers (EXPORT_COOKIES single, EXPORT_ALL_COOKIES) shape // outcome so the two callers (EXPORT_COOKIES single, EXPORT_ALL_COOKIES) shape
// their own response + skip semantics. Verifies the captured cookies are // their own response + skip semantics. Verifies the captured cookies are
@@ -277,8 +188,6 @@ browser.runtime.onMessage.addListener(async (msg) => {
} }
} else if (key === 'discord') { } else if (key === 'discord') {
status[key] = { hasToken: !!discordToken, capturedAt: discordTokenCapturedAt }; status[key] = { hasToken: !!discordToken, capturedAt: discordTokenCapturedAt };
} else if (key === 'pixiv') {
status[key] = { hasToken: !!pixivRefreshToken, capturedAt: pixivTokenCapturedAt };
} else { } else {
status[key] = {}; status[key] = {};
} }
@@ -306,13 +215,6 @@ browser.runtime.onMessage.addListener(async (msg) => {
await api.uploadCredentials('discord', 'token', discordToken); await api.uploadCredentials('discord', 'token', discordToken);
return { success: true }; return { success: true };
} }
if (key === 'pixiv') {
if (!pixivRefreshToken) {
await initiatePixivOAuth();
}
await api.uploadCredentials('pixiv', 'token', pixivRefreshToken);
return { success: true };
}
return { error: 'Unsupported platform.' }; return { error: 'Unsupported platform.' };
} catch (e) { } catch (e) {
return { error: e.message }; return { error: e.message };
-9
View File
@@ -60,14 +60,6 @@ const PLATFORMS = {
urlPattern: /^https?:\/\/(www\.)?discord\.com/, urlPattern: /^https?:\/\/(www\.)?discord\.com/,
note: 'Open Discord in browser to capture token', note: 'Open Discord in browser to capture token',
}, },
pixiv: {
name: 'Pixiv',
domains: ['.pixiv.net', 'www.pixiv.net', 'pixiv.net'],
authType: 'token',
color: '#0096FA',
urlPattern: /^https?:\/\/(www\.)?pixiv\.net/,
note: 'Click to authenticate via OAuth',
},
}; };
/** /**
@@ -88,7 +80,6 @@ const PLATFORM_ARTIST_PATTERNS = {
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i, patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i, subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i, hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
pixiv: /^https?:\/\/(www\.)?pixiv\.net\/(en\/)?users\/\d+/i,
}; };
function getPlatformFromUrl(url) { function getPlatformFromUrl(url) {
+1 -5
View File
@@ -32,9 +32,6 @@
"*://*.subscribestar.adult/*", "*://*.subscribestar.adult/*",
"*://*.hentai-foundry.com/*", "*://*.hentai-foundry.com/*",
"*://*.discord.com/*", "*://*.discord.com/*",
"*://*.pixiv.net/*",
"*://app-api.pixiv.net/*",
"*://oauth.secure.pixiv.net/*",
"*://*/*" "*://*/*"
], ],
@@ -59,8 +56,7 @@
"*://*.patreon.com/*", "*://*.patreon.com/*",
"*://*.subscribestar.com/*", "*://*.subscribestar.com/*",
"*://*.subscribestar.adult/*", "*://*.subscribestar.adult/*",
"*://*.hentai-foundry.com/*", "*://*.hentai-foundry.com/*"
"*://*.pixiv.net/*"
], ],
"js": ["lib/platforms.js", "content/content-script.js"], "js": ["lib/platforms.js", "content/content-script.js"],
"css": ["content/content-script.css"], "css": ["content/content-script.css"],
+1 -3
View File
@@ -108,7 +108,7 @@ function createPlatformCard(key, platform, status) {
card.className = 'platform-card'; card.className = 'platform-card';
card.dataset.platform = key; card.dataset.platform = key;
const isTokenOnly = platform.authType === 'token' && !['discord', 'pixiv'].includes(key); const isTokenOnly = platform.authType === 'token' && key !== 'discord';
const discordNeedsToken = key === 'discord' && !status.hasToken; const discordNeedsToken = key === 'discord' && !status.hasToken;
if (isTokenOnly || discordNeedsToken) card.classList.add('disabled'); if (isTokenOnly || discordNeedsToken) card.classList.add('disabled');
@@ -141,7 +141,6 @@ function createPlatformCard(key, platform, status) {
function statusText(s, platform, key) { function statusText(s, platform, key) {
if (key === 'discord') return s.hasToken ? 'Token captured — ready' : 'Open Discord to capture token'; if (key === 'discord') return s.hasToken ? 'Token captured — ready' : 'Open Discord to capture token';
if (key === 'pixiv') return s.hasToken ? 'Token captured — ready' : 'Click to authenticate via OAuth';
if (platform.authType === 'token') return 'Manual token entry required'; if (platform.authType === 'token') return 'Manual token entry required';
if (s.error) return 'Error checking cookies'; if (s.error) return 'Error checking cookies';
if (!s.hasCookies || !s.cookieCount) return 'No cookies — log in first'; if (!s.hasCookies || !s.cookieCount) return 'No cookies — log in first';
@@ -149,7 +148,6 @@ function statusText(s, platform, key) {
} }
function statusClass(s, platform, key) { function statusClass(s, platform, key) {
if (key === 'discord') return s.hasToken ? 'ready' : 'no-cookies'; if (key === 'discord') return s.hasToken ? 'ready' : 'no-cookies';
if (key === 'pixiv') return s.hasToken ? 'ready' : 'no-cookies';
if (platform.authType === 'token') return 'no-cookies'; if (platform.authType === 'token') return 'no-cookies';
if (s.error) return 'error'; if (s.error) return 'error';
if (!s.hasCookies || !s.cookieCount) return 'no-cookies'; if (!s.hasCookies || !s.cookieCount) return 'no-cookies';
+13 -11
View File
@@ -18,7 +18,6 @@ describe('getPlatformFromUrl', () => {
expect(getPlatformFromUrl('https://subscribestar.adult/someone')).toBe('subscribestar') expect(getPlatformFromUrl('https://subscribestar.adult/someone')).toBe('subscribestar')
expect(getPlatformFromUrl('https://www.hentai-foundry.com/user/someone')).toBe('hentaifoundry') expect(getPlatformFromUrl('https://www.hentai-foundry.com/user/someone')).toBe('hentaifoundry')
expect(getPlatformFromUrl('https://discord.com/channels/@me')).toBe('discord') expect(getPlatformFromUrl('https://discord.com/channels/@me')).toBe('discord')
expect(getPlatformFromUrl('https://www.pixiv.net/en/users/123')).toBe('pixiv')
}) })
it('accepts http as well as https, with or without www', () => { it('accepts http as well as https, with or without www', () => {
@@ -32,6 +31,15 @@ describe('getPlatformFromUrl', () => {
expect(getPlatformFromUrl('')).toBe(null) expect(getPlatformFromUrl('')).toBe(null)
}) })
it('returns null for pixiv, retired at milestone #406', () => {
// Retired on the operator's platform-focus decision (rule #171). Same guard
// as deviantart's below, and for the same reason: an absence nothing asserts
// is an absence a later edit can quietly undo.
expect(getPlatformFromUrl('https://www.pixiv.net/en/users/12345')).toBe(null)
expect(PLATFORMS.pixiv).toBeUndefined()
expect(PLATFORM_ARTIST_PATTERNS.pixiv).toBeUndefined()
})
it('returns null for deviantart, retired at #3069', () => { it('returns null for deviantart, retired at #3069', () => {
// The 2026-07-05 product decision (FC downloaders = art-dedicated services // The 2026-07-05 product decision (FC downloaders = art-dedicated services
// only) left deviantart wired for seven weeks. Asserting the negative is // only) left deviantart wired for seven weeks. Asserting the negative is
@@ -80,12 +88,6 @@ describe('isArtistPage', () => {
) )
}) })
it('matches Pixiv numeric user pages, with or without the /en/ prefix', () => {
expect(isArtistPage('https://www.pixiv.net/users/12345', 'pixiv')).toBe(true)
expect(isArtistPage('https://www.pixiv.net/en/users/12345', 'pixiv')).toBe(true)
expect(isArtistPage('https://www.pixiv.net/en/artworks/999', 'pixiv')).toBe(false)
})
it('returns false for a platform with no artist pattern (discord)', () => { it('returns false for a platform with no artist pattern (discord)', () => {
expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false) expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false)
}) })
@@ -123,8 +125,7 @@ describe('platform table integrity', () => {
const samples = { const samples = {
patreon: 'https://www.patreon.com/cw/Atole', patreon: 'https://www.patreon.com/cw/Atole',
subscribestar: 'https://subscribestar.adult/someone', subscribestar: 'https://subscribestar.adult/someone',
hentaifoundry: 'https://www.hentai-foundry.com/user/someone', hentaifoundry: 'https://www.hentai-foundry.com/user/someone'
pixiv: 'https://www.pixiv.net/en/users/12345'
} }
for (const [key, url] of Object.entries(samples)) { for (const [key, url] of Object.entries(samples)) {
expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true) expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true)
@@ -179,8 +180,9 @@ describe('manifest.json agrees with the platform table', () => {
for (const h of manifest.host_permissions) { for (const h of manifest.host_permissions) {
if (h === '*://*/*') continue if (h === '*://*/*') continue
const host = hostOf(h) const host = hostOf(h)
// pixiv's OAuth/API hosts are pixiv infrastructure, not creator pages, // Suffix matching lets a platform's infrastructure subdomains belong to
// so they are matched by suffix rather than by the domains list. // it without listing each one. (It was added for pixiv's OAuth hosts,
// which left with pixiv at milestone #406; the rule itself is general.)
const claimed = Object.values(PLATFORMS).some( const claimed = Object.values(PLATFORMS).some(
(p) => p.domains.includes(host) || p.domains.some((d) => host.endsWith(d)) (p) => p.domains.includes(host) || p.domains.some((d) => host.endsWith(d))
) )
@@ -19,7 +19,7 @@
<v-card-text> <v-card-text>
<p class="fc-muted text-body-2"> <p class="fc-muted text-body-2">
Pushes session cookies from supported platforms Pushes session cookies from supported platforms
(patreon, subscribestar, hentaifoundry, discord, pixiv) (patreon, subscribestar, hentaifoundry, discord)
into FabledCurator, and lets you add a creator as a source from into FabledCurator, and lets you add a creator as a source from
their page in one click. their page in one click.
</p> </p>
@@ -77,7 +77,7 @@ const recapturing = computed(() => !!props.source.backfill_recapture)
// Recover / recapture are native-ingester features (ledger-bypass re-walk and // Recover / recapture are native-ingester features (ledger-bypass re-walk and
// post-text re-grab), available to every native platform — not just Patreon. // post-text re-grab), available to every native platform — not just Patreon.
// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. // Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS.
const NATIVE_PLATFORMS = ['patreon', 'subscribestar', 'pixiv'] const NATIVE_PLATFORMS = ['patreon', 'subscribestar']
const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))
</script> </script>
+16 -2
View File
@@ -129,8 +129,6 @@ async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch):
("https://www.subscribestar.com/foobar", "subscribestar", "foobar"), ("https://www.subscribestar.com/foobar", "subscribestar", "foobar"),
("https://subscribestar.adult/foobar", "subscribestar", "foobar"), ("https://subscribestar.adult/foobar", "subscribestar", "foobar"),
("https://www.hentai-foundry.com/user/Foo/profile", "hentaifoundry", "Foo"), ("https://www.hentai-foundry.com/user/Foo/profile", "hentaifoundry", "Foo"),
("https://www.pixiv.net/users/12345", "pixiv", "12345"),
("https://www.pixiv.net/en/users/12345", "pixiv", "12345"),
]) ])
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_quick_add_source_url_patterns(client, ext_key, url, platform, slug): async def test_quick_add_source_url_patterns(client, ext_key, url, platform, slug):
@@ -176,6 +174,22 @@ async def test_quick_add_source_rejects_retired_deviantart(client, ext_key):
assert "deviantart" not in body["known"] assert "deviantart" not in body["known"]
@pytest.mark.asyncio
async def test_quick_add_source_rejects_retired_pixiv(client, ext_key):
"""Milestone #406: the same shape as deviantart's retirement above. An
un-updated extension can still offer the button on a pixiv creator page, so
the backend refuses rather than creating a source nothing can download."""
resp = await client.post(
"/api/extension/quick-add-source",
json={"url": "https://www.pixiv.net/users/12345"},
headers={"X-Extension-Key": ext_key},
)
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "unknown_platform"
assert "pixiv" not in body["known"]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_quick_add_source_invalid_url_400(client, ext_key): async def test_quick_add_source_invalid_url_400(client, ext_key):
resp = await client.post( resp = await client.post(
+3 -4
View File
@@ -6,17 +6,17 @@ pytestmark = pytest.mark.integration
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_platforms_returns_gs_five(client): async def test_platforms_returns_the_supported_four(client):
resp = await client.get("/api/platforms") resp = await client.get("/api/platforms")
assert resp.status_code == 200 assert resp.status_code == 200
body = await resp.get_json() body = await resp.get_json()
platforms = body["platforms"] platforms = body["platforms"]
assert set(platforms.keys()) == { assert set(platforms.keys()) == {
"patreon", "subscribestar", "hentaifoundry", "patreon", "subscribestar", "hentaifoundry", "discord",
"discord", "pixiv",
} }
assert "fanbox" not in platforms assert "fanbox" not in platforms
assert "deviantart" not in platforms # retired at #3069 assert "deviantart" not in platforms # retired at #3069
assert "pixiv" not in platforms # retired at milestone #406
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -37,5 +37,4 @@ async def test_platforms_record_shape(client):
async def test_platform_auth_types_match_gs(client): async def test_platform_auth_types_match_gs(client):
body = await (await client.get("/api/platforms")).get_json() body = await (await client.get("/api/platforms")).get_json()
assert body["platforms"]["discord"]["auth_type"] == "token" assert body["platforms"]["discord"]["auth_type"] == "token"
assert body["platforms"]["pixiv"]["auth_type"] == "token"
assert body["platforms"]["patreon"]["auth_type"] == "cookies" assert body["platforms"]["patreon"]["auth_type"] == "cookies"
+87 -1
View File
@@ -1,21 +1,107 @@
"""download_backends — the single predicate that routes a platform to the """download_backends — the single predicate that routes a platform to the
native ingester vs. gallery-dl. Pure, no DB.""" native ingester vs. gallery-dl. Pure, no DB."""
from pathlib import Path
import pytest
from backend.app.services.download_backends import ( from backend.app.services.download_backends import (
NATIVE_INGESTER_PLATFORMS, NATIVE_INGESTER_PLATFORMS,
_campaign_resolution_error, _campaign_resolution_error,
_native_ingester_cls, _native_ingester_cls,
_unsupported_platform_message,
run_download,
uses_native_ingester, uses_native_ingester,
verify_source_credential,
) )
from backend.app.services.gallery_dl import ErrorType
from backend.app.services.pixiv_ingester import PixivIngester from backend.app.services.pixiv_ingester import PixivIngester
def test_native_platforms(): def test_native_platforms():
for platform in ("patreon", "subscribestar", "pixiv"): for platform in ("patreon", "subscribestar"):
assert uses_native_ingester(platform) is True assert uses_native_ingester(platform) is True
assert platform in NATIVE_INGESTER_PLATFORMS assert platform in NATIVE_INGESTER_PLATFORMS
def test_pixiv_is_no_longer_native():
"""Retired at milestone #406. The refusal below is what stops it falling
through to gallery-dl now that it is not native."""
assert uses_native_ingester("pixiv") is False
assert "pixiv" not in NATIVE_INGESTER_PLATFORMS
# --- the retired-platform guard (#406 phase 1) -----------------------------
class _RecordingGalleryDL:
"""Stands in for GalleryDLService: records whether a download was attempted."""
def __init__(self):
self.calls = []
async def download(self, **kwargs):
self.calls.append(kwargs["platform"])
return "reached gallery-dl"
def _ctx(platform):
return {
"platform": platform, "url": f"https://example.invalid/{platform}",
"artist_slug": "someone", "cookies_path": None, "auth_token": None,
}
@pytest.mark.asyncio
async def test_a_retired_platform_never_reaches_a_downloader():
"""An enabled source on a retired platform is data that survives a deploy.
Unguarded, pixiv — no longer native — would fall straight through to the
gallery-dl branch, which still has a pixiv extractor."""
gdl = _RecordingGalleryDL()
result, campaign_id = await run_download(
ctx=_ctx("pixiv"), source_config=None, skip_value=False, mode=None,
gdl=gdl, sync_session_factory=None,
)
assert gdl.calls == []
assert result.success is False
assert result.error_type == ErrorType.UNSUPPORTED_URL
assert "pixiv" in result.error_message
assert campaign_id is None
@pytest.mark.asyncio
async def test_a_supported_gallery_dl_platform_still_reaches_gallery_dl():
"""The positive control. Without it, a guard that refused EVERY platform
would pass the test above just as well (rule #167)."""
gdl = _RecordingGalleryDL()
result, _ = await run_download(
ctx=_ctx("hentaifoundry"), source_config=None, skip_value=False, mode=None,
gdl=gdl, sync_session_factory=None,
)
assert gdl.calls == ["hentaifoundry"]
assert result == "reached gallery-dl"
def test_the_guard_discriminates_by_registration():
assert _unsupported_platform_message("hentaifoundry") is None
assert _unsupported_platform_message("patreon") is None
assert _unsupported_platform_message("pixiv") is not None
assert _unsupported_platform_message("deviantart") is not None
@pytest.mark.asyncio
async def test_verifying_a_retired_platform_is_inconclusive_not_rejected():
"""Nothing is probed, so nothing is rejected — returning False would tell the
operator their credential is bad when the platform is simply gone."""
ok, message = await verify_source_credential(
platform="pixiv", url="https://www.pixiv.net/users/1", artist_slug="someone",
config_overrides=None, cookies_path=None, auth_token=None,
images_root=Path("/nonexistent"),
)
assert ok is None
assert "pixiv" in message
def test_gallery_dl_platforms_are_not_native(): def test_gallery_dl_platforms_are_not_native():
# The platforms still served by gallery-dl must NOT route to the native # The platforms still served by gallery-dl must NOT route to the native
# ingester — guards an accidental over-broad migration. # ingester — guards an accidental over-broad migration.
+11 -3
View File
@@ -11,13 +11,21 @@ from backend.app.services.platforms import (
) )
def test_known_platform_keys_is_gs_five(): def test_known_platform_keys_are_the_supported_four():
# GS's original five, less pixiv (retired at milestone #406, rule #171).
assert known_platform_keys() == frozenset({ assert known_platform_keys() == frozenset({
"patreon", "subscribestar", "hentaifoundry", "patreon", "subscribestar", "hentaifoundry", "discord",
"discord", "pixiv",
}) })
def test_pixiv_is_retired():
# Milestone #406 phase 1. Unregistering is what switches pixiv off: the
# registry feeds /api/platforms, the source validator and the download
# guard, so this one absence is load-bearing everywhere else.
assert "pixiv" not in PLATFORMS
assert "pixiv" not in known_platform_keys()
def test_fanbox_not_in_registry(): def test_fanbox_not_in_registry():
# Sanity check — FC-3a added 'fanbox' by mistake; it's not a GS platform. # Sanity check — FC-3a added 'fanbox' by mistake; it's not a GS platform.
assert "fanbox" not in PLATFORMS assert "fanbox" not in PLATFORMS
-13
View File
@@ -183,19 +183,6 @@ def test_parse_subscribestar_post_url_derived_and_post_id_wins():
assert sd.post_url == "https://www.subscribestar.com/posts/360360" assert sd.post_url == "https://www.subscribestar.com/posts/360360"
def test_parse_pixiv_post_url_derived():
"""Pixiv's `url` is the image URL (i.pximg.net); must be replaced
with the post permalink under /artworks/<id>."""
sd = parse_sidecar({
"category": "pixiv",
"id": 140466853,
"url": "https://i.pximg.net/img-original/img/2026/01/28/10/28/24/140466853_p0.jpg",
"title": "Nerissa x Jailbird",
})
assert sd.external_post_id == "140466853"
assert sd.post_url == "https://www.pixiv.net/artworks/140466853"
def test_parse_hentaifoundry_post_url_derived(): def test_parse_hentaifoundry_post_url_derived():
"""HF sidecars omit `url` entirely and use `index`+`user` for the """HF sidecars omit `url` entirely and use `index`+`user` for the
post's natural key. Synthesize the canonical /pictures/user/<u>/<i> post's natural key. Synthesize the canonical /pictures/user/<u>/<i>
+4 -3
View File
@@ -23,14 +23,15 @@ async def _artist(db, name="Alice"):
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_known_platforms_is_gs_five(db): async def test_known_platforms_are_the_supported_four(db):
assert KNOWN_PLATFORMS == frozenset({ assert KNOWN_PLATFORMS == frozenset({
"patreon", "subscribestar", "hentaifoundry", "patreon", "subscribestar", "hentaifoundry", "discord",
"discord", "pixiv",
}) })
assert "fanbox" not in KNOWN_PLATFORMS assert "fanbox" not in KNOWN_PLATFORMS
# Retired at #3069 — a source can no longer be created on it. # Retired at #3069 — a source can no longer be created on it.
assert "deviantart" not in KNOWN_PLATFORMS assert "deviantart" not in KNOWN_PLATFORMS
# Retired at milestone #406 — likewise.
assert "pixiv" not in KNOWN_PLATFORMS
@pytest.mark.asyncio @pytest.mark.asyncio