From ad8392b790be61b9c64c54c71e558acbd6549dd8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 2 Sep 2026 19:45:54 -0400 Subject: [PATCH 01/37] fix: system health is a Settings tab, not a page only the dot reached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The surface shipped at /system with no nav entry, reachable only by clicking the health dot beside the brand — a target you have to already suspect something is wrong to go looking for. Operator-flagged: it needs a path someone can walk to. Settings is where you go to ask the instance about itself, so the view becomes a tab there, beside Activity — Activity answers "what is the queue doing", System answers "is anything left to do it". - SystemView.vue moves to components/settings/SystemHealthTab.vue; the content is unchanged apart from shedding its own container and h1. - SettingsView adopts useTabQuery (the composable Browse and Subscriptions already use) so a tab can be linked TO. The health dot now points at ?tab=system, and /system redirects there so the previous build's link and any bookmark still land. - The tab drops its own 10s poll. v-window keeps a visited item mounted rather than destroyed, so that timer would have gone on firing behind Maintenance — and TopNav already polls the same store every 15s for the dot. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01TTjbZZ6JirCMSaJzQV1RhA --- frontend/src/components/TopNav.vue | 9 ++--- .../settings/SystemHealthTab.vue} | 33 +++++++++---------- frontend/src/router.js | 14 ++++---- frontend/src/views/SettingsView.vue | 18 ++++++++-- frontend/test/router.spec.js | 8 +++++ 5 files changed, 52 insertions(+), 30 deletions(-) rename frontend/src/{views/SystemView.vue => components/settings/SystemHealthTab.vue} (80%) diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index 8b99c2b..31a9645 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -6,7 +6,8 @@ FabledCurator {{ health.icon }} @@ -282,9 +283,9 @@ onUnmounted(() => { if (healthTimer) clearInterval(healthTimer) }) display: flex; align-items: center; flex-shrink: 0; - /* A RouterLink since milestone 365 — it is the path to /system, not just an - indicator. Reset the anchor so turning a span into a link changed nothing - about how the nav reads. */ + /* A RouterLink since milestone 365 — it is the path to the Settings System + tab, not just an indicator. Reset the anchor so turning a span into a + link changed nothing about how the nav reads. */ text-decoration: none; color: inherit; border-radius: 50%; diff --git a/frontend/src/views/SystemView.vue b/frontend/src/components/settings/SystemHealthTab.vue similarity index 80% rename from frontend/src/views/SystemView.vue rename to frontend/src/components/settings/SystemHealthTab.vue index 18ead8a..c1853aa 100644 --- a/frontend/src/views/SystemView.vue +++ b/frontend/src/components/settings/SystemHealthTab.vue @@ -1,7 +1,10 @@ diff --git a/frontend/src/components/subscriptions/SubscriptionsTab.vue b/frontend/src/components/subscriptions/SubscriptionsTab.vue index 1894e77..b5ef2b2 100644 --- a/frontend/src/components/subscriptions/SubscriptionsTab.vue +++ b/frontend/src/components/subscriptions/SubscriptionsTab.vue @@ -368,7 +368,19 @@ const platformsStore = usePlatformsStore() const importStore = useImportStore() const search = ref('') -const statusFilter = ref('all') +// URL-addressable (#387 B3) so the front-door status ribbon can link straight +// to "the sources this number is about" — a count that lands you on an +// unfiltered list makes the reader do the filtering the ribbon just did. +// Mirrors how artistFilter already reads from route.query below. +const statusFilter = computed({ + get: () => route.query.status || 'all', + set: (v) => { + const q = { ...route.query } + if (!v || v === 'all') delete q.status + else q.status = v + router.replace({ query: q }) + }, +}) const needsAttention = ref(false) const expanded = ref([]) const selected = ref([]) diff --git a/frontend/src/router.js b/frontend/src/router.js index 3de94dd..00bd714 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -35,7 +35,10 @@ const routes = [ // // No stickyChrome: unlike Browse/Gallery/Settings this view has no sticky // sub-header for the nav to butt against, so the nav keeps its normal fade. - { path: '/latest', name: 'latest', component: PostsView, meta: { title: 'Latest', navOrder: 5 } }, + // `props` turns on the ingestion-status ribbon (#387 B3). Only here: inside + // Browse's Posts tab the same view renders without it. + { path: '/latest', name: 'latest', component: PostsView, props: { statusRibbon: true }, + meta: { title: 'Latest', navOrder: 5 } }, // FC-2: image backbone { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } }, diff --git a/frontend/src/views/PostsView.vue b/frontend/src/views/PostsView.vue index 30bbb96..ee5b760 100644 --- a/frontend/src/views/PostsView.vue +++ b/frontend/src/views/PostsView.vue @@ -49,6 +49,8 @@ @@ -68,18 +68,12 @@ diff --git a/frontend/test/components/feedEmptyState.spec.js b/frontend/test/components/feedEmptyState.spec.js index 2b6d11d..904e12c 100644 --- a/frontend/test/components/feedEmptyState.spec.js +++ b/frontend/test/components/feedEmptyState.spec.js @@ -2,6 +2,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import FeedEmptyState from '../../src/components/posts/FeedEmptyState.vue' +import { useCredentialsStore } from '../../src/stores/credentials.js' +import { useMembershipReconcileStore } from '../../src/stores/membershipReconcile.js' +import { useMembershipSyncStore } from '../../src/stores/membershipSync.js' import { useSourcesStore } from '../../src/stores/sources.js' import { mountWithStore } from '../support/mountComponent.js' @@ -10,9 +13,45 @@ import { mountWithStore } from '../support/mountComponent.js' // "add a source" when they already have three and are mid-backfill is worse // than saying nothing — it reads as the app not knowing its own state. -const mountWith = (status) => mountWithStore(FeedEmptyState, () => { - useSourcesStore().scheduleStatus = status -}) +// The second argument is C6's world. Omitting it means "no credential", which +// is both the fresh-install default and the rung every B4 case above asserts — +// so those keep passing unchanged rather than needing the new vocabulary. +const mountWith = (status, { credentials = [], sync = [], reconcile = [] } = {}) => + mountWithStore(FeedEmptyState, () => { + useSourcesStore().scheduleStatus = status + useCredentialsStore().byPlatform = new Map( + credentials.map((name) => [name, { platform: name }]), + ) + useMembershipSyncStore().platforms = sync + useMembershipReconcileStore().platforms = reconcile + }) + +const EMPTY = { total_sources: 0 } +const SYNCED = [{ platform: 'patreon', last_success_at: '2026-09-12T00:00:00Z' }] +const NEVER = [{ platform: 'patreon', last_success_at: null }] + +// Every rung's distinguishing phrase, so a test can assert that exactly ONE is +// on screen. Each is chosen to sit on a SINGLE template line: a phrase spanning +// a line break would never match once the renderer keeps the newline, and a +// mutual-exclusion check whose phrases never match passes vacuously — coverage +// in appearance only (rule #167). Both sides are whitespace-normalised anyway, +// so indentation changes cannot quietly break it either. +const RUNG_PHRASES = { + credential: 'Add a credential', + discover: 'Find what you already subscribe to', + adopt: 'not being followed here yet', + unavailable: "couldn't reach", + manual: 'Everything you subscribe to on', +} + +const flat = (s) => s.replace(/\s+/g, ' ').trim() + +function rungsShown (w) { + const text = flat(w.text()) + return Object.entries(RUNG_PHRASES) + .filter(([, phrase]) => text.includes(flat(phrase))) + .map(([name]) => name) +} describe('FeedEmptyState', () => { beforeEach(() => { @@ -58,4 +97,120 @@ describe('FeedEmptyState', () => { // Decorative: the surrounding prose already carries the meaning. expect(img.attributes('alt')).toBe('') }) + + // --- #387 C6: the discovery rung ----------------------------------------- + // + // The loop this closes: a new installer has already told Patreon which + // creators they follow, and making them retype that list is the friction the + // whole milestone is about. These pin that the rung shown matches what is + // actually POSSIBLE — offering discovery before a credential exists, or + // after it has proven unreachable, is a button that cannot work. + + it('offers discovery once a credential exists and nothing has synced', () => { + const w = mountWith(EMPTY, { credentials: ['patreon'], sync: NEVER }) + expect(w.text()).toContain('Find what you already subscribe to') + expect(w.text()).toContain('patreon') + // Nothing is added for them — the offer/never-auto-add line from C4. + expect(w.text()).toContain('Nothing is added automatically') + }) + + it('never offers discovery before a credential exists', () => { + // The button would 404 against an unconnected platform. Absence beats a + // dead control on the first screen anyone sees. + const w = mountWith(EMPTY, { sync: NEVER }) + expect(w.text()).not.toContain('Find what you already subscribe to') + expect(w.text()).toContain('Add a credential') + }) + + it('a sweep that has never worked reads as unavailable, not as a retry', () => { + // Rule #164: an install with no outbound network must reach this screen and + // be told plainly. Not a spinner, not a crash, not a button that will fail + // the same way — and the manual path stays open. + const w = mountWith(EMPTY, { + credentials: ['patreon'], + sync: [{ platform: 'patreon', last_success_at: null, last_error_type: 'ConnectionError' }], + }) + expect(w.text()).toContain("couldn't reach") + expect(w.text()).toContain('ConnectionError') + expect(w.text()).toContain('Add a source') + expect(w.text()).not.toContain('Find what you already subscribe to') + }) + + it('a roster that synced once and failed since is NOT unavailable', () => { + // It has a roster, just an ageing one — C4's freshness gate handles that. + // Collapsing the two would hide a usable roster behind an error banner. + const w = mountWith(EMPTY, { + credentials: ['patreon'], + sync: [{ + platform: 'patreon', + last_success_at: '2026-09-12T00:00:00Z', + last_error_type: 'PatreonAuthError', + }], + reconcile: [{ platform: 'patreon', subscribed_not_tracked: [{ id: 1 }] }], + }) + expect(w.text()).not.toContain("couldn't reach") + expect(w.text()).toContain('not being followed here yet') + }) + + it('counts what there is to adopt, and sends them to the picker', () => { + const w = mountWith(EMPTY, { + credentials: ['patreon'], + sync: SYNCED, + reconcile: [{ platform: 'patreon', subscribed_not_tracked: [{ id: 1 }, { id: 2 }] }], + }) + expect(w.text()).toContain('2 creators you subscribe to are') + expect(w.text()).toContain('Choose who to follow') + }) + + it('singularises a lone unmatched creator', () => { + const w = mountWith(EMPTY, { + credentials: ['patreon'], + sync: SYNCED, + reconcile: [{ platform: 'patreon', subscribed_not_tracked: [{ id: 1 }] }], + }) + expect(w.text()).toContain('1 creator you subscribe to is') + expect(w.text()).not.toContain('1 creators') + }) + + it('a fully-tracked roster stops offering discovery', () => { + // Nothing left to find, so the manual path is the honest last rung rather + // than a discovery button that would return an empty list. + const w = mountWith(EMPTY, { + credentials: ['patreon'], + sync: SYNCED, + reconcile: [{ platform: 'patreon', subscribed_not_tracked: [] }], + }) + expect(w.text()).toContain('Everything you subscribe to on') + expect(w.text()).toContain('Add a source') + expect(w.text()).not.toContain('Find what you already subscribe to') + expect(w.text()).not.toContain('not being followed here yet') + }) + + it('shows exactly one rung in every state', () => { + // The component claims the rungs are mutually exclusive by construction. + // This is that claim, asserted — two on screen at once would be worst + // exactly here, on the first screen a new installer ever sees. + const states = [ + [EMPTY, {}], + [EMPTY, { credentials: ['patreon'], sync: NEVER }], + [EMPTY, { credentials: ['patreon'], sync: [{ platform: 'patreon', last_success_at: null, last_error_type: 'ConnectionError' }] }], + [EMPTY, { credentials: ['patreon'], sync: SYNCED, reconcile: [{ platform: 'patreon', subscribed_not_tracked: [{ id: 1 }] }] }], + [EMPTY, { credentials: ['patreon'], sync: SYNCED, reconcile: [{ platform: 'patreon', subscribed_not_tracked: [] }] }], + ] + for (const [status, world] of states) { + expect(rungsShown(mountWith(status, world))).toHaveLength(1) + } + }) + + it('an install that is already fetching never sees an on-ramp rung', () => { + // hasSources wins over everything C6 added — someone mid-backfill is not + // onboarding, whatever their roster says. + const w = mountWith({ total_sources: 3 }, { + credentials: ['patreon'], + sync: SYNCED, + reconcile: [{ platform: 'patreon', subscribed_not_tracked: [{ id: 1 }] }], + }) + expect(w.text()).toContain("See what's running") + expect(rungsShown(w)).toHaveLength(0) + }) }) -- 2.54.0 From 529d4bff578958811a6438a4274400e9fce8d52f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 12 Sep 2026 21:31:21 -0400 Subject: [PATCH 32/37] test: tie the install docs to the code they quote (3422 follow-up) 3422 was already fixed. Commit 86abaf0 applied option 1 in full and it is on main: .env.example carries the bootstrap section with the backup warning, README explains the refusal and why it is deliberate, docker-compose forwards the variable, and the milestone-362 smoke gate that FOUND the bug now sets it (build.yml:1159) and passes. The issue's premise - "CURATOR_BOOTSTRAP_NEW_KEY appears nowhere outside backend/" - is stale. What was left is the dependency that fix created. README.md and .env.example now both print the literal error text, the literal key path and the variable name, because a stranger greps for the string their terminal showed them. That is the right call and it means two user-facing files now depend on this module's wording with nothing connecting them - the install surface's characteristic defect, one rename away from a README that sends strangers to a path that does not exist. Three guards, all presence checks on both sides. An absence check against prose would pass for the wrong reason the moment a sentence were reworded (snippet 3352): - the raised message still contains the sentence README reproduces, the variable both docs say to set, and the restore-rather-than-mint alternative the whole refusal rests on; - both docs still name _CREDENTIAL_KEY_PATH and the variable, read from the code rather than retyped, so a rename fails here; - compose still forwards the variable - without that line the docs' "set it in .env" is silently inert and fails identically to not setting it. Option 2 (mint when the credential table is empty) is deliberately NOT done. The issue's own guidance is "(1) now, (2) if the friction proves annoying", and the friction has not been reported. Worth recording that its predicate checks out exactly: the Fernet key protects Credential.encrypted_blob and nothing else - no other Fernet user exists - so "no credential rows means nothing can be made undecryptable" is provable rather than probable. The cost is placement: create_app() is sync and constructs the key before any engine exists, so the check cannot live where the failure is. entrypoint.sh, which already runs alembic against the DB, is the natural seam. Only the web role is affected; the Celery roles build the key lazily inside tasks. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- tests/test_credential_crypto.py | 66 +++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_credential_crypto.py b/tests/test_credential_crypto.py index 6274bb5..2ca31b0 100644 --- a/tests/test_credential_crypto.py +++ b/tests/test_credential_crypto.py @@ -69,3 +69,69 @@ def test_missing_key_with_env_var_bootstraps(tmp_path, monkeypatch): key_path = tmp_path / "bootstrap.b64" CredentialCrypto(key_path) # no bootstrap_ok kwarg — relies on env assert key_path.exists() + + +# --- #3422: the install docs quote this module, so they must keep agreeing -- +# +# README.md and .env.example both print the literal failure a new installer +# hits, and the literal path and env var they must act on. That is the right +# call — a stranger greps for the string their terminal showed them — but it +# means those two files now DEPEND on this module's wording, with nothing +# connecting them. The characteristic defect of the install surface is exactly +# this: the documented behaviour and the code drift apart, and the code is the +# one that is right. +# +# Presence checks on both sides, deliberately: an absence check against prose +# would pass for the wrong reason the moment a sentence were reworded +# (snippet #3352). + +_REPO_ROOT = Path(__file__).resolve().parents[1] +_INSTALL_DOCS = ("README.md", ".env.example") + + +def test_the_bootstrap_refusal_still_reads_the_way_the_docs_quote_it(tmp_path, monkeypatch): + """The three things a reader is told to look for, in the raised message.""" + from backend.app.services.credential_crypto import ( + _BOOTSTRAP_ENV_VAR, + MissingCredentialKey, + ) + monkeypatch.delenv(_BOOTSTRAP_ENV_VAR, raising=False) + with pytest.raises(MissingCredentialKey) as exc: + CredentialCrypto(tmp_path / "absent.b64") + message = str(exc.value) + # The sentence README.md reproduces verbatim. + assert "Fernet key file not found at" in message + # The variable both docs tell the operator to set. + assert _BOOTSTRAP_ENV_VAR in message + # The alternative the docs lean on — that a restored instance restores the + # key rather than minting one. Losing this line loses the whole point. + assert "restore the key file" in message + + +def test_the_install_docs_name_the_real_key_path_and_variable(): + """Pins the two literals, read from the code rather than retyped here. + + Changing `_CREDENTIAL_KEY_PATH` or the env var name without updating the + docs fails this — which is the only thing standing between a rename and a + README that sends strangers to a path that does not exist. + """ + from backend.app import _CREDENTIAL_KEY_PATH + from backend.app.services.credential_crypto import _BOOTSTRAP_ENV_VAR + + for name in _INSTALL_DOCS: + text = (_REPO_ROOT / name).read_text() + assert str(_CREDENTIAL_KEY_PATH) in text, f"{name} does not name the key path" + assert _BOOTSTRAP_ENV_VAR in text, f"{name} does not name the bootstrap variable" + + +def test_compose_passes_the_bootstrap_variable_through(): + """The docs say "set it in .env"; that only works if compose forwards it. + + Without this line the instruction is silently inert — the operator sets the + variable, the container never sees it, and the failure is identical to not + having set it at all. + """ + from backend.app.services.credential_crypto import _BOOTSTRAP_ENV_VAR + + compose = (_REPO_ROOT / "docker-compose.yml").read_text() + assert f"{_BOOTSTRAP_ENV_VAR}: ${{{_BOOTSTRAP_ENV_VAR}" in compose -- 2.54.0 From ef91fcfd26879798f765d9bd2642943337ac2b47 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 13 Sep 2026 10:53:01 -0400 Subject: [PATCH 33/37] feat: SubscribeStar joins the membership roster (387 D1) The second platform through the seam note 3970 contracted, characterized first from a live capture of the account's /subscriptions page (note 3989). The capture lives in the gitignored captures dir; the committed fixture is hand-built with invented values and was verified tag-for-tag against it - card wrappers, both table heads, and every distinct row shape - before any code depended on it. What the page is, and the three decisions it forced: The table IS the status. SubscribeStar has no per-row status word: a creator is either in the active_subscriptions card or the cancelled_subscriptions one. The card's data-identifier is stored verbatim as Membership.status and mapped in MEMBERSHIP_STATUS, keyed on the identifier rather than the table class because the cancelled table's class names the same list differently (for-unsubscribed_users). The creator's numeric data-user-id is the key, not the slug. A slug re-keys when a creator renames; the old row stops appearing; and a disappearance is exactly what reconciliation reads as a lapse. Keyed on the slug, a rename would have told a paying subscriber they had cancelled. The slug rides as vanity, where the identity join already looks for a handle. Price is kept as text, never parsed into amount_cents. A bare $ names no currency and a page price is not proven to be the charge - 3970 finding 4. Tier names live behind a per-row modal and are not fetched. Refusals, because SubscribeStar offers nothing like Patreon's meta.pagination.total and every conclusion downstream is drawn from absence. The parser raises when: the active card is missing (auth error on a login/age wall, drift otherwise); a row lacks a numeric creator id or a creator link; anything renders after a card's table; or the page carries a page= link. Both cards are paginatable (app#embed_pagination) and the captured account was too small to show what pagination looks like, so possible pagination is a roster FC cannot prove complete. A loud error on a larger account beats a quiet half-list. A missing cancelled card is not drift, and a creator in both tables is reported once, as active. Fetched from subscribestar.adult, not the .art the capture came from: FC's requests never clear the .art age wall with the 18+ cookie (1259, 1284). Whether /subscriptions on .adult authenticates exactly as .art did in the browser is untested - if not, the sweep records a visible error and C6 shows its unavailable rung. The seam leak D1 found. Note 3970 promised a second platform would be one builders line plus the client method. The sweep instead called current_user_id() on every client, which only Patreon's has, so SubscribeStar would have raised AttributeError on the first sweep. roster_user_id probes it with getattr, the same way the sweep already probes iter_memberships. Two existing tests were passing for the wrong reason and now can fail: - "a platform that has never been characterised says nothing" named SubscribeStar, and stayed green only because active_patron is not a SubscribeStar word. Now uses hentaifoundry, with a positive SubscribeStar test beside it. - the freshness test gave SubscribeStar a Patreon word, so the vocabulary excluded it and deleting the freshness gate outright would have left it green. It now uses cancelled_subscriptions, making the gate the only thing that excludes it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- backend/app/services/membership_roster.py | 28 ++ backend/app/services/subscribestar_client.py | 208 ++++++++++++++ backend/app/tasks/maintenance.py | 14 +- .../subscribestar_subscriptions_page1.html | 3 + tests/test_gated_reason.py | 23 +- tests/test_membership_roster.py | 4 + tests/test_subscribestar_memberships.py | 261 ++++++++++++++++++ 7 files changed, 532 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/subscribestar_subscriptions_page1.html create mode 100644 tests/test_subscribestar_memberships.py diff --git a/backend/app/services/membership_roster.py b/backend/app/services/membership_roster.py index 2c52888..8cf4945 100644 --- a/backend/app/services/membership_roster.py +++ b/backend/app/services/membership_roster.py @@ -60,11 +60,21 @@ log = logging.getLogger(__name__) # Unknown words are NOT an error: an unrecognised status means the roster # records evidence it cannot yet interpret, which is a better state than # dropping the row or asserting a meaning for it. +# +# subscribestar: from a live capture of the account's /subscriptions page, +# 2026-09-13 (Scribe note #3989). SubscribeStar gives NO per-row status word — +# a membership's state is which of two tables it sits in — so the "word" stored +# is the table card's own `data-identifier`, verbatim. Those two identifiers are +# the whole vocabulary; there is nothing further to characterise later. MEMBERSHIP_STATUS: dict[str, dict[str, bool]] = { "patreon": { "active_patron": True, "former_patron": False, }, + "subscribestar": { + "active_subscriptions": True, + "cancelled_subscriptions": False, + }, } @@ -194,6 +204,24 @@ async def _record_sync(session: AsyncSession, platform: str, **values) -> None: )) +def roster_user_id(client) -> str | None: + """The account id a client's roster walk needs, if that client needs one. + + Patreon's members endpoint filters on the account's own user id, so the + sweep has to resolve it first. SubscribeStar's /subscriptions page is simply + the logged-in account's, with nothing to resolve. Probed with `getattr`, + the same way the sweep probes `iter_memberships` itself (rule #169), rather + than called unconditionally. + + Calling `current_user_id()` unconditionally was the one place the membership + seam was still Patreon-shaped: note #3970 promised a second platform would be + one `builders` line plus the client method, and D1 found the sweep would + instead have crashed on the first client without that method. + """ + resolve = getattr(client, "current_user_id", None) + return resolve() if resolve is not None else None + + async def sync_platform( session: AsyncSession, *, diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index ffdedba..0a1751b 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -43,6 +43,7 @@ import requests from ..utils.paths import filehash_from_url from .native_ingest_common import ( _MAX_429_RETRIES, + Membership, NativeAuthError, NativeDriftError, NativeIngestError, @@ -297,6 +298,196 @@ def _extract_creator_name(html: str) -> str | None: return name or None +# -- membership roster (#387 D1) ------------------------------------------ +# +# Characterized from a live operator capture of the account's /subscriptions +# page, 2026-09-13 — Scribe note #3989. Read that note before changing any of +# this; each constant below is a finding from it, not a guess. + +# The account page is fetched from `.adult`. The `.art` age wall never clears +# with the 18+ cookie for FC's requests (see _normalize_ss_host, issues #1259 / +# #1284). The capture itself came from `.art` only because a human had clicked +# through the gate in the browser. +_ROSTER_BASE = "https://subscribestar.adult" +_ROSTER_URL = f"{_ROSTER_BASE}/subscriptions" + +# Two tables, and WHICH table a creator sits in is the only status the page +# gives — there is no per-row status word. Keyed on each card's +# `data-identifier`, the one vocabulary that names a state: the table class +# inside the cancelled card says `for-unsubscribed_users`, a different word for +# the same list (note #3989, CORRECTION 1). The identifier is stored verbatim as +# Membership.status and mapped in membership_roster.MEMBERSHIP_STATUS. +_ROSTER_ACTIVE = "active_subscriptions" +_ROSTER_CANCELLED = "cancelled_subscriptions" + +_ROSTER_ROW_OPEN = '' +# Active rows nest a second `` INSIDE the row's own +# — a narrow-screen duplicate of the actions cell. Its s are not +# columns, so every row is cut here before its cells are read. +_ROSTER_NESTED_ROW = ']*>([^<]*)") +_ROSTER_HEAD_RE = re.compile(r']*>(.*?)', re.DOTALL) +_ROSTER_CELL_RE = re.compile(r']*>(.*?)', re.DOTALL) +_ROSTER_PAGE_LINK_RE = re.compile(r'href="[^"]*[?&]page=\d') +_TAG_RE = re.compile(r"<[^>]+>") + +# Columns that hold identity or controls rather than facts about the +# subscription, so they stay out of `details`. Matched on the header's own text, +# lowercased — the page's words, not ours. +_ROSTER_SKIP_COLUMNS = frozenset({"profile", "updates", "actions"}) + + +def _cell_text(fragment: str) -> str: + """Visible text of a cell: tags dropped, entities decoded, whitespace folded. + + Decoding matters here specifically: an active row with no Discord link + renders its cell as the entity `—`, not as an empty cell. + """ + return " ".join(unescape(_TAG_RE.sub(" ", fragment)).split()) + + +def _roster_table(html: str, identifier: str) -> tuple[str, str] | None: + """One roster card: (its table markup, whatever trails `` inside it). + + None when the card is absent. The trailing part is returned rather than + discarded because it is the pagination check: in the characterized page a + card closes the moment its table does. + """ + start = html.find(f'data-identifier="{identifier}"') + if start < 0: + return None + end = html.find("", start) + if end < 0: + raise SubscribeStarDriftError( + f"SubscribeStar roster card {identifier!r} has no table" + ) + close = html.find("", end) + trailing = html[end + len(""): close if close >= 0 else len(html)] + return html[start:end], trailing + + +def _roster_rows(table: str, identifier: str, base: str) -> list[Membership]: + labels = [_cell_text(h).lower() for h in _ROSTER_HEAD_RE.findall(table)] + body = table[table.find(""):] if "" in table else "" + starts = [m.start() for m in re.finditer(re.escape(_ROSTER_ROW_OPEN), body)] + rows = [] + for n, start in enumerate(starts): + row = body[start: starts[n + 1] if n + 1 < len(starts) else len(body)] + row = row.split(_ROSTER_NESTED_ROW, 1)[0] + + href = _ROSTER_HREF_RE.search(row) + if href is None: + raise SubscribeStarDriftError( + f"SubscribeStar roster row in {identifier!r} has no creator link" + ) + # The creator's numeric id, NOT the slug, is the key (note #3989, + # CORRECTION 2). A slug re-keys when a creator renames; the old row then + # stops appearing, and a disappearance is exactly what reconciliation + # reads as a lapse. The id survives a rename. + user_id = _ROSTER_USER_ID_RE.search(row) + if user_id is None or not user_id.group(1).isdigit(): + raise SubscribeStarDriftError( + f"SubscribeStar roster row in {identifier!r} has no numeric " + f"data-user-id — a membership that cannot be attributed to a " + f"creator is not usable" + ) + name = _ROSTER_NAME_RE.search(row) + slug = unescape(href.group(1)) + cells = _ROSTER_CELL_RE.findall(row) + rows.append(Membership( + campaign_id=user_id.group(1), + display_name=(_cell_text(name.group(1)) if name else "") or None, + url=f"{base}/{slug}", + vanity=slug, + status=identifier, + # No free-follow concept on this page (#3970 §2: False when a + # platform has none). + is_free_member=False, + # Tier names live behind a per-row modal, not inline. Fetching every + # modal would be N authenticated requests for a field nothing reads. + tier_names=[], + # Deliberately NOT parsed from the price cell: a bare `$` names no + # currency, and a page price is not proven to be the charge (#3970 + # finding 4). None keeps "unknown" distinct from zero. The raw text + # is kept in `details`. + amount_cents=None, + currency=None, + details={ + # Paired with the header text by POSITION: two columns share the + # `for-date` class, and the updates column's does not carry + # its 's class at all. + "columns": { + label: _cell_text(cell) + for label, cell in zip(labels, cells) + if label not in _ROSTER_SKIP_COLUMNS + }, + }, + )) + return rows + + +def parse_subscriptions_page(html: str, *, base: str = _ROSTER_BASE) -> list[Membership]: + """Every membership on the account's /subscriptions page. + + Refuses rather than guessing, because every conclusion downstream is drawn + from ABSENCE — a roster that comes back short reads as "you cancelled + those". So this raises when: + + * the active card is missing — as SubscribeStarAuthError if the page is a + login or age wall (the fix is credentials), otherwise as drift; + * a row has no creator link or no numeric creator id; + * anything renders after a card's table, or the page carries a `page=` link. + Both cards are paginatable (`data-view="app#embed_pagination"`), and the + characterized account was too small to show what pagination looks like — + so possible pagination is treated as a roster FC cannot prove complete. + + A missing cancelled card is NOT drift: an account that has never cancelled + plausibly has no such table. A creator present in both tables is reported + once, as active — a current subscription is the fact that matters. + """ + active = _roster_table(html, _ROSTER_ACTIVE) + if active is None: + if any(marker in html for marker in _LOGIN_MARKERS): + raise SubscribeStarAuthError( + "SubscribeStar served a login/age wall instead of the " + "subscriptions page (cookies expired or age cookie missing)" + ) + raise SubscribeStarDriftError( + f"SubscribeStar subscriptions page has no {_ROSTER_ACTIVE!r} card " + f"— {_describe_page(html)}" + ) + roster_region = html[html.find(f'data-identifier="{_ROSTER_ACTIVE}"'):] + if _ROSTER_PAGE_LINK_RE.search(roster_region): + raise SubscribeStarDriftError( + "SubscribeStar subscriptions page carries a page= link — the roster " + "may be paginated, and FC cannot prove it is complete (note #3989)" + ) + + memberships: list[Membership] = [] + seen: set[str] = set() + for identifier, found in ( + (_ROSTER_ACTIVE, active), + (_ROSTER_CANCELLED, _roster_table(html, _ROSTER_CANCELLED)), + ): + if found is None: + continue + table, trailing = found + if trailing.strip(): + raise SubscribeStarDriftError( + f"SubscribeStar roster card {identifier!r} renders content after " + f"its table — possibly pagination, so the roster cannot be " + f"proven complete (note #3989)" + ) + for membership in _roster_rows(table, identifier, base): + if membership.campaign_id in seen: + continue + seen.add(membership.campaign_id) + memberships.append(membership) + return memberships + + class SubscribeStarClient: """Synchronous SubscribeStar HTML-scrape read client. Construct with a path to a Netscape cookies.txt (the same file CredentialService.get_cookies_path @@ -645,6 +836,23 @@ class SubscribeStarClient: return None return _extract_creator_name(html) + # -- membership roster (#387 D1) ---------------------------------------- + + def iter_memberships(self, user_id: str | None = None) -> Iterator[Membership]: + """Yield every subscription the account holds (note #3989). + + `user_id` exists for the seam's signature (note #3970) and is ignored: + the page is the logged-in account's own, so there is nothing to resolve. + The sweep only resolves an id for a client that exposes + `current_user_id`, which this one does not. + + One request, and the whole page is parsed before anything is yielded, so + a drift error can never leave a caller holding part of a roster. + """ + self._session.headers["Referer"] = f"{_ROSTER_BASE}/" + resp = self._get(_ROSTER_URL) + yield from parse_subscriptions_page(resp.text or "", base=_ROSTER_BASE) + # -- verify ------------------------------------------------------------ def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]: diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 2af679c..6e2b47e 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -1247,17 +1247,20 @@ def sync_memberships() -> str: from ..services.artist_membership_service import rescan as membership_rescan from ..services.credential_crypto import CredentialCrypto from ..services.credential_service import CredentialService - from ..services.membership_roster import sync_platform + from ..services.membership_roster import roster_user_id, sync_platform from ..services.patreon_client import PatreonClient + from ..services.subscribestar_client import SubscribeStarClient from ._async_session import async_session_factory key_path = IMAGES_ROOT / "secrets" / "credential_key.b64" # platform -> how to build a client from a cookies path. A platform is in # the sweep only if it is here AND its client exposes `iter_memberships` - # AND a credential exists — three independent gates, each silent, so - # adding SubscribeStar (D1) is one line here and nothing else. - builders = {"patreon": PatreonClient} + # AND a credential exists — three independent gates, each silent, so a + # platform is added with one line here and nothing else. SubscribeStar (D1) + # was the second; the only other change it needed was `roster_user_id` + # replacing an unconditional Patreon-only call below. + builders = {"patreon": PatreonClient, "subscribestar": SubscribeStarClient} async def _run() -> dict: async_factory, engine = async_session_factory() @@ -1287,8 +1290,7 @@ def sync_memberships() -> str: # slow roster does not block the event loop, and bound the # whole walk rather than only its individual requests. def _walk(): - user_id = _client.current_user_id() - return list(_client.iter_memberships(user_id)) + return list(_client.iter_memberships(roster_user_id(_client))) return await asyncio.wait_for( asyncio.to_thread(_walk), diff --git a/tests/fixtures/subscribestar_subscriptions_page1.html b/tests/fixtures/subscribestar_subscriptions_page1.html new file mode 100644 index 0000000..138080c --- /dev/null +++ b/tests/fixtures/subscribestar_subscriptions_page1.html @@ -0,0 +1,3 @@ +My Subscriptions | SubscribeStar.adult

Active Subscriptions

ProfileUpdatesSubscribedRenewedPausedPriceDiscordActions
Creator AlphaCreator Alpha
Jan 2026Mar 2026-$5 +
Manage options
Creator Beta StudioCreator Beta Studio
Feb 2025Mar 2026-$12 +—
Manage options

Cancelled subscriptions

ProfileUnsubscribedPriceActions
Creator GammaCreator Gamma
Dec 2025$3
Creator DeltaCreator Delta
Aug 2025$10
diff --git a/tests/test_gated_reason.py b/tests/test_gated_reason.py index fb179d5..8242a1b 100644 --- a/tests/test_gated_reason.py +++ b/tests/test_gated_reason.py @@ -74,8 +74,22 @@ def test_an_unrecognised_status_says_nothing(status): def test_a_platform_that_has_never_been_characterised_says_nothing(): - """SubscribeStar and FANBOX (D1) inherit silence, not a Patreon guess.""" - assert gated_reason("subscribestar", "active_patron") is None + """An uncharacterised platform inherits silence, not a Patreon guess. + + This named SubscribeStar until D1 characterised it (note #3989). Left as it + was, it would have kept passing only because `active_patron` is not a + SubscribeStar word, testing nothing its name claims. + """ + assert gated_reason("hentaifoundry", "active_patron") is None + + +def test_subscribestar_explains_the_gate_in_its_own_words(): + """SubscribeStar's status is which table the creator sits in, and those + identifiers reach the same three reasons as Patreon's words.""" + assert gated_reason("subscribestar", "active_subscriptions") == GATED_TIER + assert gated_reason("subscribestar", "cancelled_subscriptions") == GATED_LAPSED + # Patreon's vocabulary does not leak across platforms. + assert gated_reason("subscribestar", "former_patron") is None # --- the join, against the database ---------------------------------------- @@ -178,8 +192,11 @@ async def test_one_platforms_fresh_sweep_does_not_vouch_for_anothers(db): url="https://subscribestar.adult/maewix", ) await _membership(db, status="former_patron") + # SubscribeStar's REAL word, so that the freshness gate is the only thing + # excluding this membership. With a Patreon word (as this test first had), + # the vocabulary excluded it and removing the gate would have left it green. await _membership(db, platform="subscribestar", campaign="s1", - status="former_patron", + status="cancelled_subscriptions", url="https://subscribestar.adult/maewix", details={"campaign": {"vanity": "maewix"}}) await _synced(db) # patreon only diff --git a/tests/test_membership_roster.py b/tests/test_membership_roster.py index 6d5b2fa..1e99855 100644 --- a/tests/test_membership_roster.py +++ b/tests/test_membership_roster.py @@ -170,8 +170,12 @@ def test_the_status_map_contains_only_characterised_values(): contain. Adding it because it "obviously" belongs is precisely the guess this test exists to stop. """ + # subscribestar added with its own capture (note #3989, 2026-09-13). Its + # "words" are the two table identifiers, because the page carries no + # per-row status at all. assert MEMBERSHIP_STATUS == { "patreon": {"active_patron": True, "former_patron": False}, + "subscribestar": {"active_subscriptions": True, "cancelled_subscriptions": False}, } diff --git a/tests/test_subscribestar_memberships.py b/tests/test_subscribestar_memberships.py new file mode 100644 index 0000000..5c68704 --- /dev/null +++ b/tests/test_subscribestar_memberships.py @@ -0,0 +1,261 @@ +"""SubscribeStarClient.iter_memberships — parsing only, no network (#387 D1). + +The fixture mirrors a REAL capture of the operator's /subscriptions page +(Scribe note #3989) with every value invented. Its structure was checked +against the capture tag for tag before this file was written: the card +wrappers, both table heads, and every distinct row shape — including the two +forms the Discord cell takes, and the narrow-screen `` +that active rows nest INSIDE their own row. + +Most of what follows pins refusals. SubscribeStar gives no completeness signal +the way Patreon's `meta.pagination.total` does, and every conclusion drawn from +this roster is drawn from absence, so the parser's job is to raise whenever it +cannot vouch for the whole list. + +`_get` is stubbed rather than mocked at the socket: these tests are about what +the client does with a page, and the HTTP path is covered by +test_subscribestar_client.py. +""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from backend.app.services.membership_roster import has_paid_access, roster_user_id +from backend.app.services.native_ingest_common import Membership +from backend.app.services.subscribestar_client import ( + SubscribeStarAuthError, + SubscribeStarClient, + SubscribeStarDriftError, + parse_subscriptions_page, +) + +_FIXTURE = Path(__file__).parent / "fixtures" / "subscribestar_subscriptions_page1.html" + + +@pytest.fixture +def page(): + return _FIXTURE.read_text() + + +@pytest.fixture +def client(): + return SubscribeStarClient(cookies_path=None) + + +def _serve(client, html): + calls = [] + + def fake(url, *, headers=None): + calls.append(url) + return SimpleNamespace(text=html) + + client._get = fake + return calls + + +def _by_id(memberships): + return {m.campaign_id: m for m in memberships} + + +# --- the request ----------------------------------------------------------- + + +def test_the_roster_is_fetched_from_adult_not_art(client, page): + """The capture came from `.art`, but FC's requests never clear the `.art` + age wall with the 18+ cookie (#1259, #1284). Copying the browser's host is + the obvious move and would turn every sweep into an auth error.""" + calls = _serve(client, page) + list(client.iter_memberships()) + assert calls == ["https://subscribestar.adult/subscriptions"] + + +def test_one_request_for_the_whole_roster(client, page): + calls = _serve(client, page) + list(client.iter_memberships(user_id="ignored")) + assert len(calls) == 1 + + +# --- what a membership contains -------------------------------------------- + + +def test_both_tables_are_read(page): + rows = parse_subscriptions_page(page) + assert all(isinstance(m, Membership) for m in rows) + assert len(rows) == 4 + statuses = sorted(m.status for m in rows) + assert statuses == [ + "active_subscriptions", "active_subscriptions", + "cancelled_subscriptions", "cancelled_subscriptions", + ] + + +def test_the_creator_id_is_the_key_not_the_slug(page): + """A slug re-keys when a creator renames; the old row then stops appearing, + and reconciliation reads a disappearance as a lapse. The numeric id does not + change, so a rename can never surface as "you cancelled this creator" + (note #3989, CORRECTION 2).""" + rows = _by_id(parse_subscriptions_page(page)) + assert set(rows) == {"1001", "2002002", "3003003", "4004"} + alpha = rows["1001"] + assert alpha.vanity == "creator-alpha" + # The handle is carried as a URL so `match_kind`'s URL-tail fallback reaches + # it — which is what a SubscribeStar Source.url ends in. + assert alpha.url == "https://subscribestar.adult/creator-alpha" + assert alpha.display_name == "Creator Alpha" + + +def test_the_table_is_the_status(page): + """No per-row status word exists. The card's `data-identifier` is stored + verbatim — and is exactly what MEMBERSHIP_STATUS maps, so a renamed + identifier fails here rather than silently reading as unknown.""" + rows = _by_id(parse_subscriptions_page(page)) + assert rows["1001"].status == "active_subscriptions" + assert rows["3003003"].status == "cancelled_subscriptions" + assert has_paid_access("subscribestar", rows["1001"].status) is True + assert has_paid_access("subscribestar", rows["3003003"].status) is False + + +def test_price_is_kept_as_text_and_never_parsed_into_money(page): + """A bare `$` names no currency and a page price is not proven to be the + charge. None keeps "unknown" distinct from zero (#3970 finding 4).""" + rows = _by_id(parse_subscriptions_page(page)) + alpha = rows["1001"] + assert alpha.amount_cents is None + assert alpha.currency is None + assert alpha.details["columns"]["price"] == "$5" + assert alpha.tier_names == [] + assert alpha.is_free_member is False + + +def test_columns_are_paired_by_position_not_by_class(page): + """Two active columns share the `for-date` class, and the updates column's + does not carry its 's class — pairing by class would merge or drop + them. Controls and identity stay out of details.""" + cols = _by_id(parse_subscriptions_page(page))["1001"].details["columns"] + assert cols == { + "subscribed": "Jan 2026", + "renewed": "Mar 2026", + "paused": "-", + "price": "$5", + "discord": "Joined", + } + cancelled = _by_id(parse_subscriptions_page(page))["3003003"].details["columns"] + assert cancelled == {"unsubscribed": "Dec 2025", "price": "$3"} + + +def test_both_discord_cell_forms_read_as_text(page): + """The real page renders a linked Discord as a button with an icon, and an + unlinked one as the ENTITY `—`. Undecoded, the second would store the + literal string "—".""" + rows = _by_id(parse_subscriptions_page(page)) + assert rows["1001"].details["columns"]["discord"] == "Joined" + assert rows["2002002"].details["columns"]["discord"] == "—" + + +def test_the_nested_actions_row_is_not_read_as_columns(page): + """Active rows nest a narrow-screen `` inside their + own row. Its cells would otherwise zip onto the header as extra columns.""" + cols = _by_id(parse_subscriptions_page(page))["2002002"].details["columns"] + assert "Manage options" not in cols.values() + assert set(cols) == {"subscribed", "renewed", "paused", "price", "discord"} + + +# --- refusals -------------------------------------------------------------- + + +def test_a_creator_in_both_tables_is_reported_once_as_active(page): + """A current subscription is the fact that matters. Written the other way + round, the cancelled row would overwrite the active one in the upsert and + tell a paying subscriber they had lapsed.""" + duplicated = page.replace('data-user-id="3003003"', 'data-user-id="1001"') + rows = parse_subscriptions_page(duplicated) + matches = [m for m in rows if m.campaign_id == "1001"] + assert len(matches) == 1 + assert matches[0].status == "active_subscriptions" + + +def test_an_account_that_never_cancelled_has_no_cancelled_card(page): + """Not drift — such an account plausibly renders no cancelled table.""" + start = page.index('
", start) + len("") + rows = parse_subscriptions_page(page[:start] + page[end:]) + assert {m.status for m in rows} == {"active_subscriptions"} + + +def test_a_login_wall_is_auth_not_drift(): + """The fix for this is a fresh credential, not a scraper change — and the + distinction survives into membership_sync.last_error_type, where it is what + makes the UI's advice correct.""" + wall = '
' + with pytest.raises(SubscribeStarAuthError): + parse_subscriptions_page(wall) + + +def test_an_unrecognised_page_is_drift(): + with pytest.raises(SubscribeStarDriftError): + parse_subscriptions_page("Something else") + + +def test_a_row_without_a_creator_id_refuses_the_whole_roster(page): + """Identity is the point (#3970 §4). Skipping the row instead would return a + roster one creator short, which downstream reads as a cancellation.""" + with pytest.raises(SubscribeStarDriftError, match="data-user-id"): + parse_subscriptions_page(page.replace(' data-user-id="2002002"', "")) + + +def test_a_non_numeric_creator_id_is_drift(page): + with pytest.raises(SubscribeStarDriftError, match="data-user-id"): + parse_subscriptions_page(page.replace('data-user-id="2002002"', 'data-user-id="beta"')) + + +def test_a_row_without_a_creator_link_is_drift(page): + with pytest.raises(SubscribeStarDriftError, match="creator link"): + parse_subscriptions_page(page.replace('', "")) + + +@pytest.mark.parametrize("card", ["active_subscriptions", "cancelled_subscriptions"]) +def test_content_after_a_table_is_treated_as_possible_pagination(page, card): + """Both cards are paginatable (`app#embed_pagination`), and the capture was + too short to show what pagination looks like. Anything rendered after a + table is therefore a roster FC cannot prove complete — raised, never + returned short.""" + start = page.index(f'data-identifier="{card}"') + close = page.index("", start) + len("") + paginated = page[:close] + '' + page[close:] + with pytest.raises(SubscribeStarDriftError, match="cannot be proven complete"): + parse_subscriptions_page(paginated) + + +def test_a_page_link_anywhere_in_the_roster_is_drift(page): + """The same concern for a paginator rendered outside the card.""" + paged = page.replace("", 'Next') + with pytest.raises(SubscribeStarDriftError, match="page="): + parse_subscriptions_page(paged) + + +def test_a_drift_error_mid_page_yields_nothing(client, page): + """The whole page is parsed before anything is yielded, so a caller can never + be left holding the rows that came before the bad one.""" + _serve(client, page.replace(' data-user-id="4004"', "")) + got = [] + with pytest.raises(SubscribeStarDriftError): + for m in client.iter_memberships(): + got.append(m) + assert got == [] + + +# --- the sweep's user-id probe --------------------------------------------- + + +def test_a_client_without_current_user_id_needs_no_id(): + """The seam leak D1 found: the sweep called `current_user_id()` on every + client, which exists only on Patreon's. Note #3970 promised a second platform + would be one `builders` line and the client method; without this probe it + would have raised AttributeError on the first sweep.""" + assert roster_user_id(SubscribeStarClient(cookies_path=None)) is None + + +def test_a_client_with_current_user_id_is_asked_for_it(): + assert roster_user_id(SimpleNamespace(current_user_id=lambda: "248453")) == "248453" -- 2.54.0 From 0835da8a91780beedb9651a26cadfc72a2d1fcaa Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 13 Sep 2026 10:56:18 -0400 Subject: [PATCH 34/37] fix: give the roster's column zip an explicit strict=False (387 D1, B905) ef91fcf failed ruff's B905 lane on one zip(labels, cells) without strict=. Tests, integration and the frontend were already green on that SHA. strict=False is the deliberate side, not the quiet one. strict=True raises a bare ValueError - not SubscribeStarDriftError - and would fail the whole roster sync over a column mismatch in `details`, which nothing reads yet. That would take down reconciliation and the gated-post reasons over a cosmetic markup change, while creator identity (id, slug) never depended on the columns at all. But a shifted column would mislabel details silently (a price filed under "discord"), so a count mismatch now logs a canary warning, mirroring the feed parser's existing parse canary: diagnosable from the worker log, never fatal. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- backend/app/services/subscribestar_client.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/app/services/subscribestar_client.py b/backend/app/services/subscribestar_client.py index 0a1751b..e51e026 100644 --- a/backend/app/services/subscribestar_client.py +++ b/backend/app/services/subscribestar_client.py @@ -396,6 +396,16 @@ def _roster_rows(table: str, identifier: str, base: str) -> list[Membership]: name = _ROSTER_NAME_RE.search(row) slug = unescape(href.group(1)) cells = _ROSTER_CELL_RE.findall(row) + if len(cells) != len(labels): + # Canary, not a refusal. Identity above does not depend on columns, + # so a shifted column must not fail the whole roster — but it would + # silently mislabel `details` (a price filed under "discord"), so + # say so in the worker log where it is diagnosable. + log.warning( + "SubscribeStar roster %r: %d cells against %d headers — column " + "details may be mislabelled; markup likely changed (note #3989)", + identifier, len(cells), len(labels), + ) rows.append(Membership( campaign_id=user_id.group(1), display_name=(_cell_text(name.group(1)) if name else "") or None, @@ -420,7 +430,7 @@ def _roster_rows(table: str, identifier: str, base: str) -> list[Membership]: # its 's class at all. "columns": { label: _cell_text(cell) - for label, cell in zip(labels, cells) + for label, cell in zip(labels, cells, strict=False) if label not in _ROSTER_SKIP_COLUMNS }, }, -- 2.54.0 From e3fd8c67d4e3b38691382ac41ea5d8360d67230d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 13 Sep 2026 11:45:49 -0400 Subject: [PATCH 35/37] =?UTF-8?q?feat:=20switch=20pixiv=20off=20=E2=80=94?= =?UTF-8?q?=20unregistered,=20unreachable,=20and=20refused=20at=20dispatch?= =?UTF-8?q?=20(406=20phase=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 406 retires pixiv (rule 171) in two phases at the operator's explicit ask: switch it off, then later delete its code. This is the switch-off. Steps 2 and 3 ship together because each is a half-state of the other: unregistered but still in the extension, pixiv creator pages would offer a button the backend then refuses. Reachability removed, never gated (rule 22 - no flag, no `if platform == "pixiv"`): - platforms registry: pixiv unregistered, so /api/platforms, the source validator and quick-add all refuse it through their existing unknown-platform paths. - NATIVE_INGESTER_PLATFORMS: pixiv removed. - extension_service: pixiv's quick-add URL pattern removed (the Python half of the JS mirror). - extension: pixiv's host permissions, content-script match, platform entry and artist pattern removed; popup's pixiv branches removed; and the whole pixiv PKCE OAuth flow cut out of background.js. That last one could not wait for phase 2 - a webRequest listener on a host the manifest no longer grants is at best dead and at worst a startup failure for the entire background script. On startup the extension now also removes any pixiv refresh token a browser still holds in storage, for the same reason as the server-side credential cleanup (3980). - frontend: the extension card stops listing pixiv; SourceActions' copy of the native list drops it. platformColor keeps rendering a pixiv key so existing pixiv posts do not look broken. The guard, and why a registry change alone was not enough. A source outlives its platform: the live instance still had one ENABLED pixiv source (step 1). Tracing it: the scheduler only selects enabled rows and every platform lookup uses .get(), so a disabled row is inert - but re-enabling it and pressing Check would have routed pixiv, no longer native, straight into the gallery-dl branch, which still has a pixiv extractor. And a worker can pick up a still-enabled row before a deploy's migration runs. So run_download and verify_source_credential - the two functions every download and credential probe pass through - now refuse any platform not in the registry: an unsupported_url failure for downloads, and an inconclusive (None, not False) verify, since nothing was probed so nothing was rejected. Generic by registration, so it covers deviantart's leftovers too. Positive-controlled: a supported gallery-dl platform must still reach gallery-dl, or a guard that refused everything would pass (rule 167). Migration 0097 disables sources on retired platforms (pixiv, deviantart) and clears their failure state exactly as disabling through the app does (1285), so the stale row stops being scheduled and stops showing as failing. Nothing is deleted: removing a source can collide with uq_post_artist_external_id_null_source on real data, which is phase 2's step 6 to check. No post or image is touched. Tests: the known-platform lists drop pixiv and gain retirement assertions beside deviantart's; pixiv's positive extension cases become negative guards; the pixiv sidecar post-URL test is deleted with the behaviour it tested; quick-add rejects a pixiv URL. The pixiv client/downloader/ingester suites stay - that code stays until phase 2. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- .../0097_disable_retired_platform_sources.py | 68 +++++++++ backend/app/services/download_backends.py | 36 ++++- backend/app/services/extension_service.py | 4 - backend/app/services/platforms/__init__.py | 8 +- extension/background/background.js | 132 +++--------------- extension/lib/platforms.js | 9 -- extension/manifest.json | 6 +- extension/popup/popup.js | 4 +- extension/test/platforms.spec.js | 24 ++-- .../settings/BrowserExtensionCard.vue | 2 +- .../subscriptions/SourceActions.vue | 2 +- tests/test_api_extension.py | 18 ++- tests/test_api_platforms.py | 7 +- tests/test_download_backends.py | 88 +++++++++++- tests/test_platforms_registry.py | 14 +- tests/test_sidecar_util.py | 13 -- tests/test_source_service.py | 7 +- 17 files changed, 262 insertions(+), 180 deletions(-) create mode 100644 alembic/versions/0097_disable_retired_platform_sources.py diff --git a/alembic/versions/0097_disable_retired_platform_sources.py b/alembic/versions/0097_disable_retired_platform_sources.py new file mode 100644 index 0000000..54f3ed5 --- /dev/null +++ b/alembic/versions/0097_disable_retired_platform_sources.py @@ -0,0 +1,68 @@ +"""Disable sources on retired platforms, so the scheduler stops selecting them. + +Milestone #406, phase 1 (switch pixiv off). Rule #171 records the scope decision. + +## Why this is a migration and not a button + +The live instance had one pixiv source still ENABLED when pixiv was retired +(read 2026-09-13, step 1) even though the operator believed it gone. Unregistering +a platform removes it from code; it does not touch the `source` rows that name it. +Left enabled, that row keeps being picked by the scheduler every interval, and +`download_backends` now refuses it with `unsupported_url` — forever, as a +climbing failure count on a source the operator has already given up. + +A migration reaches the live instance on deploy without depending on anyone +finding the row and clicking it. The `run_download` guard is what makes a stale +enabled row SAFE; this is what makes it QUIET. + +## Deliberately NOT done here + +- **No rows are deleted.** Deleting a source sets its posts' `source_id` to NULL + (FK `ON DELETE SET NULL`), and `uq_post_artist_external_id_null_source` can + reject that if a source-less copy of one of those posts already exists. That + needs checking against real data first, which is phase 2's job (step 6). A + disable cannot collide with anything. +- **No posts or images are touched.** The art stays. +- **deviantart is included** because #3069 retired it and nothing disabled its + rows either. The read found none, so for it this is a no-op — written anyway, + so the statement names every retired platform rather than just the latest one. + +## Hardcoded platform names + +A migration is a record of one event, frozen in time, so it names the platforms +it acted on rather than importing today's registry — the registry will keep +changing and this revision must not. + +Revision ID: 0097 +Revises: 0096 +Create Date: 2026-09-13 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0097" +down_revision: Union[str, None] = "0096" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Clears the failure state the same way `SourceService.update` does when a + # source is disabled through the app (issue #1285), so a retired source + # does not keep showing as failing after it stops being polled. A disable + # done here and one done by clicking must leave identical rows. + op.execute( + "UPDATE source SET enabled = false, last_error = NULL, " + "error_type = NULL, consecutive_failures = 0 " + "WHERE enabled AND platform IN ('pixiv', 'deviantart')" + ) + + +def downgrade() -> None: + # Irreversible by design: which of these rows were enabled before is not + # recorded, and re-enabling every retired-platform source would resume + # polling services the product no longer supports. Rule #22 owes no + # migration story back to a dropped platform. + pass diff --git a/backend/app/services/download_backends.py b/backend/app/services/download_backends.py index 49c510a..6d34c07 100644 --- a/backend/app/services/download_backends.py +++ b/backend/app/services/download_backends.py @@ -28,12 +28,32 @@ from .patreon_ingester import PatreonIngester from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source from .pixiv_client import user_id_from_url from .pixiv_ingester import PixivIngester +from .platforms import known_platform_keys from .subscribestar_ingester import SubscribeStarIngester # Platforms whose download + verify go through the native ingester rather than # gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until -# they migrate too. -NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"}) +# they migrate too. pixiv left this set when it was retired (milestone #406). +NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"}) + + +def _unsupported_platform_message(platform: str) -> str | None: + """Why `platform` may not be downloaded or verified, or None if it may. + + A source can outlive its platform. Retiring one (DeviantArt #3069, pixiv + #406) unregisters it, but its `Source` rows — and the `enabled` flag on + them — are data, and data survives a deploy. So this refuses at the two + functions every download and every credential probe pass through, instead + of trusting the scheduler's `enabled` filter and every future caller to + agree. + + Without it a retired platform does not fail: it falls through to the + gallery-dl branch, which is precisely where a platform lands once it is no + longer native — and gallery-dl still has an extractor for it. + """ + if platform in known_platform_keys(): + return None + return f"{platform!r} is not a supported platform (retired or unknown)" # Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure # messages so the operator sees the exact lookup endpoint that was hit. @@ -80,6 +100,13 @@ async def run_download( backfill state machine and owns phase 3. """ platform = ctx["platform"] + refusal = _unsupported_platform_message(platform) + if refusal is not None: + return DownloadResult( + success=False, url=ctx["url"], artist_slug=ctx["artist_slug"], + platform=platform, + error_type=ErrorType.UNSUPPORTED_URL, error_message=refusal, + ), None if uses_native_ingester(platform): return await _run_native_ingester( ctx, source_config, mode, gdl, sync_session_factory @@ -217,6 +244,11 @@ async def verify_source_credential( network / nothing to test). Callers don't branch on platform — they call this and render the result. """ + refusal = _unsupported_platform_message(platform) + if refusal is not None: + # Inconclusive rather than False: nothing was probed, so nothing was + # rejected. False would tell the operator their credential is bad. + return None, refusal if uses_native_ingester(platform): # Native ingester platforms verify via their own lightweight auth probe. # SubscribeStar's probe takes the creator URL directly; Patreon's diff --git a/backend/app/services/extension_service.py b/backend/app/services/extension_service.py index 7a629ab..4b437e4 100644 --- a/backend/app/services/extension_service.py +++ b/backend/app/services/extension_service.py @@ -55,10 +55,6 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P[^/?#]+)", re.IGNORECASE, )), - ("pixiv", re.compile( - r"^https?://(?:www\.)?pixiv\.net/(?:en/)?users/(?P\d+)", - re.IGNORECASE, - )), ] diff --git a/backend/app/services/platforms/__init__.py b/backend/app/services/platforms/__init__.py index be94f3d..4220dfc 100644 --- a/backend/app/services/platforms/__init__.py +++ b/backend/app/services/platforms/__init__.py @@ -11,7 +11,11 @@ Lifted from GallerySubscriber's and ~/.../extension/lib/platforms.js. Five platforms; auth_type and URL patterns match GS exactly so the existing browser extension hits FC unmodified. deviantart was dropped at #3069 (2026-08-27) — -FC downloaders are art-dedicated services only. +FC downloaders are art-dedicated services only. pixiv was retired at +milestone #406 (2026-09-13, rule #171): unregistered here first, which +switches it off everywhere this registry is consulted; `pixiv.py` and the +pixiv client/downloader/ingester stay in the tree, uncalled, until the +milestone's phase 2 deletes them. """ from .base import ( @@ -22,7 +26,6 @@ from .base import ( from .discord import INFO as _DISCORD from .hentaifoundry import INFO as _HENTAIFOUNDRY from .patreon import INFO as _PATREON -from .pixiv import INFO as _PIXIV from .subscribestar import INFO as _SUBSCRIBESTAR PLATFORMS: dict[str, PlatformInfo] = { @@ -32,7 +35,6 @@ PLATFORMS: dict[str, PlatformInfo] = { _SUBSCRIBESTAR, _HENTAIFOUNDRY, _DISCORD, - _PIXIV, ) } diff --git a/extension/background/background.js b/extension/background/background.js index edbc2a0..dd5155a 100644 --- a/extension/background/background.js +++ b/extension/background/background.js @@ -1,32 +1,35 @@ /** - * Background script — message router + Discord token capture - * (webRequest) + Pixiv PKCE OAuth. Direct port of GS background.js; - * api.js client points at FC instead of GS. + * Background script — message router + Discord token capture (webRequest). + * Direct port of GS background.js; api.js client points at FC instead of GS. + * + * pixiv's PKCE OAuth flow lived here until FC retired pixiv (milestone #406). + * It was removed together with pixiv's host permissions rather than left + * behind: a webRequest listener on a host the manifest no longer grants is at + * best dead and at worst a startup failure for the whole background script. */ let discordToken = null; let discordTokenCapturedAt = null; -let pixivRefreshToken = null; -let pixivTokenCapturedAt = null; -let pixivOAuthPending = null; - -const PIXIV_CLIENT_ID = 'MOBrBDS8blbauoSck0ZfDbtuzpyT'; -const PIXIV_CLIENT_SECRET = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj'; -const PIXIV_OAUTH_URL = 'https://app-api.pixiv.net/web/v1/login'; -const PIXIV_TOKEN_URL = 'https://oauth.secure.pixiv.net/auth/token'; -const PIXIV_REDIRECT_URI = 'https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback'; - let initialized = false; async function ensureInitialized() { if (initialized) return; await api.init(); await loadDiscordToken(); - await loadPixivToken(); + await forgetRetiredPixivToken(); initialized = true; } +// A browser that authenticated pixiv before the retirement still holds a live +// OAuth refresh token in extension storage. Nothing reads it any more, and a +// credential for a service FC no longer uses is a liability with no benefit +// (the same reasoning as the server-side cleanup, issue #3980). Removing keys +// that are absent is a no-op, so this is safe on every startup. +async function forgetRetiredPixivToken() { + await browser.storage.local.remove(['pixivRefreshToken', 'pixivTokenCapturedAt']); +} + browser.runtime.onInstalled.addListener(() => ensureInitialized()); browser.runtime.onStartup.addListener(() => ensureInitialized()); ensureInitialized().catch(e => console.error('init failed:', e)); @@ -141,98 +144,6 @@ async function saveDiscordToken(token) { await browser.storage.local.set({ discordToken: token, discordTokenCapturedAt }); } -// ---- Pixiv PKCE OAuth ---- - -async function loadPixivToken() { - const s = await browser.storage.local.get(['pixivRefreshToken', 'pixivTokenCapturedAt']); - pixivRefreshToken = s.pixivRefreshToken || null; - pixivTokenCapturedAt = s.pixivTokenCapturedAt || null; -} - -async function savePixivToken(token) { - pixivRefreshToken = token; - pixivTokenCapturedAt = new Date().toISOString(); - await browser.storage.local.set({ pixivRefreshToken: token, pixivTokenCapturedAt }); -} - -function generateCodeVerifier() { - const a = new Uint8Array(32); - crypto.getRandomValues(a); - return base64UrlEncode(a); -} - -async function generateCodeChallenge(verifier) { - const data = new TextEncoder().encode(verifier); - const hash = await crypto.subtle.digest('SHA-256', data); - return base64UrlEncode(new Uint8Array(hash)); -} - -function base64UrlEncode(buf) { - return btoa(String.fromCharCode(...buf)).replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, ''); -} - -async function initiatePixivOAuth() { - const codeVerifier = generateCodeVerifier(); - const codeChallenge = await generateCodeChallenge(codeVerifier); - - const params = new URLSearchParams({ - code_challenge: codeChallenge, - code_challenge_method: 'S256', - client: 'pixiv-android', - }); - const tab = await browser.tabs.create({ url: `${PIXIV_OAUTH_URL}?${params}` }); - - return new Promise((resolve, reject) => { - pixivOAuthPending = { codeVerifier, tabId: tab.id, resolve, reject }; - setTimeout(() => { - if (pixivOAuthPending && pixivOAuthPending.tabId === tab.id) { - pixivOAuthPending = null; - reject(new Error('Pixiv OAuth timed out (5 min)')); - } - }, 5 * 60 * 1000); - }); -} - -browser.webRequest.onBeforeRedirect.addListener( - async (details) => { - if (!pixivOAuthPending) return; - if (details.tabId !== pixivOAuthPending.tabId) return; - const url = new URL(details.redirectUrl); - const code = url.searchParams.get('code'); - if (!code) return; - const verifier = pixivOAuthPending.codeVerifier; - const resolve = pixivOAuthPending.resolve; - const reject = pixivOAuthPending.reject; - pixivOAuthPending = null; - try { - const tokenResp = await fetch(PIXIV_TOKEN_URL, { - method: 'POST', - headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, - body: new URLSearchParams({ - client_id: PIXIV_CLIENT_ID, - client_secret: PIXIV_CLIENT_SECRET, - code, - code_verifier: verifier, - grant_type: 'authorization_code', - include_policy: 'true', - redirect_uri: PIXIV_REDIRECT_URI, - }), - }); - const body = await tokenResp.json(); - if (!body.refresh_token) { - reject(new Error(`Pixiv token exchange failed: ${JSON.stringify(body)}`)); - return; - } - await savePixivToken(body.refresh_token); - try { await browser.tabs.remove(details.tabId); } catch {} - resolve(body.refresh_token); - } catch (e) { - reject(e); - } - }, - { urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] }, -); - // Extract → verify → upload one cookie-auth platform. Returns a structured // outcome so the two callers (EXPORT_COOKIES single, EXPORT_ALL_COOKIES) shape // their own response + skip semantics. Verifies the captured cookies are @@ -277,8 +188,6 @@ browser.runtime.onMessage.addListener(async (msg) => { } } else if (key === 'discord') { status[key] = { hasToken: !!discordToken, capturedAt: discordTokenCapturedAt }; - } else if (key === 'pixiv') { - status[key] = { hasToken: !!pixivRefreshToken, capturedAt: pixivTokenCapturedAt }; } else { status[key] = {}; } @@ -306,13 +215,6 @@ browser.runtime.onMessage.addListener(async (msg) => { await api.uploadCredentials('discord', 'token', discordToken); return { success: true }; } - if (key === 'pixiv') { - if (!pixivRefreshToken) { - await initiatePixivOAuth(); - } - await api.uploadCredentials('pixiv', 'token', pixivRefreshToken); - return { success: true }; - } return { error: 'Unsupported platform.' }; } catch (e) { return { error: e.message }; diff --git a/extension/lib/platforms.js b/extension/lib/platforms.js index c3f0c5a..86972d9 100644 --- a/extension/lib/platforms.js +++ b/extension/lib/platforms.js @@ -60,14 +60,6 @@ const PLATFORMS = { urlPattern: /^https?:\/\/(www\.)?discord\.com/, note: 'Open Discord in browser to capture token', }, - pixiv: { - name: 'Pixiv', - domains: ['.pixiv.net', 'www.pixiv.net', 'pixiv.net'], - authType: 'token', - color: '#0096FA', - urlPattern: /^https?:\/\/(www\.)?pixiv\.net/, - note: 'Click to authenticate via OAuth', - }, }; /** @@ -88,7 +80,6 @@ const PLATFORM_ARTIST_PATTERNS = { patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i, subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i, hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i, - pixiv: /^https?:\/\/(www\.)?pixiv\.net\/(en\/)?users\/\d+/i, }; function getPlatformFromUrl(url) { diff --git a/extension/manifest.json b/extension/manifest.json index b488988..3227775 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -32,9 +32,6 @@ "*://*.subscribestar.adult/*", "*://*.hentai-foundry.com/*", "*://*.discord.com/*", - "*://*.pixiv.net/*", - "*://app-api.pixiv.net/*", - "*://oauth.secure.pixiv.net/*", "*://*/*" ], @@ -59,8 +56,7 @@ "*://*.patreon.com/*", "*://*.subscribestar.com/*", "*://*.subscribestar.adult/*", - "*://*.hentai-foundry.com/*", - "*://*.pixiv.net/*" + "*://*.hentai-foundry.com/*" ], "js": ["lib/platforms.js", "content/content-script.js"], "css": ["content/content-script.css"], diff --git a/extension/popup/popup.js b/extension/popup/popup.js index 771bb6a..7becdba 100644 --- a/extension/popup/popup.js +++ b/extension/popup/popup.js @@ -108,7 +108,7 @@ function createPlatformCard(key, platform, status) { card.className = 'platform-card'; card.dataset.platform = key; - const isTokenOnly = platform.authType === 'token' && !['discord', 'pixiv'].includes(key); + const isTokenOnly = platform.authType === 'token' && key !== 'discord'; const discordNeedsToken = key === 'discord' && !status.hasToken; if (isTokenOnly || discordNeedsToken) card.classList.add('disabled'); @@ -141,7 +141,6 @@ function createPlatformCard(key, platform, status) { function statusText(s, platform, key) { if (key === 'discord') return s.hasToken ? 'Token captured — ready' : 'Open Discord to capture token'; - if (key === 'pixiv') return s.hasToken ? 'Token captured — ready' : 'Click to authenticate via OAuth'; if (platform.authType === 'token') return 'Manual token entry required'; if (s.error) return 'Error checking cookies'; if (!s.hasCookies || !s.cookieCount) return 'No cookies — log in first'; @@ -149,7 +148,6 @@ function statusText(s, platform, key) { } function statusClass(s, platform, key) { if (key === 'discord') return s.hasToken ? 'ready' : 'no-cookies'; - if (key === 'pixiv') return s.hasToken ? 'ready' : 'no-cookies'; if (platform.authType === 'token') return 'no-cookies'; if (s.error) return 'error'; if (!s.hasCookies || !s.cookieCount) return 'no-cookies'; diff --git a/extension/test/platforms.spec.js b/extension/test/platforms.spec.js index df1e53a..f869751 100644 --- a/extension/test/platforms.spec.js +++ b/extension/test/platforms.spec.js @@ -18,7 +18,6 @@ describe('getPlatformFromUrl', () => { expect(getPlatformFromUrl('https://subscribestar.adult/someone')).toBe('subscribestar') expect(getPlatformFromUrl('https://www.hentai-foundry.com/user/someone')).toBe('hentaifoundry') expect(getPlatformFromUrl('https://discord.com/channels/@me')).toBe('discord') - expect(getPlatformFromUrl('https://www.pixiv.net/en/users/123')).toBe('pixiv') }) it('accepts http as well as https, with or without www', () => { @@ -32,6 +31,15 @@ describe('getPlatformFromUrl', () => { expect(getPlatformFromUrl('')).toBe(null) }) + it('returns null for pixiv, retired at milestone #406', () => { + // Retired on the operator's platform-focus decision (rule #171). Same guard + // as deviantart's below, and for the same reason: an absence nothing asserts + // is an absence a later edit can quietly undo. + expect(getPlatformFromUrl('https://www.pixiv.net/en/users/12345')).toBe(null) + expect(PLATFORMS.pixiv).toBeUndefined() + expect(PLATFORM_ARTIST_PATTERNS.pixiv).toBeUndefined() + }) + it('returns null for deviantart, retired at #3069', () => { // The 2026-07-05 product decision (FC downloaders = art-dedicated services // only) left deviantart wired for seven weeks. Asserting the negative is @@ -80,12 +88,6 @@ describe('isArtistPage', () => { ) }) - it('matches Pixiv numeric user pages, with or without the /en/ prefix', () => { - expect(isArtistPage('https://www.pixiv.net/users/12345', 'pixiv')).toBe(true) - expect(isArtistPage('https://www.pixiv.net/en/users/12345', 'pixiv')).toBe(true) - expect(isArtistPage('https://www.pixiv.net/en/artworks/999', 'pixiv')).toBe(false) - }) - it('returns false for a platform with no artist pattern (discord)', () => { expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false) }) @@ -123,8 +125,7 @@ describe('platform table integrity', () => { const samples = { patreon: 'https://www.patreon.com/cw/Atole', subscribestar: 'https://subscribestar.adult/someone', - hentaifoundry: 'https://www.hentai-foundry.com/user/someone', - pixiv: 'https://www.pixiv.net/en/users/12345' + hentaifoundry: 'https://www.hentai-foundry.com/user/someone' } for (const [key, url] of Object.entries(samples)) { expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true) @@ -179,8 +180,9 @@ describe('manifest.json agrees with the platform table', () => { for (const h of manifest.host_permissions) { if (h === '*://*/*') continue const host = hostOf(h) - // pixiv's OAuth/API hosts are pixiv infrastructure, not creator pages, - // so they are matched by suffix rather than by the domains list. + // Suffix matching lets a platform's infrastructure subdomains belong to + // it without listing each one. (It was added for pixiv's OAuth hosts, + // which left with pixiv at milestone #406; the rule itself is general.) const claimed = Object.values(PLATFORMS).some( (p) => p.domains.includes(host) || p.domains.some((d) => host.endsWith(d)) ) diff --git a/frontend/src/components/settings/BrowserExtensionCard.vue b/frontend/src/components/settings/BrowserExtensionCard.vue index 4226eba..50b6a6a 100644 --- a/frontend/src/components/settings/BrowserExtensionCard.vue +++ b/frontend/src/components/settings/BrowserExtensionCard.vue @@ -19,7 +19,7 @@

Pushes session cookies from supported platforms - (patreon, subscribestar, hentaifoundry, discord, pixiv) + (patreon, subscribestar, hentaifoundry, discord) into FabledCurator, and lets you add a creator as a source from their page in one click.

diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index 5c87393..e026aed 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -77,7 +77,7 @@ const recapturing = computed(() => !!props.source.backfill_recapture) // Recover / recapture are native-ingester features (ledger-bypass re-walk and // post-text re-grab), available to every native platform — not just Patreon. // Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. -const NATIVE_PLATFORMS = ['patreon', 'subscribestar', 'pixiv'] +const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) diff --git a/tests/test_api_extension.py b/tests/test_api_extension.py index 65a8a77..dffa7dc 100644 --- a/tests/test_api_extension.py +++ b/tests/test_api_extension.py @@ -129,8 +129,6 @@ async def test_resolve_artist_name_dispatches_per_platform(db, monkeypatch): ("https://www.subscribestar.com/foobar", "subscribestar", "foobar"), ("https://subscribestar.adult/foobar", "subscribestar", "foobar"), ("https://www.hentai-foundry.com/user/Foo/profile", "hentaifoundry", "Foo"), - ("https://www.pixiv.net/users/12345", "pixiv", "12345"), - ("https://www.pixiv.net/en/users/12345", "pixiv", "12345"), ]) @pytest.mark.asyncio async def test_quick_add_source_url_patterns(client, ext_key, url, platform, slug): @@ -176,6 +174,22 @@ async def test_quick_add_source_rejects_retired_deviantart(client, ext_key): assert "deviantart" not in body["known"] +@pytest.mark.asyncio +async def test_quick_add_source_rejects_retired_pixiv(client, ext_key): + """Milestone #406: the same shape as deviantart's retirement above. An + un-updated extension can still offer the button on a pixiv creator page, so + the backend refuses rather than creating a source nothing can download.""" + resp = await client.post( + "/api/extension/quick-add-source", + json={"url": "https://www.pixiv.net/users/12345"}, + headers={"X-Extension-Key": ext_key}, + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_platform" + assert "pixiv" not in body["known"] + + @pytest.mark.asyncio async def test_quick_add_source_invalid_url_400(client, ext_key): resp = await client.post( diff --git a/tests/test_api_platforms.py b/tests/test_api_platforms.py index f65817a..04074d1 100644 --- a/tests/test_api_platforms.py +++ b/tests/test_api_platforms.py @@ -6,17 +6,17 @@ pytestmark = pytest.mark.integration @pytest.mark.asyncio -async def test_platforms_returns_gs_five(client): +async def test_platforms_returns_the_supported_four(client): resp = await client.get("/api/platforms") assert resp.status_code == 200 body = await resp.get_json() platforms = body["platforms"] assert set(platforms.keys()) == { - "patreon", "subscribestar", "hentaifoundry", - "discord", "pixiv", + "patreon", "subscribestar", "hentaifoundry", "discord", } assert "fanbox" not in platforms assert "deviantart" not in platforms # retired at #3069 + assert "pixiv" not in platforms # retired at milestone #406 @pytest.mark.asyncio @@ -37,5 +37,4 @@ async def test_platforms_record_shape(client): async def test_platform_auth_types_match_gs(client): body = await (await client.get("/api/platforms")).get_json() assert body["platforms"]["discord"]["auth_type"] == "token" - assert body["platforms"]["pixiv"]["auth_type"] == "token" assert body["platforms"]["patreon"]["auth_type"] == "cookies" diff --git a/tests/test_download_backends.py b/tests/test_download_backends.py index ce7dda1..8a1ff20 100644 --- a/tests/test_download_backends.py +++ b/tests/test_download_backends.py @@ -1,21 +1,107 @@ """download_backends — the single predicate that routes a platform to the native ingester vs. gallery-dl. Pure, no DB.""" +from pathlib import Path + +import pytest + from backend.app.services.download_backends import ( NATIVE_INGESTER_PLATFORMS, _campaign_resolution_error, _native_ingester_cls, + _unsupported_platform_message, + run_download, uses_native_ingester, + verify_source_credential, ) +from backend.app.services.gallery_dl import ErrorType from backend.app.services.pixiv_ingester import PixivIngester def test_native_platforms(): - for platform in ("patreon", "subscribestar", "pixiv"): + for platform in ("patreon", "subscribestar"): assert uses_native_ingester(platform) is True assert platform in NATIVE_INGESTER_PLATFORMS +def test_pixiv_is_no_longer_native(): + """Retired at milestone #406. The refusal below is what stops it falling + through to gallery-dl now that it is not native.""" + assert uses_native_ingester("pixiv") is False + assert "pixiv" not in NATIVE_INGESTER_PLATFORMS + + +# --- the retired-platform guard (#406 phase 1) ----------------------------- + + +class _RecordingGalleryDL: + """Stands in for GalleryDLService: records whether a download was attempted.""" + + def __init__(self): + self.calls = [] + + async def download(self, **kwargs): + self.calls.append(kwargs["platform"]) + return "reached gallery-dl" + + +def _ctx(platform): + return { + "platform": platform, "url": f"https://example.invalid/{platform}", + "artist_slug": "someone", "cookies_path": None, "auth_token": None, + } + + +@pytest.mark.asyncio +async def test_a_retired_platform_never_reaches_a_downloader(): + """An enabled source on a retired platform is data that survives a deploy. + Unguarded, pixiv — no longer native — would fall straight through to the + gallery-dl branch, which still has a pixiv extractor.""" + gdl = _RecordingGalleryDL() + result, campaign_id = await run_download( + ctx=_ctx("pixiv"), source_config=None, skip_value=False, mode=None, + gdl=gdl, sync_session_factory=None, + ) + assert gdl.calls == [] + assert result.success is False + assert result.error_type == ErrorType.UNSUPPORTED_URL + assert "pixiv" in result.error_message + assert campaign_id is None + + +@pytest.mark.asyncio +async def test_a_supported_gallery_dl_platform_still_reaches_gallery_dl(): + """The positive control. Without it, a guard that refused EVERY platform + would pass the test above just as well (rule #167).""" + gdl = _RecordingGalleryDL() + result, _ = await run_download( + ctx=_ctx("hentaifoundry"), source_config=None, skip_value=False, mode=None, + gdl=gdl, sync_session_factory=None, + ) + assert gdl.calls == ["hentaifoundry"] + assert result == "reached gallery-dl" + + +def test_the_guard_discriminates_by_registration(): + assert _unsupported_platform_message("hentaifoundry") is None + assert _unsupported_platform_message("patreon") is None + assert _unsupported_platform_message("pixiv") is not None + assert _unsupported_platform_message("deviantart") is not None + + +@pytest.mark.asyncio +async def test_verifying_a_retired_platform_is_inconclusive_not_rejected(): + """Nothing is probed, so nothing is rejected — returning False would tell the + operator their credential is bad when the platform is simply gone.""" + ok, message = await verify_source_credential( + platform="pixiv", url="https://www.pixiv.net/users/1", artist_slug="someone", + config_overrides=None, cookies_path=None, auth_token=None, + images_root=Path("/nonexistent"), + ) + assert ok is None + assert "pixiv" in message + + def test_gallery_dl_platforms_are_not_native(): # The platforms still served by gallery-dl must NOT route to the native # ingester — guards an accidental over-broad migration. diff --git a/tests/test_platforms_registry.py b/tests/test_platforms_registry.py index c94341c..df62db1 100644 --- a/tests/test_platforms_registry.py +++ b/tests/test_platforms_registry.py @@ -11,13 +11,21 @@ from backend.app.services.platforms import ( ) -def test_known_platform_keys_is_gs_five(): +def test_known_platform_keys_are_the_supported_four(): + # GS's original five, less pixiv (retired at milestone #406, rule #171). assert known_platform_keys() == frozenset({ - "patreon", "subscribestar", "hentaifoundry", - "discord", "pixiv", + "patreon", "subscribestar", "hentaifoundry", "discord", }) +def test_pixiv_is_retired(): + # Milestone #406 phase 1. Unregistering is what switches pixiv off: the + # registry feeds /api/platforms, the source validator and the download + # guard, so this one absence is load-bearing everywhere else. + assert "pixiv" not in PLATFORMS + assert "pixiv" not in known_platform_keys() + + def test_fanbox_not_in_registry(): # Sanity check — FC-3a added 'fanbox' by mistake; it's not a GS platform. assert "fanbox" not in PLATFORMS diff --git a/tests/test_sidecar_util.py b/tests/test_sidecar_util.py index 8ba6ab9..f6773c5 100644 --- a/tests/test_sidecar_util.py +++ b/tests/test_sidecar_util.py @@ -183,19 +183,6 @@ def test_parse_subscribestar_post_url_derived_and_post_id_wins(): assert sd.post_url == "https://www.subscribestar.com/posts/360360" -def test_parse_pixiv_post_url_derived(): - """Pixiv's `url` is the image URL (i.pximg.net); must be replaced - with the post permalink under /artworks/.""" - sd = parse_sidecar({ - "category": "pixiv", - "id": 140466853, - "url": "https://i.pximg.net/img-original/img/2026/01/28/10/28/24/140466853_p0.jpg", - "title": "Nerissa x Jailbird", - }) - assert sd.external_post_id == "140466853" - assert sd.post_url == "https://www.pixiv.net/artworks/140466853" - - def test_parse_hentaifoundry_post_url_derived(): """HF sidecars omit `url` entirely and use `index`+`user` for the post's natural key. Synthesize the canonical /pictures/user// diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 09ffdd2..5fb32e9 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -23,14 +23,15 @@ async def _artist(db, name="Alice"): @pytest.mark.asyncio -async def test_known_platforms_is_gs_five(db): +async def test_known_platforms_are_the_supported_four(db): assert KNOWN_PLATFORMS == frozenset({ - "patreon", "subscribestar", "hentaifoundry", - "discord", "pixiv", + "patreon", "subscribestar", "hentaifoundry", "discord", }) assert "fanbox" not in KNOWN_PLATFORMS # Retired at #3069 — a source can no longer be created on it. assert "deviantart" not in KNOWN_PLATFORMS + # Retired at milestone #406 — likewise. + assert "pixiv" not in KNOWN_PLATFORMS @pytest.mark.asyncio -- 2.54.0 From c2f9e9cc0829dd72c1bdbb94179f93cb15781201 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 13 Sep 2026 11:45:49 -0400 Subject: [PATCH 36/37] docs: stop claiming pixiv, and stop claiming everything gallery-dl supports (406 step 4) Ships with the switch-off rather than with the code removal: a doc that promises a platform the product refuses is the Install and Public Surface area's characteristic defect. pixiv comes out of README (twice), SECURITY.md (twice), .env.example and the compose header. The stored-credential warnings now name Patreon and SubscribeStar - the accounts that usually carry a payment method. One correction beyond pixiv. README said FabledCurator follows creators on Patreon, SubscribeStar, Pixiv "and anything gallery-dl supports". That was already false: a platform not in the registry is rejected, however capable gallery-dl is. It now names the real set, which rule 171 records: Patreon, SubscribeStar, Discord and HentaiFoundry. The 3422 docs guards still hold - the key path and bootstrap variable are untouched. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- .env.example | 2 +- README.md | 10 +++++----- SECURITY.md | 4 ++-- docker-compose.yml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index fe2cb12..a000892 100644 --- a/.env.example +++ b/.env.example @@ -93,7 +93,7 @@ DB_NAME=fabledcurator # # FabledCurator has no login, no accounts and no permission model. Anything # that can reach PORT is an administrator and can read the platform session -# cookies the app stores for Patreon, SubscribeStar and Pixiv. +# cookies the app stores for Patreon and SubscribeStar. # # Bind it to a trusted network. See "Before you expose it" in README.md and # the deployment posture section of SECURITY.md. diff --git a/README.md b/README.md index 4c2fee2..8c94e40 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ through afterwards. artist, tag, post and series. A newest-first feed of what just arrived as the front page, a random Showcase, a filterable gallery, a similarity-driven Explore view, and a page-turning reader for series. -- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Pixiv and - anything `gallery-dl` supports, on a schedule. Handles paywalled posts using - your own logged-in session. +- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Discord and + HentaiFoundry, on a schedule. Handles paywalled posts using your own + logged-in session. - **ML tagging.** Runs image models in-container to suggest tags, group characters, find near-duplicates and power similarity search. Suggestions are reviewable — it proposes, you confirm, and it learns which proposals you keep @@ -39,8 +39,8 @@ is no config file to edit beyond a handful of bootstrap environment variables. permission model. Anything that can reach the port is an administrator. That matters more here than it would in most self-hosted apps, because of what -this one stores: **live platform session cookies for Patreon, SubscribeStar and -Pixiv** — accounts that usually have a payment method attached. Whoever reaches +this one stores: **live platform session cookies for Patreon and +SubscribeStar** — accounts that usually have a payment method attached. Whoever reaches the port can read them, alongside your entire library. So: diff --git a/SECURITY.md b/SECURITY.md index f59adcf..bad7648 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -26,7 +26,7 @@ FabledCurator is self-hosted and holds things worth stating plainly, because they shape what counts as a serious bug here: - **Platform credentials.** The app captures and stores session cookies for - third-party subscription sites (Patreon, SubscribeStar, Pixiv) so it can + third-party subscription sites (Patreon, SubscribeStar) so it can download on the operator's behalf. These are live credentials for accounts that usually carry a payment method. Anything that discloses them, decrypts them, or lets one user of a shared instance read another's is high severity. @@ -56,7 +56,7 @@ reverse proxy. It also does not authenticate anyone — see above. These are documented design decisions, not oversights. Putting this on the public internet, with or without TLS, hands whoever finds -it your Patreon, SubscribeStar and Pixiv sessions. A reverse proxy that adds +it your Patreon and SubscribeStar sessions. A reverse proxy that adds TLS but not an authentication layer does not change that. Reports that reduce to "the application is served over HTTP", "there is no diff --git a/docker-compose.yml b/docker-compose.yml index 0bca4f2..e7d919d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ # explicitly skips the override and pulls the published :latest images. # # FabledCurator has no authentication. Whatever can reach ${PORT} is an -# administrator, including over the stored Patreon/SubscribeStar/Pixiv session +# administrator, including over the stored Patreon/SubscribeStar session # cookies. Do not publish this port beyond a network you trust — see # "Before you expose it" in README.md. -- 2.54.0 From e3c516d6be6431be482d9fbd6d2faf3250200626 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 13 Sep 2026 11:52:06 -0400 Subject: [PATCH 37/37] fix: a retired platform leaves the chip maps, and a fixture stops creating pixiv sources (406) e3fd8c6 failed two lanes, both on things pixiv's retirement correctly started refusing. Unit - test_fe_be_contract pins platformColor.js's ICONS keys to known_platform_keys(). I had kept pixiv's icon, colour and label "so existing pixiv posts don't look broken". That was wrong, and the file's own header already said why: unknown platforms fall back to a grey chip with the raw key, and that fallback is deliberately what a retired platform hits - it is how a pre-3069 deviantart row already renders. So pixiv leaves all three maps, and its posts show as a grey "pixiv" chip rather than a broken one. The header now says outright that a retired platform is removed, never kept, since the milestone plan itself got this backwards. Integration - four reassign tests built their fixture source through SourceService.create with platform="pixiv", which the validator now rejects. Reassign never reads the platform and never moves files, so any registered platform serves; the fixture uses hentaifoundry. 1294 other integration tests passed on e3fd8c6, so nothing else used pixiv through a validating path. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SHQB1YukL3VyvMK8rcbmV9 --- frontend/src/utils/platformColor.js | 13 +++++++------ tests/test_source_service.py | 9 ++++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/frontend/src/utils/platformColor.js b/frontend/src/utils/platformColor.js index ef759b6..8d57cb1 100644 --- a/frontend/src/utils/platformColor.js +++ b/frontend/src/utils/platformColor.js @@ -1,9 +1,13 @@ // Single source of truth for platform → color + icon mapping. Used by -// PlatformChip and any other GS-style platform-tagged surface. The five +// PlatformChip and any other GS-style platform-tagged surface. The four // platforms FC supports map 1:1 to the GS palette; unknown platforms fall // back to grey + mdi-web — which is deliberately what a retired platform -// hits: a pre-#3069 deviantart source row still renders, as its raw key on -// a grey chip. Operator-confirmed scope 2026-05-27. The ICONS key set is +// hits: a pre-#3069 deviantart source row, or a post from pixiv (retired at +// milestone #406), still renders, as its raw key on a grey chip. So a +// retired platform is REMOVED from these maps, never kept "so old rows +// look right" — the fallback is what makes old rows look right, and keeping +// the entry would break the contract pin below. Operator-confirmed scope +// 2026-05-27. The ICONS key set is // pinned against backend known_platform_keys() by // tests/test_fe_be_contract.py. @@ -12,7 +16,6 @@ const ICONS = { subscribestar: 'mdi-star', hentaifoundry: 'mdi-palette', discord: 'mdi-discord', - pixiv: 'mdi-alpha-p-box', } const COLORS = { @@ -20,7 +23,6 @@ const COLORS = { subscribestar: 'amber', hentaifoundry: 'purple', discord: 'indigo', - pixiv: 'blue', } const LABELS = { @@ -28,7 +30,6 @@ const LABELS = { subscribestar: 'SubscribeStar', hentaifoundry: 'HentaiFoundry', discord: 'Discord', - pixiv: 'Pixiv', } export function platformIcon(platform) { diff --git a/tests/test_source_service.py b/tests/test_source_service.py index 5fb32e9..9f8428b 100644 --- a/tests/test_source_service.py +++ b/tests/test_source_service.py @@ -388,14 +388,17 @@ async def test_update_while_enabled_keeps_failure_state(db): async def _source_with_content(db, svc, artist): """A source under `artist` with one post + one image it contributed.""" from backend.app.models import ImageProvenance, ImageRecord, Post + # Any registered platform will do — reassign never reads the platform, and + # never moves files (the storage path is immutable). This used pixiv until + # pixiv was retired (milestone #406) and `create` began refusing it. rec = await svc.create( - artist_id=artist.id, platform="pixiv", - url=f"https://www.pixiv.net/users/{artist.id}", + artist_id=artist.id, platform="hentaifoundry", + url=f"https://www.hentai-foundry.com/user/{artist.slug}/profile", ) post = Post(source_id=rec.id, artist_id=artist.id, external_post_id="p1") db.add(post) img = ImageRecord( - path=f"/images/{artist.slug}/pixiv/pixiv/1_a_00.jpg", + path=f"/images/{artist.slug}/hentaifoundry/hentaifoundry/1_a_00.jpg", sha256=str(artist.id).rjust(64, "0"), size_bytes=1, mime="image/jpeg", width=1, height=1, origin="imported_filesystem", integrity_status="unknown", artist_id=artist.id, -- 2.54.0