"""The learned membership roster: what the account actually subscribes to. Milestone 387, phase C. Sibling of `service_roster` (milestone 365) and built on the same insight — an absence is only observable against a record of presence. There, a stopped worker; here, a subscription that lapsed. ## Nothing calls this yet `touch_membership` is written before its caller because the caller (the sweep, C3) needs a client seam (C2) that needs Patreon's real response characterised from a captured sample (C0), and that capture needs the operator's browser session. The write side does not depend on any of it: an upsert keyed on (platform, external_campaign_id) is the same regardless of what the payload turns out to look like, and `details` carries whatever C0 finds. ## Why the whitelist lives here and not in the column `platform_membership.status` is an unconstrained String holding the PLATFORM's own word — `active_patron`, not some normalised FC value. The mapping from those words to FC's meaning is a read-site concern and belongs in code that can be corrected without a migration, because the vocabulary comes from whatever each platform says and will be discovered per platform rather than designed up front. `native_ingest_common.MEMBERSHIP_STATUS` is where that knowledge accumulates as platforms are characterised, and it holds no guesses. It lives there rather than here because platform clients need it, and a client may not import this module (test_gated_reason.py). """ from __future__ import annotations import logging from collections.abc import Awaitable, Callable from datetime import UTC, datetime, timedelta from sqlalchemy import func, select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from ..models import MembershipSync, PlatformMembership, Source from .native_ingest_common import has_paid_access log = logging.getLogger(__name__) async def touch_membership( session: AsyncSession, *, platform: str, external_campaign_id: str, display_name: str | None = None, url: str | None = None, status: str | None = None, tier_names: list | None = None, amount_cents: int | None = None, currency: str | None = None, details: dict | None = None, ) -> None: """Record that this membership was observed just now. Upsert rather than read-modify-write, for the same reason as `service_roster.touch_service`: a sweep may overlap its own previous run, and the last writer is simply the most recent sighting. `first_seen_at` is deliberately NOT in the update set. It is the one field that answers "has this ever been true", which is what makes a membership's later DISAPPEARANCE readable as a lapse rather than indistinguishable from a creator FC never knew about. Every other column is last-writer-wins, including status — a membership that goes from active to former must move. """ stmt = pg_insert(PlatformMembership).values( platform=platform, external_campaign_id=external_campaign_id, display_name=display_name, url=url, status=status, tier_names=tier_names, amount_cents=amount_cents, currency=currency, details=details or {}, ) stmt = stmt.on_conflict_do_update( constraint="uq_platform_membership_platform_campaign", set_={ "display_name": stmt.excluded.display_name, "url": stmt.excluded.url, "status": stmt.excluded.status, "tier_names": stmt.excluded.tier_names, "amount_cents": stmt.excluded.amount_cents, "currency": stmt.excluded.currency, "details": stmt.excluded.details, "last_seen_at": func.now(), }, ) await session.execute(stmt) # --------------------------------------------------------------------------- # The sweep, and the state that makes its failures readable (#387 C3) # --------------------------------------------------------------------------- # # How long a successful sync stays trustworthy. Beyond this the roster is # STALE, and C4 must refuse to draw conclusions from it — "you are tracking 12 # sources you no longer subscribe to", computed from a roster that stopped # syncing a week ago, is an invitation to cancel things the operator is still # paying for. # # Generous relative to the daily cadence: a few missed runs are a blip, not a # reason to stop trusting a roster that changes on a billing cycle. ROSTER_STALE_AFTER = timedelta(days=3) async def get_sync_state(session: AsyncSession, platform: str) -> MembershipSync | None: return (await session.execute( select(MembershipSync).where(MembershipSync.platform == platform) )).scalar_one_or_none() def roster_is_fresh(state: MembershipSync | None, *, now: datetime | None = None) -> bool: """May a caller draw CONCLUSIONS from this roster? False for never-synced and for stale, and those are deliberately the same answer here even though the UI must tell them apart: both mean the roster is not evidence. The asymmetry that matters is that `False` never means "you subscribe to nothing" — it means "we do not know", and a caller that cannot represent "we do not know" must not be asking this question. """ if state is None or state.last_success_at is None: return False now = now or datetime.now(UTC) return (now - state.last_success_at) <= ROSTER_STALE_AFTER async def _record_sync(session: AsyncSession, platform: str, **values) -> None: stmt = pg_insert(MembershipSync).values(platform=platform, **values) await session.execute(stmt.on_conflict_do_update( constraint="uq_membership_sync_platform", set_={**values, "updated_at": func.now()}, )) def roster_user_id(client) -> str | None: """The account id a client's roster walk needs, if that client needs one. Patreon's members endpoint filters on the account's own user id, so the sweep has to resolve it first. SubscribeStar's /subscriptions page is simply the logged-in account's, with nothing to resolve. Probed with `getattr`, the same way the sweep probes `iter_memberships` itself (rule #169), rather than called unconditionally. Calling `current_user_id()` unconditionally was the one place the membership seam was still Patreon-shaped: note #3970 promised a second platform would be one `builders` line plus the client method, and D1 found the sweep would instead have crashed on the first client without that method. """ resolve = getattr(client, "current_user_id", None) return resolve() if resolve is not None else None async def sync_platform( session: AsyncSession, *, platform: str, fetch: Callable[[], Awaitable[list]], now: datetime | None = None, ) -> dict: """Walk one platform's roster and record what happened. `fetch` is injected rather than built here so the error-to-state mapping — the part with the consequences — is testable without a credential, and so this service needs to know nothing about how any particular client is constructed. THE FETCH COMPLETES BEFORE ANYTHING IS WRITTEN. That ordering is the whole safety property: a walk that dies half way through pagination writes nothing, so a failure can never leave a roster that is partly this week's and partly last week's. (`touch_membership` never deletes, so a failure cannot empty the roster either — but "intact" should mean intact, not merely non-empty.) Returns a summary dict; never raises for a platform failure, because one platform failing must not abort the others. """ now = now or datetime.now(UTC) await _record_sync(session, platform, last_attempt_at=now) await session.commit() try: memberships = await fetch() except Exception as exc: # noqa: BLE001 - deliberately broad, see below # Broad on purpose: a sweep is a background job, and ANY escape here # kills the run for every other platform too. The exception's class # name is recorded so the distinction the client drew (auth vs drift # vs transport) survives into the UI, which is where it is actionable. # # EXCEPT the worker asking us to stop. Celery raises its soft time # limit as an ordinary Exception subclass, so a broad catch swallows # the shutdown request and lets the sweep run on into the HARD limit, # where it is SIGKILLed mid-transaction. A sweep that cannot be stopped # is worse than one that fails. (KeyboardInterrupt and SystemExit are # BaseException and pass through this clause already.) from celery.exceptions import SoftTimeLimitExceeded if isinstance(exc, SoftTimeLimitExceeded): raise await session.rollback() await _record_sync( session, platform, last_error_type=type(exc).__name__, last_error_message=str(exc)[:2000], ) await session.commit() log.warning("membership sync failed for %s: %s", platform, exc) return {"platform": platform, "ok": False, "error": type(exc).__name__} for m in memberships: await touch_membership( session, platform=platform, external_campaign_id=m.campaign_id, display_name=m.display_name, url=m.url, status=m.status, tier_names=m.tier_names or None, amount_cents=m.amount_cents, currency=m.currency, details={**(m.details or {}), "is_free_member": m.is_free_member}, ) await _record_sync( session, platform, last_success_at=now, last_count=len(memberships), # Cleared on success — a stale error beside a fresh success would read # as "still broken" forever. last_error_type=None, last_error_message=None, ) await session.commit() log.info("membership sync ok for %s: %d membership(s)", platform, len(memberships)) return {"platform": platform, "ok": True, "count": len(memberships)} # --------------------------------------------------------------------------- # Membership <-> Source identity (#387 C4) # --------------------------------------------------------------------------- # # "Is this membership already tracked?" is asked by TWO features — C4's # reconciliation buckets and E4's creator suggestions — and it lives here, once, # on purpose. Built inline in C4 it would have looked finished while leaving E4 # matching on name similarity alone, so the two would answer the same question # differently and only one of them would be right. # # E4 and C4 use it from opposite sides: E4 as the NEGATIVE check (propose only # where nothing matches) and C4 as the join itself. # Any platform that caches its creator id does so under this suffix; see # `download_service._phase3_persist`, which writes `patreon_campaign_id`. _CAMPAIGN_KEY_SUFFIX = "_campaign_id" def identity_keys_for_source(source: Source) -> set[str]: """Every platform-side creator id cached on this source. Reads ANY `_campaign_id` override rather than naming Patreon's, so a second platform participates by caching its id under the same suffix — no registry, no `if platform ==` branch (rule 169). A source that has never been walked has cached nothing and simply contributes no exact key, which is what makes the handle fallback below necessary rather than merely tolerated. """ keys = set() for name, value in (source.config_overrides or {}).items(): if name.endswith(_CAMPAIGN_KEY_SUFFIX) and isinstance(value, str) and value: keys.add(value) return keys def url_tail(url: str | None) -> str | None: """The creator handle at the end of a source URL, lowercased. Deliberately the same derivation as `PlatformMembership.vanity_or_none`'s own fallback, so both sides of the comparison reduce a URL to a handle the same way. Query strings and fragments are stripped first; Patreon's `/c/` and `/cw/` forms both end in the vanity, so they need no special case (the missing-`/c/` regex is what broke creator detection in #1485). Returns None for the pre-0030 `sidecar:` synthetic anchors, which are not feeds and must never match anything. """ if not url or url.startswith("sidecar:"): return None cleaned = url.split("?", 1)[0].split("#", 1)[0] tail = cleaned.rstrip("/").rsplit("/", 1)[-1] return tail.lower() or None def match_kind(source: Source, membership: PlatformMembership) -> str | None: """How this source and this membership are known to be the same creator. Returns "campaign" for an exact platform-id match, "vanity" for agreeing URL handles, or None for no evidence. THE ORDER MUST NOT BE INVERTED. The campaign id is exact and the handle is not, but the id is only written AFTER a source has been walked at least once — so checking the handle first would let a stale or renamed URL outvote the authoritative id on every source FC has actually polled. """ if source.platform != membership.platform: return None if membership.external_campaign_id in identity_keys_for_source(source): return "campaign" vanity = membership.vanity_or_none() tail = url_tail(source.url) if vanity and tail and vanity.strip().lower() == tail: return "vanity" return None def pair_sources_with_memberships( sources: list[Source], memberships: list[PlatformMembership], ) -> dict[int, tuple[PlatformMembership, str]]: """Source id -> the membership it is the same creator as, and how we know. Extracted from C4's reconcile loop when C5 became its second caller. It is a nested loop rather than a SQL join because the match is a predicate over a JSON blob and a derived URL handle, neither of which is indexable, and both sides are tens of rows on any real library. Keeping it in Python means ONE definition of identity (`match_kind`) instead of a second one in SQL that could drift from it. First match wins, which is `match_kind`'s ordering doing its job: a source with a cached campaign id can only pair with the membership holding that id, so an ambiguous handle never outvotes it. """ pairs: dict[int, tuple[PlatformMembership, str]] = {} for source in sources: for m in memberships: kind = match_kind(source, m) if kind: pairs[source.id] = (m, kind) break return pairs # --------------------------------------------------------------------------- # Why the posts are invisible (#387 C5) # --------------------------------------------------------------------------- # # A3 made a tier-gated source say "47 posts you can't see". These are the words # the roster is allowed to add to that count — and ONLY to that count. # # THE LINE: the roster ANNOTATES the gated flag, it never produces it. # `current_user_can_view` (read per post by `patreon_client.post_is_gated`) is # the authoritative per-post signal, and entitled-tier data cannot stand in for # it — a creator can gate a post behind an access rule that maps onto no tier # name at all. So nothing here may suppress a download, skip a walk, or decide # a post is inaccessible. It explains a skip that ALREADY happened. Getting # that backwards would make FC silently stop fetching content the operator is # paying for, which is the worst failure available in this milestone. # `test_no_fetch_path_can_read_the_roster` pins that structurally. GATED_LAPSED = "lapsed" # the membership ended — resubscribe, or disable GATED_TIER = "tier" # paying, but this tier doesn't reach these posts GATED_FREE = "free" # a current FREE follow — nobody is paying for access def gated_reason( platform: str, status: str | None, *, is_free_member: bool = False, ) -> str | None: """Why a tier-gated source's posts are out of reach, if the roster knows. None means "no words beyond the count" and is the answer for every case where the roster is not evidence: a status this code has not been taught, and (at the call site) a campaign absent from the roster or a roster too stale to trust. Absence is not evidence — the same discipline as `test_post_is_gated_only_on_explicit_false`. `is_free_member` is read AFTER the status axis, not folded into it, which is why `has_paid_access` is called here with it forced off. The two axes are independent in Patreon's payload, and collapsing them loses a real distinction: a current free follower has not lost anything, so telling them "you're not a patron any more" would be a false sentence about a state they were never in. """ by_status = has_paid_access(platform, status, is_free_member=False) if by_status is None: return None if not by_status: return GATED_LAPSED return GATED_FREE if is_free_member else GATED_TIER async def gated_reasons_for_sources( session: AsyncSession, sources: list[Source], *, now: datetime | None = None, ) -> dict[int, str]: """The reason word for each of these sources, where the roster has one. Callers pass ONLY the sources already known to be tier-gated: the question "why can't I see these posts" is meaningless for a source whose posts are all visible, and asking it anyway would put roster data on rows that have no gated state for it to annotate. Sources with no entry in the result get A3's bare count, which is the correct degraded rendering for all three of: platform never swept, roster stale, campaign not in the roster. """ if not sources: return {} platforms = {s.platform for s in sources} # Per platform, because freshness is per platform: a working Patreon sweep # must not lend its credibility to a SubscribeStar roster that has never # run. Same gate as C4's `tracked_not_subscribed`, for the same reason. fresh = { p for p in platforms if roster_is_fresh(await get_sync_state(session, p), now=now) } if not fresh: return {} memberships = (await session.execute( select(PlatformMembership).where( PlatformMembership.platform.in_(sorted(fresh)) ) )).scalars().all() pairs = pair_sources_with_memberships( [s for s in sources if s.platform in fresh], memberships, ) reasons: dict[int, str] = {} for source_id, (m, _kind) in pairs.items(): reason = gated_reason( m.platform, m.status, is_free_member=bool((m.details or {}).get("is_free_member")), ) if reason is not None: reasons[source_id] = reason return reasons async def source_for_membership( session: AsyncSession, membership: PlatformMembership, ) -> Source | None: """The source FC already tracks for this membership, if there is one. Scoped to the membership's own platform, so a creator tracked on Discord and subscribed to on Patreon does not read as already-tracked — that pairing is E4's suggestion to make, not an identity. """ rows = (await session.execute( select(Source).where(Source.platform == membership.platform) )).scalars().all() for source in rows: if match_kind(source, membership): return source return None