FabledCurator/backend/app/services/membership_reconcile.py

"""Reconciling the learned roster against the sources FC actually tracks.

Milestone 387, step C4. The step the operator asked for; C0-C3 are what make it
trustworthy enough to act on.

## The buckets

1. `subscribed_not_tracked` — you pay for this and FC does not follow it. The
   adoption win, and the only bucket carrying an action.
2. `tracked_not_subscribed` — FC follows this and the roster does not show you
   paying for it. No longer shown on the card: the operator reversed the
   2026-09-11 "report only" call on 2026-09-13. The lapsed half of it now ACTS,
   in `apply_membership_lapses` below (#3995). The absent half still only
   reports, because absence proves nothing.
3. `matched` — the healthy set. Counted, not listed loudly.
4. `unidentified` — sources this join cannot speak to at all. Reported as
   exactly that, because the alternative is filing them under a verdict.

## Why absence is the dangerous direction

Bucket 1 is safe to be wrong about: the cost of offering a source the operator
does not want is one ignored row. Bucket 2 is not. It is computed from an
ABSENCE — no membership matched — and three different things produce that
absence: the subscription genuinely lapsed, the sweep failed, or the creator
renamed and this source has never been walked so no exact id was ever cached.

Two guards follow from that, and they are the substance of this module:

* the whole bucket is gated on `roster_is_fresh`, so a failed or never-run sweep
  yields an empty list rather than a confident accusation (C3 built the state
  this reads);
* every row carries the BASIS for its claim, so "your membership says former
  patron" and "we know this creator's id and it is not in your roster" and "we
  only have a URL handle to go on" are three different sentences rather than one
  overconfident one.

`has_paid_access` returning None is honoured throughout: unknown is never
rendered as lapsed. That is the whole reason it returns a tri-state.
"""

from __future__ import annotations

from datetime import UTC, datetime

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from ..models import Artist, MembershipSync, PlatformMembership, Source
from .membership_roster import (
    get_sync_state,
    identity_keys_for_source,
    pair_sources_with_memberships,
    roster_is_fresh,
    url_tail,
)
from .native_ingest_common import has_paid_access

# Why a source appears in `tracked_not_subscribed`. Ordered strongest first —
# the UI renders a different sentence per basis, because collapsing them into
# one would make the weakest claim sound like the strongest.
BASIS_LAPSED = "lapsed"           # a matched membership says access ended
BASIS_ABSENT_EXACT = "absent_exact"    # exact id known, not in a fresh roster
BASIS_ABSENT_HANDLE = "absent_handle"  # only a URL handle to go on


def _membership_row(m: PlatformMembership) -> dict:
    return {
        "id": m.id,
        "platform": m.platform,
        "external_campaign_id": m.external_campaign_id,
        "display_name": m.display_name or m.vanity_or_none(),
        "url": m.url,
        "vanity": m.vanity_or_none(),
        "status": m.status,
        "tier_names": m.tier_names,
        "amount_cents": m.amount_cents,
        "currency": m.currency,
        "paid_access": has_paid_access(
            m.platform, m.status,
            is_free_member=bool((m.details or {}).get("is_free_member")),
        ),
    }


def _source_row(source: Source, artist: Artist) -> dict:
    return {
        "id": source.id,
        "platform": source.platform,
        "url": source.url,
        "enabled": source.enabled,
        "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
    }


async def reconcile(
    session: AsyncSession, *, platform: str, now: datetime | None = None,
) -> dict:
    """Sort one platform's memberships and sources into the four buckets.

    Always returns the COMPLETE shape, including when the roster is not fresh —
    a caller reading `len(result["tracked_not_subscribed"])` must not have to
    check which keys exist first. `fresh` is what says whether the emptiness
    means anything.
    """
    state = await get_sync_state(session, platform)
    fresh = roster_is_fresh(state, now=now)

    memberships = (await session.execute(
        select(PlatformMembership).where(PlatformMembership.platform == platform)
    )).scalars().all()
    rows = (await session.execute(
        select(Source, Artist)
        .join(Artist, Artist.id == Source.artist_id)
        .where(Source.platform == platform)
    )).all()

    # The join itself lives in `membership_roster` beside `match_kind`, so C5's
    # gated-reason annotation pairs sources with memberships by exactly the same
    # rule this card sorts them by. Two copies would let the Subscriptions row
    # and this card disagree about which creator a source IS.
    pairs = pair_sources_with_memberships([s for s, _a in rows], memberships)
    matched_membership_ids = {m.id for m, _kind in pairs.values()}

    subscribed_not_tracked = []
    for m in memberships:
        if m.id in matched_membership_ids:
            continue
        paid = has_paid_access(
            m.platform, m.status,
            is_free_member=bool((m.details or {}).get("is_free_member")),
        )
        # A membership FC knows has ENDED is not an adoption opportunity —
        # adding it would start a walk that can only fetch what is already
        # public. Unknown (None) is still offered: the operator can judge it,
        # and refusing to show it would hide a real subscription behind a word
        # this code has not been taught.
        if paid is False:
            continue
        subscribed_not_tracked.append(_membership_row(m))

    tracked_not_subscribed = []
    matched = []
    unidentified = []
    for source, artist in rows:
        pair = pairs.get(source.id)
        if pair is not None:
            m, kind = pair
            paid = has_paid_access(
                m.platform, m.status,
                is_free_member=bool((m.details or {}).get("is_free_member")),
            )
            if paid is False:
                if not source.enabled:
                    # Already off. Reporting a source the operator has already
                    # stopped following is noise, not a finding.
                    continue
                row = _source_row(source, artist)
                row["basis"] = BASIS_LAPSED
                row["matched_by"] = kind
                row["membership"] = _membership_row(m)
                tracked_not_subscribed.append(row)
            else:
                row = _source_row(source, artist)
                row["matched_by"] = kind
                row["membership"] = _membership_row(m)
                matched.append(row)
            continue

        # No membership matched. Whether that MEANS anything depends entirely on
        # how well this source can be identified at all.
        has_exact = bool(identity_keys_for_source(source))
        if not has_exact and url_tail(source.url) is None:
            # Nothing to match on — a sidecar anchor or a URL with no handle.
            # Reported as unidentified rather than silently dropped, so the
            # counts add up to the source list the operator can see.
            unidentified.append(_source_row(source, artist))
            continue
        if not source.enabled:
            # Already off. Telling the operator to stop following something they
            # have stopped following is noise, not a finding.
            continue
        row = _source_row(source, artist)
        row["basis"] = BASIS_ABSENT_EXACT if has_exact else BASIS_ABSENT_HANDLE
        row["matched_by"] = None
        row["membership"] = None
        tracked_not_subscribed.append(row)

    # THE GATE. Everything above computed the bucket; this decides whether it may
    # be shown. A stale or never-run roster makes every absence meaningless, and
    # an absence rendered as a verdict is how this feature would tell the
    # operator to cancel something they are still paying for.
    if not fresh:
        tracked_not_subscribed = []

    return {
        "platform": platform,
        "fresh": fresh,
        # How many sources exist on this platform at all. The UI needs it to
        # decide whether an untrustworthy roster is worth mentioning: with no
        # sources here there is nothing to reconcile, and a stale-roster warning
        # would be noise on an install that simply has not started yet (that
        # empty-install case is C6's, not this card's).
        "tracked_total": len(rows),
        "last_success_at": (
            state.last_success_at.isoformat()
            if state is not None and state.last_success_at else None
        ),
        "subscribed_not_tracked": subscribed_not_tracked,
        "tracked_not_subscribed": tracked_not_subscribed,
        "matched": matched,
        "unidentified": unidentified,
    }


async def reconcile_all(session: AsyncSession, now: datetime | None = None) -> dict:
    """Every platform the roster knows about, in one payload for the UI.

    The platform list is the UNION of platforms with memberships and platforms
    with sync state, not just the former. A sweep that has never succeeded has
    recorded zero memberships, and deriving the list from memberships alone
    would drop exactly that platform from the payload — making a broken
    credential indistinguishable from a platform FC was never asked about. That
    distinction is the whole reason C3 records sync state.
    """
    with_memberships = (await session.execute(
        select(PlatformMembership.platform).distinct()
    )).scalars().all()
    with_state = (await session.execute(
        select(MembershipSync.platform)
    )).scalars().all()
    platforms = set(with_memberships) | set(with_state)
    return {
        "platforms": [
            await reconcile(session, platform=p, now=now) for p in sorted(platforms)
        ]
    }


# ---------------------------------------------------------------------------
# Stop pulling what the account no longer pays for (#3995)
# ---------------------------------------------------------------------------
#
# Operator decision, 2026-09-13, reversing the 2026-09-11 "report only" call
# for this direction: "if I kill a subscription on patreon I would like the
# pulling to stop on curator as well", with automatic resume on resubscribing.
#
# This is a SOURCE-level action taken by the daily sweep, visible on the source
# row and reversible there. It is not a fetch-path decision. The line C5 draws,
# that the roster never decides a POST is inaccessible, still holds: nothing
# here reads per-post access, and no download path reads the roster
# (`test_no_fetch_path_can_read_the_roster`). The scheduler keeps selecting on
# `enabled` alone.
#
# Acts ONLY on positive evidence. A source whose matched membership says access
# has ended is stopped. A source with NO matched membership is left alone,
# because absence has innocent causes: a creator rename, a source never walked
# so no id is cached, a membership the platform stopped listing. Stopping on
# absence would switch off things the operator still pays for.
#
# Two app-managed config_overrides keys carry the state. The `_` prefix is
# already the "FC writes this, an operator edit preserves it" family.
#   _membership_stopped  set when the sweep stops a source; the sweep resumes
#                        ONLY sources carrying it, so a source the operator
#                        switched off by hand is never switched back on
#   _membership_kept     set by SourceService.update when the operator turns a
#                        stopped source back ON: a deliberate choice to keep
#                        pulling a lapsed creator, which the next sweep must
#                        not undo. Cleared when the membership is paid again.
STOPPED_KEY = "_membership_stopped"
KEPT_KEY = "_membership_kept"


def _access_expires_at(m: PlatformMembership) -> datetime | None:
    """When paid access actually ends, if the platform says.

    Patreon keeps a cancelled membership's access until the end of the billing
    period and reports that date (`member.access_expires_at`, note #3992).
    SubscribeStar's page gives no such date, so a cancelled SubscribeStar
    membership stops at once. Returns None when there is no usable date.
    """
    details = m.details or {}
    raw = details.get("access_expires_at") or (details.get("member") or {}).get("access_expires_at")
    if not isinstance(raw, str) or not raw:
        return None
    try:
        parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except ValueError:
        return None
    return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)


async def apply_membership_lapses(
    session: AsyncSession, *, platform: str, now: datetime | None = None,
) -> dict:
    """Stop sources whose paid access has ended; resume the ones this stopped.

    Refuses to act on a roster that isn't fresh, for the same reason C4 refuses
    to draw conclusions from one.
    """
    now = now or datetime.now(UTC)
    state = await get_sync_state(session, platform)
    if not roster_is_fresh(state, now=now):
        return {"platform": platform, "skipped": "roster not fresh", "stopped": 0, "resumed": 0}

    memberships = (await session.execute(
        select(PlatformMembership).where(PlatformMembership.platform == platform)
    )).scalars().all()
    sources = (await session.execute(
        select(Source).where(Source.platform == platform)
    )).scalars().all()
    pairs = pair_sources_with_memberships(list(sources), list(memberships))

    stopped: list[int] = []
    resumed: list[int] = []
    for source in sources:
        pair = pairs.get(source.id)
        if pair is None:
            continue  # absence is never acted on, see above
        m, _kind = pair
        paid = has_paid_access(
            m.platform, m.status,
            is_free_member=bool((m.details or {}).get("is_free_member")),
        )
        co = dict(source.config_overrides or {})

        if paid is True:
            changed = co.pop(KEPT_KEY, None) is not None
            if STOPPED_KEY in co:
                co.pop(STOPPED_KEY)
                source.enabled = True
                resumed.append(source.id)
                changed = True
            if changed:
                source.config_overrides = co
            continue

        # Unknown status: never a reason to stop something (has_paid_access's
        # tri-state exists for exactly this).
        if paid is None:
            continue
        if not source.enabled or co.get(KEPT_KEY):
            continue
        expires = _access_expires_at(m)
        if expires is not None and expires > now:
            continue  # still inside the paid-through period

        co[STOPPED_KEY] = {"at": now.isoformat(), "status": m.status}
        source.config_overrides = co
        source.enabled = False
        # The same clean slate a manual disable gives (SourceService.update,
        # #1285), so a stopped source doesn't linger as failing or gated.
        source.last_error = None
        source.error_type = None
        source.consecutive_failures = 0
        stopped.append(source.id)

    await session.commit()
    return {"platform": platform, "stopped": len(stopped), "resumed": len(resumed)}