FabledCurator/tests/test_source_service.py

import pytest
from sqlalchemy import select

from backend.app.models import Artist, Source
from backend.app.services.source_service import (
    KNOWN_PLATFORMS,
    ArtistNotFoundError,
    DuplicateSourceError,
    EmptyUrlError,
    InvalidConfigError,
    SourceService,
    UnknownPlatformError,
)

pytestmark = pytest.mark.integration


async def _artist(db, name="Alice"):
    a = Artist(name=name, slug=name.lower())
    db.add(a)
    await db.flush()
    return a


@pytest.mark.asyncio
async def test_known_platforms_are_the_supported_four(db):
    assert KNOWN_PLATFORMS == frozenset({
        "patreon", "subscribestar", "hentaifoundry", "discord",
    })
    assert "fanbox" not in KNOWN_PLATFORMS
    # Retired at #3069 — a source can no longer be created on it.
    assert "deviantart" not in KNOWN_PLATFORMS
    # Retired at milestone #406 — likewise.
    assert "pixiv" not in KNOWN_PLATFORMS


@pytest.mark.asyncio
async def test_create_flips_is_subscription_on_first_source(db):
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/alice",
    )
    assert rec.id is not None
    is_sub = (await db.execute(
        select(Artist.is_subscription).where(Artist.id == artist.id)
    )).scalar_one()
    assert is_sub is True


@pytest.mark.asyncio
async def test_delete_last_source_flips_is_subscription_off(db):
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/alice",
    )
    await svc.delete(rec.id)
    is_sub = (await db.execute(
        select(Artist.is_subscription).where(Artist.id == artist.id)
    )).scalar_one()
    assert is_sub is False


@pytest.mark.asyncio
async def test_create_rejects_unknown_platform(db):
    artist = await _artist(db)
    svc = SourceService(db)
    with pytest.raises(UnknownPlatformError):
        await svc.create(
            artist_id=artist.id, platform="myspace", url="https://m/x",
        )


@pytest.mark.asyncio
async def test_create_rejects_non_dict_config(db):
    artist = await _artist(db)
    svc = SourceService(db)
    with pytest.raises(InvalidConfigError):
        await svc.create(
            artist_id=artist.id, platform="patreon",
            url="https://patreon.com/alice", config_overrides=[1, 2, 3],
        )


@pytest.mark.asyncio
async def test_create_rejects_empty_url(db):
    artist = await _artist(db)
    svc = SourceService(db)
    with pytest.raises(EmptyUrlError):
        await svc.create(artist_id=artist.id, platform="patreon", url="   ")


@pytest.mark.asyncio
async def test_create_rejects_unknown_artist(db):
    svc = SourceService(db)
    with pytest.raises(ArtistNotFoundError):
        await svc.create(artist_id=99999, platform="patreon", url="https://x/y")


@pytest.mark.asyncio
async def test_create_duplicate_raises_with_existing_id(db):
    artist = await _artist(db)
    svc = SourceService(db)
    first = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/alice",
    )
    with pytest.raises(DuplicateSourceError) as exc:
        await svc.create(
            artist_id=artist.id, platform="patreon",
            url="https://patreon.com/alice",
        )
    assert exc.value.existing_id == first.id


@pytest.mark.asyncio
async def test_list_filters_by_artist(db):
    a = await _artist(db, "Alice")
    b = await _artist(db, "Bob")
    svc = SourceService(db)
    await svc.create(artist_id=a.id, platform="patreon", url="https://patreon.com/a")
    await svc.create(artist_id=b.id, platform="patreon", url="https://patreon.com/b")
    only_a = await svc.list(artist_id=a.id)
    assert [s.artist_id for s in only_a] == [a.id]
    all_rows = await svc.list()
    assert len(all_rows) == 2


@pytest.mark.asyncio
async def test_list_joins_tier_gated_count_from_the_latest_event(db):
    """A no-access source carries the count from its most recent walk.

    The number lives on the DownloadEvent's run_stats, not on the source, so
    `list()` joins it in. Two events are seeded deliberately: the newest must
    win, or the row would show a stale figure from a walk where the operator
    still held the tier.
    """
    from datetime import UTC, datetime, timedelta

    from backend.app.models import DownloadEvent

    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/gated",
    )
    src = (await db.execute(
        select(Source).where(Source.id == rec.id)
    )).scalar_one()
    src.error_type = "tier_limited"
    src.last_checked_at = datetime.now(UTC)

    now = datetime.now(UTC)
    db.add(DownloadEvent(
        source_id=rec.id, status="ok", started_at=now - timedelta(hours=2),
        metadata_={"run_stats": {"tier_gated_count": 3}},
    ))
    db.add(DownloadEvent(
        source_id=rec.id, status="ok", started_at=now,
        metadata_={"run_stats": {"tier_gated_count": 47}},
    ))
    await db.commit()

    rows = await svc.list(artist_id=artist.id)
    assert rows[0].tier_gated_count == 47
    assert rows[0].to_dict()["tier_gated_count"] == 47


@pytest.mark.asyncio
async def test_list_leaves_tier_gated_count_none_for_ordinary_sources(db):
    """The join is scoped to tier-gated sources, so a healthy row reports None
    rather than 0 — the UI distinguishes 'no count available' from 'zero
    gated', and must not print a fabricated number."""
    artist = await _artist(db)
    svc = SourceService(db)
    await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/fine",
    )

    rows = await svc.list(artist_id=artist.id)
    assert rows[0].tier_gated_count is None


@pytest.mark.asyncio
async def test_update_changes_fields(db):
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/a",
    )
    updated = await svc.update(rec.id, enabled=False, config_overrides={"videos": False})
    assert updated.enabled is False
    # The operator's key lands, and FC's own state is not collateral. This
    # assertion used to read `== {"videos": False}` — it was DOCUMENTING the bug
    # the tests below now pin: `create` arms `_backfill_state` (#693), and every
    # edit was silently discarding it.
    assert updated.config_overrides["videos"] is False
    assert updated.config_overrides["_backfill_state"] == "running"


# --- config_overrides: the operator's keys and FC's own share one column ---
#
# `SourceFormDialog` posts the WHOLE object back, and its structured tab rebuilds
# that object from two fields — so saving the dialog without touching anything
# used to discard the resolved campaign id and the backfill position. The id
# matters twice over: `patreon_resolver` reads it BEFORE attempting any lookup,
# and the membership join (#387 C4, snippet #3947) keys on it.


async def _source_with_state(db, svc, artist, **overrides):
    """A source carrying both kinds of key: FC's own state and the operator's."""
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/mixed",
    )
    await db.execute(
        Source.__table__.update().where(Source.id == rec.id).values(
            config_overrides={
                "_backfill_state": "running",
                "_backfill_cursor": "03:RESUME",
                "patreon_campaign_id": "camp-1",
                "videos": True,
                **overrides,
            },
        )
    )
    await db.commit()
    return rec


@pytest.mark.asyncio
async def test_an_operator_edit_keeps_fc_managed_state(db):
    """THE bug. One dialog save used to cost the campaign id and the walk."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist)

    updated = await svc.update(rec.id, config_overrides={"videos": False})

    assert updated.config_overrides["patreon_campaign_id"] == "camp-1"
    assert updated.config_overrides["_backfill_state"] == "running"
    assert updated.config_overrides["_backfill_cursor"] == "03:RESUME"
    assert updated.config_overrides["videos"] is False


@pytest.mark.asyncio
async def test_an_operator_key_left_out_of_the_edit_is_still_removed(db):
    """The other half, and why this is a merge rather than a blanket keep: the
    operator's own settings must still replace wholesale, or removing a key in
    the dialog could never remove it."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist, since="2026-01-01")

    updated = await svc.update(rec.id, config_overrides={"videos": False})

    assert "since" not in updated.config_overrides


@pytest.mark.asyncio
async def test_a_null_config_clears_operator_settings_but_not_fc_state(db):
    """`_parseConfig` sends null for an empty object, so this is the dialog's
    real "I cleared the box" path — not a request to forget the walk."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist)

    updated = await svc.update(rec.id, config_overrides=None)

    assert "videos" not in updated.config_overrides
    assert updated.config_overrides["_backfill_state"] == "running"
    assert updated.config_overrides["patreon_campaign_id"] == "camp-1"


@pytest.mark.asyncio
async def test_a_stale_echoed_value_cannot_overwrite_what_the_walk_wrote(db):
    """The dialog round-trips whatever it last READ. If the backfill advanced in
    the meantime, saving that stale copy must not roll the walk backwards — so
    FC's keys are applied last and win."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist)

    updated = await svc.update(rec.id, config_overrides={
        "videos": False,
        "_backfill_cursor": "00:STALE",
        "patreon_campaign_id": "wrong-campaign",
    })

    assert updated.config_overrides["_backfill_cursor"] == "03:RESUME"
    assert updated.config_overrides["patreon_campaign_id"] == "camp-1"


@pytest.mark.asyncio
async def test_repointing_the_url_drops_the_resolved_campaign_id(db):
    """A different creator behind the same row makes the cached id WRONG, not
    stale. Preserving it above is exactly what makes this invalidation this
    service's job — the old wholesale overwrite used to wipe the id by accident.
    """
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist)

    updated = await svc.update(rec.id, url="https://patreon.com/someone-else")

    assert "patreon_campaign_id" not in updated.config_overrides
    # The walk's position survives: it is expensive to rebuild, the walk has its
    # own stall guard, and a cosmetic URL edit must not restart a long backfill.
    assert updated.config_overrides["_backfill_cursor"] == "03:RESUME"


@pytest.mark.asyncio
async def test_rewriting_the_same_url_keeps_the_campaign_id(db):
    """The dialog posts the URL on every save. An unchanged URL is not a
    repoint, and treating it as one would re-resolve after every edit."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await _source_with_state(db, svc, artist)

    updated = await svc.update(rec.id, url="https://patreon.com/mixed")

    assert updated.config_overrides["patreon_campaign_id"] == "camp-1"


def test_the_campaign_id_suffix_agrees_with_the_membership_join():
    """Two modules must agree on the key shape: this one preserves and
    invalidates it, `membership_roster` matches on it (snippet #3947). They
    deliberately do not import across — a core CRUD service should not depend on
    a membership feature — so the agreement is pinned here rather than hoped for.
    Do NOT fix a failure by editing one side; decide which shape is right.
    """
    from backend.app.services.membership_roster import _CAMPAIGN_KEY_SUFFIX
    from backend.app.services.source_service import _CAMPAIGN_ID_SUFFIX

    assert _CAMPAIGN_ID_SUFFIX == _CAMPAIGN_KEY_SUFFIX


@pytest.mark.asyncio
async def test_disable_clears_failure_state(db):
    """Disabling a source wipes its failure state so it stops showing as
    'failing' (operator: disable subs you're not paying for)."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/a",
    )
    source = (await db.execute(
        select(Source).where(Source.id == rec.id)
    )).scalar_one()
    source.last_error = "auth failed"
    source.error_type = "auth_error"
    source.consecutive_failures = 5
    await db.commit()

    updated = await svc.update(rec.id, enabled=False)
    assert updated.enabled is False
    assert updated.last_error is None
    assert updated.consecutive_failures == 0
    refetched = (await db.execute(
        select(Source).where(Source.id == rec.id)
    )).scalar_one()
    assert refetched.error_type is None


@pytest.mark.asyncio
async def test_update_while_enabled_keeps_failure_state(db):
    """A non-disable edit must NOT wipe failure state — only the explicit
    disable clears it (else a config tweak would hide a real failure)."""
    artist = await _artist(db)
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon", url="https://patreon.com/a",
    )
    source = (await db.execute(
        select(Source).where(Source.id == rec.id)
    )).scalar_one()
    source.last_error = "auth failed"
    source.consecutive_failures = 3
    await db.commit()

    await svc.update(rec.id, config_overrides={"videos": False})
    refetched = (await db.execute(
        select(Source).where(Source.id == rec.id)
    )).scalar_one()
    assert refetched.last_error == "auth failed"
    assert refetched.consecutive_failures == 3


async def _source_with_content(db, svc, artist):
    """A source under `artist` with one post + one image it contributed."""
    from backend.app.models import ImageProvenance, ImageRecord, Post
    rec = await svc.create(
        artist_id=artist.id, platform="pixiv",
        url=f"https://www.pixiv.net/users/{artist.id}",
    )
    post = Post(source_id=rec.id, artist_id=artist.id, external_post_id="p1")
    db.add(post)
    img = ImageRecord(
        path=f"/images/{artist.slug}/pixiv/pixiv/1_a_00.jpg",
        sha256=str(artist.id).rjust(64, "0"), size_bytes=1, mime="image/jpeg",
        width=1, height=1, origin="imported_filesystem",
        integrity_status="unknown", artist_id=artist.id,
    )
    db.add(img)
    await db.flush()
    db.add(ImageProvenance(image_record_id=img.id, post_id=post.id, source_id=rec.id))
    await db.commit()
    return rec, post, img


@pytest.mark.asyncio
async def test_reassign_moves_source_posts_images(db):
    from backend.app.models import ImageRecord, Post
    old = await _artist(db, "OldOwner")
    new = await _artist(db, "NewOwner")
    svc = SourceService(db)
    rec, post, img = await _source_with_content(db, svc, old)

    await svc.reassign(rec.id, new.id)

    assert (await db.execute(
        select(Source.artist_id).where(Source.id == rec.id)
    )).scalar_one() == new.id
    assert (await db.execute(
        select(Post.artist_id).where(Post.id == post.id)
    )).scalar_one() == new.id
    assert (await db.execute(
        select(ImageRecord.artist_id).where(ImageRecord.id == img.id)
    )).scalar_one() == new.id
    # Old artist is now empty → deleted.
    assert (await db.execute(
        select(Artist).where(Artist.id == old.id)
    )).scalar_one_or_none() is None


@pytest.mark.asyncio
async def test_reassign_keeps_nonempty_old_artist(db):
    from backend.app.models import ImageRecord
    old = await _artist(db, "OldMulti")
    new = await _artist(db, "NewMulti")
    svc = SourceService(db)
    rec, _post, _img = await _source_with_content(db, svc, old)
    # A second, unrelated image keeps `old` non-empty after the move.
    db.add(ImageRecord(
        path="/images/oldmulti/loose.jpg", sha256="e" * 64, size_bytes=1,
        mime="image/jpeg", width=1, height=1, origin="imported_filesystem",
        integrity_status="unknown", artist_id=old.id,
    ))
    await db.commit()

    await svc.reassign(rec.id, new.id)
    still = (await db.execute(
        select(Artist).where(Artist.id == old.id)
    )).scalar_one()
    assert still.is_subscription is False        # lost its last source


@pytest.mark.asyncio
async def test_reassign_same_artist_is_noop(db):
    a = await _artist(db, "Solo")
    svc = SourceService(db)
    rec, _post, _img = await _source_with_content(db, svc, a)
    out = await svc.reassign(rec.id, a.id)
    assert out.artist_id == a.id


@pytest.mark.asyncio
async def test_reassign_unknown_target_raises(db):
    from backend.app.services.source_service import ArtistNotFoundError
    a = await _artist(db, "Whom")
    svc = SourceService(db)
    rec, _post, _img = await _source_with_content(db, svc, a)
    with pytest.raises(ArtistNotFoundError):
        await svc.reassign(rec.id, 999999)


@pytest.mark.asyncio
async def test_list_hides_sidecar_synthetic_anchors(db):
    """Filesystem-import synthetic Sources (url='sidecar:<platform>:<slug>',
    enabled=False — historical pre-alembic-0030 artifact) used to leak into the
    Subscriptions UI as phantom subscriptions because list() didn't filter
    them. They aren't pollable feeds; hide by default."""
    artist = await _artist(db, "Alice")
    real = Source(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice", enabled=True, config_overrides={},
    )
    synthetic = Source(
        artist_id=artist.id, platform="patreon",
        url="sidecar:patreon:alice", enabled=False, config_overrides={},
    )
    db.add_all([real, synthetic])
    await db.commit()

    svc = SourceService(db)
    visible = await svc.list()
    visible_urls = {s.url for s in visible}
    assert "https://patreon.com/alice" in visible_urls
    assert "sidecar:patreon:alice" not in visible_urls

    # Same filter applies to the artist-scoped list path (the artist detail
    # page hits /api/sources?artist_id=N).
    artist_scoped = await svc.list(artist_id=artist.id)
    assert {s.url for s in artist_scoped} == {"https://patreon.com/alice"}

    # include_synthetic=True opts back in for admin tooling.
    everything = await svc.list(include_synthetic=True)
    assert {s.url for s in everything} >= {
        "https://patreon.com/alice", "sidecar:patreon:alice",
    }


# --- Plan #544: backfill counter -------------------------------------------


@pytest.mark.asyncio
async def test_start_backfill_arms_run_until_done(db):
    """start_backfill sets state=running + the chunk cap and clears any prior
    cursor/chunk state, returning the updated record for the API to echo."""
    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS

    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
    # Simulate a prior, finished walk leaving stale checkpoint state.
    await db.execute(
        Source.__table__.update().where(Source.id == rec.id).values(
            config_overrides={"_backfill_state": "complete", "_backfill_cursor": "old",
                              "_backfill_chunks": 7},
        )
    )
    await db.commit()

    updated = await svc.start_backfill(rec.id)
    assert updated.backfill_state == "running"
    assert updated.backfill_chunks == 0
    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS

    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert co.get("_backfill_state") == "running"
    assert "_backfill_cursor" not in co


@pytest.mark.asyncio
async def test_stop_backfill_returns_to_idle(db):
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
    await svc.start_backfill(rec.id)
    updated = await svc.stop_backfill(rec.id)
    assert updated.backfill_state is None
    assert updated.backfill_runs_remaining == 0
    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert "_backfill_state" not in (co or {})


@pytest.mark.asyncio
async def test_start_recovery_arms_bypass_flag(db):
    """Plan #697: start_recovery arms the backfill state machine PLUS the
    _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge
    can label it 'Recovering'. Clears any prior checkpoint state."""
    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS

    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
    # Simulate a prior walk's posts counter — start must clear it (plan #704).
    await db.execute(
        Source.__table__.update().where(Source.id == rec.id).values(
            config_overrides={"_backfill_posts": 42},
        )
    )
    await db.commit()

    updated = await svc.start_recovery(rec.id)
    assert updated.backfill_state == "running"
    assert updated.backfill_bypass_seen is True
    assert updated.backfill_posts == 0  # cleared for a fresh walk
    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS

    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert co.get("_backfill_bypass_seen") is True
    assert "_backfill_posts" not in co

    # Stop clears the bypass flag too (shared lifecycle).
    stopped = await svc.stop_backfill(rec.id)
    assert stopped.backfill_bypass_seen is False
    co2 = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert "_backfill_bypass_seen" not in (co2 or {})


@pytest.mark.asyncio
async def test_start_recapture_arms_recapture_flag(db):
    """#830: start_recapture arms the backfill state machine PLUS the
    _backfill_recapture flag, surfaced on the record so the badge can label it
    'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop
    clears it."""
    from backend.app.services.source_service import BACKFILL_MAX_CHUNKS

    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice",
    )
    # Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion).
    await svc.start_recovery(rec.id)

    updated = await svc.start_recapture(rec.id)
    assert updated.backfill_state == "running"
    assert updated.backfill_recapture is True
    assert updated.backfill_bypass_seen is False
    assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS

    co = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert co.get("_backfill_recapture") is True
    assert "_backfill_bypass_seen" not in co

    # to_dict carries the new field for the API/UI.
    assert updated.to_dict()["backfill_recapture"] is True

    # Stop clears the recapture flag too.
    stopped = await svc.stop_backfill(rec.id)
    assert stopped.backfill_recapture is False
    co2 = (await db.execute(
        select(Source.config_overrides).where(Source.id == rec.id)
    )).scalar_one()
    assert "_backfill_recapture" not in (co2 or {})


@pytest.mark.asyncio
async def test_start_backfill_raises_when_source_missing(db):
    svc = SourceService(db)
    with pytest.raises(LookupError):
        await svc.start_backfill(99999)


@pytest.mark.asyncio
async def test_new_enabled_source_starts_in_backfill_mode(db):
    """Plan #693: freshly added enabled sources have no archive yet, so they
    arm run-until-done backfill — state 'running' — to walk the full history
    on the first ticks instead of blowing the wall-clock cap in tick mode."""
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-new",
    )
    assert rec.backfill_state == "running"


@pytest.mark.asyncio
async def test_new_disabled_source_skips_backfill(db):
    """Disabled sources (incl. sidecar synthetics that arrive disabled) are
    never polled, so don't burn a backfill budget on them."""
    artist = await _artist(db, "Alice")
    svc = SourceService(db)
    rec = await svc.create(
        artist_id=artist.id, platform="patreon",
        url="https://patreon.com/alice-disabled",
        enabled=False,
    )
    assert rec.backfill_runs_remaining == 0