import pytest from sqlalchemy import select from backend.app.models import Artist, Source from backend.app.services.source_service import ( KNOWN_PLATFORMS, ArtistNotFoundError, DuplicateSourceError, EmptyUrlError, InvalidConfigError, SourceService, UnknownPlatformError, ) pytestmark = pytest.mark.integration async def _artist(db, name="Alice"): a = Artist(name=name, slug=name.lower()) db.add(a) await db.flush() return a @pytest.mark.asyncio async def test_known_platforms_is_gs_five(db): assert KNOWN_PLATFORMS == frozenset({ "patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", }) assert "fanbox" not in KNOWN_PLATFORMS # Retired at #3069 — a source can no longer be created on it. assert "deviantart" not in KNOWN_PLATFORMS @pytest.mark.asyncio async def test_create_flips_is_subscription_on_first_source(db): artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) assert rec.id is not None is_sub = (await db.execute( select(Artist.is_subscription).where(Artist.id == artist.id) )).scalar_one() assert is_sub is True @pytest.mark.asyncio async def test_delete_last_source_flips_is_subscription_off(db): artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) await svc.delete(rec.id) is_sub = (await db.execute( select(Artist.is_subscription).where(Artist.id == artist.id) )).scalar_one() assert is_sub is False @pytest.mark.asyncio async def test_create_rejects_unknown_platform(db): artist = await _artist(db) svc = SourceService(db) with pytest.raises(UnknownPlatformError): await svc.create( artist_id=artist.id, platform="myspace", url="https://m/x", ) @pytest.mark.asyncio async def test_create_rejects_non_dict_config(db): artist = await _artist(db) svc = SourceService(db) with pytest.raises(InvalidConfigError): await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", config_overrides=[1, 2, 3], ) @pytest.mark.asyncio async def test_create_rejects_empty_url(db): artist = await _artist(db) svc = SourceService(db) with pytest.raises(EmptyUrlError): await svc.create(artist_id=artist.id, platform="patreon", url=" ") @pytest.mark.asyncio async def test_create_rejects_unknown_artist(db): svc = SourceService(db) with pytest.raises(ArtistNotFoundError): await svc.create(artist_id=99999, platform="patreon", url="https://x/y") @pytest.mark.asyncio async def test_create_duplicate_raises_with_existing_id(db): artist = await _artist(db) svc = SourceService(db) first = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) with pytest.raises(DuplicateSourceError) as exc: await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) assert exc.value.existing_id == first.id @pytest.mark.asyncio async def test_list_filters_by_artist(db): a = await _artist(db, "Alice") b = await _artist(db, "Bob") svc = SourceService(db) await svc.create(artist_id=a.id, platform="patreon", url="https://patreon.com/a") await svc.create(artist_id=b.id, platform="patreon", url="https://patreon.com/b") only_a = await svc.list(artist_id=a.id) assert [s.artist_id for s in only_a] == [a.id] all_rows = await svc.list() assert len(all_rows) == 2 @pytest.mark.asyncio async def test_list_joins_tier_gated_count_from_the_latest_event(db): """A no-access source carries the count from its most recent walk. The number lives on the DownloadEvent's run_stats, not on the source, so `list()` joins it in. Two events are seeded deliberately: the newest must win, or the row would show a stale figure from a walk where the operator still held the tier. """ from datetime import UTC, datetime, timedelta from backend.app.models import DownloadEvent artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/gated", ) src = (await db.execute( select(Source).where(Source.id == rec.id) )).scalar_one() src.error_type = "tier_limited" src.last_checked_at = datetime.now(UTC) now = datetime.now(UTC) db.add(DownloadEvent( source_id=rec.id, status="ok", started_at=now - timedelta(hours=2), metadata_={"run_stats": {"tier_gated_count": 3}}, )) db.add(DownloadEvent( source_id=rec.id, status="ok", started_at=now, metadata_={"run_stats": {"tier_gated_count": 47}}, )) await db.commit() rows = await svc.list(artist_id=artist.id) assert rows[0].tier_gated_count == 47 assert rows[0].to_dict()["tier_gated_count"] == 47 @pytest.mark.asyncio async def test_list_leaves_tier_gated_count_none_for_ordinary_sources(db): """The join is scoped to tier-gated sources, so a healthy row reports None rather than 0 — the UI distinguishes 'no count available' from 'zero gated', and must not print a fabricated number.""" artist = await _artist(db) svc = SourceService(db) await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/fine", ) rows = await svc.list(artist_id=artist.id) assert rows[0].tier_gated_count is None @pytest.mark.asyncio async def test_update_changes_fields(db): artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/a", ) updated = await svc.update(rec.id, enabled=False, config_overrides={"videos": False}) assert updated.enabled is False # The operator's key lands, and FC's own state is not collateral. This # assertion used to read `== {"videos": False}` — it was DOCUMENTING the bug # the tests below now pin: `create` arms `_backfill_state` (#693), and every # edit was silently discarding it. assert updated.config_overrides["videos"] is False assert updated.config_overrides["_backfill_state"] == "running" # --- config_overrides: the operator's keys and FC's own share one column --- # # `SourceFormDialog` posts the WHOLE object back, and its structured tab rebuilds # that object from two fields — so saving the dialog without touching anything # used to discard the resolved campaign id and the backfill position. The id # matters twice over: `patreon_resolver` reads it BEFORE attempting any lookup, # and the membership join (#387 C4, snippet #3947) keys on it. async def _source_with_state(db, svc, artist, **overrides): """A source carrying both kinds of key: FC's own state and the operator's.""" rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/mixed", ) await db.execute( Source.__table__.update().where(Source.id == rec.id).values( config_overrides={ "_backfill_state": "running", "_backfill_cursor": "03:RESUME", "patreon_campaign_id": "camp-1", "videos": True, **overrides, }, ) ) await db.commit() return rec @pytest.mark.asyncio async def test_an_operator_edit_keeps_fc_managed_state(db): """THE bug. One dialog save used to cost the campaign id and the walk.""" artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist) updated = await svc.update(rec.id, config_overrides={"videos": False}) assert updated.config_overrides["patreon_campaign_id"] == "camp-1" assert updated.config_overrides["_backfill_state"] == "running" assert updated.config_overrides["_backfill_cursor"] == "03:RESUME" assert updated.config_overrides["videos"] is False @pytest.mark.asyncio async def test_an_operator_key_left_out_of_the_edit_is_still_removed(db): """The other half, and why this is a merge rather than a blanket keep: the operator's own settings must still replace wholesale, or removing a key in the dialog could never remove it.""" artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist, since="2026-01-01") updated = await svc.update(rec.id, config_overrides={"videos": False}) assert "since" not in updated.config_overrides @pytest.mark.asyncio async def test_a_null_config_clears_operator_settings_but_not_fc_state(db): """`_parseConfig` sends null for an empty object, so this is the dialog's real "I cleared the box" path — not a request to forget the walk.""" artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist) updated = await svc.update(rec.id, config_overrides=None) assert "videos" not in updated.config_overrides assert updated.config_overrides["_backfill_state"] == "running" assert updated.config_overrides["patreon_campaign_id"] == "camp-1" @pytest.mark.asyncio async def test_a_stale_echoed_value_cannot_overwrite_what_the_walk_wrote(db): """The dialog round-trips whatever it last READ. If the backfill advanced in the meantime, saving that stale copy must not roll the walk backwards — so FC's keys are applied last and win.""" artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist) updated = await svc.update(rec.id, config_overrides={ "videos": False, "_backfill_cursor": "00:STALE", "patreon_campaign_id": "wrong-campaign", }) assert updated.config_overrides["_backfill_cursor"] == "03:RESUME" assert updated.config_overrides["patreon_campaign_id"] == "camp-1" @pytest.mark.asyncio async def test_repointing_the_url_drops_the_resolved_campaign_id(db): """A different creator behind the same row makes the cached id WRONG, not stale. Preserving it above is exactly what makes this invalidation this service's job — the old wholesale overwrite used to wipe the id by accident. """ artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist) updated = await svc.update(rec.id, url="https://patreon.com/someone-else") assert "patreon_campaign_id" not in updated.config_overrides # The walk's position survives: it is expensive to rebuild, the walk has its # own stall guard, and a cosmetic URL edit must not restart a long backfill. assert updated.config_overrides["_backfill_cursor"] == "03:RESUME" @pytest.mark.asyncio async def test_rewriting_the_same_url_keeps_the_campaign_id(db): """The dialog posts the URL on every save. An unchanged URL is not a repoint, and treating it as one would re-resolve after every edit.""" artist = await _artist(db) svc = SourceService(db) rec = await _source_with_state(db, svc, artist) updated = await svc.update(rec.id, url="https://patreon.com/mixed") assert updated.config_overrides["patreon_campaign_id"] == "camp-1" def test_the_campaign_id_suffix_agrees_with_the_membership_join(): """Two modules must agree on the key shape: this one preserves and invalidates it, `membership_roster` matches on it (snippet #3947). They deliberately do not import across — a core CRUD service should not depend on a membership feature — so the agreement is pinned here rather than hoped for. Do NOT fix a failure by editing one side; decide which shape is right. """ from backend.app.services.membership_roster import _CAMPAIGN_KEY_SUFFIX from backend.app.services.source_service import _CAMPAIGN_ID_SUFFIX assert _CAMPAIGN_ID_SUFFIX == _CAMPAIGN_KEY_SUFFIX @pytest.mark.asyncio async def test_disable_clears_failure_state(db): """Disabling a source wipes its failure state so it stops showing as 'failing' (operator: disable subs you're not paying for).""" artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/a", ) source = (await db.execute( select(Source).where(Source.id == rec.id) )).scalar_one() source.last_error = "auth failed" source.error_type = "auth_error" source.consecutive_failures = 5 await db.commit() updated = await svc.update(rec.id, enabled=False) assert updated.enabled is False assert updated.last_error is None assert updated.consecutive_failures == 0 refetched = (await db.execute( select(Source).where(Source.id == rec.id) )).scalar_one() assert refetched.error_type is None @pytest.mark.asyncio async def test_update_while_enabled_keeps_failure_state(db): """A non-disable edit must NOT wipe failure state — only the explicit disable clears it (else a config tweak would hide a real failure).""" artist = await _artist(db) svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/a", ) source = (await db.execute( select(Source).where(Source.id == rec.id) )).scalar_one() source.last_error = "auth failed" source.consecutive_failures = 3 await db.commit() await svc.update(rec.id, config_overrides={"videos": False}) refetched = (await db.execute( select(Source).where(Source.id == rec.id) )).scalar_one() assert refetched.last_error == "auth failed" assert refetched.consecutive_failures == 3 async def _source_with_content(db, svc, artist): """A source under `artist` with one post + one image it contributed.""" from backend.app.models import ImageProvenance, ImageRecord, Post rec = await svc.create( artist_id=artist.id, platform="pixiv", url=f"https://www.pixiv.net/users/{artist.id}", ) post = Post(source_id=rec.id, artist_id=artist.id, external_post_id="p1") db.add(post) img = ImageRecord( path=f"/images/{artist.slug}/pixiv/pixiv/1_a_00.jpg", sha256=str(artist.id).rjust(64, "0"), size_bytes=1, mime="image/jpeg", width=1, height=1, origin="imported_filesystem", integrity_status="unknown", artist_id=artist.id, ) db.add(img) await db.flush() db.add(ImageProvenance(image_record_id=img.id, post_id=post.id, source_id=rec.id)) await db.commit() return rec, post, img @pytest.mark.asyncio async def test_reassign_moves_source_posts_images(db): from backend.app.models import ImageRecord, Post old = await _artist(db, "OldOwner") new = await _artist(db, "NewOwner") svc = SourceService(db) rec, post, img = await _source_with_content(db, svc, old) await svc.reassign(rec.id, new.id) assert (await db.execute( select(Source.artist_id).where(Source.id == rec.id) )).scalar_one() == new.id assert (await db.execute( select(Post.artist_id).where(Post.id == post.id) )).scalar_one() == new.id assert (await db.execute( select(ImageRecord.artist_id).where(ImageRecord.id == img.id) )).scalar_one() == new.id # Old artist is now empty → deleted. assert (await db.execute( select(Artist).where(Artist.id == old.id) )).scalar_one_or_none() is None @pytest.mark.asyncio async def test_reassign_keeps_nonempty_old_artist(db): from backend.app.models import ImageRecord old = await _artist(db, "OldMulti") new = await _artist(db, "NewMulti") svc = SourceService(db) rec, _post, _img = await _source_with_content(db, svc, old) # A second, unrelated image keeps `old` non-empty after the move. db.add(ImageRecord( path="/images/oldmulti/loose.jpg", sha256="e" * 64, size_bytes=1, mime="image/jpeg", width=1, height=1, origin="imported_filesystem", integrity_status="unknown", artist_id=old.id, )) await db.commit() await svc.reassign(rec.id, new.id) still = (await db.execute( select(Artist).where(Artist.id == old.id) )).scalar_one() assert still.is_subscription is False # lost its last source @pytest.mark.asyncio async def test_reassign_same_artist_is_noop(db): a = await _artist(db, "Solo") svc = SourceService(db) rec, _post, _img = await _source_with_content(db, svc, a) out = await svc.reassign(rec.id, a.id) assert out.artist_id == a.id @pytest.mark.asyncio async def test_reassign_unknown_target_raises(db): from backend.app.services.source_service import ArtistNotFoundError a = await _artist(db, "Whom") svc = SourceService(db) rec, _post, _img = await _source_with_content(db, svc, a) with pytest.raises(ArtistNotFoundError): await svc.reassign(rec.id, 999999) @pytest.mark.asyncio async def test_list_hides_sidecar_synthetic_anchors(db): """Filesystem-import synthetic Sources (url='sidecar::', enabled=False — historical pre-alembic-0030 artifact) used to leak into the Subscriptions UI as phantom subscriptions because list() didn't filter them. They aren't pollable feeds; hide by default.""" artist = await _artist(db, "Alice") real = Source( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", enabled=True, config_overrides={}, ) synthetic = Source( artist_id=artist.id, platform="patreon", url="sidecar:patreon:alice", enabled=False, config_overrides={}, ) db.add_all([real, synthetic]) await db.commit() svc = SourceService(db) visible = await svc.list() visible_urls = {s.url for s in visible} assert "https://patreon.com/alice" in visible_urls assert "sidecar:patreon:alice" not in visible_urls # Same filter applies to the artist-scoped list path (the artist detail # page hits /api/sources?artist_id=N). artist_scoped = await svc.list(artist_id=artist.id) assert {s.url for s in artist_scoped} == {"https://patreon.com/alice"} # include_synthetic=True opts back in for admin tooling. everything = await svc.list(include_synthetic=True) assert {s.url for s in everything} >= { "https://patreon.com/alice", "sidecar:patreon:alice", } # --- Plan #544: backfill counter ------------------------------------------- @pytest.mark.asyncio async def test_start_backfill_arms_run_until_done(db): """start_backfill sets state=running + the chunk cap and clears any prior cursor/chunk state, returning the updated record for the API to echo.""" from backend.app.services.source_service import BACKFILL_MAX_CHUNKS artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) # Simulate a prior, finished walk leaving stale checkpoint state. await db.execute( Source.__table__.update().where(Source.id == rec.id).values( config_overrides={"_backfill_state": "complete", "_backfill_cursor": "old", "_backfill_chunks": 7}, ) ) await db.commit() updated = await svc.start_backfill(rec.id) assert updated.backfill_state == "running" assert updated.backfill_chunks == 0 assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS co = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert co.get("_backfill_state") == "running" assert "_backfill_cursor" not in co @pytest.mark.asyncio async def test_stop_backfill_returns_to_idle(db): artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) await svc.start_backfill(rec.id) updated = await svc.stop_backfill(rec.id) assert updated.backfill_state is None assert updated.backfill_runs_remaining == 0 co = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert "_backfill_state" not in (co or {}) @pytest.mark.asyncio async def test_start_recovery_arms_bypass_flag(db): """Plan #697: start_recovery arms the backfill state machine PLUS the _backfill_bypass_seen flag (recovery), surfaced on the record so the UI badge can label it 'Recovering'. Clears any prior checkpoint state.""" from backend.app.services.source_service import BACKFILL_MAX_CHUNKS artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) # Simulate a prior walk's posts counter — start must clear it (plan #704). await db.execute( Source.__table__.update().where(Source.id == rec.id).values( config_overrides={"_backfill_posts": 42}, ) ) await db.commit() updated = await svc.start_recovery(rec.id) assert updated.backfill_state == "running" assert updated.backfill_bypass_seen is True assert updated.backfill_posts == 0 # cleared for a fresh walk assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS co = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert co.get("_backfill_bypass_seen") is True assert "_backfill_posts" not in co # Stop clears the bypass flag too (shared lifecycle). stopped = await svc.stop_backfill(rec.id) assert stopped.backfill_bypass_seen is False co2 = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert "_backfill_bypass_seen" not in (co2 or {}) @pytest.mark.asyncio async def test_start_recapture_arms_recapture_flag(db): """#830: start_recapture arms the backfill state machine PLUS the _backfill_recapture flag, surfaced on the record so the badge can label it 'Recapturing'. Mutually exclusive with recovery (clears bypass_seen); stop clears it.""" from backend.app.services.source_service import BACKFILL_MAX_CHUNKS artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", ) # Pre-arm recovery, then recapture must clear bypass_seen (mutual exclusion). await svc.start_recovery(rec.id) updated = await svc.start_recapture(rec.id) assert updated.backfill_state == "running" assert updated.backfill_recapture is True assert updated.backfill_bypass_seen is False assert updated.backfill_runs_remaining == BACKFILL_MAX_CHUNKS co = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert co.get("_backfill_recapture") is True assert "_backfill_bypass_seen" not in co # to_dict carries the new field for the API/UI. assert updated.to_dict()["backfill_recapture"] is True # Stop clears the recapture flag too. stopped = await svc.stop_backfill(rec.id) assert stopped.backfill_recapture is False co2 = (await db.execute( select(Source.config_overrides).where(Source.id == rec.id) )).scalar_one() assert "_backfill_recapture" not in (co2 or {}) @pytest.mark.asyncio async def test_start_backfill_raises_when_source_missing(db): svc = SourceService(db) with pytest.raises(LookupError): await svc.start_backfill(99999) @pytest.mark.asyncio async def test_new_enabled_source_starts_in_backfill_mode(db): """Plan #693: freshly added enabled sources have no archive yet, so they arm run-until-done backfill — state 'running' — to walk the full history on the first ticks instead of blowing the wall-clock cap in tick mode.""" artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice-new", ) assert rec.backfill_state == "running" @pytest.mark.asyncio async def test_new_disabled_source_skips_backfill(db): """Disabled sources (incl. sidecar synthetics that arrive disabled) are never polled, so don't burn a backfill budget on them.""" artist = await _artist(db, "Alice") svc = SourceService(db) rec = await svc.create( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice-disabled", enabled=False, ) assert rec.backfill_runs_remaining == 0