"""DownloadService orchestrator tests. GalleryDLService is mocked to return a canned DownloadResult; tests provide on-disk files for the importer to attach_in_place. """ from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest from sqlalchemy import select from backend.app.models import Artist, DownloadEvent, ImportSettings, Source from backend.app.services.credential_crypto import CredentialCrypto from backend.app.services.credential_service import CredentialService from backend.app.services.thumbnailer import Thumbnailer pytestmark = pytest.mark.integration @pytest.fixture async def seed_artist_and_source(db, db_sync): artist = Artist(name="Alice", slug="alice") db.add(artist) await db.flush() source = Source( artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", enabled=True, config_overrides={}, ) db.add(source) await db.commit() return artist, source def _make_jpg(path: Path, split: str = "h"): """Write a structured JPEG so phash isn't degenerate. Solid-color images all phash-collapse (DCT of a flat image is zero). Use `split='h'` (top/bottom halves) or `split='v'` (left/right) to give each fixture file a distinct phash. """ from PIL import Image path.parent.mkdir(parents=True, exist_ok=True) im = Image.new("RGB", (64, 64), (255, 255, 255)) px = im.load() if split == "h": for y in range(32): for x in range(64): px[x, y] = (0, 0, 0) else: # 'v' for y in range(64): for x in range(32): px[x, y] = (0, 0, 0) im.save(path, "JPEG") def _make_fake_dl_result( *, success=True, written_paths=None, quarantined_paths=None, files_downloaded=0, error_type=None, error_message=None, stdout="", stderr="", ): return SimpleNamespace( success=success, url="https://patreon.com/alice", artist_slug="alice", platform="patreon", files_downloaded=files_downloaded, files_quarantined=len(quarantined_paths or []), quarantined_paths=quarantined_paths or [], written_paths=written_paths or [], stdout=stdout, stderr=stderr, return_code=0 if success else 1, error_type=error_type, error_message=error_message, duration_seconds=1.23, started_at="2026-05-20T14:00:00+00:00", completed_at="2026-05-20T14:01:00+00:00", ) def _fake_gdl_with_result(result): fake = MagicMock() fake.download = AsyncMock(return_value=result) fake._compute_run_stats = lambda *a, **k: { "exit_code": 0, "downloaded_count": len(result.written_paths), "skipped_count": 0, "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, } fake._extract_errors_warnings = lambda *a, **k: "" fake._truncate_log = lambda x, **k: x return fake @pytest.mark.asyncio async def test_download_source_attaches_written_files( db, db_sync, tmp_path, seed_artist_and_source, ): from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer _artist, source = seed_artist_and_source images_root = tmp_path / "images" f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" f2 = images_root / "alice" / "patreon" / "post" / "b.jpg" # Structured h/v split → distinct sha256 AND distinct phash; # also pin phash_threshold=0 so even tiny perceptual similarity # between the two doesn't trigger duplicate_phash. _make_jpg(f1, split="h") _make_jpg(f2, split="v") fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( success=True, written_paths=[str(f1), str(f2)], files_downloaded=2, stdout=f"{f1}\n{f2}\n", )) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() sync_settings.phash_threshold = 0 # only exact phash collisions dedup importer = Importer( session=db_sync, images_root=images_root, import_root=images_root, thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings, ) crypto = CredentialCrypto(tmp_path / "key.b64") cred_service = CredentialService(db, crypto) svc = DownloadService( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) event_id = await svc.download_source(source.id) ev = (await db.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() assert ev.status == "ok" assert ev.files_count == 2 assert ev.metadata_["import_summary"]["attached"] == 2 assert ev.metadata_["run_stats"]["downloaded_count"] == 2 @pytest.mark.asyncio async def test_download_source_skipped_when_disabled( db, db_sync, tmp_path, seed_artist_and_source, ): from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer _artist, source = seed_artist_and_source source.enabled = False await db.commit() fake_gdl = MagicMock() fake_gdl.download = AsyncMock() sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() importer = Importer( session=db_sync, images_root=tmp_path, import_root=tmp_path, thumbnailer=Thumbnailer(images_root=tmp_path), settings=sync_settings, ) cred_service = CredentialService(db, MagicMock()) svc = DownloadService( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) event_id = await svc.download_source(source.id) ev = (await db.execute( select(DownloadEvent).where(DownloadEvent.id == event_id) )).scalar_one() assert ev.status == "skipped" assert ev.error == "source disabled" fake_gdl.download.assert_not_called() @pytest.mark.asyncio async def test_in_flight_idempotency_returns_existing_event( db, db_sync, tmp_path, seed_artist_and_source, ): from backend.app.services.download_service import DownloadService _artist, source = seed_artist_and_source running = DownloadEvent(source_id=source.id, status="running") db.add(running) await db.commit() await db.refresh(running) fake_gdl = MagicMock() fake_gdl.download = AsyncMock() importer_stub = MagicMock() cred_service = CredentialService(db, MagicMock()) svc = DownloadService( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer_stub, cred_service=cred_service, ) event_id = await svc.download_source(source.id) assert event_id == running.id fake_gdl.download.assert_not_called() @pytest.mark.asyncio async def test_patreon_retry_caches_campaign_id( db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, ): from backend.app.services.download_service import DownloadService from backend.app.services.importer import Importer _artist, source = seed_artist_and_source failed = _make_fake_dl_result( success=False, written_paths=[], stdout="", stderr="[patreon][error] Failed to extract campaign ID for alice\n", ) succeeded = _make_fake_dl_result(success=True, written_paths=[]) download_calls = [] async def fake_download(url, **k): download_calls.append(url) return failed if len(download_calls) == 1 else succeeded fake_gdl = MagicMock() fake_gdl.download = fake_download fake_gdl._compute_run_stats = lambda *a, **k: { "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, } fake_gdl._extract_errors_warnings = lambda *a, **k: "" fake_gdl._truncate_log = lambda x, **k: x monkeypatch.setattr( "backend.app.services.download_service.resolve_campaign_id", AsyncMock(return_value="99"), ) sync_settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() importer = Importer( session=db_sync, images_root=tmp_path, import_root=tmp_path, thumbnailer=Thumbnailer(images_root=tmp_path), settings=sync_settings, ) cred_service = CredentialService(db, MagicMock()) svc = DownloadService( async_session=db, sync_session=db_sync, gdl=fake_gdl, importer=importer, cred_service=cred_service, ) await svc.download_source(source.id) assert len(download_calls) == 2 assert "id:99" in download_calls[1] overrides = db_sync.execute( select(Source.config_overrides).where(Source.id == source.id) ).scalar_one() assert overrides.get("patreon_campaign_id") == "99" # --- FC-3d: finalize hook updates Source health columns ------------------- async def _seed_source_with_health(db, *, failures=0, last_error=None, suffix="fz"): artist = Artist(name=f"alice-{suffix}", slug=f"alice-{suffix}") db.add(artist) await db.flush() source = Source( artist_id=artist.id, platform="patreon", url=f"https://patreon.com/alice-{suffix}", enabled=True, consecutive_failures=failures, last_error=last_error, ) db.add(source) await db.flush() event = DownloadEvent(source_id=source.id, status="running") db.add(event) await db.commit() return source.id, event.id @pytest.mark.asyncio async def test_finalize_ok_resets_failures_and_clears_error(db): from backend.app.services.download_service import DownloadService source_id, event_id = await _seed_source_with_health( db, failures=3, last_error="prev", suffix="ok", ) svc = DownloadService( async_session=db, sync_session=None, gdl=None, importer=None, cred_service=None, ) await svc._finalize_event_and_source( event_id=event_id, source_id=source_id, status="ok", error_message=None, ) row = (await db.execute( select( Source.consecutive_failures, Source.last_error, Source.last_checked_at, ).where(Source.id == source_id) )).one() assert row.consecutive_failures == 0 assert row.last_error is None assert row.last_checked_at is not None @pytest.mark.asyncio async def test_finalize_error_increments_failures_and_sets_error(db): from backend.app.services.download_service import DownloadService source_id, event_id = await _seed_source_with_health( db, failures=2, last_error=None, suffix="err", ) svc = DownloadService( async_session=db, sync_session=None, gdl=None, importer=None, cred_service=None, ) await svc._finalize_event_and_source( event_id=event_id, source_id=source_id, status="error", error_message="boom", ) row = (await db.execute( select( Source.consecutive_failures, Source.last_error, Source.last_checked_at, ).where(Source.id == source_id) )).one() assert row.consecutive_failures == 3 assert row.last_error == "boom" assert row.last_checked_at is not None @pytest.mark.asyncio async def test_finalize_skipped_preserves_failures_clears_error(db): from backend.app.services.download_service import DownloadService source_id, event_id = await _seed_source_with_health( db, failures=4, last_error="stale", suffix="skip", ) svc = DownloadService( async_session=db, sync_session=None, gdl=None, importer=None, cred_service=None, ) await svc._finalize_event_and_source( event_id=event_id, source_id=source_id, status="skipped", error_message=None, ) row = (await db.execute( select( Source.consecutive_failures, Source.last_error, Source.last_checked_at, ).where(Source.id == source_id) )).one() assert row.consecutive_failures == 4 assert row.last_error is None assert row.last_checked_at is not None