refactor(downloads): unify phase-2 dispatch in download_backends — A5 (plan #707)
Mirror verify_source_credential: download_backends.run_download is now the single download entry, so that module is the ONE registry of how each platform both downloads AND verifies — the seam that makes adding a platform a bounded job (write its adapter construction next to its verify). The native-ingester construction + campaign-id resolution moves out of download_service into download_backends._run_native_ingester. download_service.download_source drops its `if uses_native_ingester ... else gdl.download` branch and calls one `self._run_download(...)` (a thin delegate to run_download passing the service's gdl + sync sessionmaker). mode (tick/backfill/ recovery) is still chosen there from the backfill state machine and threaded through. Removed the now-unused PatreonIngester / resolve_campaign_id_for_source imports from download_service. Tests: the phase-2 stub seam moves from svc._run_patreon_ingester to svc._run_download (helper + the db-release test); the two native-construction tests repoint to download_backends.run_download (patching download_backends.resolve_campaign_id_for_source / PatreonIngester). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -89,7 +89,7 @@ def _make_fake_dl_result(
|
||||
def _fake_gdl_with_result(result):
|
||||
fake = MagicMock()
|
||||
fake.download = AsyncMock(return_value=result)
|
||||
# Real values for the attrs _run_patreon_ingester reads off the gdl service
|
||||
# Real values for the attrs run_download's native branch reads off the gdl
|
||||
# (the native pacing config, plan #703) — a MagicMock would break the
|
||||
# arithmetic (max(0.5, rate_limit/4)).
|
||||
fake._rate_limit = 3.0
|
||||
@@ -105,19 +105,24 @@ def _fake_gdl_with_result(result):
|
||||
|
||||
|
||||
def _stub_patreon_ingester(svc, result, resolved_campaign_id=None):
|
||||
"""Route the Patreon phase-2 branch (plan #697 native ingester) to a canned
|
||||
DownloadResult so these tests exercise download_service's
|
||||
phase-2-result→phase-3 handling (status mapping, backfill lifecycle, health)
|
||||
without the real ingester's network/DB. The ingester itself is covered by
|
||||
test_patreon_ingester.py. Returns a list capturing (ctx, source_config, mode)
|
||||
per call so a test can assert mode / resume_cursor threading."""
|
||||
"""Route the phase-2 dispatch (download_backends.run_download via
|
||||
DownloadService._run_download, plan #707 A5) to a canned DownloadResult so
|
||||
these tests exercise download_service's phase-2-result→phase-3 handling
|
||||
(status mapping, backfill lifecycle, health) without the real ingester's
|
||||
network/DB. The ingester itself is covered by test_patreon_ingester.py; the
|
||||
native construction/resolution by the run_download tests below. Returns a list
|
||||
capturing (ctx, source_config, skip_value, mode) per call so a test can assert
|
||||
mode / resume_cursor threading."""
|
||||
calls = []
|
||||
|
||||
async def fake(ctx, source_config, mode):
|
||||
calls.append({"ctx": ctx, "source_config": source_config, "mode": mode})
|
||||
async def fake(*, ctx, source_config, skip_value, mode):
|
||||
calls.append({
|
||||
"ctx": ctx, "source_config": source_config,
|
||||
"skip_value": skip_value, "mode": mode,
|
||||
})
|
||||
return result, resolved_campaign_id
|
||||
|
||||
svc._run_patreon_ingester = fake
|
||||
svc._run_download = fake
|
||||
return calls
|
||||
|
||||
|
||||
@@ -285,20 +290,19 @@ async def test_patreon_resolved_campaign_id_is_cached(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_patreon_ingester_resolves_vanity_and_runs(
|
||||
async def test_run_download_native_resolves_vanity_and_runs(
|
||||
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||
):
|
||||
"""_run_patreon_ingester: with no cached campaign id, resolve the vanity and
|
||||
pass the resolved id straight to the ingester; report it back so phase 3 can
|
||||
cache it."""
|
||||
from backend.app.services import download_service as dl_mod
|
||||
from backend.app.services.download_service import DownloadService
|
||||
"""run_download (native branch, plan #707 A5): with no cached campaign id,
|
||||
resolve the vanity and pass the resolved id straight to the ingester; report
|
||||
it back so phase 3 can cache it."""
|
||||
from backend.app.services import download_backends as db_mod
|
||||
from backend.app.services.gallery_dl import SourceConfig
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
|
||||
monkeypatch.setattr(
|
||||
dl_mod, "resolve_campaign_id_for_source",
|
||||
db_mod, "resolve_campaign_id_for_source",
|
||||
AsyncMock(return_value=("4242", "4242")),
|
||||
)
|
||||
run_kwargs = {}
|
||||
@@ -311,21 +315,19 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
|
||||
run_kwargs.update(kw)
|
||||
return _make_fake_dl_result(success=True, written_paths=[])
|
||||
|
||||
monkeypatch.setattr(dl_mod, "PatreonIngester", _FakeIngester)
|
||||
monkeypatch.setattr(db_mod, "PatreonIngester", _FakeIngester)
|
||||
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
|
||||
importer=MagicMock(), cred_service=MagicMock(),
|
||||
sync_session_factory=MagicMock(),
|
||||
)
|
||||
ctx = {
|
||||
"platform": "patreon",
|
||||
"source_id": source.id, "url": "https://patreon.com/alice",
|
||||
"artist_slug": "alice", "cookies_path": None,
|
||||
"config_overrides": {},
|
||||
}
|
||||
result, resolved = await svc._run_patreon_ingester(
|
||||
ctx, SourceConfig.from_dict({}), "tick",
|
||||
result, resolved = await db_mod.run_download(
|
||||
ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
|
||||
mode="tick",
|
||||
gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
|
||||
sync_session_factory=MagicMock(),
|
||||
)
|
||||
assert result.success is True
|
||||
assert resolved == "4242"
|
||||
@@ -334,33 +336,29 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_patreon_ingester_unresolvable_fails_loud(
|
||||
async def test_run_download_native_unresolvable_fails_loud(
|
||||
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||
):
|
||||
"""A campaign id we can't resolve is a loud NOT_FOUND failure, never a
|
||||
silent empty success."""
|
||||
from backend.app.services import download_service as dl_mod
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services import download_backends as db_mod
|
||||
from backend.app.services.gallery_dl import ErrorType, SourceConfig
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
|
||||
monkeypatch.setattr(
|
||||
dl_mod, "resolve_campaign_id_for_source",
|
||||
db_mod, "resolve_campaign_id_for_source",
|
||||
AsyncMock(return_value=(None, None)),
|
||||
)
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
|
||||
sync_session_factory=MagicMock(),
|
||||
)
|
||||
ctx = {
|
||||
"platform": "patreon",
|
||||
"source_id": source.id, "url": "https://patreon.com/alice",
|
||||
"artist_slug": "alice", "cookies_path": None,
|
||||
"config_overrides": {},
|
||||
}
|
||||
result, resolved = await svc._run_patreon_ingester(
|
||||
ctx, SourceConfig.from_dict({}), "tick",
|
||||
result, resolved = await db_mod.run_download(
|
||||
ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
|
||||
mode="tick", gdl=MagicMock(), sync_session_factory=MagicMock(),
|
||||
)
|
||||
assert result.success is False
|
||||
assert result.error_type == ErrorType.NOT_FOUND
|
||||
@@ -926,11 +924,11 @@ async def test_releases_db_connections_before_subprocess(
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
|
||||
async def fake_ingest(ctx, source_config, mode):
|
||||
async def fake_ingest(*, ctx, source_config, skip_value, mode):
|
||||
seen["closed_before_phase2"] = closed["async"]
|
||||
return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None
|
||||
|
||||
svc._run_patreon_ingester = fake_ingest
|
||||
svc._run_download = fake_ingest
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
assert seen["closed_before_phase2"] is True
|
||||
|
||||
Reference in New Issue
Block a user