refactor(downloads): unify phase-2 dispatch in download_backends — A5 (plan #707)
CI / lint (push) Successful in 3s
CI / backend-lint-and-test (push) Successful in 29s
CI / frontend-build (push) Successful in 31s
CI / integration (push) Successful in 2m59s

Mirror verify_source_credential: download_backends.run_download is now the single
download entry, so that module is the ONE registry of how each platform both
downloads AND verifies — the seam that makes adding a platform a bounded job
(write its adapter construction next to its verify). The native-ingester
construction + campaign-id resolution moves out of download_service into
download_backends._run_native_ingester.

download_service.download_source drops its `if uses_native_ingester ... else
gdl.download` branch and calls one `self._run_download(...)` (a thin delegate to
run_download passing the service's gdl + sync sessionmaker). mode (tick/backfill/
recovery) is still chosen there from the backfill state machine and threaded
through. Removed the now-unused PatreonIngester / resolve_campaign_id_for_source
imports from download_service.

Tests: the phase-2 stub seam moves from svc._run_patreon_ingester to
svc._run_download (helper + the db-release test); the two native-construction
tests repoint to download_backends.run_download (patching
download_backends.resolve_campaign_id_for_source / PatreonIngester).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-06 11:22:48 -04:00
parent b211900390
commit e47fa0cf4b
3 changed files with 167 additions and 134 deletions
+37 -39
View File
@@ -89,7 +89,7 @@ def _make_fake_dl_result(
def _fake_gdl_with_result(result):
fake = MagicMock()
fake.download = AsyncMock(return_value=result)
# Real values for the attrs _run_patreon_ingester reads off the gdl service
# Real values for the attrs run_download's native branch reads off the gdl
# (the native pacing config, plan #703) — a MagicMock would break the
# arithmetic (max(0.5, rate_limit/4)).
fake._rate_limit = 3.0
@@ -105,19 +105,24 @@ def _fake_gdl_with_result(result):
def _stub_patreon_ingester(svc, result, resolved_campaign_id=None):
"""Route the Patreon phase-2 branch (plan #697 native ingester) to a canned
DownloadResult so these tests exercise download_service's
phase-2-result→phase-3 handling (status mapping, backfill lifecycle, health)
without the real ingester's network/DB. The ingester itself is covered by
test_patreon_ingester.py. Returns a list capturing (ctx, source_config, mode)
per call so a test can assert mode / resume_cursor threading."""
"""Route the phase-2 dispatch (download_backends.run_download via
DownloadService._run_download, plan #707 A5) to a canned DownloadResult so
these tests exercise download_service's phase-2-result→phase-3 handling
(status mapping, backfill lifecycle, health) without the real ingester's
network/DB. The ingester itself is covered by test_patreon_ingester.py; the
native construction/resolution by the run_download tests below. Returns a list
capturing (ctx, source_config, skip_value, mode) per call so a test can assert
mode / resume_cursor threading."""
calls = []
async def fake(ctx, source_config, mode):
calls.append({"ctx": ctx, "source_config": source_config, "mode": mode})
async def fake(*, ctx, source_config, skip_value, mode):
calls.append({
"ctx": ctx, "source_config": source_config,
"skip_value": skip_value, "mode": mode,
})
return result, resolved_campaign_id
svc._run_patreon_ingester = fake
svc._run_download = fake
return calls
@@ -285,20 +290,19 @@ async def test_patreon_resolved_campaign_id_is_cached(
@pytest.mark.asyncio
async def test_run_patreon_ingester_resolves_vanity_and_runs(
async def test_run_download_native_resolves_vanity_and_runs(
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
):
"""_run_patreon_ingester: with no cached campaign id, resolve the vanity and
pass the resolved id straight to the ingester; report it back so phase 3 can
cache it."""
from backend.app.services import download_service as dl_mod
from backend.app.services.download_service import DownloadService
"""run_download (native branch, plan #707 A5): with no cached campaign id,
resolve the vanity and pass the resolved id straight to the ingester; report
it back so phase 3 can cache it."""
from backend.app.services import download_backends as db_mod
from backend.app.services.gallery_dl import SourceConfig
_artist, source = seed_artist_and_source
monkeypatch.setattr(
dl_mod, "resolve_campaign_id_for_source",
db_mod, "resolve_campaign_id_for_source",
AsyncMock(return_value=("4242", "4242")),
)
run_kwargs = {}
@@ -311,21 +315,19 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
run_kwargs.update(kw)
return _make_fake_dl_result(success=True, written_paths=[])
monkeypatch.setattr(dl_mod, "PatreonIngester", _FakeIngester)
monkeypatch.setattr(db_mod, "PatreonIngester", _FakeIngester)
svc = DownloadService(
async_session=db, sync_session=db_sync,
gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
importer=MagicMock(), cred_service=MagicMock(),
sync_session_factory=MagicMock(),
)
ctx = {
"platform": "patreon",
"source_id": source.id, "url": "https://patreon.com/alice",
"artist_slug": "alice", "cookies_path": None,
"config_overrides": {},
}
result, resolved = await svc._run_patreon_ingester(
ctx, SourceConfig.from_dict({}), "tick",
result, resolved = await db_mod.run_download(
ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
mode="tick",
gdl=_fake_gdl_with_result(_make_fake_dl_result(success=True)),
sync_session_factory=MagicMock(),
)
assert result.success is True
assert resolved == "4242"
@@ -334,33 +336,29 @@ async def test_run_patreon_ingester_resolves_vanity_and_runs(
@pytest.mark.asyncio
async def test_run_patreon_ingester_unresolvable_fails_loud(
async def test_run_download_native_unresolvable_fails_loud(
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
):
"""A campaign id we can't resolve is a loud NOT_FOUND failure, never a
silent empty success."""
from backend.app.services import download_service as dl_mod
from backend.app.services.download_service import DownloadService
from backend.app.services import download_backends as db_mod
from backend.app.services.gallery_dl import ErrorType, SourceConfig
_artist, source = seed_artist_and_source
monkeypatch.setattr(
dl_mod, "resolve_campaign_id_for_source",
db_mod, "resolve_campaign_id_for_source",
AsyncMock(return_value=(None, None)),
)
svc = DownloadService(
async_session=db, sync_session=db_sync,
gdl=MagicMock(), importer=MagicMock(), cred_service=MagicMock(),
sync_session_factory=MagicMock(),
)
ctx = {
"platform": "patreon",
"source_id": source.id, "url": "https://patreon.com/alice",
"artist_slug": "alice", "cookies_path": None,
"config_overrides": {},
}
result, resolved = await svc._run_patreon_ingester(
ctx, SourceConfig.from_dict({}), "tick",
result, resolved = await db_mod.run_download(
ctx=ctx, source_config=SourceConfig.from_dict({}), skip_value=True,
mode="tick", gdl=MagicMock(), sync_session_factory=MagicMock(),
)
assert result.success is False
assert result.error_type == ErrorType.NOT_FOUND
@@ -926,11 +924,11 @@ async def test_releases_db_connections_before_subprocess(
gdl=fake_gdl, importer=importer, cred_service=cred_service,
)
async def fake_ingest(ctx, source_config, mode):
async def fake_ingest(*, ctx, source_config, skip_value, mode):
seen["closed_before_phase2"] = closed["async"]
return _make_fake_dl_result(success=True, written_paths=[], stdout=""), None
svc._run_patreon_ingester = fake_ingest
svc._run_download = fake_ingest
event_id = await svc.download_source(source.id)
assert seen["closed_before_phase2"] is True