feat(downloads): serialize same-platform downloads (Patreon concurrency cap)
Two concurrent Patreon walks could trip the server rate limit even with each
source pacing its own requests. The platform-cooldown handled the aftermath of
a 429; this adds the preventive half — a per-platform Redis lock so only one
Patreon walk runs at a time. Different platforms still run concurrently up to
the worker concurrency; only a second walk on the SAME serialized platform
waits.
download_source acquires fc:download_lock:<platform> (non-blocking) before the
run. On contention it re-enqueues itself with a short countdown (the pending
event stays — no new event, no log spam), bounded to ~15 min then runs uncapped
as a safety valve. The lock TTL sits just past the hard kill so a SIGKILL'd
worker auto-releases; a backfill chunk only holds it ~10 min, well under the
30-min DownloadEvent recovery sweep. A broker hiccup degrades to uncapped
(prior behaviour) rather than stalling downloads. SERIALIZED_PLATFORMS={patreon};
gallery-dl platforms are left uncapped (self-pacing subprocesses).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -162,6 +162,12 @@ async def test_download_source_catches_soft_limit_and_salvages_event(
|
||||
|
||||
monkeypatch.setattr(celery.conf, "task_always_eager", True)
|
||||
monkeypatch.setattr(celery.conf, "task_eager_propagates", False)
|
||||
# This test exercises the soft-limit salvage path, not the per-platform
|
||||
# serialization. Neutralize the Redis lock so it can't defer/recurse under
|
||||
# eager mode (and so it doesn't contend with test_platform_lock).
|
||||
monkeypatch.setattr(
|
||||
"backend.app.services.platform_lock.platform_lock", lambda *a, **k: None,
|
||||
)
|
||||
|
||||
source, event_id = _seed_running_event(
|
||||
db_sync, slug="anduowire", backfill=1, failures=0,
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Per-platform download concurrency cap (Redis lock)."""
|
||||
import pytest
|
||||
|
||||
from backend.app.services.platform_lock import platform_lock
|
||||
|
||||
# Needs the broker (Redis) the lock lives in.
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def test_non_serialized_platform_has_no_lock():
|
||||
# gallery-dl platforms aren't capped — they get no lock at all.
|
||||
assert platform_lock("subscribestar", ttl_seconds=60) is None
|
||||
assert platform_lock("deviantart", ttl_seconds=60) is None
|
||||
|
||||
|
||||
def test_patreon_serializes_to_one_holder():
|
||||
first = platform_lock("patreon", ttl_seconds=60)
|
||||
assert first is not None
|
||||
assert first.acquire(blocking=False) is True
|
||||
try:
|
||||
# A second acquirer is refused while the first holds it.
|
||||
second = platform_lock("patreon", ttl_seconds=60)
|
||||
assert second.acquire(blocking=False) is False
|
||||
finally:
|
||||
first.release()
|
||||
|
||||
# Once released, the platform is acquirable again.
|
||||
third = platform_lock("patreon", ttl_seconds=60)
|
||||
assert third.acquire(blocking=False) is True
|
||||
third.release()
|
||||
Reference in New Issue
Block a user