"""Per-platform download concurrency cap. Some platforms (Patreon) are API-rate-sensitive enough that two *simultaneous* walks can trip the server's rate limit even with each source pacing its own requests. The platform-cooldown handles the AFTERMATH of a 429; this is the preventive half — it serializes downloads PER PLATFORM to one at a time. Different platforms still run concurrently up to the worker's concurrency; only a second walk on the SAME serialized platform waits. The lock lives in Redis (the Celery broker) with a TTL, so a SIGKILL'd worker can't wedge a platform — the lock auto-expires shortly after the download hard time limit. """ from __future__ import annotations import logging import redis from ..config import get_config log = logging.getLogger(__name__) # Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT # here: each runs as a self-pacing subprocess and they're lower-volume. The # native-ingester platforms are serialized (one paced scrape/API walk at a time). # Add a platform here to cap it to a single concurrent walk. Discord most of # all: every source walks on the operator's ONE user token, and parallel walks # on a user account are both how its rate limit trips and what gets it flagged. SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"}) _LOCK_PREFIX = "fc:download_lock:" _client: redis.Redis | None = None def _redis() -> redis.Redis: # One client per worker process (Celery prefork forks before tasks run, so # each process lazily builds its own). redis-py pools connections. global _client if _client is None: _client = redis.from_url(get_config().celery_broker_url) return _client def platform_lock(platform: str, *, ttl_seconds: int): """A non-blocking Redis lock for `platform`, or None when the platform is not serialized. Caller does `.acquire(blocking=False)` / `.release()`. Returns None (rather than raising) on any Redis error so a broker hiccup degrades to the prior behaviour (uncapped) instead of stalling downloads. """ if platform not in SERIALIZED_PLATFORMS: return None try: return _redis().lock( f"{_LOCK_PREFIX}{platform}", timeout=ttl_seconds, blocking=False, ) except redis.RedisError as exc: # pragma: no cover - broker outage log.warning("platform_lock unavailable for %s: %s", platform, exc) return None def release_all_platform_locks() -> int: """Drop every serialized platform's lock. Returns how many were held. Only for the download lane's boot (#4433). A worker that restarts mid-walk is SIGKILLed past its stop grace, so its `finally` never releases the lock, and the TTL keeps every other source on that platform bouncing for up to 27 minutes after the new worker is ready. At boot no walk of ours can be running, so a held lock names a dead one. Assumes one download consumer — the only shape FC deploys; a second replica booting would free a live walk's lock (not corrupt it: the walk runs on, a second walk may overlap it). """ try: client = _redis() return int(client.delete(*(f"{_LOCK_PREFIX}{p}" for p in SERIALIZED_PLATFORMS))) except redis.RedisError as exc: # pragma: no cover - broker outage log.warning("could not release platform locks at boot: %s", exc) return 0