feat(scheduler): platform-wide cooldown on RATE_LIMITED — burst prevention
The scan tick fired download_source.delay() for every due source without
grouping by platform; with multiple download workers, N due Patreon
sources could all hit Patreon's API in parallel and rate-limit each
other. Per-source consecutive_failures backoff REACTS to that (slows the
offender across cycles) but didn't PREVENT the first-tick burst.
When DownloadService._update_source_health sees a source error
classified as ErrorType.RATE_LIMITED, it now stamps an AppSetting row
`platform_cooldown:<platform>` with the cooldown expiry (now + 15 min,
PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS). select_due_sources queries every
platform_cooldown:* key at the start of each tick and excludes every
source whose platform is in active cooldown. scheduler_status surfaces
active cooldowns as platform_cooldowns: {platform: expires_iso} so the
TopNav pipeline chip / activity summary can display them.
INSERT...ON CONFLICT DO UPDATE for the upsert so two workers racing
RATE_LIMITED responses on the same platform don't let one's
IntegrityError roll back the other's event-finalize transaction
(stranding the event for the recovery sweep). Atomic at the SQL level.
Tests cover: select_due_sources skips a platform in cooldown; other
platforms unaffected during single-platform cooldown; expired cooldown
rows don't filter; set_platform_cooldown is upsert-safe under repeated
calls.
Operator-flagged 2026-05-30 ("running multiple workers I don't know how
we'd keep the downloader from hitting a rate limit on a source").
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -147,3 +147,113 @@ async def test_select_includes_past_due(db):
|
||||
await db.commit()
|
||||
due = await select_due_sources(db)
|
||||
assert any(s.url == "https://sel-past" for s in due)
|
||||
|
||||
|
||||
# --- platform-rate-limit cooldown -----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_skips_sources_on_platform_in_cooldown(db):
|
||||
"""After set_platform_cooldown('patreon'), select_due_sources excludes
|
||||
every patreon source — even ones that would otherwise be past-due."""
|
||||
from backend.app.services.scheduler_service import set_platform_cooldown
|
||||
|
||||
artist = await _seed_artist(db, interval=60, name="cooldown-p")
|
||||
db.add(Source(
|
||||
artist_id=artist.id, platform="patreon", url="https://cd-p",
|
||||
enabled=True, consecutive_failures=0,
|
||||
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
# Sanity-pre: due before the cooldown is set.
|
||||
due = await select_due_sources(db)
|
||||
assert any(s.url == "https://cd-p" for s in due)
|
||||
|
||||
await set_platform_cooldown(db, "patreon")
|
||||
await db.commit()
|
||||
|
||||
due = await select_due_sources(db)
|
||||
assert all(s.url != "https://cd-p" for s in due)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cooldown_is_per_platform_other_platforms_unaffected(db):
|
||||
"""A cooldown on patreon doesn't block subscribestar sources from
|
||||
being due — cooldowns are scoped to the platform that hit the limit."""
|
||||
from backend.app.services.scheduler_service import set_platform_cooldown
|
||||
|
||||
artist = await _seed_artist(db, interval=60, name="cooldown-mixed")
|
||||
db.add_all([
|
||||
Source(
|
||||
artist_id=artist.id, platform="patreon", url="https://mix-p",
|
||||
enabled=True, consecutive_failures=0,
|
||||
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
||||
),
|
||||
Source(
|
||||
artist_id=artist.id, platform="subscribestar", url="https://mix-ss",
|
||||
enabled=True, consecutive_failures=0,
|
||||
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
||||
),
|
||||
])
|
||||
await db.commit()
|
||||
|
||||
await set_platform_cooldown(db, "patreon")
|
||||
await db.commit()
|
||||
|
||||
due = await select_due_sources(db)
|
||||
urls = {s.url for s in due}
|
||||
assert "https://mix-ss" in urls
|
||||
assert "https://mix-p" not in urls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_select_ignores_expired_cooldown(db):
|
||||
"""An expired cooldown row doesn't filter — routing treats expired
|
||||
same as not-set. The stale row stays around until a future sweep
|
||||
prunes it."""
|
||||
from backend.app.models import AppSetting
|
||||
from backend.app.services.scheduler_service import (
|
||||
PLATFORM_COOLDOWN_KEY_PREFIX,
|
||||
)
|
||||
|
||||
artist = await _seed_artist(db, interval=60, name="cooldown-expired")
|
||||
db.add(Source(
|
||||
artist_id=artist.id, platform="patreon", url="https://cd-exp",
|
||||
enabled=True, consecutive_failures=0,
|
||||
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
||||
))
|
||||
db.add(AppSetting(
|
||||
key=f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon",
|
||||
value=(datetime.now(UTC) - timedelta(seconds=10)).isoformat(),
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
due = await select_due_sources(db)
|
||||
assert any(s.url == "https://cd-exp" for s in due)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_platform_cooldown_upserts(db):
|
||||
"""Calling set_platform_cooldown twice on the same platform updates
|
||||
the row in place rather than inserting a duplicate (AppSetting's PK
|
||||
is `key`)."""
|
||||
from sqlalchemy import select as sa_select
|
||||
|
||||
from backend.app.models import AppSetting
|
||||
from backend.app.services.scheduler_service import (
|
||||
PLATFORM_COOLDOWN_KEY_PREFIX,
|
||||
set_platform_cooldown,
|
||||
)
|
||||
|
||||
await set_platform_cooldown(db, "patreon", seconds=60)
|
||||
await db.commit()
|
||||
await set_platform_cooldown(db, "patreon", seconds=600)
|
||||
await db.commit()
|
||||
|
||||
rows = (await db.execute(
|
||||
sa_select(AppSetting).where(
|
||||
AppSetting.key == f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon"
|
||||
)
|
||||
)).scalars().all()
|
||||
assert len(rows) == 1
|
||||
|
||||
Reference in New Issue
Block a user