feat(scheduler): platform-wide cooldown on RATE_LIMITED — burst prevention

The scan tick fired download_source.delay() for every due source without
grouping by platform; with multiple download workers, N due Patreon
sources could all hit Patreon's API in parallel and rate-limit each
other. Per-source consecutive_failures backoff REACTS to that (slows the
offender across cycles) but didn't PREVENT the first-tick burst.

When DownloadService._update_source_health sees a source error
classified as ErrorType.RATE_LIMITED, it now stamps an AppSetting row
`platform_cooldown:<platform>` with the cooldown expiry (now + 15 min,
PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS). select_due_sources queries every
platform_cooldown:* key at the start of each tick and excludes every
source whose platform is in active cooldown. scheduler_status surfaces
active cooldowns as platform_cooldowns: {platform: expires_iso} so the
TopNav pipeline chip / activity summary can display them.

INSERT...ON CONFLICT DO UPDATE for the upsert so two workers racing
RATE_LIMITED responses on the same platform don't let one's
IntegrityError roll back the other's event-finalize transaction
(stranding the event for the recovery sweep). Atomic at the SQL level.

Tests cover: select_due_sources skips a platform in cooldown; other
platforms unaffected during single-platform cooldown; expired cooldown
rows don't filter; set_platform_cooldown is upsert-safe under repeated
calls.

Operator-flagged 2026-05-30 ("running multiple workers I don't know how
we'd keep the downloader from hitting a rate limit on a source").

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-30 11:01:40 -04:00
parent d28db32012
commit 61ce1ce13c
3 changed files with 196 additions and 1 deletions
+110
View File
@@ -147,3 +147,113 @@ async def test_select_includes_past_due(db):
await db.commit()
due = await select_due_sources(db)
assert any(s.url == "https://sel-past" for s in due)
# --- platform-rate-limit cooldown -----------------------------------------
@pytest.mark.asyncio
async def test_select_skips_sources_on_platform_in_cooldown(db):
"""After set_platform_cooldown('patreon'), select_due_sources excludes
every patreon source — even ones that would otherwise be past-due."""
from backend.app.services.scheduler_service import set_platform_cooldown
artist = await _seed_artist(db, interval=60, name="cooldown-p")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cd-p",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
))
await db.commit()
# Sanity-pre: due before the cooldown is set.
due = await select_due_sources(db)
assert any(s.url == "https://cd-p" for s in due)
await set_platform_cooldown(db, "patreon")
await db.commit()
due = await select_due_sources(db)
assert all(s.url != "https://cd-p" for s in due)
@pytest.mark.asyncio
async def test_cooldown_is_per_platform_other_platforms_unaffected(db):
"""A cooldown on patreon doesn't block subscribestar sources from
being due — cooldowns are scoped to the platform that hit the limit."""
from backend.app.services.scheduler_service import set_platform_cooldown
artist = await _seed_artist(db, interval=60, name="cooldown-mixed")
db.add_all([
Source(
artist_id=artist.id, platform="patreon", url="https://mix-p",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
),
Source(
artist_id=artist.id, platform="subscribestar", url="https://mix-ss",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
),
])
await db.commit()
await set_platform_cooldown(db, "patreon")
await db.commit()
due = await select_due_sources(db)
urls = {s.url for s in due}
assert "https://mix-ss" in urls
assert "https://mix-p" not in urls
@pytest.mark.asyncio
async def test_select_ignores_expired_cooldown(db):
"""An expired cooldown row doesn't filter — routing treats expired
same as not-set. The stale row stays around until a future sweep
prunes it."""
from backend.app.models import AppSetting
from backend.app.services.scheduler_service import (
PLATFORM_COOLDOWN_KEY_PREFIX,
)
artist = await _seed_artist(db, interval=60, name="cooldown-expired")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cd-exp",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
))
db.add(AppSetting(
key=f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon",
value=(datetime.now(UTC) - timedelta(seconds=10)).isoformat(),
))
await db.commit()
due = await select_due_sources(db)
assert any(s.url == "https://cd-exp" for s in due)
@pytest.mark.asyncio
async def test_set_platform_cooldown_upserts(db):
"""Calling set_platform_cooldown twice on the same platform updates
the row in place rather than inserting a duplicate (AppSetting's PK
is `key`)."""
from sqlalchemy import select as sa_select
from backend.app.models import AppSetting
from backend.app.services.scheduler_service import (
PLATFORM_COOLDOWN_KEY_PREFIX,
set_platform_cooldown,
)
await set_platform_cooldown(db, "patreon", seconds=60)
await db.commit()
await set_platform_cooldown(db, "patreon", seconds=600)
await db.commit()
rows = (await db.execute(
sa_select(AppSetting).where(
AppSetting.key == f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon"
)
)).scalars().all()
assert len(rows) == 1