Files
FabledCurator/tests/test_scheduler_service.py
T
bvandeusen 61ce1ce13c feat(scheduler): platform-wide cooldown on RATE_LIMITED — burst prevention
The scan tick fired download_source.delay() for every due source without
grouping by platform; with multiple download workers, N due Patreon
sources could all hit Patreon's API in parallel and rate-limit each
other. Per-source consecutive_failures backoff REACTS to that (slows the
offender across cycles) but didn't PREVENT the first-tick burst.

When DownloadService._update_source_health sees a source error
classified as ErrorType.RATE_LIMITED, it now stamps an AppSetting row
`platform_cooldown:<platform>` with the cooldown expiry (now + 15 min,
PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS). select_due_sources queries every
platform_cooldown:* key at the start of each tick and excludes every
source whose platform is in active cooldown. scheduler_status surfaces
active cooldowns as platform_cooldowns: {platform: expires_iso} so the
TopNav pipeline chip / activity summary can display them.

INSERT...ON CONFLICT DO UPDATE for the upsert so two workers racing
RATE_LIMITED responses on the same platform don't let one's
IntegrityError roll back the other's event-finalize transaction
(stranding the event for the recovery sweep). Atomic at the SQL level.

Tests cover: select_due_sources skips a platform in cooldown; other
platforms unaffected during single-platform cooldown; expired cooldown
rows don't filter; set_platform_cooldown is upsert-safe under repeated
calls.

Operator-flagged 2026-05-30 ("running multiple workers I don't know how
we'd keep the downloader from hitting a rate limit on a source").

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-30 11:01:40 -04:00

260 lines
8.8 KiB
Python

"""FC-3d: scheduler_service unit tests.
Covers compute_effective_interval (override/artist/default precedence,
backoff exponent at failure counts 0/1/3/6/10, floor and ceiling clamp)
and select_due_sources (auto_check off, never-checked, enabled=False,
due/not-due boundary).
"""
from datetime import UTC, datetime, timedelta
import pytest
from backend.app.models import Artist, ImportSettings, Source
from backend.app.services.scheduler_service import (
compute_effective_interval,
select_due_sources,
)
pytestmark = pytest.mark.integration
# --- compute_effective_interval (pure, no DB) -----------------------------
def _src(override=None, failures=0):
return Source(
artist_id=1, platform="patreon", url="https://x", enabled=True,
check_interval_override=override, consecutive_failures=failures,
)
def _art(interval=None, auto=True):
return Artist(name="A", slug="a", auto_check=auto, check_interval_seconds=interval)
def _settings(default=28800):
return ImportSettings(id=1, download_schedule_default_seconds=default)
def test_interval_uses_source_override_when_set():
assert compute_effective_interval(_src(override=600), _art(interval=3600), _settings()) == 600
def test_interval_falls_back_to_artist_interval():
assert compute_effective_interval(_src(), _art(interval=3600), _settings()) == 3600
def test_interval_falls_back_to_global_default():
assert compute_effective_interval(_src(), _art(), _settings(default=900)) == 900
def test_interval_backoff_zero_failures_is_identity():
assert compute_effective_interval(_src(override=600), _art(), _settings()) == 600
def test_interval_backoff_one_failure_doubles():
assert compute_effective_interval(_src(override=600, failures=1), _art(), _settings()) == 1200
def test_interval_backoff_three_failures_x8():
assert compute_effective_interval(_src(override=600, failures=3), _art(), _settings()) == 4800
def test_interval_backoff_caps_at_exponent_six():
# 600 * 2^6 = 38400; 2^7 would be 76800 but exponent caps at 6.
assert compute_effective_interval(_src(override=600, failures=6), _art(), _settings()) == 38400
assert compute_effective_interval(_src(override=600, failures=10), _art(), _settings()) == 38400
def test_interval_floor_at_60_seconds():
# base 30 with 0 failures would be 30s; floor lifts to 60.
assert compute_effective_interval(_src(override=30), _art(), _settings()) == 60
def test_interval_ceiling_at_86400_seconds():
# base 28800 * 2^6 = 1843200; ceiling caps at 86400.
assert compute_effective_interval(_src(failures=6), _art(), _settings(default=28800)) == 86400
# --- select_due_sources (real session) ------------------------------------
async def _seed_artist(db, *, auto=True, interval=None, name="A"):
a = Artist(name=name, slug=name.lower(), auto_check=auto, check_interval_seconds=interval)
db.add(a)
await db.flush()
return a
@pytest.mark.asyncio
async def test_select_skips_disabled_sources(db):
artist = await _seed_artist(db, name="sel-disabled")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://sel-disabled",
enabled=False, consecutive_failures=0,
))
await db.commit()
due = await select_due_sources(db)
assert all(s.url != "https://sel-disabled" for s in due)
@pytest.mark.asyncio
async def test_select_skips_artist_auto_check_false(db):
artist = await _seed_artist(db, auto=False, name="sel-noauto")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://sel-noauto",
enabled=True, consecutive_failures=0,
))
await db.commit()
due = await select_due_sources(db)
assert all(s.url != "https://sel-noauto" for s in due)
@pytest.mark.asyncio
async def test_select_includes_never_checked(db):
artist = await _seed_artist(db, name="sel-never")
src = Source(
artist_id=artist.id, platform="patreon", url="https://sel-never",
enabled=True, consecutive_failures=0,
)
db.add(src)
await db.commit()
due = await select_due_sources(db)
assert any(s.url == "https://sel-never" for s in due)
@pytest.mark.asyncio
async def test_select_excludes_not_yet_due(db):
artist = await _seed_artist(db, interval=3600, name="sel-recent")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://sel-recent",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=60),
))
await db.commit()
due = await select_due_sources(db)
assert all(s.url != "https://sel-recent" for s in due)
@pytest.mark.asyncio
async def test_select_includes_past_due(db):
artist = await _seed_artist(db, interval=60, name="sel-past")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://sel-past",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
))
await db.commit()
due = await select_due_sources(db)
assert any(s.url == "https://sel-past" for s in due)
# --- platform-rate-limit cooldown -----------------------------------------
@pytest.mark.asyncio
async def test_select_skips_sources_on_platform_in_cooldown(db):
"""After set_platform_cooldown('patreon'), select_due_sources excludes
every patreon source — even ones that would otherwise be past-due."""
from backend.app.services.scheduler_service import set_platform_cooldown
artist = await _seed_artist(db, interval=60, name="cooldown-p")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cd-p",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
))
await db.commit()
# Sanity-pre: due before the cooldown is set.
due = await select_due_sources(db)
assert any(s.url == "https://cd-p" for s in due)
await set_platform_cooldown(db, "patreon")
await db.commit()
due = await select_due_sources(db)
assert all(s.url != "https://cd-p" for s in due)
@pytest.mark.asyncio
async def test_cooldown_is_per_platform_other_platforms_unaffected(db):
"""A cooldown on patreon doesn't block subscribestar sources from
being due — cooldowns are scoped to the platform that hit the limit."""
from backend.app.services.scheduler_service import set_platform_cooldown
artist = await _seed_artist(db, interval=60, name="cooldown-mixed")
db.add_all([
Source(
artist_id=artist.id, platform="patreon", url="https://mix-p",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
),
Source(
artist_id=artist.id, platform="subscribestar", url="https://mix-ss",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
),
])
await db.commit()
await set_platform_cooldown(db, "patreon")
await db.commit()
due = await select_due_sources(db)
urls = {s.url for s in due}
assert "https://mix-ss" in urls
assert "https://mix-p" not in urls
@pytest.mark.asyncio
async def test_select_ignores_expired_cooldown(db):
"""An expired cooldown row doesn't filter — routing treats expired
same as not-set. The stale row stays around until a future sweep
prunes it."""
from backend.app.models import AppSetting
from backend.app.services.scheduler_service import (
PLATFORM_COOLDOWN_KEY_PREFIX,
)
artist = await _seed_artist(db, interval=60, name="cooldown-expired")
db.add(Source(
artist_id=artist.id, platform="patreon", url="https://cd-exp",
enabled=True, consecutive_failures=0,
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
))
db.add(AppSetting(
key=f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon",
value=(datetime.now(UTC) - timedelta(seconds=10)).isoformat(),
))
await db.commit()
due = await select_due_sources(db)
assert any(s.url == "https://cd-exp" for s in due)
@pytest.mark.asyncio
async def test_set_platform_cooldown_upserts(db):
"""Calling set_platform_cooldown twice on the same platform updates
the row in place rather than inserting a duplicate (AppSetting's PK
is `key`)."""
from sqlalchemy import select as sa_select
from backend.app.models import AppSetting
from backend.app.services.scheduler_service import (
PLATFORM_COOLDOWN_KEY_PREFIX,
set_platform_cooldown,
)
await set_platform_cooldown(db, "patreon", seconds=60)
await db.commit()
await set_platform_cooldown(db, "patreon", seconds=600)
await db.commit()
rows = (await db.execute(
sa_select(AppSetting).where(
AppSetting.key == f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon"
)
)).scalars().all()
assert len(rows) == 1