61ce1ce13c
The scan tick fired download_source.delay() for every due source without
grouping by platform; with multiple download workers, N due Patreon
sources could all hit Patreon's API in parallel and rate-limit each
other. Per-source consecutive_failures backoff REACTS to that (slows the
offender across cycles) but didn't PREVENT the first-tick burst.
When DownloadService._update_source_health sees a source error
classified as ErrorType.RATE_LIMITED, it now stamps an AppSetting row
`platform_cooldown:<platform>` with the cooldown expiry (now + 15 min,
PLATFORM_RATE_LIMIT_COOLDOWN_SECONDS). select_due_sources queries every
platform_cooldown:* key at the start of each tick and excludes every
source whose platform is in active cooldown. scheduler_status surfaces
active cooldowns as platform_cooldowns: {platform: expires_iso} so the
TopNav pipeline chip / activity summary can display them.
INSERT...ON CONFLICT DO UPDATE for the upsert so two workers racing
RATE_LIMITED responses on the same platform don't let one's
IntegrityError roll back the other's event-finalize transaction
(stranding the event for the recovery sweep). Atomic at the SQL level.
Tests cover: select_due_sources skips a platform in cooldown; other
platforms unaffected during single-platform cooldown; expired cooldown
rows don't filter; set_platform_cooldown is upsert-safe under repeated
calls.
Operator-flagged 2026-05-30 ("running multiple workers I don't know how
we'd keep the downloader from hitting a rate limit on a source").
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
260 lines
8.8 KiB
Python
260 lines
8.8 KiB
Python
"""FC-3d: scheduler_service unit tests.
|
|
|
|
Covers compute_effective_interval (override/artist/default precedence,
|
|
backoff exponent at failure counts 0/1/3/6/10, floor and ceiling clamp)
|
|
and select_due_sources (auto_check off, never-checked, enabled=False,
|
|
due/not-due boundary).
|
|
"""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from backend.app.models import Artist, ImportSettings, Source
|
|
from backend.app.services.scheduler_service import (
|
|
compute_effective_interval,
|
|
select_due_sources,
|
|
)
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
# --- compute_effective_interval (pure, no DB) -----------------------------
|
|
|
|
|
|
def _src(override=None, failures=0):
|
|
return Source(
|
|
artist_id=1, platform="patreon", url="https://x", enabled=True,
|
|
check_interval_override=override, consecutive_failures=failures,
|
|
)
|
|
|
|
|
|
def _art(interval=None, auto=True):
|
|
return Artist(name="A", slug="a", auto_check=auto, check_interval_seconds=interval)
|
|
|
|
|
|
def _settings(default=28800):
|
|
return ImportSettings(id=1, download_schedule_default_seconds=default)
|
|
|
|
|
|
def test_interval_uses_source_override_when_set():
|
|
assert compute_effective_interval(_src(override=600), _art(interval=3600), _settings()) == 600
|
|
|
|
|
|
def test_interval_falls_back_to_artist_interval():
|
|
assert compute_effective_interval(_src(), _art(interval=3600), _settings()) == 3600
|
|
|
|
|
|
def test_interval_falls_back_to_global_default():
|
|
assert compute_effective_interval(_src(), _art(), _settings(default=900)) == 900
|
|
|
|
|
|
def test_interval_backoff_zero_failures_is_identity():
|
|
assert compute_effective_interval(_src(override=600), _art(), _settings()) == 600
|
|
|
|
|
|
def test_interval_backoff_one_failure_doubles():
|
|
assert compute_effective_interval(_src(override=600, failures=1), _art(), _settings()) == 1200
|
|
|
|
|
|
def test_interval_backoff_three_failures_x8():
|
|
assert compute_effective_interval(_src(override=600, failures=3), _art(), _settings()) == 4800
|
|
|
|
|
|
def test_interval_backoff_caps_at_exponent_six():
|
|
# 600 * 2^6 = 38400; 2^7 would be 76800 but exponent caps at 6.
|
|
assert compute_effective_interval(_src(override=600, failures=6), _art(), _settings()) == 38400
|
|
assert compute_effective_interval(_src(override=600, failures=10), _art(), _settings()) == 38400
|
|
|
|
|
|
def test_interval_floor_at_60_seconds():
|
|
# base 30 with 0 failures would be 30s; floor lifts to 60.
|
|
assert compute_effective_interval(_src(override=30), _art(), _settings()) == 60
|
|
|
|
|
|
def test_interval_ceiling_at_86400_seconds():
|
|
# base 28800 * 2^6 = 1843200; ceiling caps at 86400.
|
|
assert compute_effective_interval(_src(failures=6), _art(), _settings(default=28800)) == 86400
|
|
|
|
|
|
# --- select_due_sources (real session) ------------------------------------
|
|
|
|
|
|
async def _seed_artist(db, *, auto=True, interval=None, name="A"):
|
|
a = Artist(name=name, slug=name.lower(), auto_check=auto, check_interval_seconds=interval)
|
|
db.add(a)
|
|
await db.flush()
|
|
return a
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_skips_disabled_sources(db):
|
|
artist = await _seed_artist(db, name="sel-disabled")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://sel-disabled",
|
|
enabled=False, consecutive_failures=0,
|
|
))
|
|
await db.commit()
|
|
due = await select_due_sources(db)
|
|
assert all(s.url != "https://sel-disabled" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_skips_artist_auto_check_false(db):
|
|
artist = await _seed_artist(db, auto=False, name="sel-noauto")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://sel-noauto",
|
|
enabled=True, consecutive_failures=0,
|
|
))
|
|
await db.commit()
|
|
due = await select_due_sources(db)
|
|
assert all(s.url != "https://sel-noauto" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_includes_never_checked(db):
|
|
artist = await _seed_artist(db, name="sel-never")
|
|
src = Source(
|
|
artist_id=artist.id, platform="patreon", url="https://sel-never",
|
|
enabled=True, consecutive_failures=0,
|
|
)
|
|
db.add(src)
|
|
await db.commit()
|
|
due = await select_due_sources(db)
|
|
assert any(s.url == "https://sel-never" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_excludes_not_yet_due(db):
|
|
artist = await _seed_artist(db, interval=3600, name="sel-recent")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://sel-recent",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=60),
|
|
))
|
|
await db.commit()
|
|
due = await select_due_sources(db)
|
|
assert all(s.url != "https://sel-recent" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_includes_past_due(db):
|
|
artist = await _seed_artist(db, interval=60, name="sel-past")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://sel-past",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
|
))
|
|
await db.commit()
|
|
due = await select_due_sources(db)
|
|
assert any(s.url == "https://sel-past" for s in due)
|
|
|
|
|
|
# --- platform-rate-limit cooldown -----------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_skips_sources_on_platform_in_cooldown(db):
|
|
"""After set_platform_cooldown('patreon'), select_due_sources excludes
|
|
every patreon source — even ones that would otherwise be past-due."""
|
|
from backend.app.services.scheduler_service import set_platform_cooldown
|
|
|
|
artist = await _seed_artist(db, interval=60, name="cooldown-p")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://cd-p",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
|
))
|
|
await db.commit()
|
|
|
|
# Sanity-pre: due before the cooldown is set.
|
|
due = await select_due_sources(db)
|
|
assert any(s.url == "https://cd-p" for s in due)
|
|
|
|
await set_platform_cooldown(db, "patreon")
|
|
await db.commit()
|
|
|
|
due = await select_due_sources(db)
|
|
assert all(s.url != "https://cd-p" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cooldown_is_per_platform_other_platforms_unaffected(db):
|
|
"""A cooldown on patreon doesn't block subscribestar sources from
|
|
being due — cooldowns are scoped to the platform that hit the limit."""
|
|
from backend.app.services.scheduler_service import set_platform_cooldown
|
|
|
|
artist = await _seed_artist(db, interval=60, name="cooldown-mixed")
|
|
db.add_all([
|
|
Source(
|
|
artist_id=artist.id, platform="patreon", url="https://mix-p",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
|
),
|
|
Source(
|
|
artist_id=artist.id, platform="subscribestar", url="https://mix-ss",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
|
),
|
|
])
|
|
await db.commit()
|
|
|
|
await set_platform_cooldown(db, "patreon")
|
|
await db.commit()
|
|
|
|
due = await select_due_sources(db)
|
|
urls = {s.url for s in due}
|
|
assert "https://mix-ss" in urls
|
|
assert "https://mix-p" not in urls
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_select_ignores_expired_cooldown(db):
|
|
"""An expired cooldown row doesn't filter — routing treats expired
|
|
same as not-set. The stale row stays around until a future sweep
|
|
prunes it."""
|
|
from backend.app.models import AppSetting
|
|
from backend.app.services.scheduler_service import (
|
|
PLATFORM_COOLDOWN_KEY_PREFIX,
|
|
)
|
|
|
|
artist = await _seed_artist(db, interval=60, name="cooldown-expired")
|
|
db.add(Source(
|
|
artist_id=artist.id, platform="patreon", url="https://cd-exp",
|
|
enabled=True, consecutive_failures=0,
|
|
last_checked_at=datetime.now(UTC) - timedelta(seconds=3600),
|
|
))
|
|
db.add(AppSetting(
|
|
key=f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon",
|
|
value=(datetime.now(UTC) - timedelta(seconds=10)).isoformat(),
|
|
))
|
|
await db.commit()
|
|
|
|
due = await select_due_sources(db)
|
|
assert any(s.url == "https://cd-exp" for s in due)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_platform_cooldown_upserts(db):
|
|
"""Calling set_platform_cooldown twice on the same platform updates
|
|
the row in place rather than inserting a duplicate (AppSetting's PK
|
|
is `key`)."""
|
|
from sqlalchemy import select as sa_select
|
|
|
|
from backend.app.models import AppSetting
|
|
from backend.app.services.scheduler_service import (
|
|
PLATFORM_COOLDOWN_KEY_PREFIX,
|
|
set_platform_cooldown,
|
|
)
|
|
|
|
await set_platform_cooldown(db, "patreon", seconds=60)
|
|
await db.commit()
|
|
await set_platform_cooldown(db, "patreon", seconds=600)
|
|
await db.commit()
|
|
|
|
rows = (await db.execute(
|
|
sa_select(AppSetting).where(
|
|
AppSetting.key == f"{PLATFORM_COOLDOWN_KEY_PREFIX}patreon"
|
|
)
|
|
)).scalars().all()
|
|
assert len(rows) == 1
|