171c486939
The `select(ImportSettings).where(id == 1)).scalar_one()` singleton load was repeated 15× across services, API, and 5 task modules. Added async load() + sync load_sync() classmethods on the model and migrated all 15 full-row sites (callers already imported ImportSettings, so no new imports; dropped download's now-orphaned select import). Left maintenance.py's deliberate column-select (import_scan_path only) as-is. Rest of the service layer was already adequately DRY — the Record/to_dict pattern is only 2 instances and the savepoint find-or-create recovery is correctly per-entity, so neither was forced into a shared abstraction. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
143 lines
4.9 KiB
Python
143 lines
4.9 KiB
Python
"""FC-3d: pure logic for deciding which sources are due for a check.
|
|
|
|
The Celery tick task wraps `select_due_sources` and fires
|
|
`download_source.delay()` per result; this module knows nothing about
|
|
Celery, gallery-dl, or any side effect.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from ..models import AppSetting, Artist, ImportSettings, Source
|
|
|
|
MIN_INTERVAL_SECONDS = 60
|
|
MAX_INTERVAL_SECONDS = 86400
|
|
MAX_BACKOFF_EXPONENT = 6
|
|
|
|
# AppSetting key stamped every time the Beat tick fires (see scan.py). The
|
|
# tick runs every 60s; the UI flags the scheduler as stalled if the last
|
|
# stamp is older than a few minutes.
|
|
SCHEDULER_LAST_TICK_KEY = "scheduler_last_tick_at"
|
|
|
|
|
|
def compute_effective_interval(
|
|
source: Source, artist: Artist, settings: ImportSettings,
|
|
) -> int:
|
|
"""Return the seconds between scheduled checks for one source.
|
|
|
|
Precedence: source.check_interval_override > artist.check_interval_seconds
|
|
> settings.download_schedule_default_seconds. Multiplied by
|
|
2 ** min(consecutive_failures, MAX_BACKOFF_EXPONENT) and clamped to
|
|
[MIN_INTERVAL_SECONDS, MAX_INTERVAL_SECONDS].
|
|
"""
|
|
base = (
|
|
source.check_interval_override
|
|
or artist.check_interval_seconds
|
|
or settings.download_schedule_default_seconds
|
|
)
|
|
exponent = min(max(0, source.consecutive_failures or 0), MAX_BACKOFF_EXPONENT)
|
|
factor = 2 ** exponent
|
|
raw = base * factor
|
|
return max(MIN_INTERVAL_SECONDS, min(MAX_INTERVAL_SECONDS, raw))
|
|
|
|
|
|
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
|
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
|
|
|
Never-checked sources (last_checked_at IS NULL) are always due.
|
|
"""
|
|
rows = (await session.execute(
|
|
select(Source)
|
|
.options(selectinload(Source.artist))
|
|
.join(Artist, Source.artist_id == Artist.id)
|
|
.where(Source.enabled.is_(True))
|
|
.where(Artist.auto_check.is_(True))
|
|
)).scalars().all()
|
|
|
|
settings = await ImportSettings.load(session)
|
|
|
|
now = datetime.now(UTC)
|
|
due: list[Source] = []
|
|
for s in rows:
|
|
interval = compute_effective_interval(s, s.artist, settings)
|
|
if s.last_checked_at is None:
|
|
due.append(s)
|
|
continue
|
|
elapsed = (now - s.last_checked_at).total_seconds()
|
|
if elapsed >= interval:
|
|
due.append(s)
|
|
return due
|
|
|
|
|
|
def compute_next_check_at(
|
|
source: Source, artist: Artist, settings: ImportSettings,
|
|
) -> datetime | None:
|
|
"""Return the projected datetime of the next check, or None if never checked."""
|
|
if source.last_checked_at is None:
|
|
return None
|
|
interval = compute_effective_interval(source, artist, settings)
|
|
return source.last_checked_at + timedelta(seconds=interval)
|
|
|
|
|
|
async def record_tick(session: AsyncSession) -> None:
|
|
"""Stamp the current time on the SCHEDULER_LAST_TICK_KEY AppSetting.
|
|
|
|
Called once per Beat tick so the UI can prove the scheduler is alive.
|
|
Commits its own write so the stamp survives even if the rest of the
|
|
tick errors out.
|
|
"""
|
|
now_iso = datetime.now(UTC).isoformat()
|
|
row = (await session.execute(
|
|
select(AppSetting).where(AppSetting.key == SCHEDULER_LAST_TICK_KEY)
|
|
)).scalar_one_or_none()
|
|
if row is None:
|
|
session.add(AppSetting(key=SCHEDULER_LAST_TICK_KEY, value=now_iso))
|
|
else:
|
|
row.value = now_iso
|
|
await session.commit()
|
|
|
|
|
|
async def scheduler_status(session: AsyncSession) -> dict:
|
|
"""Summarise scheduler health for the dashboard.
|
|
|
|
Returns last_tick_at (when Beat last fired), next_due_at (earliest
|
|
upcoming scheduled check across enabled auto-check sources), due_now
|
|
(how many are due right now), and auto_sources (total under schedule).
|
|
"""
|
|
last_tick_at = (await session.execute(
|
|
select(AppSetting.value).where(AppSetting.key == SCHEDULER_LAST_TICK_KEY)
|
|
)).scalar_one_or_none()
|
|
|
|
rows = (await session.execute(
|
|
select(Source)
|
|
.options(selectinload(Source.artist))
|
|
.join(Artist, Source.artist_id == Artist.id)
|
|
.where(Source.enabled.is_(True))
|
|
.where(Artist.auto_check.is_(True))
|
|
)).scalars().all()
|
|
settings = await ImportSettings.load(session)
|
|
|
|
now = datetime.now(UTC)
|
|
due_now = 0
|
|
next_due_at: datetime | None = None
|
|
for s in rows:
|
|
if s.last_checked_at is None:
|
|
due_now += 1
|
|
continue
|
|
nca = compute_next_check_at(s, s.artist, settings)
|
|
if nca is None or nca <= now:
|
|
due_now += 1
|
|
elif next_due_at is None or nca < next_due_at:
|
|
next_due_at = nca
|
|
|
|
return {
|
|
"last_tick_at": last_tick_at,
|
|
"next_due_at": next_due_at.isoformat() if next_due_at else None,
|
|
"due_now": due_now,
|
|
"auto_sources": len(rows),
|
|
}
|