Files
FabledCurator/backend/app/api/sources.py
T
bvandeusen a5101494b6 feat(downloads): bulk retry respects cooldown; single-source RETRY overrides
Today's platform-cooldown commit (61ce1ce) only filtered the scan tick
— manual /api/sources/<id>/check still bypassed it. Operator-flagged
2026-05-30: clicked "Retry failed" on a Patreon failure pile and saw
every one go 'queued' without realising the cooldown wasn't in the
loop. Bulk retry with N sources on a cooled-down platform bowls right
back into the rate limit the cooldown is trying to prevent.

**Backend (`/api/sources/<id>/check`):**
- Reads optional `?force=true` query flag.
- Without force: queries `active_platform_cooldowns` (renamed from the
  private `_platforms_in_cooldown` since it's now a cross-module API).
  If the source's platform is in cooldown, returns **202** with
  `{status: 'deferred', platform, cooldown_until}` — no event created,
  no dispatch.
- With force: cooldown skipped entirely.
- In-flight guard always applies (no point creating duplicate pendings).

**Frontend (`sourcesStore.checkNow(id, {force=false})`):** new optional
`force` flag → adds `?force=true` to the URL.

**Frontend (`DownloadsTab`):**
- `onRetrySource` (single-source RETRY click): passes `force: true` →
  explicit operator override, useful for rapid auth-fix testing.
- `onRetryAll` (RETRY ALL + MaintenanceMenu "Retry failed"): no force →
  cooldown respected. Tallies `deferred` alongside `queued` /
  `already_running`; toast reads e.g. *"5 queued, 12 deferred
  (cooldown), 3 already running"*. That count is the operator's
  diagnostic answer for "is rate-limit the cause of most failures?"
  (12-of-20 deferred → yes; 0 deferred → no).

**Auto-resume:** no new sweep needed. Deferred sources still have stale
`last_checked_at`, so the next scan tick after the cooldown AppSetting
expires picks them up via `select_due_sources` (which already filters
on `active_platform_cooldowns`).

Tests: two new — deferred-on-cooldown returns 202 with the right body
and no dispatch; force=true overrides the cooldown and creates the
event normally.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-30 23:15:07 -04:00

181 lines
7.0 KiB
Python

"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
from quart import Blueprint, jsonify, request
from sqlalchemy import select
from ..extensions import get_session
from ..models import DownloadEvent, Source
from ..services.scheduler_service import active_platform_cooldowns, scheduler_status
from ..services.source_service import (
KNOWN_PLATFORMS,
ArtistNotFoundError,
DuplicateSourceError,
EmptyUrlError,
InvalidConfigError,
SourceService,
UnknownPlatformError,
)
from ._responses import error_response as _bad
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
@sources_bp.route("", methods=["GET"])
async def list_sources():
artist_id_raw = request.args.get("artist_id")
artist_id = None
if artist_id_raw is not None:
try:
artist_id = int(artist_id_raw)
except ValueError:
return _bad("invalid_artist_id", detail="artist_id must be an integer")
failing = request.args.get("failing", "").lower() in ("1", "true", "yes")
async with get_session() as session:
records = await SourceService(session).list(artist_id=artist_id, failing=failing)
return jsonify([r.to_dict() for r in records])
@sources_bp.route("/schedule-status", methods=["GET"])
async def schedule_status():
"""FC-dashboards: scheduler health for the Subscriptions hub."""
async with get_session() as session:
return jsonify(await scheduler_status(session))
@sources_bp.route("/<int:source_id>", methods=["GET"])
async def get_source(source_id: int):
async with get_session() as session:
record = await SourceService(session).get(source_id)
if record is None:
return _bad("not_found", status=404, detail=f"source id={source_id}")
return jsonify(record.to_dict())
@sources_bp.route("", methods=["POST"])
async def create_source():
body = await request.get_json()
if not isinstance(body, dict):
return _bad("invalid_body", detail="body must be a JSON object")
try:
artist_id = int(body["artist_id"])
platform = body["platform"]
url = body["url"]
except (KeyError, TypeError, ValueError):
return _bad("invalid_body", detail="artist_id, platform, url are required")
optional = {
k: body[k]
for k in ("enabled", "config_overrides", "check_interval_override")
if k in body
}
async with get_session() as session:
svc = SourceService(session)
try:
record = await svc.create(
artist_id=artist_id, platform=platform, url=url, **optional,
)
except ArtistNotFoundError:
return _bad("artist_not_found", status=404, detail=f"artist id={artist_id}")
except UnknownPlatformError as exc:
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
except InvalidConfigError as exc:
return _bad("invalid_config", detail=str(exc))
except EmptyUrlError as exc:
return _bad("empty_url", detail=str(exc))
except DuplicateSourceError as exc:
return _bad("duplicate", status=409, existing_id=exc.existing_id)
return jsonify(record.to_dict()), 201
@sources_bp.route("/<int:source_id>", methods=["PATCH"])
async def patch_source(source_id: int):
body = await request.get_json()
if not isinstance(body, dict):
return _bad("invalid_body", detail="body must be a JSON object")
async with get_session() as session:
svc = SourceService(session)
try:
record = await svc.update(source_id, **body)
except LookupError:
return _bad("not_found", status=404)
except UnknownPlatformError as exc:
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
except InvalidConfigError as exc:
return _bad("invalid_config", detail=str(exc))
except EmptyUrlError as exc:
return _bad("empty_url", detail=str(exc))
except DuplicateSourceError as exc:
return _bad("duplicate", status=409, existing_id=exc.existing_id)
return jsonify(record.to_dict())
@sources_bp.route("/<int:source_id>", methods=["DELETE"])
async def delete_source(source_id: int):
async with get_session() as session:
try:
await SourceService(session).delete(source_id)
except LookupError:
return _bad("not_found", status=404)
return "", 204
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
async def check_source(source_id: int):
"""FC-3c: enqueue a download for this source.
Returns 202 with the new DownloadEvent id. If a pending/running
event already exists for this source, returns 409 with that id. If
the source's platform is currently in a rate-limit cooldown, returns
**202 with `{status: "deferred", cooldown_until, platform}`** and
does NOT create an event or dispatch — the bulk retry path uses this
to avoid bowling N sources right back into the rate limit the
cooldown is preventing. Single-click "retry this one source" passes
`?force=true` to override the cooldown (operator-explicit, useful
for rapid auth-fix testing). The in-flight guard always applies.
"""
force = (request.args.get("force") or "").lower() in ("1", "true", "yes")
async with get_session() as session:
source = (await session.execute(
select(Source).where(Source.id == source_id)
)).scalar_one_or_none()
if source is None:
return _bad("not_found", status=404, detail=f"source id={source_id}")
if not source.enabled:
return _bad("source_disabled", detail="enable the source first")
# Cooldown gate (unless explicitly overridden). Checked before the
# in-flight guard because a deferred retry doesn't need to create
# or check for an event at all.
if not force:
cooldowns = await active_platform_cooldowns(session)
expires_at = cooldowns.get(source.platform)
if expires_at is not None:
return jsonify({
"status": "deferred",
"platform": source.platform,
"cooldown_until": expires_at.isoformat(),
}), 202
in_flight = (await session.execute(
select(DownloadEvent.id).where(
DownloadEvent.source_id == source_id,
DownloadEvent.status.in_(["pending", "running"]),
).order_by(DownloadEvent.id.desc()).limit(1)
)).scalar_one_or_none()
if in_flight is not None:
return jsonify(
{"download_event_id": in_flight, "status": "already_running"}
), 409
event = DownloadEvent(source_id=source_id, status="pending")
session.add(event)
await session.commit()
await session.refresh(event)
event_id = event.id
from ..tasks.download import download_source
download_source.delay(source_id)
return jsonify({"download_event_id": event_id, "status": "pending"}), 202