96fffaff64
Plan #693. Large-catalog backfill (Anduo) no longer sprints to the timeout wall and dies as an error each run. Builds on the cursor checkpoint (#689). - Time-boxed chunks: BACKFILL_TIMEOUT_SECONDS(1170)→BACKFILL_CHUNK_SECONDS(600), far under the 1350 soft limit. Hitting it = normal chunk boundary (the TimeoutExpired path already captures partial output + the cursor), not a near-wall death. - Run-until-done state machine driven by config_overrides[_backfill_state] (running/complete/stalled). A running backfill auto-continues in chunks across ticks until gallery-dl exits cleanly (rc=0 = reached the bottom → 'complete'); a safety-cap (BACKFILL_MAX_CHUNKS=200) + the #689 stall-guard pause a pathological walk as 'stalled'. Replaces the N-runs counter (backfill_runs_remaining repurposed as the cap countdown). - Progress, not error: a chunk that timed out but advanced (cursor moved and/or files written) is reclassified TIMEOUT→PARTIAL (status 'ok'). - Retry storm tamed: gallery-dl retries 3→2, downloader timeout 120→60s, so one stuck CDN file fails in ~1-2 min not ~10 (Anduo #40838). - API: POST /sources/{id}/backfill now takes {action: start|stop}; service start_backfill/stop_backfill; new enabled sources auto-arm run-until-done; source dict exposes backfill_state + backfill_chunks. Frontend (Start/Stop control + state badge) lands in the next push. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
205 lines
8.1 KiB
Python
205 lines
8.1 KiB
Python
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
|
|
|
from quart import Blueprint, jsonify, request
|
|
from sqlalchemy import select
|
|
|
|
from ..extensions import get_session
|
|
from ..models import DownloadEvent, Source
|
|
from ..services.scheduler_service import active_platform_cooldowns, scheduler_status
|
|
from ..services.source_service import (
|
|
KNOWN_PLATFORMS,
|
|
ArtistNotFoundError,
|
|
DuplicateSourceError,
|
|
EmptyUrlError,
|
|
InvalidConfigError,
|
|
SourceService,
|
|
UnknownPlatformError,
|
|
)
|
|
from ._responses import error_response as _bad
|
|
|
|
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
|
|
|
|
|
|
@sources_bp.route("", methods=["GET"])
|
|
async def list_sources():
|
|
artist_id_raw = request.args.get("artist_id")
|
|
artist_id = None
|
|
if artist_id_raw is not None:
|
|
try:
|
|
artist_id = int(artist_id_raw)
|
|
except ValueError:
|
|
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
|
failing = request.args.get("failing", "").lower() in ("1", "true", "yes")
|
|
async with get_session() as session:
|
|
records = await SourceService(session).list(artist_id=artist_id, failing=failing)
|
|
return jsonify([r.to_dict() for r in records])
|
|
|
|
|
|
@sources_bp.route("/schedule-status", methods=["GET"])
|
|
async def schedule_status():
|
|
"""FC-dashboards: scheduler health for the Subscriptions hub."""
|
|
async with get_session() as session:
|
|
return jsonify(await scheduler_status(session))
|
|
|
|
|
|
@sources_bp.route("/<int:source_id>", methods=["GET"])
|
|
async def get_source(source_id: int):
|
|
async with get_session() as session:
|
|
record = await SourceService(session).get(source_id)
|
|
if record is None:
|
|
return _bad("not_found", status=404, detail=f"source id={source_id}")
|
|
return jsonify(record.to_dict())
|
|
|
|
|
|
@sources_bp.route("", methods=["POST"])
|
|
async def create_source():
|
|
body = await request.get_json()
|
|
if not isinstance(body, dict):
|
|
return _bad("invalid_body", detail="body must be a JSON object")
|
|
try:
|
|
artist_id = int(body["artist_id"])
|
|
platform = body["platform"]
|
|
url = body["url"]
|
|
except (KeyError, TypeError, ValueError):
|
|
return _bad("invalid_body", detail="artist_id, platform, url are required")
|
|
|
|
optional = {
|
|
k: body[k]
|
|
for k in ("enabled", "config_overrides", "check_interval_override")
|
|
if k in body
|
|
}
|
|
|
|
async with get_session() as session:
|
|
svc = SourceService(session)
|
|
try:
|
|
record = await svc.create(
|
|
artist_id=artist_id, platform=platform, url=url, **optional,
|
|
)
|
|
except ArtistNotFoundError:
|
|
return _bad("artist_not_found", status=404, detail=f"artist id={artist_id}")
|
|
except UnknownPlatformError as exc:
|
|
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
|
|
except InvalidConfigError as exc:
|
|
return _bad("invalid_config", detail=str(exc))
|
|
except EmptyUrlError as exc:
|
|
return _bad("empty_url", detail=str(exc))
|
|
except DuplicateSourceError as exc:
|
|
return _bad("duplicate", status=409, existing_id=exc.existing_id)
|
|
return jsonify(record.to_dict()), 201
|
|
|
|
|
|
@sources_bp.route("/<int:source_id>", methods=["PATCH"])
|
|
async def patch_source(source_id: int):
|
|
body = await request.get_json()
|
|
if not isinstance(body, dict):
|
|
return _bad("invalid_body", detail="body must be a JSON object")
|
|
async with get_session() as session:
|
|
svc = SourceService(session)
|
|
try:
|
|
record = await svc.update(source_id, **body)
|
|
except LookupError:
|
|
return _bad("not_found", status=404)
|
|
except UnknownPlatformError as exc:
|
|
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
|
|
except InvalidConfigError as exc:
|
|
return _bad("invalid_config", detail=str(exc))
|
|
except EmptyUrlError as exc:
|
|
return _bad("empty_url", detail=str(exc))
|
|
except DuplicateSourceError as exc:
|
|
return _bad("duplicate", status=409, existing_id=exc.existing_id)
|
|
return jsonify(record.to_dict())
|
|
|
|
|
|
@sources_bp.route("/<int:source_id>", methods=["DELETE"])
|
|
async def delete_source(source_id: int):
|
|
async with get_session() as session:
|
|
try:
|
|
await SourceService(session).delete(source_id)
|
|
except LookupError:
|
|
return _bad("not_found", status=404)
|
|
return "", 204
|
|
|
|
|
|
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
|
async def set_backfill(source_id: int):
|
|
"""Plan #693: start or stop a run-until-done backfill. Body:
|
|
`{"action": "start" | "stop"}` (default "start"). 'start' walks the full
|
|
post history in time-boxed chunks until it reaches the bottom (then the
|
|
source shows 'complete'); 'stop' cancels back to tick mode. Returns the
|
|
updated source dict (incl. backfill_state / backfill_chunks)."""
|
|
payload = await request.get_json(silent=True) or {}
|
|
action = payload.get("action", "start")
|
|
if action not in ("start", "stop"):
|
|
return _bad("invalid_action", detail="action must be 'start' or 'stop'")
|
|
async with get_session() as session:
|
|
try:
|
|
svc = SourceService(session)
|
|
record = (
|
|
await svc.start_backfill(source_id)
|
|
if action == "start"
|
|
else await svc.stop_backfill(source_id)
|
|
)
|
|
except LookupError:
|
|
return _bad("not_found", status=404)
|
|
return jsonify(record.to_dict())
|
|
|
|
|
|
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
|
async def check_source(source_id: int):
|
|
"""FC-3c: enqueue a download for this source.
|
|
|
|
Returns 202 with the new DownloadEvent id. If a pending/running
|
|
event already exists for this source, returns 409 with that id. If
|
|
the source's platform is currently in a rate-limit cooldown, returns
|
|
**202 with `{status: "deferred", cooldown_until, platform}`** and
|
|
does NOT create an event or dispatch — the bulk retry path uses this
|
|
to avoid bowling N sources right back into the rate limit the
|
|
cooldown is preventing. Single-click "retry this one source" passes
|
|
`?force=true` to override the cooldown (operator-explicit, useful
|
|
for rapid auth-fix testing). The in-flight guard always applies.
|
|
"""
|
|
force = (request.args.get("force") or "").lower() in ("1", "true", "yes")
|
|
async with get_session() as session:
|
|
source = (await session.execute(
|
|
select(Source).where(Source.id == source_id)
|
|
)).scalar_one_or_none()
|
|
if source is None:
|
|
return _bad("not_found", status=404, detail=f"source id={source_id}")
|
|
if not source.enabled:
|
|
return _bad("source_disabled", detail="enable the source first")
|
|
|
|
# Cooldown gate (unless explicitly overridden). Checked before the
|
|
# in-flight guard because a deferred retry doesn't need to create
|
|
# or check for an event at all.
|
|
if not force:
|
|
cooldowns = await active_platform_cooldowns(session)
|
|
expires_at = cooldowns.get(source.platform)
|
|
if expires_at is not None:
|
|
return jsonify({
|
|
"status": "deferred",
|
|
"platform": source.platform,
|
|
"cooldown_until": expires_at.isoformat(),
|
|
}), 202
|
|
|
|
in_flight = (await session.execute(
|
|
select(DownloadEvent.id).where(
|
|
DownloadEvent.source_id == source_id,
|
|
DownloadEvent.status.in_(["pending", "running"]),
|
|
).order_by(DownloadEvent.id.desc()).limit(1)
|
|
)).scalar_one_or_none()
|
|
if in_flight is not None:
|
|
return jsonify(
|
|
{"download_event_id": in_flight, "status": "already_running"}
|
|
), 409
|
|
|
|
event = DownloadEvent(source_id=source_id, status="pending")
|
|
session.add(event)
|
|
await session.commit()
|
|
await session.refresh(event)
|
|
event_id = event.id
|
|
|
|
from ..tasks.download import download_source
|
|
download_source.delay(source_id)
|
|
|
|
return jsonify({"download_event_id": event_id, "status": "pending"}), 202
|