"""FC-3a: CRUD over Source rows. FC-3c adds POST //check.""" from quart import Blueprint, jsonify, request from sqlalchemy import func, select from ..extensions import get_session from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source from ..services.artist_membership_service import ArtistMembershipService from ..services.artist_membership_service import rescan as membership_rescan from ..services.membership_roster import roster_is_fresh from ..services.scheduler_service import active_platform_cooldowns, scheduler_status from ..services.source_service import ( KNOWN_PLATFORMS, ArtistNotFoundError, DuplicateSourceError, EmptyUrlError, InvalidConfigError, SourceService, UnknownPlatformError, ) from ._responses import error_response as _bad sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources") @sources_bp.route("", methods=["GET"]) async def list_sources(): artist_id_raw = request.args.get("artist_id") artist_id = None if artist_id_raw is not None: try: artist_id = int(artist_id_raw) except ValueError: return _bad("invalid_artist_id", detail="artist_id must be an integer") failing = request.args.get("failing", "").lower() in ("1", "true", "yes") async with get_session() as session: records = await SourceService(session).list(artist_id=artist_id, failing=failing) return jsonify([r.to_dict() for r in records]) @sources_bp.route("/schedule-status", methods=["GET"]) async def schedule_status(): """FC-dashboards: scheduler health for the Subscriptions hub.""" async with get_session() as session: return jsonify(await scheduler_status(session)) @sources_bp.route("/", methods=["GET"]) async def get_source(source_id: int): async with get_session() as session: record = await SourceService(session).get(source_id) if record is None: return _bad("not_found", status=404, detail=f"source id={source_id}") return jsonify(record.to_dict()) @sources_bp.route("", methods=["POST"]) async def create_source(): body = await request.get_json() if not isinstance(body, dict): return _bad("invalid_body", detail="body must be a JSON object") try: artist_id = int(body["artist_id"]) platform = body["platform"] url = body["url"] except (KeyError, TypeError, ValueError): return _bad("invalid_body", detail="artist_id, platform, url are required") optional = { k: body[k] for k in ("enabled", "config_overrides", "check_interval_override") if k in body } async with get_session() as session: svc = SourceService(session) try: record = await svc.create( artist_id=artist_id, platform=platform, url=url, **optional, ) except ArtistNotFoundError: return _bad("artist_not_found", status=404, detail=f"artist id={artist_id}") except UnknownPlatformError as exc: return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS)) except InvalidConfigError as exc: return _bad("invalid_config", detail=str(exc)) except EmptyUrlError as exc: return _bad("empty_url", detail=str(exc)) except DuplicateSourceError as exc: return _bad("duplicate", status=409, existing_id=exc.existing_id) # Immediate kickoff: a new enabled source is armed for backfill (#693) # but would otherwise sit idle until the next scheduler tick (~60s). # Enqueue the first walk now, skipping only if the platform is in a # rate-limit cooldown (the scheduler picks it up when that clears). dispatch_id = None if record.enabled: cooldowns = await active_platform_cooldowns(session) if record.platform not in cooldowns: session.add(DownloadEvent(source_id=record.id, status="pending")) await session.commit() dispatch_id = record.id if dispatch_id is not None: from ..tasks.download import download_source download_source.delay(dispatch_id) return jsonify(record.to_dict()), 201 @sources_bp.route("/", methods=["PATCH"]) async def patch_source(source_id: int): body = await request.get_json() if not isinstance(body, dict): return _bad("invalid_body", detail="body must be a JSON object") async with get_session() as session: svc = SourceService(session) try: record = await svc.update(source_id, **body) except LookupError: return _bad("not_found", status=404) except UnknownPlatformError as exc: return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS)) except InvalidConfigError as exc: return _bad("invalid_config", detail=str(exc)) except EmptyUrlError as exc: return _bad("empty_url", detail=str(exc)) except DuplicateSourceError as exc: return _bad("duplicate", status=409, existing_id=exc.existing_id) return jsonify(record.to_dict()) @sources_bp.route("/", methods=["DELETE"]) async def delete_source(source_id: int): async with get_session() as session: try: await SourceService(session).delete(source_id) except LookupError: return _bad("not_found", status=404) return "", 204 @sources_bp.route("//reassign", methods=["POST"]) async def reassign_source(source_id: int): """Move this source (and the content it brought in) to another artist (#130). Files don't move — the slug is immutable — so this just re-attributes the source, its posts, and its images. Body: {target_artist_id}.""" body = await request.get_json(silent=True) or {} target = body.get("target_artist_id") if not isinstance(target, int): return _bad("invalid_body", detail="target_artist_id (int) required") async with get_session() as session: try: record = await SourceService(session).reassign(source_id, target) except LookupError: return _bad("not_found", status=404) except ArtistNotFoundError: return _bad("artist_not_found", detail="target artist not found", status=404) return jsonify(record.to_dict()) @sources_bp.route("//backfill", methods=["POST"]) async def set_backfill(source_id: int): """Plan #693/#697 + #830: start/stop a backfill, or start a recovery / recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}` (default "start"). 'start' walks the full post history in time-boxed chunks until it reaches the bottom (then the source shows 'complete'); 'recover' is the same walk but bypasses the Patreon seen-ledger to re-fetch dropped-and-deleted near-dups under the current pHash threshold; 'recapture' re-grabs EVERY post's body + external links and localizes on-disk inline images WITHOUT re-downloading media; 'stop' cancels any back to tick mode. Returns the updated source dict (incl. backfill_state / backfill_chunks / backfill_bypass_seen / backfill_recapture).""" from pathlib import Path from ..services.credential_service import CredentialService from ..services.download_backends import ( uses_native_ingester, verify_source_credential, ) from .credentials import _get_crypto payload = await request.get_json(silent=True) or {} action = payload.get("action", "start") if action not in ("start", "stop", "recover", "recapture"): return _bad( "invalid_action", detail="action must be 'start', 'stop', 'recover', or 'recapture'", ) # Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester # platform (where verify is one cheap API page), refuse if the credential is # DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed # on valid OR inconclusive (a network blip shouldn't block). Gated to native # platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for # an arm action. The credential read happens in a session that's CLOSED # before the verify network call (don't hold a DB conn across the request). if action in ("start", "recover", "recapture"): async with get_session() as session: rec = await SourceService(session).get(source_id) if rec is None: return _bad("not_found", status=404) native = uses_native_ingester(rec.platform) if native: cred = CredentialService(session, _get_crypto()) cookies_path = await cred.get_cookies_path(rec.platform) auth_token = await cred.get_token(rec.platform) if native: ok, message = await verify_source_credential( platform=rec.platform, url=rec.url, artist_slug=rec.artist_slug, config_overrides=rec.config_overrides or {}, cookies_path=str(cookies_path) if cookies_path else None, auth_token=auth_token, images_root=Path("/images"), ) if ok is False: return _bad("credential_rejected", detail=message, status=409) async with get_session() as session: try: svc = SourceService(session) if action == "start": record = await svc.start_backfill(source_id) elif action == "recover": record = await svc.start_recovery(source_id) elif action == "recapture": record = await svc.start_recapture(source_id) else: record = await svc.stop_backfill(source_id) except LookupError: return _bad("not_found", status=404) return jsonify(record.to_dict()) @sources_bp.route("//check", methods=["POST"]) async def check_source(source_id: int): """FC-3c: enqueue a download for this source. Returns 202 with the new DownloadEvent id. If a pending/running event already exists for this source, returns 409 with that id. If the source's platform is currently in a rate-limit cooldown, returns **202 with `{status: "deferred", cooldown_until, platform}`** and does NOT create an event or dispatch — the bulk retry path uses this to avoid bowling N sources right back into the rate limit the cooldown is preventing. Single-click "retry this one source" passes `?force=true` to override the cooldown (operator-explicit, useful for rapid auth-fix testing). The in-flight guard always applies. """ force = (request.args.get("force") or "").lower() in ("1", "true", "yes") async with get_session() as session: source = (await session.execute( select(Source).where(Source.id == source_id) )).scalar_one_or_none() if source is None: return _bad("not_found", status=404, detail=f"source id={source_id}") if not source.enabled: return _bad("source_disabled", detail="enable the source first") # Cooldown gate (unless explicitly overridden). Checked before the # in-flight guard because a deferred retry doesn't need to create # or check for an event at all. if not force: cooldowns = await active_platform_cooldowns(session) expires_at = cooldowns.get(source.platform) if expires_at is not None: return jsonify({ "status": "deferred", "platform": source.platform, "cooldown_until": expires_at.isoformat(), }), 202 in_flight = (await session.execute( select(DownloadEvent.id).where( DownloadEvent.source_id == source_id, DownloadEvent.status.in_(["pending", "running"]), ).order_by(DownloadEvent.id.desc()).limit(1) )).scalar_one_or_none() if in_flight is not None: return jsonify( {"download_event_id": in_flight, "status": "already_running"} ), 409 event = DownloadEvent(source_id=source_id, status="pending") session.add(event) await session.commit() await session.refresh(event) event_id = event.id from ..tasks.download import download_source download_source.delay(source_id) return jsonify({"download_event_id": event_id, "status": "pending"}), 202 # --- #387 C3: the membership roster's sync state -------------------------- # # Rule 164's visibility requirement lives here. A roster that failed to sync, # or never has, must be DISTINGUISHABLE from an account that subscribes to # nothing — otherwise the reconciliation this unlocks would tell the operator # to cancel sources they are actively paying for. @sources_bp.route("/membership-sync", methods=["GET"]) async def membership_sync_status(): async with get_session() as session: rows = (await session.execute(select(MembershipSync))).scalars().all() counts = dict( (await session.execute( select(PlatformMembership.platform, func.count()) .group_by(PlatformMembership.platform) )).all() ) return jsonify({"platforms": [ { "platform": r.platform, "last_attempt_at": r.last_attempt_at.isoformat() if r.last_attempt_at else None, # NULL here means NEVER, and the UI must say so in words. Rendering # it as 0 or as "-" is the exact conflation this endpoint exists to # prevent. "last_success_at": r.last_success_at.isoformat() if r.last_success_at else None, "last_count": r.last_count, "last_error_type": r.last_error_type, "last_error_message": r.last_error_message, # Whether a CONCLUSION may be drawn from this roster — not merely # whether it looks recent. C4 gates on this, and it is computed # server-side so no caller can forget to. "fresh": roster_is_fresh(r), "known_memberships": counts.get(r.platform, 0), } for r in sorted(rows, key=lambda r: r.platform) ]}) @sources_bp.route("/membership-sync", methods=["POST"]) async def trigger_membership_sync(): """Run the roster sweep now. The beat schedule runs daily, which is right for a billing-cycle fact but far too slow when the operator has just connected a credential and wants to see whether it works. Queued rather than run inline: it crosses the network to an external service and the request path is not where that belongs. """ from ..tasks.maintenance import sync_memberships sync_memberships.delay() return jsonify({"queued": True}) # --- #388 E4: creator/membership suggestions ------------------------------ # # Confirm-only. Accepting ADDS A SOURCE under the existing artist — it never # merges two artists, because adding a source is trivially undone and a wrong # merge silently mixes two creators' work with nothing left to separate them by. @sources_bp.route("/membership-suggestions", methods=["GET"]) async def list_membership_suggestions(): async with get_session() as session: return jsonify({"items": await ArtistMembershipService(session).list_pending()}) @sources_bp.route("/membership-suggestions//accept", methods=["POST"]) async def accept_membership_suggestion(sid: int): async with get_session() as session: result = await ArtistMembershipService(session).accept(sid) if result is None: return _bad("suggestion_not_found", status=404) await session.commit() return jsonify(result) @sources_bp.route("/membership-suggestions//dismiss", methods=["POST"]) async def dismiss_membership_suggestion(sid: int): async with get_session() as session: result = await ArtistMembershipService(session).dismiss(sid) if result is None: return _bad("suggestion_not_found", status=404) await session.commit() return jsonify(result) @sources_bp.route("/membership-suggestions/rescan", methods=["POST"]) async def rescan_membership_suggestions(): async with get_session() as session: result = await membership_rescan(session) await session.commit() return jsonify(result)