FabledCurator/backend/app/api/gpu.py

"""GPU-job API (#114): the HTTP surface the desktop agent pulls work from.

The agent stays HTTP-only — it leases jobs, fetches image pixels via the normal
FC image URLs, and submits embeddings/regions back, all over this API. Redis and
Postgres are never exposed. The agent endpoints are gated by a bearer token
(Authorization: Bearer <token>) stored in AppSetting; the admin endpoints
(token / backfill / status) ride the browser session like the rest of FC's
homelab admin.
"""

import secrets

from quart import Blueprint, jsonify, request
from sqlalchemy import func, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert

from ..extensions import get_session
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
from ..services.gallery_service import image_url
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
from ..services.ml.regions import RegionService

gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")

_TOKEN_KEY = "gpu_agent_token"


def _bearer() -> str | None:
    h = request.headers.get("Authorization", "")
    return h[7:].strip() if h.startswith("Bearer ") else None


async def _agent_authed(session) -> bool:
    supplied = _bearer()
    if not supplied:
        return False
    stored = (
        await session.execute(
            select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
        )
    ).scalar_one_or_none()
    return stored is not None and secrets.compare_digest(supplied, stored)


# --- Admin (browser): token + backfill + status -------------------------

@gpu_bp.route("/token", methods=["GET"])
async def get_token():
    async with get_session() as session:
        tok = (
            await session.execute(
                select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
            )
        ).scalar_one_or_none()
    return jsonify({"token": tok, "configured": tok is not None})


@gpu_bp.route("/token/rotate", methods=["POST"])
async def rotate_token():
    token = secrets.token_urlsafe(32)
    async with get_session() as session:
        await session.execute(
            pg_insert(AppSetting)
            .values(key=_TOKEN_KEY, value=token)
            .on_conflict_do_update(index_elements=["key"], set_={"value": token})
        )
        await session.commit()
    return jsonify({"token": token})


@gpu_bp.route("/status", methods=["GET"])
async def status():
    async with get_session() as session:
        rows = (
            await session.execute(
                select(GpuJob.status, func.count()).group_by(GpuJob.status)
            )
        ).all()
    counts = dict(rows)
    return jsonify({
        "pending": counts.get("pending", 0),
        "leased": counts.get("leased", 0),
        "done": counts.get("done", 0),
        "error": counts.get("error", 0),
    })


@gpu_bp.route("/backfill", methods=["POST"])
async def backfill():
    """Enqueue a job for every image that doesn't already have one for `task`."""
    body = await request.get_json(silent=True) or {}
    task = str(body.get("task") or "ccip")
    from ..tasks.ml import enqueue_gpu_backfill

    r = enqueue_gpu_backfill.delay(task)
    return jsonify({"celery_task_id": r.id, "task": task}), 202


@gpu_bp.route("/reprocess", methods=["POST"])
async def reprocess():
    """Reset every done/error job of `task` back to pending so the agent re-runs
    the WHOLE library under the current pipeline (e.g. after adding crop
    detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
    body = await request.get_json(silent=True) or {}
    task = str(body.get("task") or "ccip")
    from ..tasks.ml import reprocess_gpu_jobs

    r = reprocess_gpu_jobs.delay(task)
    return jsonify({"celery_task_id": r.id, "task": task}), 202


@gpu_bp.route("/retry_errors", methods=["POST"])
async def retry_errors():
    """Requeue every ERRORED job (all task types) back to pending — the scoped
    recovery after an agent-side fix (e.g. the short-video sampler), where
    /reprocess would needlessly re-run the whole done library too. Attempts and
    the stored error reset so each job gets its full retry budget under the
    fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
    rows a later success made moot — the same statements the backfills run), so
    one failing file requeues as ONE job, never a fan-out of duplicates. Small
    row count (errors only) → inline statements; the response carries both
    counts for the UI toast."""
    async with get_session() as session:
        pruned = 0
        for stmt in error_dedupe_statements():
            pruned += (await session.execute(stmt)).rowcount or 0
        res = await session.execute(
            update(GpuJob)
            .where(GpuJob.status == "error")
            .values(
                status="pending", attempts=0, error=None, lease_token=None,
                leased_at=None, lease_expires_at=None, updated_at=func.now(),
            )
        )
        await session.commit()
    return jsonify({"requeued": res.rowcount or 0, "pruned": pruned})


# --- Agent (bearer token): lease / submit / heartbeat / fail ------------

@gpu_bp.route("/jobs/lease", methods=["POST"])
async def lease():
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    try:
        batch = min(max(int(body.get("batch_size", 8)), 1), 64)
    except (TypeError, ValueError):
        batch = 8
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
        ml = (
            await session.execute(select(MLSettings).where(MLSettings.id == 1))
        ).scalar_one()
        # image rows for url/mime in one shot
        ids = [j.image_record_id for j in jobs]
        imgs = {
            i.id: i for i in (
                await session.execute(
                    select(ImageRecord).where(ImageRecord.id.in_(ids))
                )
            ).scalars()
        } if ids else {}
        await session.commit()
        out = []
        for j in jobs:
            img = imgs.get(j.image_record_id)
            if img is None:
                continue
            out.append({
                "job_id": j.id,
                "image_id": j.image_record_id,
                "task": j.task,
                "mime": img.mime,
                "image_url": image_url(img.path),
                # For video/animated: the agent samples at this cadence.
                "frame_interval_seconds": ml.video_frame_interval_seconds,
                "max_frames": ml.video_max_frames,
                # The embedding model the agent must use for concept crops + the
                # whole-image 'embed' task, so its vectors land in the SAME space
                # the heads trained in. Server-announced FROM THE SETTING → the
                # agent stays model-agnostic; an operator swap is a setting + a
                # re-embed, never an agent change.
                "embed_model_name": ml.embedder_model_name,
                "embed_version": ml.embedder_model_version,
            })
    return jsonify({"jobs": out})


@gpu_bp.route("/jobs/heartbeat", methods=["POST"])
async def heartbeat():
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    job_ids = [int(x) for x in (body.get("job_ids") or [])]
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        n = await GpuJobService(session).heartbeat(agent_id, job_ids)
        await session.commit()
    return jsonify({"extended": n})


@gpu_bp.route("/jobs/submit", methods=["POST"])
async def submit():
    """Store a job's regions + close it. regions: [{kind, bbox:[x,y,w,h],
    frame_time?, score?, *_version?, ccip_embedding?, siglip_embedding?}].
    replace_kinds defaults to the kinds present in the submitted regions."""
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    job_id = body.get("job_id")
    regions = body.get("regions") or []
    if job_id is None:
        return jsonify({"error": "job_id required"}), 400
    kinds = body.get("replace_kinds") or sorted({r["kind"] for r in regions})
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        job = await session.get(GpuJob, int(job_id))
        if job is None or job.status != "leased" or job.lease_token != agent_id:
            return jsonify({"error": "lease_invalid"}), 409
        if kinds:
            await RegionService(session).replace_regions(
                job.image_record_id, kinds, regions
            )
        await GpuJobService(session).complete(agent_id, int(job_id))
        await session.commit()
    return jsonify({"ok": True, "stored": len(regions)})


@gpu_bp.route("/jobs/submit_embedding", methods=["POST"])
async def submit_embedding():
    """Store a whole-image SigLIP embedding (the 'embed' task) on image_record +
    close the job. Body: {agent_id, job_id, embedding:[...], embedding_version}.
    This is how the GPU agent re-embeds the library under a new model (#1190) —
    much faster than the CPU ml-worker at higher resolutions."""
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    job_id = body.get("job_id")
    embedding = body.get("embedding")
    version = body.get("embedding_version")
    if job_id is None or not embedding or not version:
        return jsonify({"error": "job_id, embedding, embedding_version required"}), 400
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        job = await session.get(GpuJob, int(job_id))
        if job is None or job.status != "leased" or job.lease_token != agent_id:
            return jsonify({"error": "lease_invalid"}), 409
        img = await session.get(ImageRecord, job.image_record_id)
        if img is not None:
            img.siglip_embedding = embedding
            img.siglip_model_version = version
        await GpuJobService(session).complete(agent_id, int(job_id))
        await session.commit()
    return jsonify({"ok": True})


@gpu_bp.route("/jobs/fail", methods=["POST"])
async def fail():
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    job_id = body.get("job_id")
    if job_id is None:
        return jsonify({"error": "job_id required"}), 400
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        ok = await GpuJobService(session).fail(
            agent_id, int(job_id), str(body.get("error") or "")
        )
        await session.commit()
    return jsonify({"ok": ok})


@gpu_bp.route("/jobs/release", methods=["POST"])
async def release():
    """Graceful stop: the agent hands its still-leased jobs back to pending so
    they're picked up immediately instead of waiting out the lease."""
    body = await request.get_json(silent=True) or {}
    agent_id = str(body.get("agent_id") or "agent")
    job_ids = [int(x) for x in (body.get("job_ids") or [])]
    async with get_session() as session:
        if not await _agent_authed(session):
            return jsonify({"error": "unauthorized"}), 401
        n = await GpuJobService(session).release(agent_id, job_ids)
        await session.commit()
    return jsonify({"released": n})