FabledCurator/backend/app/api/system_activity.py

"""FC-3i: system activity dashboard endpoints.

Read-only. Combines Redis-broker queue depths (LLEN per queue),
Celery worker introspection (celery inspect), and the task_run DB
history into the surfaces the SystemActivityTab UI consumes.

All filesystem/sync-client work goes through asyncio.to_thread per
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
"""

from __future__ import annotations

import asyncio
import time
from datetime import UTC, datetime, timedelta

from quart import Blueprint, jsonify, request
from sqlalchemy import desc, func, select

from ..config import get_config
from ..extensions import get_session
from ..models import TaskRun
from ..services.scheduler_service import scheduler_status
from ..services.worker_lanes import LANES

system_activity_bp = Blueprint(
    "system_activity", __name__, url_prefix="/api/system/activity",
)

# Every queue, grouped by the lane that consumes it. DERIVED from
# `worker_lanes.LANES` (milestone 422 step 1) rather than written out:
# this was a hand-kept third copy of "which queues exist", alongside
# celery_app.task_routes and service_roster.ROLE_NAMES, and its own comment
# admitted the coupling — "must match celery_app.task_routes".
#
# The rendered ORDER changes with this: lane order rather than the previous
# hand-chosen one. That is the better grouping for a lane-oriented UI, and
# queues with no LLEN response still show as null rather than absent.
_QUEUE_NAMES = tuple(q for lane in LANES for q in lane.queues)

# Cache module-level so all requests share the cache between polls.
# Tests can reset via direct dict mutation if needed.
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
_QUEUE_CACHE_TTL = 2.0
_WORKER_CACHE_TTL = 5.0


def _read_queues_sync() -> dict:
    """Reads each queue's LLEN from the broker. Sync — caller wraps in
    asyncio.to_thread. Per-queue try/except returns None on failure so
    one bad queue doesn't break the whole response."""
    import redis  # local import; only this endpoint needs it

    cfg = get_config()
    client = redis.Redis.from_url(cfg.celery_broker_url)
    out: dict = {}
    for name in _QUEUE_NAMES:
        try:
            out[name] = int(client.llen(name))
        except Exception:  # noqa: BLE001 — broker hiccup shouldn't break UI
            out[name] = None
    return {
        "queues": out,
        "fetched_at": datetime.now(UTC).isoformat(),
    }


def _read_workers_sync() -> dict:
    """celery inspect active_queues + active. Returns per-worker info."""
    from ..celery_app import celery as celery_app

    insp = celery_app.control.inspect(timeout=2.0)
    active_queues = insp.active_queues() or {}
    active_tasks = insp.active() or {}

    workers: dict = {}
    for hostname, queues in active_queues.items():
        workers[hostname] = {
            "queues": sorted({q["name"] for q in queues}),
            "active_count": len(active_tasks.get(hostname, [])),
        }
    return {
        "workers": workers,
        "fetched_at": datetime.now(UTC).isoformat(),
    }


async def _queues_cached() -> dict:
    """Per-queue Redis LLEN, cached 2s. Shared by /queues and /summary."""
    now = time.time()
    if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
        _QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
        _QUEUE_CACHE["ts"] = now
    return _QUEUE_CACHE["data"]


@system_activity_bp.route("/queues", methods=["GET"])
async def get_queues():
    """Per-queue Redis LLEN. Cached 2s.

    Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
    """
    return jsonify(await _queues_cached())


@system_activity_bp.route("/workers", methods=["GET"])
async def get_workers():
    """Live celery inspect. Cached 5s.

    Response: {workers: {hostname: {queues, active_count}}, fetched_at}
    """
    now = time.time()
    if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
        _WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
        _WORKER_CACHE["ts"] = now
    return jsonify(_WORKER_CACHE["data"])


@system_activity_bp.route("/summary", methods=["GET"])
async def get_summary():
    """One-call rollup for the always-on TopNav pipeline indicator:
    scheduler health, per-queue pending depths, currently-running count, and
    recent (24h) failure count. Cheap — cached queue LLENs + two TaskRun
    counts — so it's safe to poll app-wide."""
    queues_data = await _queues_cached()
    depths = queues_data.get("queues", {})
    queued_total = sum(v for v in depths.values() if isinstance(v, int))
    since = datetime.now(UTC) - timedelta(hours=24)
    async with get_session() as session:
        scheduler = await scheduler_status(session)
        running = (await session.execute(
            select(func.count(TaskRun.id)).where(TaskRun.status == "running")
        )).scalar_one()
        failing = (await session.execute(
            select(func.count(TaskRun.id))
            .where(TaskRun.status.in_(["error", "timeout"]))
            .where(TaskRun.finished_at >= since)
        )).scalar_one()
    return jsonify({
        "scheduler": scheduler,
        "queues": depths,
        "queued_total": queued_total,
        "running": int(running),
        "failing": int(failing),
    })


@system_activity_bp.route("/runs", methods=["GET"])
async def list_runs():
    """Paginated task_run history. Query params:
      queue=<name>     filter to one queue
      status=<status>  filter to one status (running/ok/error/timeout/retry)
      task=<substr>    case-insensitive substring match on task_name
      celery_task_id=<id>  exactly one run — how a page follows a job it
                           started without having to know its lane
      limit=<int>      default 50, max 200
      before_id=<int>  cursor for keyset pagination

    Response: {runs: [...], next_cursor: id|null}
    """
    try:
        limit = min(int(request.args.get("limit", "50")), 200)
    except ValueError:
        return jsonify({"error": "invalid_limit"}), 400
    if limit < 1:
        return jsonify({"error": "invalid_limit"}), 400

    queue = request.args.get("queue")
    status = request.args.get("status")
    task = request.args.get("task")
    celery_task_id = request.args.get("celery_task_id")
    before_id_raw = request.args.get("before_id")
    before_id = int(before_id_raw) if before_id_raw else None

    async with get_session() as session:
        stmt = select(TaskRun).order_by(desc(TaskRun.id))
        if queue:
            stmt = stmt.where(TaskRun.queue == queue)
        if status:
            stmt = stmt.where(TaskRun.status == status)
        if celery_task_id:
            stmt = stmt.where(TaskRun.celery_task_id == celery_task_id)
        if task:
            # Task names contain literal underscores (download_source,
            # vacuum_analyze) — escape LIKE wildcards so a search for
            # "vacuum_analyze" doesn't treat "_" as a single-char match.
            stmt = stmt.where(TaskRun.task_name.ilike(f"%{_escape_like(task)}%", escape="\\"))
        if before_id is not None:
            stmt = stmt.where(TaskRun.id < before_id)
        stmt = stmt.limit(limit + 1)
        rows = (await session.execute(stmt)).scalars().all()

    has_more = len(rows) > limit
    rows = rows[:limit]
    return jsonify({
        "runs": [_row_to_dict(r) for r in rows],
        "next_cursor": rows[-1].id if has_more and rows else None,
    })


@system_activity_bp.route("/failures", methods=["GET"])
async def list_failures():
    """Recent failures across all lanes (24h window).

    Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
    """
    try:
        limit = min(int(request.args.get("limit", "50")), 200)
    except ValueError:
        return jsonify({"error": "invalid_limit"}), 400

    since = datetime.now(UTC) - timedelta(hours=24)

    async with get_session() as session:
        recent_stmt = (
            select(TaskRun)
            .where(TaskRun.status.in_(["error", "timeout"]))
            .where(TaskRun.finished_at >= since)
            .order_by(desc(TaskRun.finished_at))
            .limit(limit)
        )
        recent = (await session.execute(recent_stmt)).scalars().all()

        count_stmt = (
            select(TaskRun.error_type, func.count(TaskRun.id))
            .where(TaskRun.status.in_(["error", "timeout"]))
            .where(TaskRun.finished_at >= since)
            .group_by(TaskRun.error_type)
            .order_by(desc(func.count(TaskRun.id)))
        )
        counts = (await session.execute(count_stmt)).all()

    return jsonify({
        "recent": [_row_to_dict(r) for r in recent],
        "count_by_type": {
            (row[0] or "Unknown"): row[1]
            for row in counts
        },
        "since": since.isoformat(),
    })


def _escape_like(value: str) -> str:
    """Escape SQL LIKE/ILIKE metacharacters so user search text is matched
    literally. Pairs with `escape="\\"` on the .ilike() call."""
    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")


def _row_to_dict(r: TaskRun) -> dict:
    return {
        "id": r.id,
        "queue": r.queue,
        "task_name": r.task_name,
        "target_id": r.target_id,
        "celery_task_id": r.celery_task_id,
        "started_at": r.started_at.isoformat() if r.started_at else None,
        "finished_at": r.finished_at.isoformat() if r.finished_at else None,
        "duration_ms": r.duration_ms,
        "status": r.status,
        "error_type": r.error_type,
        "error_message": r.error_message,
        "retry_count": r.retry_count,
        "worker_hostname": r.worker_hostname,
        "args_summary": r.args_summary,
    }