"""Per-turn tool-call telemetry for assistant generations. Captures the empirical surface for evaluating model swaps: - Which tools were available to the model on this turn? - Which did it attempt to call? - Which succeeded? - Which failed, and with what error? One row per assistant turn. Designed to answer questions like "does mistral-small actually fire record_moment when it should?" without relying on anecdote across model changes. """ from __future__ import annotations import logging from typing import Iterable from sqlalchemy import select from fabledassistant.models import async_session from fabledassistant.models.generation_tool_log import GenerationToolLog logger = logging.getLogger(__name__) async def log_tool_outcomes( *, user_id: int, conv_id: int, assistant_message_id: int | None, model: str, think_enabled: bool, tools_available: Iterable[str], tool_calls: Iterable[dict], ) -> None: """Persist one row capturing this turn's tool-call outcomes. `tool_calls` is the assembled `all_tool_calls` list from `generation_task.run_generation`. Each entry has the shape: {"function": , "arguments": ..., "result": ..., "status": ...} where `status` is "success" or "error" and `result` may contain an `error` field when status is "error". Fire-and-forget from the caller's perspective — failure here must not affect the user-facing generation flow, so exceptions are caught and logged rather than propagated. """ try: available = sorted({str(t) for t in tools_available if t}) attempted: list[str] = [] succeeded: list[str] = [] failed: list[dict] = [] for call in tool_calls: name = str(call.get("function") or call.get("name") or "").strip() if not name: continue attempted.append(name) status = call.get("status") result = call.get("result") or {} err = result.get("error") if isinstance(result, dict) else None is_error = (status == "error") or bool(err) or ( isinstance(result, dict) and result.get("success") is False ) if is_error: failed.append({"name": name, "error": str(err or "unspecified")[:500]}) else: succeeded.append(name) async with async_session() as session: session.add( GenerationToolLog( user_id=user_id, conv_id=conv_id, assistant_message_id=assistant_message_id, model=model, think_enabled=think_enabled, tools_available=available, tools_attempted=attempted, tools_succeeded=succeeded, tools_failed=failed, ) ) await session.commit() except Exception: logger.warning( "Failed to persist generation_tool_log for conv %d (msg=%s, model=%s)", conv_id, assistant_message_id, model, exc_info=True, ) async def recent_logs( *, user_id: int, limit: int = 50, model: str | None = None, ) -> list[dict]: """Return recent generation_tool_log rows for the user, newest first. Optionally filter to a specific model. Returns plain dicts so callers don't need an open session to read fields. """ async with async_session() as session: stmt = select(GenerationToolLog).where(GenerationToolLog.user_id == user_id) if model: stmt = stmt.where(GenerationToolLog.model == model) stmt = stmt.order_by(GenerationToolLog.created_at.desc()).limit(limit) result = await session.execute(stmt) return [row.to_dict() for row in result.scalars().all()]