feat(llm): per-turn tool-call telemetry (generation_tool_log)

Adds an empirical surface for evaluating model swaps. One row per
assistant turn captures: model, think_enabled, tools_available,
tools_attempted, tools_succeeded, tools_failed (with error details
as JSONB). Without this, judging whether a new model "actually fires
record_moment when it should" relies on anecdote across user-reported
sessions. With it, the data is queryable directly.

Pieces:
- Migration 0046: generation_tool_log table with user_created and
  per-conversation indexes.
- Model: SQLAlchemy GenerationToolLog with to_dict() for plain-dict
  consumption outside session scope.
- Service: log_tool_outcomes() normalizes the in-app tool-call shape
  (function/result/status) into the split buckets and persists. It
  catches its own exceptions — telemetry failure must NEVER affect
  the user-facing generation flow. recent_logs() helper for read.
- Integration in run_generation: called once per turn right after
  log_generation, fire-and-forget.
- Tests: pure-normalization unit tests using a stub session — no DB
  needed in CI. Cover the success/error split, the empty-tool-calls
  case, the exception-swallowing contract, and the success=False
  edge case where status incorrectly says "success".

No UI for the telemetry yet — internal infrastructure (the operator
is the consumer, not the journal user), which the FabledRulebook
"no UI no ship" explicitly excepts. Query via psql or extend the
Fable MCP later if direct shell access gets tiresome.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/fabledassistant/models/__init__.py

@@ -49,3 +49,4 @@ from fabledassistant.models.moment import (  # noqa: E402, F401
     moment_tasks,
     moment_notes,
 )
+from fabledassistant.models.generation_tool_log import GenerationToolLog  # noqa: E402, F401

src/fabledassistant/models/generation_tool_log.py

@@ -0,0 +1,75 @@
+from datetime import datetime, timezone
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, Text
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from fabledassistant.models import Base
+
+
+class GenerationToolLog(Base):
+    """One row per assistant turn — what tools the model could/did use.
+
+    Lets the operator answer "does model X actually fire record_moment?"
+    empirically across model swaps without relying on anecdote.
+    """
+
+    __tablename__ = "generation_tool_log"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    user_id: Mapped[int] = mapped_column(
+        Integer,
+        ForeignKey("users.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    conv_id: Mapped[int] = mapped_column(
+        Integer,
+        ForeignKey("conversations.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    # SET NULL (migration matches) so the telemetry row survives if the
+    # underlying assistant message is later deleted.
+    assistant_message_id: Mapped[int | None] = mapped_column(
+        Integer,
+        ForeignKey("messages.id", ondelete="SET NULL"),
+        nullable=True,
+    )
+    model: Mapped[str] = mapped_column(Text, nullable=False)
+    think_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False)
+    tools_available: Mapped[list[str]] = mapped_column(
+        ARRAY(Text), nullable=False, default=list
+    )
+    tools_attempted: Mapped[list[str]] = mapped_column(
+        ARRAY(Text), nullable=False, default=list
+    )
+    tools_succeeded: Mapped[list[str]] = mapped_column(
+        ARRAY(Text), nullable=False, default=list
+    )
+    # JSONB list of {name, error} dicts.
+    tools_failed: Mapped[list[dict]] = mapped_column(
+        JSONB, nullable=False, default=list
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc),
+    )
+
+    __table_args__ = (
+        Index("ix_generation_tool_log_user_created", "user_id", created_at.desc()),
+        Index("ix_generation_tool_log_conv", "conv_id"),
+    )
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "user_id": self.user_id,
+            "conv_id": self.conv_id,
+            "assistant_message_id": self.assistant_message_id,
+            "model": self.model,
+            "think_enabled": self.think_enabled,
+            "tools_available": list(self.tools_available or []),
+            "tools_attempted": list(self.tools_attempted or []),
+            "tools_succeeded": list(self.tools_succeeded or []),
+            "tools_failed": list(self.tools_failed or []),
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }

src/fabledassistant/services/generation_log.py

@@ -0,0 +1,111 @@
+"""Per-turn tool-call telemetry for assistant generations.
+
+Captures the empirical surface for evaluating model swaps:
+- Which tools were available to the model on this turn?
+- Which did it attempt to call?
+- Which succeeded?
+- Which failed, and with what error?
+
+One row per assistant turn. Designed to answer questions like
+"does mistral-small actually fire record_moment when it should?"
+without relying on anecdote across model changes.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Iterable
+
+from sqlalchemy import select
+
+from fabledassistant.models import async_session
+from fabledassistant.models.generation_tool_log import GenerationToolLog
+
+logger = logging.getLogger(__name__)
+
+
+async def log_tool_outcomes(
+    *,
+    user_id: int,
+    conv_id: int,
+    assistant_message_id: int | None,
+    model: str,
+    think_enabled: bool,
+    tools_available: Iterable[str],
+    tool_calls: Iterable[dict],
+) -> None:
+    """Persist one row capturing this turn's tool-call outcomes.
+
+    `tool_calls` is the assembled `all_tool_calls` list from
+    `generation_task.run_generation`. Each entry has the shape:
+        {"function": <name>, "arguments": ..., "result": ..., "status": ...}
+    where `status` is "success" or "error" and `result` may contain an
+    `error` field when status is "error".
+
+    Fire-and-forget from the caller's perspective — failure here must not
+    affect the user-facing generation flow, so exceptions are caught and
+    logged rather than propagated.
+    """
+    try:
+        available = sorted({str(t) for t in tools_available if t})
+        attempted: list[str] = []
+        succeeded: list[str] = []
+        failed: list[dict] = []
+        for call in tool_calls:
+            name = str(call.get("function") or call.get("name") or "").strip()
+            if not name:
+                continue
+            attempted.append(name)
+            status = call.get("status")
+            result = call.get("result") or {}
+            err = result.get("error") if isinstance(result, dict) else None
+            is_error = (status == "error") or bool(err) or (
+                isinstance(result, dict) and result.get("success") is False
+            )
+            if is_error:
+                failed.append({"name": name, "error": str(err or "unspecified")[:500]})
+            else:
+                succeeded.append(name)
+
+        async with async_session() as session:
+            session.add(
+                GenerationToolLog(
+                    user_id=user_id,
+                    conv_id=conv_id,
+                    assistant_message_id=assistant_message_id,
+                    model=model,
+                    think_enabled=think_enabled,
+                    tools_available=available,
+                    tools_attempted=attempted,
+                    tools_succeeded=succeeded,
+                    tools_failed=failed,
+                )
+            )
+            await session.commit()
+    except Exception:
+        logger.warning(
+            "Failed to persist generation_tool_log for conv %d (msg=%s, model=%s)",
+            conv_id,
+            assistant_message_id,
+            model,
+            exc_info=True,
+        )
+
+
+async def recent_logs(
+    *,
+    user_id: int,
+    limit: int = 50,
+    model: str | None = None,
+) -> list[dict]:
+    """Return recent generation_tool_log rows for the user, newest first.
+
+    Optionally filter to a specific model. Returns plain dicts so callers
+    don't need an open session to read fields.
+    """
+    async with async_session() as session:
+        stmt = select(GenerationToolLog).where(GenerationToolLog.user_id == user_id)
+        if model:
+            stmt = stmt.where(GenerationToolLog.model == model)
+        stmt = stmt.order_by(GenerationToolLog.created_at.desc()).limit(limit)
+        result = await session.execute(stmt)
+        return [row.to_dict() for row in result.scalars().all()]

src/fabledassistant/services/generation_task.py

@@ -480,6 +480,23 @@ async def run_generation(
         except Exception:
             logger.warning("Failed to persist generation timing for conv %d", conv_id, exc_info=True)
 
+        # Per-turn tool-call telemetry. Empirical surface for evaluating
+        # model swaps without needing user reports — answers "did model X
+        # actually fire record_moment when it should have?" The helper is
+        # internally try/except so this never affects the user-facing flow.
+        from fabledassistant.services.generation_log import log_tool_outcomes
+        await log_tool_outcomes(
+            user_id=user_id,
+            conv_id=conv_id,
+            assistant_message_id=msg_id,
+            model=model,
+            think_enabled=think,
+            tools_available=[
+                (t.get("function") or {}).get("name") for t in tools
+            ],
+            tool_calls=all_tool_calls,
+        )
+
         buf.state = GenerationState.COMPLETED
         buf.finished_at = time.monotonic()
         done_payload: dict = {"done": True, "message_id": msg_id, "timing": timing}