bvandeusen
bf7a29e8a0
Adds an empirical surface for evaluating model swaps. One row per assistant turn captures: model, think_enabled, tools_available, tools_attempted, tools_succeeded, tools_failed (with error details as JSONB). Without this, judging whether a new model "actually fires record_moment when it should" relies on anecdote across user-reported sessions. With it, the data is queryable directly. Pieces: - Migration 0046: generation_tool_log table with user_created and per-conversation indexes. - Model: SQLAlchemy GenerationToolLog with to_dict() for plain-dict consumption outside session scope. - Service: log_tool_outcomes() normalizes the in-app tool-call shape (function/result/status) into the split buckets and persists. It catches its own exceptions — telemetry failure must NEVER affect the user-facing generation flow. recent_logs() helper for read. - Integration in run_generation: called once per turn right after log_generation, fire-and-forget. - Tests: pure-normalization unit tests using a stub session — no DB needed in CI. Cover the success/error split, the empty-tool-calls case, the exception-swallowing contract, and the success=False edge case where status incorrectly says "success". No UI for the telemetry yet — internal infrastructure (the operator is the consumer, not the journal user), which the FabledRulebook "no UI no ship" explicitly excepts. Query via psql or extend the Fable MCP later if direct shell access gets tiresome. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
76 lines
2.7 KiB
Python
76 lines
2.7 KiB
Python
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import Boolean, DateTime, ForeignKey, Index, Integer, Text
|
|
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from fabledassistant.models import Base
|
|
|
|
|
|
class GenerationToolLog(Base):
|
|
"""One row per assistant turn — what tools the model could/did use.
|
|
|
|
Lets the operator answer "does model X actually fire record_moment?"
|
|
empirically across model swaps without relying on anecdote.
|
|
"""
|
|
|
|
__tablename__ = "generation_tool_log"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(
|
|
Integer,
|
|
ForeignKey("users.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
conv_id: Mapped[int] = mapped_column(
|
|
Integer,
|
|
ForeignKey("conversations.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
# SET NULL (migration matches) so the telemetry row survives if the
|
|
# underlying assistant message is later deleted.
|
|
assistant_message_id: Mapped[int | None] = mapped_column(
|
|
Integer,
|
|
ForeignKey("messages.id", ondelete="SET NULL"),
|
|
nullable=True,
|
|
)
|
|
model: Mapped[str] = mapped_column(Text, nullable=False)
|
|
think_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
|
tools_available: Mapped[list[str]] = mapped_column(
|
|
ARRAY(Text), nullable=False, default=list
|
|
)
|
|
tools_attempted: Mapped[list[str]] = mapped_column(
|
|
ARRAY(Text), nullable=False, default=list
|
|
)
|
|
tools_succeeded: Mapped[list[str]] = mapped_column(
|
|
ARRAY(Text), nullable=False, default=list
|
|
)
|
|
# JSONB list of {name, error} dicts.
|
|
tools_failed: Mapped[list[dict]] = mapped_column(
|
|
JSONB, nullable=False, default=list
|
|
)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True),
|
|
default=lambda: datetime.now(timezone.utc),
|
|
)
|
|
|
|
__table_args__ = (
|
|
Index("ix_generation_tool_log_user_created", "user_id", created_at.desc()),
|
|
Index("ix_generation_tool_log_conv", "conv_id"),
|
|
)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"id": self.id,
|
|
"user_id": self.user_id,
|
|
"conv_id": self.conv_id,
|
|
"assistant_message_id": self.assistant_message_id,
|
|
"model": self.model,
|
|
"think_enabled": self.think_enabled,
|
|
"tools_available": list(self.tools_available or []),
|
|
"tools_attempted": list(self.tools_attempted or []),
|
|
"tools_succeeded": list(self.tools_succeeded or []),
|
|
"tools_failed": list(self.tools_failed or []),
|
|
"created_at": self.created_at.isoformat() if self.created_at else None,
|
|
}
|