feat(llm): per-turn tool-call telemetry (generation_tool_log)

Adds an empirical surface for evaluating model swaps. One row per assistant turn captures: model, think_enabled, tools_available, tools_attempted, tools_succeeded, tools_failed (with error details as JSONB). Without this, judging whether a new model "actually fires record_moment when it should" relies on anecdote across user-reported sessions. With it, the data is queryable directly. Pieces: - Migration 0046: generation_tool_log table with user_created and per-conversation indexes. - Model: SQLAlchemy GenerationToolLog with to_dict() for plain-dict consumption outside session scope. - Service: log_tool_outcomes() normalizes the in-app tool-call shape (function/result/status) into the split buckets and persists. It catches its own exceptions — telemetry failure must NEVER affect the user-facing generation flow. recent_logs() helper for read. - Integration in run_generation: called once per turn right after log_generation, fire-and-forget. - Tests: pure-normalization unit tests using a stub session — no DB needed in CI. Cover the success/error split, the empty-tool-calls case, the exception-swallowing contract, and the success=False edge case where status incorrectly says "success". No UI for the telemetry yet — internal infrastructure (the operator is the consumer, not the journal user), which the FabledRulebook "no UI no ship" explicitly excepts. Query via psql or extend the Fable MCP later if direct shell access gets tiresome. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-21 22:04:09 -04:00
parent d345b32856
commit bf7a29e8a0
6 changed files with 487 additions and 0 deletions
@@ -0,0 +1,103 @@
+"""generation_tool_log: per-turn tool-call telemetry
+
+Revision ID: 0046
+Revises: 0045
+Create Date: 2026-05-21
+
+Captures one row per assistant turn, recording which tools the model
+could have used, which it attempted, which fired successfully, and
+which failed (with error details). The empirical surface for
+evaluating model swaps and answering "does model X actually fire
+record_moment when it should?" without relying on anecdote.
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import ARRAY, JSONB
+
+
+revision = "0046"
+down_revision = "0045"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "generation_tool_log",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column(
+            "user_id",
+            sa.Integer,
+            sa.ForeignKey("users.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "conv_id",
+            sa.Integer,
+            sa.ForeignKey("conversations.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        # SET NULL (not CASCADE) so the telemetry row survives if the
+        # underlying assistant message is later deleted — we want to keep
+        # the per-turn outcome record for retrospective analysis.
+        sa.Column(
+            "assistant_message_id",
+            sa.Integer,
+            sa.ForeignKey("messages.id", ondelete="SET NULL"),
+            nullable=True,
+        ),
+        sa.Column("model", sa.Text, nullable=False),
+        sa.Column("think_enabled", sa.Boolean, nullable=False),
+        sa.Column(
+            "tools_available",
+            ARRAY(sa.Text),
+            nullable=False,
+            server_default=sa.text("'{}'::text[]"),
+        ),
+        sa.Column(
+            "tools_attempted",
+            ARRAY(sa.Text),
+            nullable=False,
+            server_default=sa.text("'{}'::text[]"),
+        ),
+        sa.Column(
+            "tools_succeeded",
+            ARRAY(sa.Text),
+            nullable=False,
+            server_default=sa.text("'{}'::text[]"),
+        ),
+        # JSONB array of {name, error} objects so failed-tool details are
+        # queryable without a separate failure table.
+        sa.Column(
+            "tools_failed",
+            JSONB,
+            nullable=False,
+            server_default=sa.text("'[]'::jsonb"),
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            nullable=False,
+            server_default=sa.text("now()"),
+        ),
+    )
+    # Common query: "recent tool outcomes for this user, filterable by model."
+    op.create_index(
+        "ix_generation_tool_log_user_created",
+        "generation_tool_log",
+        ["user_id", sa.text("created_at DESC")],
+    )
+    # Per-conversation lookup for the journal page's own retrospection.
+    op.create_index(
+        "ix_generation_tool_log_conv",
+        "generation_tool_log",
+        ["conv_id"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_generation_tool_log_conv", table_name="generation_tool_log")
+    op.drop_index(
+        "ix_generation_tool_log_user_created", table_name="generation_tool_log"
+    )
+    op.drop_table("generation_tool_log")