feat(search): retrieval telemetry — log every semantic retrieval
CI & Build / Python lint (push) Successful in 2s
CI & Build / integration (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 44s
CI & Build / Build & push image (push) Successful in 58s
CI & Build / Python lint (push) Successful in 2s
CI & Build / integration (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 44s
CI & Build / Build & push image (push) Successful in 58s
Add retrieval_logs (migration 0068) + services/retrieval_telemetry with a fire-and-forget record_retrieval(), wired into the MCP search tool (source=mcp_search) and the REST search route (source=rest_search). Captures query, effective params, and the per-result score distribution so KB-injection thresholds can be tuned from data rather than guessed. Scribe: project 2, milestone 93, task 1032. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Xz4j1H7pjYSjKsEpgcNH5E
This commit is contained in:
@@ -0,0 +1,70 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Float, Index, Integer, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from scribe.models import Base
|
||||
|
||||
|
||||
class RetrievalLog(Base):
|
||||
"""One row per semantic-retrieval call, for KB-injection tuning.
|
||||
|
||||
Captures what a query asked for, what came back, and the score
|
||||
distribution of the results — the empirical basis for tuning the
|
||||
similarity threshold and top-k per surface. `result_ids` holds the ranked
|
||||
hits (id + score + rank) so a later pass can correlate "what we surfaced"
|
||||
against "what the agent then fetched/referenced".
|
||||
|
||||
Deliberately FK-free on user_id (mirrors AppLog): telemetry should outlive
|
||||
the row it describes, and a deleted user shouldn't cascade away history.
|
||||
"""
|
||||
|
||||
__tablename__ = "retrieval_logs"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
# Retrieval surface: 'mcp_search' | 'rest_search' | 'auto_inject' | ...
|
||||
source: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
query: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Effective parameters actually used for this call.
|
||||
threshold: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
limit_n: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
project_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
# The content-type filter as passed to semantic_search_notes: True=tasks,
|
||||
# False=notes, NULL=any.
|
||||
is_task: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
result_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
top_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
min_score: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
# [{"id": int, "score": float, "rank": int}, ...], highest-first.
|
||||
result_ids: Mapped[list] = mapped_column(JSONB, nullable=False, default=list)
|
||||
duration_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_retrieval_logs_created_at", "created_at"),
|
||||
Index("ix_retrieval_logs_user_id", "user_id"),
|
||||
Index("ix_retrieval_logs_source", "source"),
|
||||
Index("ix_retrieval_logs_source_created_at", "source", created_at.desc()),
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"user_id": self.user_id,
|
||||
"source": self.source,
|
||||
"query": self.query,
|
||||
"threshold": self.threshold,
|
||||
"limit_n": self.limit_n,
|
||||
"project_id": self.project_id,
|
||||
"is_task": self.is_task,
|
||||
"result_count": self.result_count,
|
||||
"top_score": self.top_score,
|
||||
"min_score": self.min_score,
|
||||
"result_ids": self.result_ids,
|
||||
"duration_ms": self.duration_ms,
|
||||
}
|
||||
Reference in New Issue
Block a user