bvandeusen
807f478cac
CI & Build / Python lint (push) Successful in 2s
CI & Build / integration (push) Successful in 16s
CI & Build / TypeScript typecheck (push) Successful in 32s
CI & Build / Python tests (push) Successful in 44s
CI & Build / Build & push image (push) Successful in 58s
Add retrieval_logs (migration 0068) + services/retrieval_telemetry with a fire-and-forget record_retrieval(), wired into the MCP search tool (source=mcp_search) and the REST search route (source=rest_search). Captures query, effective params, and the per-result score distribution so KB-injection thresholds can be tuned from data rather than guessed. Scribe: project 2, milestone 93, task 1032. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Xz4j1H7pjYSjKsEpgcNH5E
113 lines
4.0 KiB
Python
113 lines
4.0 KiB
Python
"""Tests for services.retrieval_telemetry.
|
|
|
|
_build_payload is pure (no DB, no loop) and gets unit coverage. The persistence
|
|
path (_insert_retrieval_log + the RetrievalLog model / JSONB roundtrip) is an
|
|
integration test against real Postgres.
|
|
"""
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
|
|
from scribe.services.retrieval_telemetry import (
|
|
_build_payload,
|
|
record_retrieval,
|
|
)
|
|
|
|
|
|
def _note(nid):
|
|
"""Minimal stand-in — _build_payload only reads .id."""
|
|
return SimpleNamespace(id=nid)
|
|
|
|
|
|
# ─── _build_payload (pure) ───────────────────────────────────────────────────
|
|
|
|
|
|
def test_build_payload_ranks_and_score_bounds():
|
|
results = [(0.91, _note(11)), (0.72, _note(22)), (0.55, _note(33))]
|
|
p = _build_payload(
|
|
user_id=7, source="mcp_search", query="hello", threshold=0.45,
|
|
limit=10, project_id=3, is_task=None, results=results, duration_ms=12.345,
|
|
)
|
|
assert p["result_count"] == 3
|
|
assert p["top_score"] == 0.91
|
|
assert p["min_score"] == 0.55
|
|
assert [it["rank"] for it in p["result_ids"]] == [0, 1, 2]
|
|
assert [it["id"] for it in p["result_ids"]] == [11, 22, 33]
|
|
assert p["duration_ms"] == 12.35 # rounded to 2dp
|
|
assert p["user_id"] == 7 and p["project_id"] == 3 and p["threshold"] == 0.45
|
|
|
|
|
|
def test_build_payload_empty_results():
|
|
p = _build_payload(
|
|
user_id=1, source="rest_search", query="x", threshold=0.3,
|
|
limit=5, project_id=None, is_task=False, results=[], duration_ms=None,
|
|
)
|
|
assert p["result_count"] == 0
|
|
assert p["top_score"] is None and p["min_score"] is None
|
|
assert p["result_ids"] == []
|
|
assert p["duration_ms"] is None
|
|
|
|
|
|
def test_build_payload_rounds_scores_to_5dp():
|
|
p = _build_payload(
|
|
user_id=1, source="mcp_search", query="q", threshold=0.45,
|
|
limit=1, project_id=None, is_task=None,
|
|
results=[(0.123456789, _note(1))], duration_ms=0.0,
|
|
)
|
|
assert p["result_ids"][0]["score"] == 0.12346
|
|
|
|
|
|
def test_record_retrieval_without_event_loop_is_safe():
|
|
"""Called from a sync context (no running loop) it must swallow and return,
|
|
never raise — telemetry can't be allowed to break a caller."""
|
|
# No event loop running in this plain sync test.
|
|
assert record_retrieval(
|
|
user_id=1, source="mcp_search", query="q", threshold=0.45,
|
|
limit=10, project_id=None, is_task=None,
|
|
results=[(0.9, _note(1))],
|
|
) is None
|
|
|
|
|
|
# ─── persistence (integration) ───────────────────────────────────────────────
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def _dispose_engine():
|
|
from scribe.models import engine
|
|
yield
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.asyncio
|
|
async def test_insert_retrieval_log_roundtrip(_dispose_engine):
|
|
from sqlalchemy import delete, select
|
|
|
|
from scribe.models import async_session
|
|
from scribe.models.retrieval_log import RetrievalLog
|
|
from scribe.services.retrieval_telemetry import _insert_retrieval_log
|
|
|
|
payload = _build_payload(
|
|
user_id=990001, source="mcp_search", query="pgvector tuning",
|
|
threshold=0.45, limit=10, project_id=None, is_task=None,
|
|
results=[(0.88, _note(501)), (0.61, _note(502))], duration_ms=9.9,
|
|
)
|
|
await _insert_retrieval_log(payload)
|
|
|
|
async with async_session() as s:
|
|
row = (
|
|
await s.execute(
|
|
select(RetrievalLog).where(RetrievalLog.user_id == 990001)
|
|
)
|
|
).scalars().first()
|
|
assert row is not None
|
|
assert row.source == "mcp_search"
|
|
assert row.result_count == 2
|
|
assert row.top_score == 0.88
|
|
# JSONB roundtrips as a list of dicts with the expected shape.
|
|
assert row.result_ids[0] == {"id": 501, "score": 0.88, "rank": 0}
|
|
assert row.created_at is not None # server_default now()
|
|
await s.execute(delete(RetrievalLog).where(RetrievalLog.user_id == 990001))
|
|
await s.commit()
|