FabledScribe/src/scribe/mcp/tools/search.py

"""search — semantic search across the user's notes and tasks.

Mirrors the existing fable-mcp contract so Claude's prior usage pattern keeps
working. Differences from fable-mcp:
  - calls services.embeddings.semantic_search_notes directly instead of HTTP
  - user_id comes from mcp.current_user_id() rather than a global API key
"""
from __future__ import annotations

import time

from scribe.mcp._context import current_user_id
from scribe.services.access import owner_names_for
from scribe.services.embeddings import (
    DEFAULT_SIMILARITY_THRESHOLD, semantic_search_notes, semantic_search_rules,
)
from scribe.services import rulebooks as rulebooks_svc
from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary


async def _search_rules(uid: int, q: str, limit: int) -> dict:
    """Rules by meaning — a separate result shape because a rule IS different.

    A rule hit carries `why` and `how_to_apply`: they are the operational half
    of a rule and the session-start payload never includes them, so a caller
    who went looking should get the whole thing rather than a summary they then
    have to re-fetch. It also carries the rule's check (`verify_with`,
    `expires_when`, `last_verified`) when it has one — a search hit is exactly
    the moment someone is about to act on a rule, and "this asserts a fact
    nobody has confirmed" is part of what the rule says.

    Rules are not project-scoped the way notes are (a family rule belongs to no
    project), so `project_id` and `system_id` do not apply here.
    """
    raw = await semantic_search_rules(uid, q, limit=limit)
    return {
        "results": [
            {
                "id": rule.id,
                "title": rule.title,
                "statement": rule.statement,
                "when_to_apply": rule.when_to_apply or "",
                "tier": rule.tier,
                "why": rule.why or "",
                "how_to_apply": rule.how_to_apply or "",
                "verify_with": rule.verify_with or "",
                "expires_when": rule.expires_when or "",
                # Only on a rule that carries a check; its absence means the
                # rule is a decision, not that nobody has looked.
                **(
                    {"last_verified": rulebooks_svc.last_verified_label(rule)}
                    if rule.verify_with else {}
                ),
                "topic_id": rule.topic_id,
                "project_id": rule.project_id,
                "similarity": float(score),
            }
            for score, rule in raw
        ],
        "total": len(raw),
    }


async def search(
    q: str,
    content_type: str = "all",
    limit: int = 10,
    project_id: int = 0,
    system_id: int = 0,
) -> dict:
    """Semantic search over the user's existing notes and tasks — Scribe's recall.

    Reach for this BEFORE answering a question about the user's work or starting
    a task: the user's second-brain almost always already holds related prior
    art. Check for an existing ticket before opening a new one (search with
    content_type='task'), and for prior notes/decisions before re-deriving them.
    Treating Scribe as the first place to look — not a place to only write — is
    the difference between it being a trustworthy record and a write-only log.

    Args:
        q: search query string.
        content_type: 'all' (default), 'note' (notes only), 'task' (tasks
            only), or 'rule' (RULES only — the operator's standing
            instructions, searchable by meaning since milestone 307).
            Reach for 'rule' when you want to know whether a standing
            instruction covers something: "is there a rule about release
            tagging?". A hit carries the rule's `why` and `how_to_apply`,
            which the session-start payload does not.
        limit: maximum number of results (1-50).
        project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
            whenever a project is in scope (the one you entered with
            enter_project) — otherwise this searches across ALL projects and
            bleeds unrelated work into the result set. 0 = search everything
            (use only when you genuinely want a cross-project sweep).
        system_id: Narrow to records tagged to one System (a named
            subsystem/area — enter_project lists them). Use when investigating
            a specific subsystem: it cuts the candidates to records someone
            deliberately filed under that area. 0 = no system filter.
            list_system_records gives the same slice unranked.

    Returns:
        {"results": [{"id", "title", "body", "is_task", "tags", "similarity"}],
         "total": int}

    A result marked `shared: true` with an `owner` belongs to another user —
    that person's suggestion, not the operator's own record or settled practice.
    Weigh it on its merits and say whose it is when you use it.
    """
    uid = current_user_id()
    limit = max(1, min(limit, 50))
    if content_type == "rule":
        return await _search_rules(uid, q, limit)
    is_task = {"note": False, "task": True}.get(content_type)  # None => any
    t0 = time.perf_counter()
    raw = await semantic_search_notes(
        uid, q, limit=limit, is_task=is_task,
        project_id=project_id or None,
        system_id=system_id or None,
        # An explicit search reaches everything the operator may read, including
        # records shared with them one-to-one.
        scope="read",
    )
    record_retrieval(
        user_id=uid, source="mcp_search", query=q,
        threshold=DEFAULT_SIMILARITY_THRESHOLD, limit=limit,
        project_id=project_id or None, is_task=is_task, results=raw,
        duration_ms=(time.perf_counter() - t0) * 1000.0,
    )
    owners = await owner_names_for(
        {int(note.user_id) for _s, note in raw if note.user_id != uid}
    )
    return {
        "results": [
            {
                "id": note.id,
                "title": note.title,
                "body": (note.body or "")[:240],
                "is_task": bool(note.is_task),
                "tags": list(note.tags or []),
                "similarity": float(score),
                **(
                    {"shared": True, "owner": owners.get(int(note.user_id))}
                    if note.user_id != uid else {}
                ),
            }
            for score, note in raw
        ],
        "total": len(raw),
    }


async def retrieval_telemetry(days: int = 30) -> dict:
    """What the retrieval telemetry says about YOUR surfaces, over a window.

    The read half of the loop the ranker's thresholds are meant to be tuned
    from (#2975). Reach for it before changing a similarity threshold, a top-k,
    or deciding whether a reranker is worth building — the alternative is
    hand-probing the live instance, which is how the last such decision had to
    be made.

    Three readouts, from the three tables built for them:

    `sources` — per retrieval surface (`auto_inject`, `write_path`,
    `mcp_search`, …), from `retrieval_logs`: `calls`, `zero_result_calls`,
    `cleared_threshold` (how often the best hit beat the threshold in force for
    that call), the `top_score` spread (p10/p50/p90/min/max), `avg_result_count`
    and `p90_duration_ms`. THE number to read first is `cleared_threshold`
    against `calls`, with the spread beside it: a surface that clears its bar
    on nearly every call is either well-tuned or too loose, and p10 says which.

    `usage` — NOTES ONLY, from `note_usage_events`, at the per-note grain
    `retrieval_logs` cannot be indexed at: `surfaced` (ranked surfacings — a
    scored surface CHOSE the record), `ambient` (the rest), `pulled` split into
    `pulled_by_agent` / `pulled_by_human`, the distinct-note counts, and
    `pull_through`. That ratio is the corpus-side precision signal: records
    surfaced often and opened never are dead weight competing for the injection
    budget every turn.

    `pull_through` is AGENT pulls over RANKED surfacings, and both halves of
    that matter. "Is this record dead weight?" is answered by any pull; "was
    that injected line useful?" — the question a threshold or a reranker is
    tuned against — only by a pull the agent made. Aggregating across the
    mcp_/rest_ prefix would silently answer the wrong one.

    `usage["by_source"]` — THE number to tune a threshold against, because the
    top-level `pull_through` is a corpus average and averages the surfaces
    together. Per surface: `notes_surfaced`, `notes_pulled`, `pull_through`,
    and `ambient: true` on surfaces whose surfacings were not scored choices
    (their ratio is null — "surfaced often, opened never" is not a judgment
    about a record nothing chose). Read it as: of the distinct notes THIS
    surface put in front of the agent, how many did the agent then open?

It is an UPPER BOUND per surface: a pull records the door it came
    through, not the surface that led there, so a note surfaced by two surfaces
    and opened once counts for both — attribution would need the session
    identity #2085 declined to invent. `by_source_failed: true` means that one
    query failed while the rest of the readout stood.

    `rule_usage` — the same question for RULES, from `rule_usage_events`:
    `surfaced`, `pulled` split into `pulled_by_agent` / `pulled_by_human`, the
    distinct-rule counts, and `pull_through` on the same definition (agent
    pulls over surfacings).

    A SEPARATE BLOCK, not folded into `usage`, and reading it as one number
    with that is the mistake to avoid. The corpora differ by orders of
    magnitude — a few dozen eligible rules against thousands of notes — so a
    blended ratio would be the note ratio with noise on it and would hide the
    rule arm entirely. It also has no `ambient` key, because nothing surfaces a
    rule un-ranked: `list_always_on_rules` and `enter_project` hand over rules
    wholesale but emit no event, so there is no ambient class to separate.

    Read it against `sources["write_path_rule"]`. That surface has never once
    declined to fire, and until this block existed there was no way to tell a
    well-tuned arm from a bar it cannot fail to clear (#3311). `pull_through`
    is the number that tells them apart.

    `rule_usage_failed: true` means that read failed while the rest of the
    readout stood. The counts are still present so a caller can render, but
    they are zeros meaning "could not find out", not "nothing happened" — do
    not report a pull-through from a block carrying that flag.

    Scoped to your own telemetry — a retrieval log records what your agent
    asked for, query text included, and is not a shared record kind.

    `read_failed: true` means the query itself failed — deliberately distinct
    from an empty window, because those two looked identical for weeks once
    (#2663) and every counter silently read zero.

    Args:
        days: window size, default 30. Clamped to at least 1.
    """
    return await retrieval_summary(current_user_id(), days=days)


def register(mcp) -> None:
    mcp.tool(name="search")(search)
    mcp.tool(name="retrieval_telemetry")(retrieval_telemetry)