FabledScribe/src/scribe/mcp/tools/search.py

"""search — semantic search across the user's notes and tasks.

Mirrors the existing fable-mcp contract so Claude's prior usage pattern keeps
working. Differences from fable-mcp:
  - calls services.embeddings.semantic_search_notes directly instead of HTTP
  - user_id comes from mcp.current_user_id() rather than a global API key
"""
from __future__ import annotations

import time

from scribe.mcp._context import current_user_id
from scribe.services.access import owner_names_for
from scribe.services.embeddings import (
    DEFAULT_SIMILARITY_THRESHOLD, semantic_search_notes, semantic_search_rules,
)
from scribe.services import rulebooks as rulebooks_svc
from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary


async def _search_rules(uid: int, q: str, limit: int) -> dict:
    """Rules by meaning — a separate result shape because a rule IS different.

    A rule hit carries `why` and `how_to_apply`: they are the operational half
    of a rule and the session-start payload never includes them, so a caller
    who went looking should get the whole thing rather than a summary they then
    have to re-fetch. It also carries the rule's check (`verify_with`,
    `expires_when`, `last_verified`) when it has one — a search hit is exactly
    the moment someone is about to act on a rule, and "this asserts a fact
    nobody has confirmed" is part of what the rule says.

    Rules are not project-scoped the way notes are (a family rule belongs to no
    project), so `project_id` and `system_id` do not apply here.
    """
    raw = await semantic_search_rules(uid, q, limit=limit)
    return {
        "results": [
            {
                "id": rule.id,
                "title": rule.title,
                "statement": rule.statement,
                "when_to_apply": rule.when_to_apply or "",
                "tier": rule.tier,
                "why": rule.why or "",
                "how_to_apply": rule.how_to_apply or "",
                "verify_with": rule.verify_with or "",
                "expires_when": rule.expires_when or "",
                # Only on a rule that carries a check; its absence means the
                # rule is a decision, not that nobody has looked.
                **(
                    {"last_verified": rulebooks_svc.last_verified_label(rule)}
                    if rule.verify_with else {}
                ),
                "topic_id": rule.topic_id,
                "project_id": rule.project_id,
                "similarity": float(score),
            }
            for score, rule in raw
        ],
        "total": len(raw),
    }


async def search(
    q: str,
    content_type: str = "all",
    limit: int = 10,
    project_id: int = 0,
    system_id: int = 0,
) -> dict:
    """Semantic search over the user's existing notes and tasks — Scribe's recall.

    Reach for this BEFORE answering a question about the user's work or starting
    a task: the user's second-brain almost always already holds related prior
    art. Check for an existing ticket before opening a new one (search with
    content_type='task'), and for prior notes/decisions before re-deriving them.
    Treating Scribe as the first place to look — not a place to only write — is
    the difference between it being a trustworthy record and a write-only log.

    Args:
        q: search query string.
        content_type: 'all' (default), 'note' (notes only), 'task' (tasks
            only), or 'rule' (RULES only — the operator's standing
            instructions, searchable by meaning since milestone 307).
            Reach for 'rule' when you want to know whether a standing
            instruction covers something: "is there a rule about release
            tagging?". A hit carries the rule's `why` and `how_to_apply`,
            which the session-start payload does not.
        limit: maximum number of results (1-50).
        project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
            whenever a project is in scope (the one you entered with
            enter_project) — otherwise this searches across ALL projects and
            bleeds unrelated work into the result set. 0 = search everything
            (use only when you genuinely want a cross-project sweep).
        system_id: Narrow to records tagged to one System (a named
            subsystem/area — enter_project lists them). Use when investigating
            a specific subsystem: it cuts the candidates to records someone
            deliberately filed under that area. 0 = no system filter.
            list_system_records gives the same slice unranked.

    Returns:
        {"results": [{"id", "title", "body", "is_task", "tags", "similarity"}],
         "total": int}

    A result marked `shared: true` with an `owner` belongs to another user —
    that person's suggestion, not the operator's own record or settled practice.
    Weigh it on its merits and say whose it is when you use it.
    """
    uid = current_user_id()
    limit = max(1, min(limit, 50))
    if content_type == "rule":
        return await _search_rules(uid, q, limit)
    is_task = {"note": False, "task": True}.get(content_type)  # None => any
    t0 = time.perf_counter()
    report: dict = {}
    raw = await semantic_search_notes(
        uid, q, limit=limit, is_task=is_task,
        project_id=project_id or None,
        system_id=system_id or None,
        # An explicit search reaches everything the operator may read, including
        # records shared with them one-to-one.
        scope="read",
        report=report,
    )
    record_retrieval(
        user_id=uid, source="mcp_search", query=q,
        threshold=DEFAULT_SIMILARITY_THRESHOLD, limit=limit,
        project_id=project_id or None, is_task=is_task, results=raw,
        duration_ms=(time.perf_counter() - t0) * 1000.0,
        best_available=report.get("best_available_score"),
        best_available_id=report.get("best_available_id"),
        searched=bool(report.get("searched", True)),
    )
    owners = await owner_names_for(
        {int(note.user_id) for _s, note in raw if note.user_id != uid}
    )
    return {
        "results": [
            {
                "id": note.id,
                "title": note.title,
                "body": (note.body or "")[:240],
                "is_task": bool(note.is_task),
                "tags": list(note.tags or []),
                "similarity": float(score),
                **(
                    {"shared": True, "owner": owners.get(int(note.user_id))}
                    if note.user_id != uid else {}
                ),
            }
            for score, note in raw
        ],
        "total": len(raw),
    }


async def retrieval_telemetry(
    days: int = 30, near_miss_samples: int = 0,
) -> dict:
    """What the retrieval telemetry says about YOUR surfaces, over a window.

    The read half of the loop the ranker's thresholds are meant to be tuned
    from (#2975). Reach for it before changing a similarity threshold, a top-k,
    or deciding whether a reranker is worth building — the alternative is
    hand-probing the live instance, which is how the last such decision had to
    be made.

    Three readouts, from the three tables built for them:

    `sources` — per retrieval surface (`auto_inject`, `write_path`,
    `mcp_search`, …), from `retrieval_logs`: `calls`, `zero_result_calls`,
    `near_misses`, the `top_score` spread (p10/p50/p90/min/max),
    `avg_result_count` and `p90_duration_ms`.

    THE NUMBER TO READ FIRST IS `near_misses.p90`, AGAINST THE THRESHOLD IN
    FORCE FOR THAT SURFACE. It is measured on the calls the BAR turned away —
    zero-result calls, minus the ones whose zero was a repeat the reader had
    already been shown — using the best score the ranker reached before the bar
    rejected it. So it is the one figure here that says something the bar
    cannot make true by construction, and `max` is always below the threshold:
    an above-bar candidate nobody excluded would have been returned. A bar at 0.72 turning away a stream of 0.71s is set too
    high by a hair and the surface is losing hits it should have had. The same
    bar turning away 0.30s is working, and the corpus simply had nothing. Both
    render as a zero-result call, and nothing else in this readout tells them
    apart.

    `near_miss_samples` (0-20, default 0) TURNS THE PERCENTILES INTO RECORDS
    YOU CAN READ. Each source then carries `near_miss_records`: its highest
    scoring declines, each with the `record_id` the bar refused and the `query`
    that asked. Reach for it whenever you are about to move a threshold.

    THE PERCENTILE CANNOT SETTLE A BAR ON ITS OWN, and this is the whole reason
    the parameter exists. `near_misses.p90` says mass is sitting just under the
    line; it says nothing about whether that mass is RELEVANT, and those are
    different questions. Lowering a bar to where the mass is, without reading
    what is there, is choosing a firing rate rather than a quality. Pull-through
    cannot referee it either — the injected rule line already carries title and
    trigger, so a session can comply without ever calling `get_rule`, which
    makes rule pull-through understate usefulness by construction. Reading the
    rejected records is the method that actually answers it.

    Off by default because it is a LISTING, not a statistic: it is for the
    moment you are making a decision, not for every readout.

    `near_misses` is `null` when no declining call in the window measured it —
    rows written before #3670 shipped cannot know. That is "not measured", not
    "nothing came close"; a 0.0 there would be a claim about the corpus
    invented out of a caller's silence.

    A NULL HERE NOW MEANS ONE THING, which it did not at first. A semantic
    search returns nothing three ways WITHOUT having run — an empty query, an
    unavailable embedder, and a failed database query — and each used to write
    a row indistinguishable from a ranker that declined (#3765). Those calls no
    longer write a row at all, on the same reasoning that already keeps a blank
    command out of the log: a row there reports a call that never happened and
    drags the clear rate down with phantom declines. So a null is "searched,
    and nothing came close", and a broken search shows up as a WARNING in the
    application log rather than as a quiet zero in here.

    THERE IS NO `cleared_threshold` ANY MORE, and if you remember one, that
    memory is of a tautology (#3670). The search applies the bar before
    returning, so every returned result cleared it by construction and a call
    with no results has no score to compare: the field was true exactly when
    `result_count > 0`, i.e. it was `calls - zero_result_calls` under a name
    that promised a second opinion. `zero_result_calls + cleared_threshold ==
    calls` held on all nineteen readings ever taken. The reading procedure
    built on it — "clears its bar on nearly every call" — asked you to compare
    a number with itself.

    CHECK `suppression` BEFORE CONCLUDING ANYTHING FROM `zero_result_calls`. A
    zero-result call is two different events wearing one number: the ranker
    found nothing above the bar, or it found only what this session had already
    been shown. Just the first is evidence about the bar. `suppression` splits
    them where the surface can tell — `zero_because_already_shown` comes off
    `zero_result_calls` to leave the true ranker declines.

    `suppression` is `null` when NO row in the window reported it, and that is
    "not measured here", NOT "none suppressed". Surfaces that pass their
    exclusions into the search never see what was dropped, so they cannot say.
    Do not read a null as a zero: reading an artifact as a measurement is how
    this surface got mis-scoped once already (#3311, #3497).

    `usage` — NOTES ONLY, from `note_usage_events`, at the per-note grain
    `retrieval_logs` cannot be indexed at: `surfaced` (ranked surfacings — a
    scored surface CHOSE the record), `ambient` (the rest), `pulled` split into
    `pulled_by_agent` / `pulled_by_human`, the distinct-note counts, and
    `pull_through`. That ratio is the corpus-side precision signal: records
    surfaced often and opened never are dead weight competing for the injection
    budget every turn.

    `pull_through` is AGENT pulls over RANKED surfacings, and both halves of
    that matter. "Is this record dead weight?" is answered by any pull; "was
    that injected line useful?" — the question a threshold or a reranker is
    tuned against — only by a pull the agent made. Aggregating across the
    mcp_/rest_ prefix would silently answer the wrong one.

    `usage["by_source"]` — THE number to tune a threshold against, because the
    top-level `pull_through` is a corpus average and averages the surfaces
    together. Per surface: `notes_surfaced`, `notes_pulled`, `pull_through`,
    and `ambient: true` on surfaces whose surfacings were not scored choices
    (their ratio is null — "surfaced often, opened never" is not a judgment
    about a record nothing chose). Read it as: of the distinct notes THIS
    surface put in front of the agent, how many did the agent then open?

It is an UPPER BOUND per surface: a pull records the door it came
    through, not the surface that led there, so a note surfaced by two surfaces
    and opened once counts for both — attribution would need the session
    identity #2085 declined to invent. `by_source_failed: true` means that one
    query failed while the rest of the readout stood.

    `rule_usage` — the same question for RULES, from `rule_usage_events`:
    `surfaced` and `ambient`, `pulled` split into `pulled_by_agent` /
    `pulled_by_human`, the distinct-rule counts, and `pull_through` on the same
    definition (agent pulls over RANKED surfacings).

    A SEPARATE BLOCK, not folded into `usage`, and reading it as one number
    with that is the mistake to avoid. The corpora differ by orders of
    magnitude — a few dozen eligible rules against thousands of notes — so a
    blended ratio would be the note ratio with noise on it and would hide the
    rule arm entirely.

    `surfaced` VS `ambient` IS THE READING THAT MATTERS HERE. `surfaced` counts
    rules a ranker chose — today only the write-path arm — and those are claims
    a pull can settle. `ambient` counts BULK DELIVERIES: the SessionStart
    preload, `list_always_on_rules`, and the `rules_payload` surfaces
    (`enter_project`, `get_project`, `get_milestone`, `start_planning`,
    `get_task`), which hand over the whole applicable set at once with nobody
    choosing anything. A large `ambient` says the resident set is big and
    arrives often — never that it is useful, and never that it is read.

    `pull_through` therefore divides by `surfaced` alone. Fold the preload in
    and growing the always-on set would depress the arm's measured precision
    while trimming it would flatter it, for reasons having nothing to do with
    the arm. To judge the PRELOAD instead, compare `ambient` against pulls of
    those same rules over time: a resident set surfaced thousands of times and
    opened never is the dead-weight signal, one tier up.

    Read it against `sources["write_path_rule"]`. That arm was once believed
    never to decline — the reading that scoped #3311 — but it was the arm's
    `retrieval_logs` row being written only on calls that FOUND something, so
    the zeros were missing rather than absent (#3497). Measured since, it
    declines the large majority of its calls like any other surface.

    EVERY COUNTER BLOCK CARRIES ITS OWN COVERAGE — `complete_from` and
    `covers_window`. `complete_from` is when the number became trustworthy:
    for one source, its first recorded row; for a section that sums several,
    the LATEST of theirs, because a total is complete only once every
    contributor was being written. `covers_window: false` means the window
    reaches back further than the recording does, so the count is a fraction
    of the period it appears to describe.

    READ IT BEFORE COMPARING TWO NUMBERS, and especially before comparing
    across a deploy. A counter added last week, read over a 30-day window,
    reports a real count against an imagined denominator — and the result is
    a plausible fraction rather than an obvious zero, which is what makes it
    dangerous. That reading cost milestone #379 five steps aimed at a defect
    that did not exist.

    `covers_window` is null, never false, when nothing was ever recorded:
    "no measurement" is not "partial measurement", the same distinction
    `suppression`'s null carries a few paragraphs up.

    A SOURCE SHOWING `calls: 0` WAS RECORDING AND MADE NO CALLS. `sources`
    lists every source the table has ever held, not only those active in the
    window, so a surface that stopped firing stays visible rather than
    disappearing — being absent is reserved for a source that has never
    recorded at all. Its score fields are null, not zero: the calls are a
    real observation, the distribution is not one.

    `rule_usage_failed: true` means that read failed while the rest of the
    readout stood. The counts are still present so a caller can render, but
    they are zeros meaning "could not find out", not "nothing happened" — do
    not report a pull-through from a block carrying that flag.

    Scoped to your own telemetry — a retrieval log records what your agent
    asked for, query text included, and is not a shared record kind.

    `read_failed: true` means the query itself failed — deliberately distinct
    from an empty window, because those two looked identical for weeks once
    (#2663) and every counter silently read zero.

    Args:
        days: window size, default 30. Clamped to at least 1.
        near_miss_samples: 0-20, default 0. How many of each source's highest
            scoring DECLINES to list by record, with the query that asked.
            Pass it when you are about to move a threshold; leave it off
            otherwise. See the near-miss section above for why a percentile
            alone cannot settle a bar.
    """
    return await retrieval_summary(
        current_user_id(), days=days, near_miss_samples=near_miss_samples,
    )


def register(mcp) -> None:
    mcp.tool(name="search")(search)
    mcp.tool(name="retrieval_telemetry")(retrieval_telemetry)