FabledScribe/src/scribe/mcp/tools/search.py

"""search — semantic search across the user's notes and tasks.

Mirrors the existing fable-mcp contract so Claude's prior usage pattern keeps
working. Differences from fable-mcp:
  - calls services.embeddings.semantic_search_notes directly instead of HTTP
  - user_id comes from mcp.current_user_id() rather than a global API key
"""
from __future__ import annotations

import time

from scribe.mcp._context import current_user_id
from scribe.services.access import owner_names_for
from scribe.services.embeddings import (
    DEFAULT_SIMILARITY_THRESHOLD, semantic_search_milestones, semantic_search_notes,
    semantic_search_rules,
)
from scribe.services import rulebooks as rulebooks_svc
from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary


async def _search_rules(uid: int, q: str, limit: int, project_id: int) -> dict:
    """Rules by meaning — a separate result shape because a rule IS different.

    A rule hit carries `why` and `how_to_apply`: they are the operational half
    of a rule and the session-start payload never includes them, so a caller
    who went looking should get the whole thing rather than a summary they then
    have to re-fetch. It also carries the rule's check (`verify_with`,
    `expires_when`, `last_verified`) when it has one — a search hit is exactly
    the moment someone is about to act on a rule, and "this asserts a fact
    nobody has confirmed" is part of what the rule says.

    `project_id` scopes the way it does for notes, with one difference: a
    GLOBAL rule (one in a rulebook) belongs to no project and applies in every
    one, so a scoped search returns global rules plus that project's own.
    Without a project it asks the whole rulebook — every rule, whatever its
    home — because that is the question an unscoped "is there a rule about
    this" is asking. `system_id` does not apply to rules.
    """
    if project_id:
        raw = await semantic_search_rules(uid, q, limit=limit, project_id=project_id)
    else:
        raw = await semantic_search_rules(uid, q, limit=limit, everywhere=True)
    return {
        "results": [
            {
                "id": rule.id,
                "title": rule.title,
                "statement": rule.statement,
                "when_to_apply": rule.when_to_apply or "",
                "why": rule.why or "",
                "how_to_apply": rule.how_to_apply or "",
                "verify_with": rule.verify_with or "",
                "expires_when": rule.expires_when or "",
                # Only on a rule that carries a check; its absence means the
                # rule is a decision, not that nobody has looked.
                **(
                    {"last_verified": rulebooks_svc.last_verified_label(rule)}
                    if rule.verify_with else {}
                ),
                "topic_id": rule.topic_id,
                "project_id": rule.project_id,
                "similarity": float(score),
            }
            for score, rule in raw
        ],
        "total": len(raw),
    }


async def _search_milestones(uid: int, q: str, limit: int, project_id: int) -> dict:
    """Milestones by meaning — "is there already a plan for this?" (milestone 415).

    Its own result shape, like rules: a milestone is a plan with progress, not
    a note with a body. The plan itself is left out — get_milestone reads it —
    because a search hit is for recognising a plan, and bodies run long.
    Not part of content_type="all", whose results are note-shaped.
    """
    raw = await semantic_search_milestones(uid, q, project_id=project_id or None, limit=limit)
    progress: dict[int, dict] = {}
    if raw:
        from scribe.services import milestones as milestones_svc

        for pid in {m.project_id for _s, m in raw}:
            for row in await milestones_svc.get_project_milestone_summary(uid, pid):
                progress[row["id"]] = row
    return {
        "results": [
            {
                "id": m.id,
                "title": m.title,
                "description": m.description or "",
                "status": m.status,
                "project_id": m.project_id,
                "total": progress.get(m.id, {}).get("total", 0),
                "completed": progress.get(m.id, {}).get("completed", 0),
                "similarity": float(score),
            }
            for score, m in raw
        ],
        "total": len(raw),
    }


async def search(
    q: str,
    content_type: str = "all",
    limit: int = 10,
    project_id: int = 0,
    system_id: int = 0,
) -> dict:
    """Semantic search over the user's existing notes and tasks — Scribe's recall.

    Reach for this BEFORE answering a question about the user's work or starting
    a task: the user's second-brain almost always already holds related prior
    art. Check for an existing ticket before opening a new one (search with
    content_type='task'), and for prior notes/decisions before re-deriving them.
    Treating Scribe as the first place to look — not a place to only write — is
    the difference between it being a trustworthy record and a write-only log.

    Args:
        q: search query string.
        content_type: 'all' (default), 'note' (notes only), 'task' (tasks
            only), or 'rule' (RULES only — the operator's standing
            instructions, searchable by meaning since milestone 307).
            Reach for 'rule' when you want to know whether a standing
            instruction covers something: "is there a rule about release
            tagging?". A hit carries the rule's `why` and `how_to_apply`,
            which the session-start payload does not. With a project_id,
            rules come back as the global rules plus that project's own;
            with 0, every rule in the rulebook. Or 'milestone' (PLANS):
            reach for it before start_planning to ask whether a plan for
            this work already exists — a match is where new steps go
            (create_records(milestone_id=…)), not a reason to open a second
            milestone. Hits carry title, description, status and progress;
            get_milestone reads the plan. Not included in 'all'.
        limit: maximum number of results (1-50).
        project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
            whenever a project is in scope (the one you entered with
            enter_project) — otherwise this searches across ALL projects and
            bleeds unrelated work into the result set. 0 = search everything
            (use only when you genuinely want a cross-project sweep).
            A LESSON is the exception and arrives whatever the scope: the kind
            records an insight that transfers, so it is reachable from a
            project it was not written on.
        system_id: Narrow to records tagged to one System (a named
            subsystem/area — enter_project lists them). Use when investigating
            a specific subsystem: it cuts the candidates to records someone
            deliberately filed under that area. 0 = no system filter.
            list_system_records gives the same slice unranked.

    Returns:
        {"results": [{"id", "title", "body", "is_task", "tags", "similarity"}],
         "total": int}

    A result marked `shared: true` with an `owner` belongs to another user —
    that person's suggestion, not the operator's own record or settled practice.
    Weigh it on its merits and say whose it is when you use it.
    """
    uid = current_user_id()
    limit = max(1, min(limit, 50))
    if content_type == "rule":
        return await _search_rules(uid, q, limit, project_id)
    if content_type == "milestone":
        return await _search_milestones(uid, q, limit, project_id)
    is_task = {"note": False, "task": True}.get(content_type)  # None => any
    t0 = time.perf_counter()
    report: dict = {}
    raw = await semantic_search_notes(
        uid, q, limit=limit, is_task=is_task,
        project_id=project_id or None,
        system_id=system_id or None,
        # A LESSON is reachable from any project (milestone 385). The kind
        # exists to carry an insight to the next project, so a project filter
        # that hid it would hide it precisely where it is worth having. Only
        # the project filter widens — everything else about the scoping holds,
        # and a caller narrowing by `content_type` still gets what it asked
        # for. This is the explicit search, where the operator asked; the
        # unasked-for arms decide their own budget separately.
        include_global_kinds=True,
        # An explicit search reaches everything the operator may read, including
        # records shared with them one-to-one.
        scope="read",
        report=report,
    )
    record_retrieval(
        user_id=uid, source="mcp_search", query=q,
        threshold=DEFAULT_SIMILARITY_THRESHOLD, limit=limit,
        project_id=project_id or None, is_task=is_task, results=raw,
        duration_ms=(time.perf_counter() - t0) * 1000.0,
        best_available=report.get("best_available_score"),
        best_available_id=report.get("best_available_id"),
        searched=bool(report.get("searched", True)),
    )
    owners = await owner_names_for(
        {int(note.user_id) for _s, note in raw if note.user_id != uid}
    )
    return {
        "results": [
            {
                "id": note.id,
                "title": note.title,
                "body": (note.body or "")[:240],
                "is_task": bool(note.is_task),
                "tags": list(note.tags or []),
                "similarity": float(score),
                **(
                    {"shared": True, "owner": owners.get(int(note.user_id))}
                    if note.user_id != uid else {}
                ),
            }
            for score, note in raw
        ],
        "total": len(raw),
    }


async def retrieval_telemetry(
    days: int = 30, near_miss_samples: int = 0,
) -> dict:
    """What the retrieval telemetry says about YOUR surfaces, over a window.

    The read half of the tuning loop, whose write half is `tune_retrieval`
    (#2975, #4102). Reach for it before moving any floor or budget, and read
    the records it names rather than its percentiles alone: this readout has
    been measured pointing the WRONG WAY — 69 consecutive declines where every
    percentile said "lower the bar" and the refused record was a false positive
    — so `near_miss_samples=5` and opening the ids it returns is the step that
    separates a real miss from a bar doing its job.

    Three readouts, from the three tables built for them:

    `sources` — per retrieval surface (`auto_inject`, `write_path`,
    `mcp_search`, …), from `retrieval_logs`: `calls`, `zero_result_calls`,
    `near_misses`, the `top_score` spread (p10/p50/p90/min/max),
    `avg_result_count` and `p90_duration_ms`.

    THE NUMBER TO READ FIRST IS `near_misses.p90`, AGAINST THE THRESHOLD IN
    FORCE FOR THAT SURFACE. It is measured on the calls the BAR turned away —
    zero-result calls, minus the ones whose zero was a repeat the reader had
    already been shown — using the best score the ranker reached before the bar
    rejected it. So it is the one figure here that says something the bar
    cannot make true by construction, and `max` is always below the threshold:
    an above-bar candidate nobody excluded would have been returned. A bar at 0.72 turning away a stream of 0.71s is set too
    high by a hair and the surface is losing hits it should have had. The same
    bar turning away 0.30s is working, and the corpus simply had nothing. Both
    render as a zero-result call, and nothing else in this readout tells them
    apart.

    `near_miss_samples` (0-20, default 0) TURNS THE PERCENTILES INTO RECORDS
    YOU CAN READ. Each source then carries `near_miss_records`: its highest
    scoring declines, each with the `record_id` the bar refused and the `query`
    that asked. Reach for it whenever you are about to move a threshold.

    THE PERCENTILE CANNOT SETTLE A BAR ON ITS OWN, and this is the whole reason
    the parameter exists. `near_misses.p90` says mass is sitting just under the
    line; it says nothing about whether that mass is RELEVANT, and those are
    different questions. Lowering a bar to where the mass is, without reading
    what is there, is choosing a firing rate rather than a quality. Pull-through
    cannot referee it either — the injected rule line already carries title and
    trigger, so a session can comply without ever calling `get_rule`, which
    makes rule pull-through understate usefulness by construction. Reading the
    rejected records is the method that actually answers it.

    Off by default because it is a LISTING, not a statistic: it is for the
    moment you are making a decision, not for every readout.

    `near_misses` is `null` when no declining call in the window measured it —
    rows written before #3670 shipped cannot know. That is "not measured", not
    "nothing came close"; a 0.0 there would be a claim about the corpus
    invented out of a caller's silence.

    A NULL HERE NOW MEANS ONE THING, which it did not at first. A semantic
    search returns nothing three ways WITHOUT having run — an empty query, an
    unavailable embedder, and a failed database query — and each used to write
    a row indistinguishable from a ranker that declined (#3765). Those calls no
    longer write a row at all, on the same reasoning that already keeps a blank
    command out of the log: a row there reports a call that never happened and
    drags the clear rate down with phantom declines. So a null is "searched,
    and nothing came close", and a broken search shows up as a WARNING in the
    application log rather than as a quiet zero in here.

    THERE IS NO `cleared_threshold` ANY MORE, and if you remember one, that
    memory is of a tautology (#3670). The search applies the bar before
    returning, so every returned result cleared it by construction and a call
    with no results has no score to compare: the field was true exactly when
    `result_count > 0`, i.e. it was `calls - zero_result_calls` under a name
    that promised a second opinion. `zero_result_calls + cleared_threshold ==
    calls` held on all nineteen readings ever taken. The reading procedure
    built on it — "clears its bar on nearly every call" — asked you to compare
    a number with itself.

    CHECK `suppression` BEFORE CONCLUDING ANYTHING FROM `zero_result_calls`. A
    zero-result call is two different events wearing one number: the ranker
    found nothing above the bar, or it found only what this session had already
    been shown. Just the first is evidence about the bar. `suppression` splits
    them where the surface can tell — `zero_because_already_shown` comes off
    `zero_result_calls` to leave the true ranker declines.

    `suppression` is `null` when NO row in the window reported it, and that is
    "not measured here", NOT "none suppressed". Surfaces that pass their
    exclusions into the search never see what was dropped, so they cannot say.
    Do not read a null as a zero: reading an artifact as a measurement is how
    this surface got mis-scoped once already (#3311, #3497).

    `usage` — NOTES ONLY, from `note_usage_events`, at the per-note grain
    `retrieval_logs` cannot be indexed at: `surfaced` (ranked surfacings — a
    scored surface CHOSE the record), `ambient` (the rest), `pulled` split into
    `pulled_by_agent` / `pulled_by_human`, the distinct-note counts, and
    `pull_through`. That ratio is the corpus-side precision signal: records
    surfaced often and opened never are dead weight competing for the injection
    budget every turn.

    `pull_through` is AGENT pulls over RANKED surfacings, and both halves of
    that matter. "Is this record dead weight?" is answered by any pull; "was
    that injected line useful?" — the question a threshold or a reranker is
    tuned against — only by a pull the agent made. Aggregating across the
    mcp_/rest_ prefix would silently answer the wrong one.

    `usage["by_source"]` — THE number to tune a threshold against, because the
    top-level `pull_through` is a corpus average and averages the surfaces
    together. Per surface: `notes_surfaced`, `notes_pulled`, `pull_through`,
    and `ambient: true` on surfaces whose surfacings were not scored choices
    (their ratio is null — "surfaced often, opened never" is not a judgment
    about a record nothing chose). Read it as: of the distinct notes THIS
    surface put in front of the agent, how many did the agent then open?

It is an UPPER BOUND per surface: a pull records the door it came
    through, not the surface that led there, so a note surfaced by two surfaces
    and opened once counts for both — attribution would need the session
    identity #2085 declined to invent. `by_source_failed: true` means that one
    query failed while the rest of the readout stood.

    `rule_usage` — the same question for RULES, from `rule_usage_events`:
    `surfaced` and `ambient`, `pulled` split into `pulled_by_agent` /
    `pulled_by_human`, the distinct-rule counts, and `pull_through` on the same
    definition (agent pulls over RANKED surfacings).

    `applied`, `departed` AND `distinct_rules_acted` ARE WHAT HAPPENED AFTER
    THE RULE WAS OPENED (#4213). A pull says the rule was read; these say it
    changed something. `applied` counts rules followed, `departed` rules
    deliberately not followed — kept apart rather than summed, because a
    departure carries the reason the agent gave and is evidence about the
    RULE, while an application is evidence about the agent. There is
    deliberately no count of rules read and quietly ignored: that state is what
    is left over when a rule was pulled and neither outcome arrived, and
    `read_and_unacted` below is where it is reported. Asking an agent to
    declare it would be asking it to notice an omission it is defined by not
    noticing.

    A SEPARATE BLOCK, not folded into `usage`, and reading it as one number
    with that is the mistake to avoid. The corpora differ by orders of
    magnitude — a few dozen eligible rules against thousands of notes — so a
    blended ratio would be the note ratio with noise on it and would hide the
    rule arm entirely.

    `surfaced` VS `ambient` IS THE READING THAT MATTERS HERE. `surfaced` counts
    rules a ranker chose — today only the write-path arm — and those are claims
    a pull can settle. `ambient` counts BULK DELIVERIES: the `rules_payload`
    surfaces (`enter_project`, `get_project`, `get_milestone`,
    `start_planning`, `get_task`), which hand over the whole applicable set at
    once with nobody choosing anything — plus, in rows older than milestone
    394, the SessionStart preload it removed. A large `ambient` says a bulk set
    is big and arrives often — never that it is useful, and never that it is
    read.

    `pull_through` therefore divides by `surfaced` alone. Fold the preload in
    and growing a bulk set would depress the arm's measured precision while
    trimming it would flatter it, for reasons having nothing to do with the
    arm. To judge a BULK surface instead, compare `ambient` against pulls of
    those same rules over time: a set surfaced thousands of times and opened
    never is the dead-weight signal, one level up.

    Read it against `sources["write_path_rule"]`. That arm was once believed
    never to decline — the reading that scoped #3311 — but it was the arm's
    `retrieval_logs` row being written only on calls that FOUND something, so
    the zeros were missing rather than absent (#3497). Measured since, it
    declines the large majority of its calls like any other surface.

    EVERY COUNTER BLOCK CARRIES ITS OWN COVERAGE — `complete_from` and
    `covers_window`. `complete_from` is when the number became trustworthy:
    for one source, its first recorded row; for a section that sums several,
    the LATEST of theirs, because a total is complete only once every
    contributor was being written. `covers_window: false` means the window
    reaches back further than the recording does, so the count is a fraction
    of the period it appears to describe.

    READ IT BEFORE COMPARING TWO NUMBERS, and especially before comparing
    across a deploy. A counter added last week, read over a 30-day window,
    reports a real count against an imagined denominator — and the result is
    a plausible fraction rather than an obvious zero, which is what makes it
    dangerous. That reading cost milestone #379 five steps aimed at a defect
    that did not exist.

    `covers_window` is null, never false, when nothing was ever recorded:
    "no measurement" is not "partial measurement", the same distinction
    `suppression`'s null carries a few paragraphs up.

    A SOURCE SHOWING `calls: 0` WAS RECORDING AND MADE NO CALLS. `sources`
    lists every source the table has ever held, not only those active in the
    window, so a surface that stopped firing stays visible rather than
    disappearing — being absent is reserved for a source that has never
    recorded at all. Its score fields are null, not zero: the calls are a
    real observation, the distribution is not one.

    `rule_usage_failed: true` means that read failed while the rest of the
    readout stood. The counts are still present so a caller can render, but
    they are zeros meaning "could not find out", not "nothing happened" — do
    not report a pull-through from a block carrying that flag.

    `warnings` IS THE PART TO READ FIRST (#3431). Everything above is a
    distribution; this is a verdict, and it exists because the same four
    checks were being redone by hand on every reading and were easy to
    forget. An EMPTY LIST means checked and clean — it is always present, so
    its emptiness is an answer rather than a gap. Each entry carries the
    numbers that triggered it, so you can disagree with the rule instead of
    having to redo the arithmetic:

    - `cannot_decline` — an arm that fires unasked answered every one of its
      calls. It cannot say nothing, which means it is not applying a floor.
      Only ever raised for unbidden arms known to log unconditionally: a
      search returning a list every time is doing its job, and an arm whose
      zeros were never written would flag a LOGGING bug while pointing you at
      a threshold, which is #3497 exactly. Nor for an arm whose query never
      changes — see the next entry.
    - `fixed_query_never_clears` — an arm that always searches the SAME query
      returned nothing on every call. Its score is one constant, so this is
      not a quiet window: the bar sits above that constant and no amount of
      further traffic will produce a different result. The arm is off rather
      than silent, and nothing else here would say so. The same property is
      why `cannot_decline` is not raised for these arms: with a constant
      score the decline rate is 0% or 100% by construction, so "never
      declined" is arithmetic and not evidence about the floor. Read the
      refused record (`near_miss_samples`) BEFORE moving the dial — the last
      time an arm sat here, every percentile said lower it and the refused
      record showed the refusal was right.
    - `band_hugs_floor` — the weakest tenth of what an arm returns sits on
      its floor. The bar is doing the selecting and the score is not, so
      moving that floor changes how MUCH you get, not how good it is.
    - `floor_moved_mid_window` — that arm's floor CHANGED inside the window,
      by a release or by a dial turn, so its calls were made under two bars
      and the band check above is suspended for it rather than answered
      wrongly. Ask again with a `days` starting after the named date. The
      warning replaces `band_hugs_floor` for that arm; it never accompanies
      it.
    - `no_duration` — rows written without timings. A logging gap, not a slow
      arm, and it devalues every other number from that source.
    - `surfaced_never_pulled` — distinct records shown and never opened, per
      corpus. Read their titles before touching a threshold: a record nobody
      opens is usually one whose title does not say when it matters.
    - `read_and_unacted` — distinct rules OPENED in the window that recorded
      no outcome, against the ones that did. The failure milestone 419 was
      opened on, and the worse sibling of `surfaced_never_pulled` above: a
      rule nobody opens is cheap, while a rule read and silently unchanged is
      indistinguishable from one that worked. It does not say which of the two
      causes it is — a rule mis-triggering, arriving where it does not apply,
      or a rule being ignored — and those want opposite fixes, so read the
      rules before moving anything.
    - `outcomes_never_recorded` — rules were opened and NOT ONE outcome exists
      anywhere in the window. Deliberately a separate code, and not a
      `read_and_unacted` with a zero in it: a window with no outcomes at all
      cannot tell "every rule was ignored" from "nothing on this install calls
      `rule_outcome` yet", and reporting the first would manufacture a finding
      out of an unwired feature. Wire the outcome call before reading this as
      a fact about the corpus.
    - `unregistered_source` — rows under a source missing from
      `retrieval_registry`. Its numbers are real; no verdict could be
      computed, because nothing says whether it was asked or fired unbidden.

    `silent_surfaces` IS THE HALF THE ROWS CANNOT SHOW YOU. Every check above
    reads rows, so an arm that produced none is invisible to all of them and
    looks exactly like an arm that does not exist. This list is driven by the
    declared registry instead: points expected to emit that emitted nothing.
    Points that are legitimately quiet — the web-UI-only sources on an install
    driven through MCP — are excluded by declaration rather than by silence,
    so a justified quiet never reads as a gap. The list stays EMPTY on a
    window with little traffic: on a fresh install every point is silent, and
    reporting all of them would be describing the emptiness.

    WARNINGS ARE COMPUTED OVER THE BLOCKS ABOVE, not over a second query, so
    one can never disagree with the numbers printed beside it. A window whose
    read failed produces none at all — a verdict over rows that did not load
    would describe the outage while appearing to describe the system.

    Two thresholds govern them, both settings so an install driven harder can
    say so: `retrieval_warn_min_calls` (default 30) is how much traffic a
    source needs before its silence means anything, and
    `retrieval_warn_floor_epsilon` (default 0.02) is how close to the bar
    counts as piled on it.

    Scoped to your own telemetry — a retrieval log records what your agent
    asked for, query text included, and is not a shared record kind.

    `read_failed: true` means the query itself failed — deliberately distinct
    from an empty window, because those two looked identical for weeks once
    (#2663) and every counter silently read zero.

    Args:
        days: window size, default 30. Clamped to at least 1.
        near_miss_samples: 0-20, default 0. How many of each source's highest
            scoring DECLINES to list by record, with the query that asked.
            Pass it when you are about to move a threshold; leave it off
            otherwise. See the near-miss section above for why a percentile
            alone cannot settle a bar.
    """
    return await retrieval_summary(
        current_user_id(), days=days, near_miss_samples=near_miss_samples,
    )


def register(mcp) -> None:
    mcp.tool(name="search")(search)
    mcp.tool(name="retrieval_telemetry")(retrieval_telemetry)