"""search — semantic search across the user's notes and tasks. Mirrors the existing fable-mcp contract so Claude's prior usage pattern keeps working. Differences from fable-mcp: - calls services.embeddings.semantic_search_notes directly instead of HTTP - user_id comes from mcp.current_user_id() rather than a global API key """ from __future__ import annotations import time from scribe.mcp._context import current_user_id from scribe.services.access import owner_names_for from scribe.services.embeddings import ( DEFAULT_SIMILARITY_THRESHOLD, semantic_search_milestones, semantic_search_notes, semantic_search_rules, ) from scribe.services import rulebooks as rulebooks_svc from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary async def _search_rules(uid: int, q: str, limit: int, project_id: int) -> dict: """Rules by meaning — a separate result shape because a rule IS different. A rule hit carries `why` and `how_to_apply`: they are the operational half of a rule and the session-start payload never includes them, so a caller who went looking should get the whole thing rather than a summary they then have to re-fetch. It also carries the rule's check (`verify_with`, `expires_when`, `last_verified`) when it has one — a search hit is exactly the moment someone is about to act on a rule, and "this asserts a fact nobody has confirmed" is part of what the rule says. `project_id` scopes the way it does for notes, with one difference: a GLOBAL rule (one in a rulebook) belongs to no project and applies in every one, so a scoped search returns global rules plus that project's own. Without a project it asks the whole rulebook — every rule, whatever its home — because that is the question an unscoped "is there a rule about this" is asking. `system_id` does not apply to rules. """ if project_id: raw = await semantic_search_rules(uid, q, limit=limit, project_id=project_id) else: raw = await semantic_search_rules(uid, q, limit=limit, everywhere=True) return { "results": [ { "id": rule.id, "title": rule.title, "statement": rule.statement, "when_to_apply": rule.when_to_apply or "", "why": rule.why or "", "how_to_apply": rule.how_to_apply or "", "verify_with": rule.verify_with or "", "expires_when": rule.expires_when or "", # Only on a rule that carries a check; its absence means the # rule is a decision, not that nobody has looked. **( {"last_verified": rulebooks_svc.last_verified_label(rule)} if rule.verify_with else {} ), "topic_id": rule.topic_id, "project_id": rule.project_id, "similarity": float(score), } for score, rule in raw ], "total": len(raw), } async def _search_milestones(uid: int, q: str, limit: int, project_id: int) -> dict: """Milestones by meaning — "is there already a plan for this?" (milestone 415). Its own result shape, like rules: a milestone is a plan with progress, not a note with a body. The plan itself is left out — get_milestone reads it — because a search hit is for recognising a plan, and bodies run long. Not part of content_type="all", whose results are note-shaped. """ raw = await semantic_search_milestones(uid, q, project_id=project_id or None, limit=limit) progress: dict[int, dict] = {} if raw: from scribe.services import milestones as milestones_svc for pid in {m.project_id for _s, m in raw}: for row in await milestones_svc.get_project_milestone_summary(uid, pid): progress[row["id"]] = row return { "results": [ { "id": m.id, "title": m.title, "description": m.description or "", "status": m.status, "project_id": m.project_id, "total": progress.get(m.id, {}).get("total", 0), "completed": progress.get(m.id, {}).get("completed", 0), "similarity": float(score), } for score, m in raw ], "total": len(raw), } async def search( q: str, content_type: str = "all", limit: int = 10, project_id: int = 0, system_id: int = 0, ) -> dict: """Semantic search over the user's existing notes and tasks — Scribe's recall. Reach for this BEFORE answering a question about the user's work or starting a task: the user's second-brain almost always already holds related prior art. Check for an existing ticket before opening a new one (search with content_type='task'), and for prior notes/decisions before re-deriving them. Treating Scribe as the first place to look — not a place to only write — is the difference between it being a trustworthy record and a write-only log. Args: q: search query string. content_type: 'all' (default), 'note' (notes only), 'task' (tasks only), or 'rule' (RULES only — the operator's standing instructions, searchable by meaning since milestone 307). Reach for 'rule' when you want to know whether a standing instruction covers something: "is there a rule about release tagging?". A hit carries the rule's `why` and `how_to_apply`, which the session-start payload does not. With a project_id, rules come back as the global rules plus that project's own; with 0, every rule in the rulebook. Or 'milestone' (PLANS): reach for it before start_planning to ask whether a plan for this work already exists — a match is where new steps go (create_records(milestone_id=…)), not a reason to open a second milestone. Hits carry title, description, status and progress; get_milestone reads the plan. Not included in 'all'. limit: maximum number of results (1-50). project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID whenever a project is in scope (the one you entered with enter_project) — otherwise this searches across ALL projects and bleeds unrelated work into the result set. 0 = search everything (use only when you genuinely want a cross-project sweep). A LESSON is the exception and arrives whatever the scope: the kind records an insight that transfers, so it is reachable from a project it was not written on. system_id: Narrow to records tagged to one System (a named subsystem/area — enter_project lists them). Use when investigating a specific subsystem: it cuts the candidates to records someone deliberately filed under that area. 0 = no system filter. list_system_records gives the same slice unranked. Returns: {"results": [{"id", "title", "body", "is_task", "tags", "similarity"}], "total": int} A result marked `shared: true` with an `owner` belongs to another user — that person's suggestion, not the operator's own record or settled practice. Weigh it on its merits and say whose it is when you use it. """ uid = current_user_id() limit = max(1, min(limit, 50)) if content_type == "rule": return await _search_rules(uid, q, limit, project_id) if content_type == "milestone": return await _search_milestones(uid, q, limit, project_id) is_task = {"note": False, "task": True}.get(content_type) # None => any t0 = time.perf_counter() report: dict = {} raw = await semantic_search_notes( uid, q, limit=limit, is_task=is_task, project_id=project_id or None, system_id=system_id or None, # A LESSON is reachable from any project (milestone 385). The kind # exists to carry an insight to the next project, so a project filter # that hid it would hide it precisely where it is worth having. Only # the project filter widens — everything else about the scoping holds, # and a caller narrowing by `content_type` still gets what it asked # for. This is the explicit search, where the operator asked; the # unasked-for arms decide their own budget separately. include_global_kinds=True, # An explicit search reaches everything the operator may read, including # records shared with them one-to-one. scope="read", report=report, ) record_retrieval( user_id=uid, source="mcp_search", query=q, threshold=DEFAULT_SIMILARITY_THRESHOLD, limit=limit, project_id=project_id or None, is_task=is_task, results=raw, duration_ms=(time.perf_counter() - t0) * 1000.0, best_available=report.get("best_available_score"), best_available_id=report.get("best_available_id"), searched=bool(report.get("searched", True)), ) owners = await owner_names_for( {int(note.user_id) for _s, note in raw if note.user_id != uid} ) return { "results": [ { "id": note.id, "title": note.title, "body": (note.body or "")[:240], "is_task": bool(note.is_task), "tags": list(note.tags or []), "similarity": float(score), **( {"shared": True, "owner": owners.get(int(note.user_id))} if note.user_id != uid else {} ), } for score, note in raw ], "total": len(raw), } async def retrieval_telemetry( days: int = 30, near_miss_samples: int = 0, ) -> dict: """What the retrieval telemetry says about YOUR surfaces, over a window. The read half of the tuning loop, whose write half is `tune_retrieval` (#2975, #4102). Reach for it before moving any floor or budget, and read the records it names rather than its percentiles alone: this readout has been measured pointing the WRONG WAY — 69 consecutive declines where every percentile said "lower the bar" and the refused record was a false positive — so `near_miss_samples=5` and opening the ids it returns is the step that separates a real miss from a bar doing its job. Three readouts, from the three tables built for them: `sources` — per retrieval surface (`auto_inject`, `write_path`, `mcp_search`, …), from `retrieval_logs`: `calls`, `zero_result_calls`, `near_misses`, the `top_score` spread (p10/p50/p90/min/max), `avg_result_count` and `p90_duration_ms`. THE NUMBER TO READ FIRST IS `near_misses.p90`, AGAINST THE THRESHOLD IN FORCE FOR THAT SURFACE. It is measured on the calls the BAR turned away — zero-result calls, minus the ones whose zero was a repeat the reader had already been shown — using the best score the ranker reached before the bar rejected it. So it is the one figure here that says something the bar cannot make true by construction, and `max` is always below the threshold: an above-bar candidate nobody excluded would have been returned. A bar at 0.72 turning away a stream of 0.71s is set too high by a hair and the surface is losing hits it should have had. The same bar turning away 0.30s is working, and the corpus simply had nothing. Both render as a zero-result call, and nothing else in this readout tells them apart. `near_miss_samples` (0-20, default 0) TURNS THE PERCENTILES INTO RECORDS YOU CAN READ. Each source then carries `near_miss_records`: its highest scoring declines, each with the `record_id` the bar refused and the `query` that asked. Reach for it whenever you are about to move a threshold. THE PERCENTILE CANNOT SETTLE A BAR ON ITS OWN, and this is the whole reason the parameter exists. `near_misses.p90` says mass is sitting just under the line; it says nothing about whether that mass is RELEVANT, and those are different questions. Lowering a bar to where the mass is, without reading what is there, is choosing a firing rate rather than a quality. Pull-through cannot referee it either — the injected rule line already carries title and trigger, so a session can comply without ever calling `get_rule`, which makes rule pull-through understate usefulness by construction. Reading the rejected records is the method that actually answers it. Off by default because it is a LISTING, not a statistic: it is for the moment you are making a decision, not for every readout. `near_misses` is `null` when no declining call in the window measured it — rows written before #3670 shipped cannot know. That is "not measured", not "nothing came close"; a 0.0 there would be a claim about the corpus invented out of a caller's silence. A NULL HERE NOW MEANS ONE THING, which it did not at first. A semantic search returns nothing three ways WITHOUT having run — an empty query, an unavailable embedder, and a failed database query — and each used to write a row indistinguishable from a ranker that declined (#3765). Those calls no longer write a row at all, on the same reasoning that already keeps a blank command out of the log: a row there reports a call that never happened and drags the clear rate down with phantom declines. So a null is "searched, and nothing came close", and a broken search shows up as a WARNING in the application log rather than as a quiet zero in here. THERE IS NO `cleared_threshold` ANY MORE, and if you remember one, that memory is of a tautology (#3670). The search applies the bar before returning, so every returned result cleared it by construction and a call with no results has no score to compare: the field was true exactly when `result_count > 0`, i.e. it was `calls - zero_result_calls` under a name that promised a second opinion. `zero_result_calls + cleared_threshold == calls` held on all nineteen readings ever taken. The reading procedure built on it — "clears its bar on nearly every call" — asked you to compare a number with itself. CHECK `suppression` BEFORE CONCLUDING ANYTHING FROM `zero_result_calls`. A zero-result call is two different events wearing one number: the ranker found nothing above the bar, or it found only what this session had already been shown. Just the first is evidence about the bar. `suppression` splits them where the surface can tell — `zero_because_already_shown` comes off `zero_result_calls` to leave the true ranker declines. `suppression` is `null` when NO row in the window reported it, and that is "not measured here", NOT "none suppressed". Surfaces that pass their exclusions into the search never see what was dropped, so they cannot say. Do not read a null as a zero: reading an artifact as a measurement is how this surface got mis-scoped once already (#3311, #3497). `usage` — NOTES ONLY, from `note_usage_events`, at the per-note grain `retrieval_logs` cannot be indexed at: `surfaced` (ranked surfacings — a scored surface CHOSE the record), `ambient` (the rest), `pulled` split into `pulled_by_agent` / `pulled_by_human`, the distinct-note counts, and `pull_through`. That ratio is the corpus-side precision signal: records surfaced often and opened never are dead weight competing for the injection budget every turn. `pull_through` is AGENT pulls over RANKED surfacings, and both halves of that matter. "Is this record dead weight?" is answered by any pull; "was that injected line useful?" — the question a threshold or a reranker is tuned against — only by a pull the agent made. Aggregating across the mcp_/rest_ prefix would silently answer the wrong one. `usage["by_source"]` — THE number to tune a threshold against, because the top-level `pull_through` is a corpus average and averages the surfaces together. Per surface: `notes_surfaced`, `notes_pulled`, `pull_through`, and `ambient: true` on surfaces whose surfacings were not scored choices (their ratio is null — "surfaced often, opened never" is not a judgment about a record nothing chose). Read it as: of the distinct notes THIS surface put in front of the agent, how many did the agent then open? It is an UPPER BOUND per surface: a pull records the door it came through, not the surface that led there, so a note surfaced by two surfaces and opened once counts for both — attribution would need the session identity #2085 declined to invent. `by_source_failed: true` means that one query failed while the rest of the readout stood. `rule_usage` — the same question for RULES, from `rule_usage_events`: `surfaced` and `ambient`, `pulled` split into `pulled_by_agent` / `pulled_by_human`, the distinct-rule counts, and `pull_through` on the same definition (agent pulls over RANKED surfacings). `applied`, `departed` AND `distinct_rules_acted` ARE WHAT HAPPENED AFTER THE RULE WAS OPENED (#4213). A pull says the rule was read; these say it changed something. `applied` counts rules followed, `departed` rules deliberately not followed — kept apart rather than summed, because a departure carries the reason the agent gave and is evidence about the RULE, while an application is evidence about the agent. There is deliberately no count of rules read and quietly ignored: that state is what is left over when a rule was pulled and neither outcome arrived, and `read_and_unacted` below is where it is reported. Asking an agent to declare it would be asking it to notice an omission it is defined by not noticing. A SEPARATE BLOCK, not folded into `usage`, and reading it as one number with that is the mistake to avoid. The corpora differ by orders of magnitude — a few dozen eligible rules against thousands of notes — so a blended ratio would be the note ratio with noise on it and would hide the rule arm entirely. `surfaced` VS `ambient` IS THE READING THAT MATTERS HERE. `surfaced` counts rules a ranker chose — today only the write-path arm — and those are claims a pull can settle. `ambient` counts BULK DELIVERIES: the `rules_payload` surfaces (`enter_project`, `get_project`, `get_milestone`, `start_planning`, `get_task`), which hand over the whole applicable set at once with nobody choosing anything — plus, in rows older than milestone 394, the SessionStart preload it removed. A large `ambient` says a bulk set is big and arrives often — never that it is useful, and never that it is read. `pull_through` therefore divides by `surfaced` alone. Fold the preload in and growing a bulk set would depress the arm's measured precision while trimming it would flatter it, for reasons having nothing to do with the arm. To judge a BULK surface instead, compare `ambient` against pulls of those same rules over time: a set surfaced thousands of times and opened never is the dead-weight signal, one level up. Read it against `sources["write_path_rule"]`. That arm was once believed never to decline — the reading that scoped #3311 — but it was the arm's `retrieval_logs` row being written only on calls that FOUND something, so the zeros were missing rather than absent (#3497). Measured since, it declines the large majority of its calls like any other surface. EVERY COUNTER BLOCK CARRIES ITS OWN COVERAGE — `complete_from` and `covers_window`. `complete_from` is when the number became trustworthy: for one source, its first recorded row; for a section that sums several, the LATEST of theirs, because a total is complete only once every contributor was being written. `covers_window: false` means the window reaches back further than the recording does, so the count is a fraction of the period it appears to describe. READ IT BEFORE COMPARING TWO NUMBERS, and especially before comparing across a deploy. A counter added last week, read over a 30-day window, reports a real count against an imagined denominator — and the result is a plausible fraction rather than an obvious zero, which is what makes it dangerous. That reading cost milestone #379 five steps aimed at a defect that did not exist. `covers_window` is null, never false, when nothing was ever recorded: "no measurement" is not "partial measurement", the same distinction `suppression`'s null carries a few paragraphs up. A SOURCE SHOWING `calls: 0` WAS RECORDING AND MADE NO CALLS. `sources` lists every source the table has ever held, not only those active in the window, so a surface that stopped firing stays visible rather than disappearing — being absent is reserved for a source that has never recorded at all. Its score fields are null, not zero: the calls are a real observation, the distribution is not one. `rule_usage_failed: true` means that read failed while the rest of the readout stood. The counts are still present so a caller can render, but they are zeros meaning "could not find out", not "nothing happened" — do not report a pull-through from a block carrying that flag. `warnings` IS THE PART TO READ FIRST (#3431). Everything above is a distribution; this is a verdict, and it exists because the same four checks were being redone by hand on every reading and were easy to forget. An EMPTY LIST means checked and clean — it is always present, so its emptiness is an answer rather than a gap. Each entry carries the numbers that triggered it, so you can disagree with the rule instead of having to redo the arithmetic: - `cannot_decline` — an arm that fires unasked answered every one of its calls. It cannot say nothing, which means it is not applying a floor. Only ever raised for unbidden arms known to log unconditionally: a search returning a list every time is doing its job, and an arm whose zeros were never written would flag a LOGGING bug while pointing you at a threshold, which is #3497 exactly. Nor for an arm whose query never changes — see the next entry. - `fixed_query_never_clears` — an arm that always searches the SAME query returned nothing on every call. Its score is one constant, so this is not a quiet window: the bar sits above that constant and no amount of further traffic will produce a different result. The arm is off rather than silent, and nothing else here would say so. The same property is why `cannot_decline` is not raised for these arms: with a constant score the decline rate is 0% or 100% by construction, so "never declined" is arithmetic and not evidence about the floor. Read the refused record (`near_miss_samples`) BEFORE moving the dial — the last time an arm sat here, every percentile said lower it and the refused record showed the refusal was right. - `band_hugs_floor` — the weakest tenth of what an arm returns sits on its floor. The bar is doing the selecting and the score is not, so moving that floor changes how MUCH you get, not how good it is. - `floor_moved_mid_window` — that arm's floor CHANGED inside the window, by a release or by a dial turn, so its calls were made under two bars and the band check above is suspended for it rather than answered wrongly. Ask again with a `days` starting after the named date. The warning replaces `band_hugs_floor` for that arm; it never accompanies it. - `no_duration` — rows written without timings. A logging gap, not a slow arm, and it devalues every other number from that source. - `surfaced_never_pulled` — distinct records shown and never opened, per corpus. Read their titles before touching a threshold: a record nobody opens is usually one whose title does not say when it matters. - `read_and_unacted` — distinct rules OPENED in the window that recorded no outcome, against the ones that did. The failure milestone 419 was opened on, and the worse sibling of `surfaced_never_pulled` above: a rule nobody opens is cheap, while a rule read and silently unchanged is indistinguishable from one that worked. It does not say which of the two causes it is — a rule mis-triggering, arriving where it does not apply, or a rule being ignored — and those want opposite fixes, so read the rules before moving anything. - `outcomes_never_recorded` — rules were opened and NOT ONE outcome exists anywhere in the window. Deliberately a separate code, and not a `read_and_unacted` with a zero in it: a window with no outcomes at all cannot tell "every rule was ignored" from "nothing on this install calls `rule_outcome` yet", and reporting the first would manufacture a finding out of an unwired feature. Wire the outcome call before reading this as a fact about the corpus. - `unregistered_source` — rows under a source missing from `retrieval_registry`. Its numbers are real; no verdict could be computed, because nothing says whether it was asked or fired unbidden. `silent_surfaces` IS THE HALF THE ROWS CANNOT SHOW YOU. Every check above reads rows, so an arm that produced none is invisible to all of them and looks exactly like an arm that does not exist. This list is driven by the declared registry instead: points expected to emit that emitted nothing. Points that are legitimately quiet — the web-UI-only sources on an install driven through MCP — are excluded by declaration rather than by silence, so a justified quiet never reads as a gap. The list stays EMPTY on a window with little traffic: on a fresh install every point is silent, and reporting all of them would be describing the emptiness. WARNINGS ARE COMPUTED OVER THE BLOCKS ABOVE, not over a second query, so one can never disagree with the numbers printed beside it. A window whose read failed produces none at all — a verdict over rows that did not load would describe the outage while appearing to describe the system. Two thresholds govern them, both settings so an install driven harder can say so: `retrieval_warn_min_calls` (default 30) is how much traffic a source needs before its silence means anything, and `retrieval_warn_floor_epsilon` (default 0.02) is how close to the bar counts as piled on it. Scoped to your own telemetry — a retrieval log records what your agent asked for, query text included, and is not a shared record kind. `read_failed: true` means the query itself failed — deliberately distinct from an empty window, because those two looked identical for weeks once (#2663) and every counter silently read zero. Args: days: window size, default 30. Clamped to at least 1. near_miss_samples: 0-20, default 0. How many of each source's highest scoring DECLINES to list by record, with the query that asked. Pass it when you are about to move a threshold; leave it off otherwise. See the near-miss section above for why a percentile alone cannot settle a bar. """ return await retrieval_summary( current_user_id(), days=days, near_miss_samples=near_miss_samples, ) def register(mcp) -> None: mcp.tool(name="search")(search) mcp.tool(name="retrieval_telemetry")(retrieval_telemetry)