FabledGrimoire/src/grimoire/services/retrieval_registry.py

"""Every point that retrieves or surfaces a record, declared.

WHY A DECLARED LIST WHEN THE ROWS ALREADY NAME THEIR SOURCE. Because the rows
can only describe arms that FIRED. A surface that emits nothing writes no row,
so it is indistinguishable from a surface that does not exist — and the two
call for opposite responses. `retrieval_telemetry` could report perfect health
for a window in which half the arms never ran, and a "silent surface"
finding could only be made by a human who happened to know which arms were
supposed to be there.

This module is that knowledge, written down. It carries the two facts the rows
structurally cannot:

  * **asked or unbidden.** `mcp_search` returning nothing is a search with no
    matches, which is its job. `auto_inject` returning nothing on every call
    for a week is an arm that has lost the ability to speak. Same row shape,
    opposite meanings, and only the caller's intent separates them.
  * **expected to emit, or deliberately quiet with a reason.** A justified
    silence must not read as a gap. `rest_*` sources fire only when a
    person opens the web UI; on an install driven entirely through MCP their
    absence is correct and permanent, and reporting it every window would
    train the reader to ignore the list that also carries the real ones.

NOT `retrieval_surfaces.SURFACES`, and the distinction is deliberate rather
than duplication. That registry answers "what can be TUNED" — it carries a
floor key, a budget key and the measurement stamp behind their defaults, and
it deliberately EXCLUDES the reserved slots (`preference_slot`, `reuse_slot`,
`lesson_slot`) because a budget of 1 is their feature and exposing it would
invite setting it to 0. This registry answers "what can be MEASURED", and the
reserved slots belong in it precisely because they are judgeable without being
tunable. One is a control panel; the other is an inventory. A test asserts
every tunable surface also appears here, so the two cannot drift apart.

ADDING A RANKED ARM MEANS WRITING ITS SPEC. The
ranked rows are read off `retrieval_pipeline.RANKED`, each spec declaring its
own (`Declared`). Any OTHER point — a lookup, an asked search, an ambient
carrier, a pull — still means adding a row here. `tests/test_retrieval_registry.py`
walks the call sites with the ast module and fails on a source it cannot find
below — deliberately not a grep, because two of the sources in this file
(`wide_net`, `preference_slot`) reach their recorder through a
module constant and a grep for `source="` misses both.
"""
from __future__ import annotations

from dataclasses import dataclass

from grimoire.services.retrieval_pipeline import (
    MOMENT_RULE_SOURCE, NOTE_SOURCES, NOTE_SURFACED_SOURCES, RANKED, RULE_ARMS,
    RULE_SOURCES,
)

# ── How a point is reached ────────────────────────────────────────────────
#
# UNBIDDEN: fires on its own, against a query the agent did not write — a
# prompt, a file being written, a command about to run. It interrupts, so it
# must be able to stay quiet, and a stream of empty calls is a real signal.
#
# ASKED: someone called a search and wants a ranked list. Returning nothing is
# an answer, not a failure, so "cannot decline" must never fire on these.
#
# AMBIENT: records travel with a reply that was going to be sent anyway (a
# project handshake, a planning read). Nothing was ranked and nothing was
# chosen, so score-shaped warnings do not apply.
#
# PULL: a reader opened a record. The terminal event of the whole system, and
# the numerator of pull-through.
UNBIDDEN = "unbidden"
ASKED = "asked"
AMBIENT = "ambient"
PULL = "pull"


@dataclass(frozen=True)
class Point:
    """One place a record reaches a reader."""

    source: str
    """The telemetry `source` value. MUST equal the string the call site passes."""

    kind: str
    """UNBIDDEN / ASKED / AMBIENT / PULL — see above."""

    what: str
    """One line, for an agent reading a warning that names this source."""

    expects_traffic: bool = True
    """Whether silence over an active window is worth reporting.

    False means "this can be quiet forever and that is correct" — and then
    `quiet_because` must say why, so a reader meets a decision rather than a
    hole.
    """

    quiet_because: str = ""

    logs_unconditionally: bool = True
    """Whether this arm writes a row even when it returns NOTHING.

    The reason this field is not merely documentation: an arm that logs only
    its hits has a zero-result count that is structurally 0, and a "cannot
    decline" warning computed over that would fire on a LOGGING BUG while
    reporting a ranking problem, sending the reader to move a floor that was
    never involved. An arm whose logging has
    not been confirmed unconditional is not eligible for that warning; it is
    excluded from the check rather than trusted.
    """


def _p(source, kind, what, **kw) -> tuple[str, Point]:
    return source, Point(source=source, kind=kind, what=what, **kw)


POINTS: dict[str, Point] = dict([
    # ── Unbidden push arms ───────────────────────────────────────────────
    #
    # The RANKED ones are read off the pipeline's specs: every arm, and every stage that runs a query of its own and records
    # under its own name, declares its row where it is defined. What stays
    # here is what no spec describes — the lookups below, and everything asked,
    # ambient or pulled.
    *(_p(spec.source, UNBIDDEN, spec.declared.what,
         expects_traffic=not spec.declared.quiet_because,
         quiet_because=spec.declared.quiet_because)
      for spec in RANKED),
    # A LOOKUP, not a ranker: a record the operator named by number
    # in the message, fetched by id. No score and no bar, so it writes no
    # retrieval_logs row and no score-shaped warning can apply; its rows are
    # in `note_usage_events`, where pull-through says whether naming a record
    # is followed by opening it.
    _p("named_ref", UNBIDDEN,
       "a record the operator named by number, looked up directly",
       expects_traffic=False,
       quiet_because="speaks only when a message names a record by its id; "
                     "a window in which none did is correctly silent here"),
    # The moment arm. A LOOKUP, not a ranker: a rule mounted on
    # a moment arrives when that moment happens, with no score, so it writes no
    # retrieval_logs row and no score-shaped warning can apply. Its rows are in
    # `rule_usage_events`, each carrying the moment in `detail`.
    _p("moment_rule", UNBIDDEN,
       "rules mounted on a moment of work, delivered when that moment happens",
       expects_traffic=False,
       quiet_because="speaks only once some rule is mounted on a moment; an "
                     "install where none is mounted is correctly silent here"),
    # The rulings arm. A LOOKUP, not a ranker: a path falls
    # under a System's patterns or it does not, so nothing here writes to
    # retrieval_logs and no score-shaped warning can apply. Its rows are in
    # `system_usage_events`, read back as `retrieval_telemetry`'s
    # `system_usage` block.
    _p("rulings_pre_tool", UNBIDDEN,
       "an area's rulings, shown because a command touched its files",
       expects_traffic=False,
       quiet_because="speaks only once a System has both path_patterns and a "
                     "Rulings section; an install where none does is "
                     "correctly silent here"),
    _p("rulings_write_path", UNBIDDEN,
       "an area's rulings, shown because an edit touched its files",
       expects_traffic=False,
       quiet_because="same as rulings_pre_tool — no System with patterns "
                     "and rulings, nothing to show"),

    # ── Asked: a caller wanted a ranked list ─────────────────────────────
    _p("mcp_search", ASKED, "an agent called search"),
    _p("wide_net", ASKED, "an agent asked what might apply, with no bar"),
    _p("rest_search", ASKED, "the web UI searched", expects_traffic=False,
       quiet_because="only fires when a person uses the web UI; an install "
                     "driven entirely through MCP is correctly silent here"),
    _p("browse_search", ASKED, "the browse listing searched", expects_traffic=False,
       quiet_because="same as rest_search — a human-only entry point"),

    # ── Ambient: carried along with a reply already being sent ───────────
    _p("enter_project", AMBIENT, "the project handshake"),
    _p("process_skill_sync", AMBIENT, "stored Processes synced into skills"),
    _p("start_planning", AMBIENT, "the rules listed when a plan is opened"),
    _p("get_task", AMBIENT, "the rules listed beside a task"),
    _p("get_project", AMBIENT, "the rules listed beside a project"),
    _p("get_milestone", AMBIENT, "the rules listed beside a milestone"),

    # ── The write-path menu's own arms ───────────────────────────────────
    #
    # These are NOT in retrieval_logs. The place arm carries no score and so
    # has no row there at all; declaring it here is what lets the arm making
    # the strongest possible claim, "there is already a canonical helper in
    # this exact file", be measured.
    _p("write_path_place", AMBIENT, "a snippet already placed in this very file"),
    _p("write_path_semantic", AMBIENT, "prior art matched by meaning"),
    _p("write_path_sync", AMBIENT,
       "records that should be updated alongside this edit"),

    # ── Pull: a reader opened the record ─────────────────────────────────
    _p("mcp_get_note", PULL, "an agent opened a note"),
    _p("mcp_get_task", PULL, "an agent opened a task"),
    _p("mcp_get_snippet", PULL, "an agent opened a snippet"),
    _p("mcp_get_process", PULL, "an agent opened a process"),
    _p("mcp_get_lesson", PULL, "an agent opened a lesson"),
    _p("mcp_get_rule", PULL, "an agent opened a rule"),
    _p("mcp_get_system", PULL, "an agent opened a System", expects_traffic=False,
       quiet_because="a System's rulings arrive in full in the rulings line, "
                     "so opening one is the exception rather than the "
                     "measure; a window without it is not a gap"),
    _p("rest_note", PULL, "a person opened a note", expects_traffic=False,
       quiet_because="web UI only"),
    _p("rest_task", PULL, "a person opened a task", expects_traffic=False,
       quiet_because="web UI only"),
    _p("rest_snippet", PULL, "a person opened a snippet", expects_traffic=False,
       quiet_because="web UI only"),
    _p("rest_lesson", PULL, "a person opened a lesson", expects_traffic=False,
       quiet_because="web UI only"),
    _p("rest_rule", PULL, "a person opened a rule", expects_traffic=False,
       quiet_because="web UI only"),
])


# Call sites that pass `source` as a VARIABLE, and what they can pass.
#
# WHY THESE ARE DECLARED RATHER THAN RESOLVED. The registry test reads the
# call sites with `ast`, which settles a literal and a module constant but not
# a value that arrives through a parameter or a dict key. Rather than let
# those sites go unchecked — the silent half of a guard that appears to cover
# everything — they are named here, and the test FAILS if it meets an
# unresolved site that is not in this list.
#
# THE HONEST LIMIT, stated so nobody over-trusts this: the test pins the SITE,
# not the values. Adding a fourth arm inside `plugin_context`'s `by_arm` fan
# out would not fail here. What catches that is `assert_registered`, which the
# recorders call as the row is written.
# Keyed by the EXACT string the extractor produces, matched by equality rather
# than by prefix. The paths are relative to `src/` and so begin `grimoire/`; a
# prefix spelled any other way silently matches nothing, and every site sails
# through a check that looks like it is running.
FAN_OUT_SITES: dict[str, tuple[str, ...]] = {
    # The write path's two LOOKUPS. Its ranked arm (`write_path_semantic`)
    # records through the pipeline's site below.
    "grimoire/services/plugin_context.py::record_surfaced(source=arm)": (
        "write_path_place", "write_path_sync",
    ),
    "grimoire/services/system_rulings.py::record_system_surfaced(source=source)": (
        "rulings_pre_tool", "rulings_write_path",
    ),
    "grimoire/services/rulebooks.py::record_rule_surfaced(source=source)": (
        "enter_project", "start_planning", "get_task", "get_project",
        "get_milestone",
    ),
    # The one retrieval pipeline: every ranked arm's call row
    # and surfacing rows — both corpora — are written by these sites, so
    # `source` arrives as a value. The values are the pipeline's own specs,
    # read rather than restated, so an arm added there is declared here by
    # construction.
    "grimoire/services/retrieval_pipeline.py::record_retrieval(source=source)": (
        RULE_SOURCES + NOTE_SOURCES
    ),
    "grimoire/services/retrieval_pipeline.py::record_surfaced(source=source)": (
        NOTE_SURFACED_SOURCES
    ),
    "grimoire/services/retrieval_pipeline.py::record_rule_surfaced(source=source)": (
        tuple(arm.source for arm in RULE_ARMS)
    ),
    # The reply moment's mounted half: it records under the
    # moment arm's source, read from the pipeline's constant so the two doors
    # cannot drift apart on the name.
    "grimoire/services/moment_delivery.py::record_rule_surfaced(source=<expression>)": (
        MOMENT_RULE_SOURCE,
    ),
}


def get_point(source: str) -> Point | None:
    """The registered point for a source, or None if it is not declared."""
    return POINTS.get(source)


def sources_expected_to_emit() -> list[str]:
    """Every point whose silence over an ACTIVE window is worth reporting."""
    return [s for s, p in POINTS.items() if p.expects_traffic]


def is_registered(source: str) -> bool:
    """Is this source declared?

    Read by the TELEMETRY READOUT, not by the recorders, and that placement is
    the design rather than an accident of where it was easy. Validating at
    write time would put a dict lookup on every retrieval to catch a mistake
    that is only ever made once, when an arm is added — and it could not
    raise anyway, because telemetry that breaks its caller is the worse bug
    (`record_retrieval` is fire-and-forget for exactly that reason). So the
    check sits where the cost is already paid and the reader is already
    looking: a source that appears in the rows and not in this file comes back
    as an `unregistered_source` warning.

    That is what covers FAN_OUT_SITES above. The static test pins those sites
    but not the values they can pass, so a fourth arm added inside one would
    slip past it — and then show up here the first time it fires.
    """
    return source in POINTS