feat(retrieval): a System's charter becomes an answer, not just a filter (#4251)

Step 2 of #4251. A System's `description` is a charter — several hundred words
saying what belongs in that area and what does not — and it is the answer to
"which part of this codebase does X live in". There was no semantic path to
one: `list_systems` enumerates, and `search(system_id=…)` uses a System as a
FILTER over notes. So a System could narrow a search and could never be the
answer to one, and an agent asking where a record belonged had to read every
charter or guess.

ITS OWN SEARCH, not a `content_type` over notes, for the reason note 3163
gives about milestones: the row could be shared, the search cannot. A charter
competing with the whole note corpus for one top-k is outranked by the records
filed under it — the right answer crowded out by its own contents — and "where
does this belong?" is a different question from "what prior art is there?",
which a caller asking one should not have to read past answers to.

So `system_embeddings` (0107) joins note_, rule_ and milestone_embeddings as
the fourth sibling, with `system_document`, `upsert_system_embedding`,
`semantic_search_systems`, a startup backfill and `search(content_type=
"system")`. Scoped like milestones: with a project_id, that project's Systems
if the caller can read the project (rule 78); without one, the caller's own.
Archived Systems are excluded — an archived area is one the operator has said
is no longer where things go, which is exactly the question being asked.

`system_document` is the plainest of the four shapes on purpose. A charter is
already written as the thing this search has to match, in the words someone
asking would use — so there is no trigger to synthesise as `rule_document`
must, and no second record to gather as `task_document` must. The stored
charter IS the sharp document, the way a snippet's is. `color`, `status` and
`order_index` stay out: presentation and bookkeeping, and a vector carrying
them would be answering a question nobody asks of a charter.

The search publishes `report["best_chunk"]` from the start rather than being
retrofitted, which is what #4251 asked of any fourth search. It matters more
here than anywhere: a charter runs long and a result shows its NAME, so a match
on the paragraph that actually decides where a record belongs would otherwise
be previewed by two words that cannot say.

The id that comes back is the one `system_id`, `system_ids` and
`list_system_records` already take, so the answer to "where does this belong?"
is directly usable as "show me what is there" and as "file it here".

`embed_system` sits beside `notes.embed_note` at the service for #2056's
reason — every door gets it by construction. Not called on delete: that is a
soft delete and the search joins through `System`, so the vectors are already
unreachable, and leaving them means a restore is findable again immediately.
`system_embeddings` is declared in backup's `_NOT_INCLUDED` as derived, beside
its three siblings.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

tests/test_mcp_tool_search.py

@@ -1,7 +1,7 @@
 """search tool — proves the tool pattern (context + service call + dict shape).
 
 Service call is mocked; no DB needed."""
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -223,7 +223,8 @@ async def test_the_docstring_names_every_kind_the_tool_accepts():
     doc = search_tool.__doc__ or ""
     for facet in FACET_TYPES:
         assert f"'{facet}'" in doc, f"{facet} is accepted but never documented"
-    assert "'rule'" in doc and "'milestone'" in doc and "'all'" in doc
+    for own in ("'rule'", "'milestone'", "'system'", "'all'"):
+        assert own in doc, f"{own} dispatches somewhere and is never documented"
 
 
 @pytest.mark.asyncio
@@ -329,3 +330,80 @@ async def test_milestone_search_is_its_own_shape_and_scopes_to_the_project():
         "status": "active", "project_id": 30, "total": 0, "completed": 0,
         "similarity": 0.81,
     }]
+
+
+# --- systems: the charter as an answer, not a filter (#4251) -----------------
+
+
+@pytest.mark.asyncio
+async def test_system_search_is_its_own_shape_and_carries_the_matched_passage():
+    """A System's charter runs to several hundred words and a result shows its
+    NAME — so a match on the paragraph that actually decides where a record
+    belongs would be previewed by two words that cannot. The passage comes
+    along, marked as the passage (#4243)."""
+    _user_id_ctx.set(7)
+    charter = (
+        "Opening sentence about the area in general. "
+        + "x" * 400
+        + " Retrieval telemetry and the floors it judges belong here."
+    )
+    fake = MagicMock(id=3, project_id=2, description=charter)
+    # `name` is MagicMock's own constructor kwarg — passed in, it names the
+    # mock and leaves `.name` a mock object, which is note #2833's whole point.
+    fake.name = "Retrieval & recall"
+
+    async def _system_search(uid, q, **kwargs):
+        kwargs["report"]["best_chunk"] = {
+            3: {"index": 2, "text": "Retrieval telemetry and the floors it judges belong here."}
+        }
+        return [(0.81, fake)]
+
+    with patch("scribe.mcp.tools.search.semantic_search_systems", _system_search):
+        out = await search(q="where does retrieval telemetry go?",
+                           content_type="system", project_id=2)
+
+    assert out["total"] == 1
+    row = out["results"][0]
+    assert row["id"] == 3
+    assert row["name"] == "Retrieval & recall"
+    assert row["project_id"] == 2
+    assert row["similarity"] == 0.81
+    # The passage that won, not the charter's opening — and SAID to be that.
+    assert row["matched"] == (
+        "Retrieval telemetry and the floors it judges belong here."
+    )
+    assert row["matched_is"] == "matched_passage"
+    assert row["body_length"] == len(charter)
+    assert "read_full" in row, "a 56-character span of a 500-character charter"
+    # The charter itself does not ride along: get_system reads it.
+    assert "description" not in row
+
+
+@pytest.mark.asyncio
+async def test_system_search_scopes_to_the_project_it_was_given():
+    """Systems are per-project and a charter from another project is not an
+    answer to "where does this belong here?"."""
+    _user_id_ctx.set(7)
+    mock = AsyncMock(return_value=[])
+    with patch("scribe.mcp.tools.search.semantic_search_systems", mock):
+        await search(q="x", content_type="system", project_id=30)
+    assert mock.call_args.kwargs["project_id"] == 30
+
+    mock.reset_mock()
+    with patch("scribe.mcp.tools.search.semantic_search_systems", mock):
+        await search(q="x", content_type="system")
+    assert mock.call_args.kwargs["project_id"] is None
+
+
+@pytest.mark.asyncio
+async def test_a_system_search_does_not_go_through_the_note_search():
+    """Its own search, not a note_type. A charter competing with the whole note
+    corpus for one top-k is outranked by the records filed under it."""
+    _user_id_ctx.set(7)
+    notes = AsyncMock(return_value=[])
+    with (
+        patch("scribe.mcp.tools.search.semantic_search_notes", notes),
+        patch("scribe.mcp.tools.search.semantic_search_systems", AsyncMock(return_value=[])),
+    ):
+        await search(q="x", content_type="system")
+    notes.assert_not_awaited()

tests/test_system_charter_search.py

@@ -0,0 +1,164 @@
+"""Systems become findable by meaning — the charter as an ANSWER (#4251).
+
+A System's `description` is a charter: several hundred words saying what
+belongs in that area and what does not. It is the answer to "where does this
+go?", and there was no semantic path to it — `list_systems` enumerates and
+`search(system_id=…)` uses a System as a FILTER over notes, so a System could
+narrow a search and could never be the answer to one.
+
+These pin the three halves: the document shape, the search's scoping and
+best-chunk contract, and that a charter edited through any door is re-indexed.
+"""
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from scribe.services import embeddings as emb
+from scribe.services import systems as systems_svc
+from tests.helpers import make_mock_session
+
+
+def _system(system_id=3, name="Retrieval & recall", description="the charter"):
+    s = MagicMock(id=system_id, project_id=2, description=description)
+    s.name = name          # never via the constructor — see note #2833
+    return s
+
+
+# --- the document shape ------------------------------------------------------
+
+
+def test_the_document_is_the_name_and_the_charter():
+    assert emb.system_document("Retrieval", "what belongs here") == (
+        "Retrieval", "what belongs here"
+    )
+
+
+def test_a_system_with_no_charter_yet_degrades_to_its_name():
+    """It still embeds, just weakly. That is an argument for writing the
+    charter, not for padding the document with whatever is to hand."""
+    assert emb.system_document("Retrieval", None) == ("Retrieval", None)
+    assert emb.system_document("Retrieval", "   ") == ("Retrieval", None)
+    assert emb.chunk_document(*emb.system_document("Retrieval", None)) == ["Retrieval"]
+
+
+def test_an_empty_system_produces_no_document_at_all():
+    """Callers gate on falsiness to clear vectors rather than embed nothing."""
+    assert emb.system_document("", "") == (None, None)
+    assert emb.chunk_document(*emb.system_document("", "")) == []
+
+
+# --- the search --------------------------------------------------------------
+
+
+def _rows_ctx(rows):
+    session = MagicMock()
+    result = MagicMock()
+    result.all.return_value = rows
+    session.execute = AsyncMock(return_value=result)
+    ctx = MagicMock()
+    ctx.__aenter__ = AsyncMock(return_value=session)
+    ctx.__aexit__ = AsyncMock(return_value=False)
+    return session, ctx
+
+
+@pytest.mark.asyncio
+async def test_the_search_collapses_to_best_chunk_and_reports_which_one_won():
+    """A charter is long, and a result shows its NAME — so the paragraph that
+    actually decides where a record belongs has to come back with it, or the
+    caller judges from two words that cannot say (#4243). Published from the
+    start here rather than retrofitted, which is what #4251 asked for."""
+    a, b = _system(3), _system(4, name="Data Model")
+    rows = [
+        (a, 0.10, 2, "retrieval telemetry belongs here"),
+        (b, 0.22, 0, "the persistence layer"),
+        (a, 0.40, 0, "a weaker paragraph of the same charter"),
+    ]
+    _session, ctx = _rows_ctx(rows)
+    report: dict = {}
+    with (
+        patch.object(emb, "async_session", return_value=ctx),
+        patch.object(emb, "get_embedding", AsyncMock(return_value=[0.0] * 384)),
+        patch.object(emb, "can_read_project", AsyncMock(return_value=True)),
+    ):
+        out = await emb.semantic_search_systems(
+            7, "where does telemetry go?",
+            project_id=2, threshold=0.0, report=report,
+        )
+
+    assert [s.id for _score, s in out] == [3, 4]
+    assert out[0][0] == 1.0 - 0.10          # the BEST chunk's score
+    assert report["best_chunk"][3] == {
+        "index": 2, "text": "retrieval telemetry belongs here",
+    }
+
+
+@pytest.mark.asyncio
+async def test_a_project_the_caller_cannot_read_returns_nothing():
+    """Rule 78 — the charter of a project someone was not given is not an
+    answer to any question they are entitled to ask."""
+    _session, ctx = _rows_ctx([])
+    with (
+        patch.object(emb, "async_session", return_value=ctx),
+        patch.object(emb, "get_embedding", AsyncMock(return_value=[0.0] * 384)),
+        patch.object(emb, "can_read_project", AsyncMock(return_value=False)),
+    ):
+        assert await emb.semantic_search_systems(7, "q", project_id=99) == []
+
+
+@pytest.mark.asyncio
+async def test_an_empty_query_never_reaches_the_embedder():
+    with patch.object(emb, "get_embedding", AsyncMock()) as embed:
+        assert await emb.semantic_search_systems(7, "   ") == []
+    embed.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_a_search_that_fails_returns_nothing_rather_than_raising():
+    """A recall aid must never break the call it serves."""
+    with patch.object(emb, "get_embedding", AsyncMock(side_effect=RuntimeError)):
+        assert await emb.semantic_search_systems(7, "q") == []
+
+
+# --- staying current ---------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_creating_a_system_indexes_its_charter():
+    session = make_mock_session()
+    with (
+        patch.object(systems_svc, "async_session", return_value=session),
+        patch.object(systems_svc.access, "can_write_project", AsyncMock(return_value=True)),
+        patch.object(systems_svc, "embed_system") as embed,
+    ):
+        await systems_svc.create_system(7, 2, "Retrieval", description="the charter")
+    embed.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_editing_a_charter_re_indexes_it():
+    """A charter edited and not re-indexed stays findable by what it used to
+    say — the stale-vector case, and the one that matters most for a record
+    whose whole job is to say where things belong now."""
+    system = _system()
+    session = make_mock_session()
+    session.get = AsyncMock(return_value=system)
+    system.deleted_at = None
+    with (
+        patch.object(systems_svc, "async_session", return_value=session),
+        patch.object(systems_svc.access, "can_write_project", AsyncMock(return_value=True)),
+        patch.object(systems_svc, "embed_system") as embed,
+    ):
+        await systems_svc.update_system(7, 3, description="a new charter")
+    embed.assert_called_once_with(system)
+
+
+def test_embedding_a_system_never_breaks_the_write_that_saved_it():
+    """A System that saved must not fail on its index refresh. ValueError
+    rather than RuntimeError deliberately: RuntimeError is also what "no
+    running loop" raises, which is an ordinary sync caller and has its own
+    branch — this has to land in the general swallow to prove it exists."""
+    with patch(
+        "scribe.services.embeddings.upsert_system_embedding",
+        side_effect=ValueError("boom"),
+    ):
+        systems_svc.embed_system(_system())   # returns quietly, does not raise