From 0c24fa392ce091943c0b13a87bd242a7070eb795 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 21 Oct 2026 20:04:59 -0400 Subject: [PATCH] docs(6205): embeddings comments state the reason, not the instance that taught it Record ids, milestone tags, dated measurements and "used to" stories are removed; the chunker version list now says what each stamped version contains. Corrected two comments that said a failed search leaves report["searched"] absent (it is stamped False first), and pointed the rule-search docstring at _RULEHINT_BAND in retrieval_pipeline instead of a tier note in plugin_context that no longer exists. plugin_context's own dangling "note on tiers" pointer is reworded to match. Co-Authored-By: Claude Opus 5.5 --- src/scribe/services/embeddings.py | 407 ++++++++++++-------------- src/scribe/services/plugin_context.py | 6 +- 2 files changed, 198 insertions(+), 215 deletions(-) diff --git a/src/scribe/services/embeddings.py b/src/scribe/services/embeddings.py index 3b7e4af7..d88fc19f 100644 --- a/src/scribe/services/embeddings.py +++ b/src/scribe/services/embeddings.py @@ -45,10 +45,8 @@ _SIMILARITY_THRESHOLD = 0.45 # The floor for INTERACTIVE, human-facing feeds — REST /api/search, Browse # search, and the list views' semantic `q`. Deliberately looser than the agent # default above: a human scanning a result list gets value from loosely-related -# hits an agent would be misled by. One constant, because this number was -# written as `0.3` in two files with the reasoning attached to only one of -# them — the exact shape where a later tuner moves one and not the other -# (#2463 finding 3). +# hits an agent would be misled by. One constant, so a tuner cannot move the +# number in one file and not another. INTERACTIVE_SEARCH_THRESHOLD = 0.3 # Public alias so callers (and telemetry) can record the effective default @@ -80,18 +78,18 @@ async def _get_model(): return _model -# ONE EMBEDDING PER QUERY PER REQUEST (milestone 456 step 1). +# ONE EMBEDDING PER QUERY PER REQUEST. # # A single plugin request runs several ranked arms over the SAME string: the # operator's message is searched by auto_inject, its reuse and lesson slots, -# prompt_rule, the preference slot and rule_via_lesson — each of which called -# `get_embedding` on its own, so one prompt cost up to six model calls for one -# vector. The arms are not wrong to search separately; they were wrong to pay -# for the vector separately. +# prompt_rule, the preference slot and rule_via_lesson — each of which calls +# `get_embedding` on its own. Without a memo one prompt costs up to six model +# calls for one vector. The arms search separately; they need not pay for the +# vector separately. # # A memo, not a cache: it lives for one request and is opened explicitly by # `query_embedding_memo()`. Outside a scope the default is None and nothing is -# remembered, so every other caller behaves exactly as before. Keyed on the +# remembered, so every other caller embeds on every call. Keyed on the # exact text — a query that differs by a character is a different query. # # Only `get_embedding` (a QUERY) consults it. `get_embeddings` embeds a @@ -170,19 +168,19 @@ def _cosine_similarity(a: list[float], b: list[float]) -> float: return dot / (mag_a * mag_b) -# How much a superseded record is pushed down the ranking (#278). +# How much a superseded record is pushed down the ranking. # -# Chosen against a measurement, not by feel. On 2026-08-07 dev-log #2420 sat at -# 0.6120 on a query made of its own title phrase, 8th, behind #1759 at 0.6506 — -# a deficit of 0.039 to the top and ~0.014 to its nearest neighbours. A penalty -# of 0.05 clears that whole band, so demoting a cluster's stale members actually -# reorders it rather than shuffling within a tie. +# Chosen against a measurement, not by feel. A dev-log queried with its own +# title phrase can sit 8th, ~0.04 below the top hit and ~0.014 from its +# nearest neighbours. A penalty of 0.05 clears that whole band, so demoting a +# cluster's stale members actually reorders it rather than shuffling within a +# tie. # # It is deliberately NOT large. Supersession is a claim about SOME of a record's # content, so a superseded note that strongly answers a question nothing else # answers should still surface — just behind anything comparable that is # current. A penalty big enough to bury it outright would be hiding by another -# name, which is the thing the operator ruled out. +# name, and a superseded record is demoted, never hidden. _SUPERSESSION_PENALTY = 0.05 # Candidates fetched per requested result when a re-rank follows. Three ranks of @@ -190,7 +188,7 @@ _SUPERSESSION_PENALTY = 0.05 # whose neighbours sit ~0.01-0.02 apart. _SUPERSESSION_OVERFETCH = 3 -# Chunk rows fetched per requested result (#280). The HNSW top-k runs at CHUNK +# Chunk rows fetched per requested result. The HNSW top-k runs at CHUNK # grain — several chunks of one strong note can occupy consecutive ranks, and # each collapses into a single result. Four ranks of headroom per result keeps # the top-k indexed while making it effectively impossible for collapsing to @@ -244,15 +242,15 @@ async def _apply_supersession_penalty( def embedding_text(title: str | None, body: str | None) -> str: """`title\\n{body}` — the atomic join every embedded document is built from. - One definition, deliberately. This was written out three times — the write - path (`notes.embed_note`), the recurring-task spawn, and the startup - backfill — and identical copies of a formatting rule are three chances to - change one and not the others. The spawn path is the dangerous one: a - recurring task embedded to a different shape than everything else would be - ranked against a corpus it doesn't match, and nothing would report it. + One definition, deliberately. Three paths embed — the write path + (`notes.embed_note`), the recurring-task spawn, and the startup backfill — + and identical copies of a formatting rule are three chances to change one + and not the others. The spawn path is the dangerous one: a recurring task + embedded to a different shape than everything else would be ranked + against a corpus it doesn't match, and nothing would report it. - Since the chunking build (#280) this is a BUILDING BLOCK, not the whole - story: the document shape a record is embedded as is `chunk_document` + This is a BUILDING BLOCK, not the whole story: the document shape a + record is embedded as is `chunk_document` below, which calls this once per chunk. Callers that want "the text this note is embedded as" want `chunk_document`; this stays public because the two functions are one contract and the guard in test_embedding_text pins @@ -272,16 +270,14 @@ TRIGGER_SEP = " — " def trigger_title(subject: str | None, trigger: str | None) -> str: """`{subject} — {trigger}` — the title half of a situation-keyed document. - ONE definition, because this join had three — rules, snippets, and a - fourth for lessons (milestone 385) — the shape #3207 records, where a fix - or an improvement then has to be found in N places by someone who does not - know N. Since milestone 427 it builds EMBEDDED titles only: `rule_document` - for rules and `document_title` for snippets and lessons. No stored title - carries it. + ONE definition for rules, snippets and lessons, so a fix or an + improvement is never hunted for in N places by someone who does not know + N. It builds EMBEDDED titles only: `rule_document` for rules and + `document_title` for snippets and lessons. No stored title carries it. - WHY THE JOIN MATTERS AT ALL, measured in note #2485: the snippet was the - only sharp record in the corpus — a 0.153 top-to-second gap against - 0.010–0.023 for everything else — and the cause was this title plus the + WHY THE JOIN MATTERS AT ALL, measured: snippets are the sharp records in + the corpus — a ~0.15 top-to-second gap against 0.010–0.023 for + everything else — and the cause is this title plus the same trigger repeated in the body, so purpose appears twice in a short document and dominates the vector. Every kind that must be findable by WHEN IT APPLIES rather than what it is about is built on this line. @@ -301,11 +297,10 @@ def trigger_title(subject: str | None, trigger: str | None) -> str: def untrigger_title(title: str | None, trigger: str | None) -> str: """The subject back out of a `trigger_title` — the inverse of the join. - Kept HERE, beside the join, for the reason the join itself was - consolidated: a separator spelled in two files is a separator that will one - day be changed in one of them. #3207 records the shape — derive it before - the third copy — and an inverse written in a caller is that third copy - wearing a different name. + Kept HERE, beside the join, for the reason the join is one function: a + separator spelled in two files is a separator that will one day be + changed in one of them, and an inverse written in a caller is another + copy wearing a different name. Needs the trigger passed in rather than guessing at the separator, because a subject may legitimately contain an em dash. Given the trigger, the @@ -337,17 +332,15 @@ def document_title( ) -> str | None: """The title a note is EMBEDDED under — its stored title, plus its trigger. - Milestone 427. A snippet's or lesson's STORED title is its subject alone; - the trigger lives in `data` (decision #4157). It still has to reach the - vector — the `subject — trigger` join is what makes these kinds rank on - the situation they apply to (#2485) — so it is joined HERE, at embed time, - rather than being carried in a title every listing then has to show. + A snippet's or lesson's STORED title is its subject alone; the trigger + lives in `data`. It still has to reach the vector — the `subject — + trigger` join is what makes these kinds rank on the situation they apply + to — so it is joined HERE, at embed time, rather than being carried in a + title every listing then has to show. - IDEMPOTENT, and that is what makes the migration safe: a title that is - already composed (a row not yet migrated, an old backup restored) is - untriggered first, so it comes out the same and never doubled. The text is - byte-identical to what these kinds were embedded as before, so no vector - moves and the floors tuned against them stay calibrated. + IDEMPOTENT: a title that is already composed (an old backup restored) is + untriggered first, so it comes out the same and never doubled, and no + vector moves. `body` is the fallback when the mirror is missing, read by the kind's own parser — the same degrade-to-the-body each kind's reader already has. @@ -370,40 +363,37 @@ def document_title( return trigger_title(untrigger_title(title, trigger), trigger) -# --- chunking (#280): the document shape ------------------------------------ +# --- chunking: the document shape ------------------------------------------- # # bge-small reads at most 512 tokens and fastembed silently truncates the rest, # so a single whole-document vector loses everything past ~400 words — for a -# long dev-log, three quarters of the record was PERMANENTLY invisible to -# search. The fix is the document shape: one vector per meaningful chunk, and a -# record is as findable as its best-matching section. +# long dev-log, three quarters of the record would be invisible to search. So +# a record is embedded as one vector per meaningful chunk, and is as findable +# as its best-matching section. # Bumped whenever THE DOCUMENT A RECORD IS EMBEDDED AS can change for a record # that itself has not changed. Stored on every note_embeddings row so the # startup backfill re-embeds exactly the stale notes — a version comparison -# instead of the table wipe migrations 0067/0077 had to do. +# instead of a table wipe. # -# Stated that way rather than as "chunk_document's output for the same input", -# which is what it used to say: `chunk_document` is only the last step, and -# version 2 moves without touching it. A task's document now carries its work -# logs (#4251), so every task that has one embeds differently than it did while -# its title, body and the chunker are all untouched — exactly the case the -# narrower wording would have read as "nothing to re-embed". +# Stated that way rather than as "chunk_document's output for the same input": +# `chunk_document` is only the last step. A task's document carries its work +# logs, so a new log changes what the task embeds as while its title, body and +# the chunker are all untouched. # -# 1 → 2: work logs joined the task document. -# 2 → 3: no chunk is a heading alone (#4784). A content-free chunk embeds -# near the centre of the space and outranked real records on any -# vague query — "## Work log — 2026-08-21" took rank 1. +# 2: a task's document includes its work logs. +# 3: no chunk is a heading alone. A content-free chunk embeds near the +# centre of the space and outranks real records on any vague query. CHUNKER_VERSION = 3 # The public name of the space every score lives in, and the two facts that -# can invalidate a tuned number (#4104). +# can invalidate a tuned number. # -# EMBEDDING_MODEL is `_MODEL_NAME` under a name other modules may read. It was -# private until this step, which is precisely why nothing outside this file -# could state what space a threshold was measured in — a floor is a distance in -# THIS model's geometry and means nothing in another's. +# EMBEDDING_MODEL is `_MODEL_NAME` under a name other modules may read, so +# anything outside this file can state what space a threshold was measured in +# — a floor is a distance in THIS model's geometry and means nothing in +# another's. # # The pair is what a stamp is made of, and the pairing is the point: a score # changes when the model changes (different geometry) OR when the document @@ -430,12 +420,12 @@ def calibration_stamp() -> dict: def is_current_stamp(table): - """The rows written by THIS chunker in THIS model's space (#4132). + """The rows written by THIS chunker in THIS model's space. Every embedding table stores both halves of `calibration_stamp()` per row, and a row is current only when both match. One predicate for all four - tables, because a backfill that checked the version alone is exactly how a - same-width model swap would have gone unnoticed. + tables, because a backfill that checked the version alone would miss a + same-width model swap. """ return and_( table.chunker_version == CHUNKER_VERSION, @@ -459,13 +449,13 @@ async def rows_off_the_live_model(table) -> int: # Character budget approximating the model window. Tokens-per-char varies by # content — ~4 chars/token for prose, closer to 3 for code and tables — so 1400 # chars sits at roughly 350-470 tokens, leaving headroom for the title prefixed -# to every chunk. Deliberately conservative: our own measurement (#2485) says +# to every chunk. Deliberately conservative: measurement says # shorter, single-topic documents embed SHARPER, so the cost of over-splitting # is a few extra cheap vectors while the cost of under-splitting is truncation — -# the exact data loss this exists to end. +# the exact data loss chunking exists to prevent. _CHUNK_CHAR_BUDGET = 1400 -# The least a chunk may say beyond its headings (#4784). Below this a chunk is +# The least a chunk may say beyond its headings. Below this a chunk is # a heading, a date or a one-line lead-in, and it embeds as text about nothing # in particular — which is close to everything, so it outranks real records on # any vague query. Such a piece is joined to its neighbour instead of being @@ -531,14 +521,14 @@ def _split_paragraphs(section: str, budget: int) -> list[str]: budget (a monster table or code block — split at line boundaries so no content is dropped, which is the entire point of this module). - No piece is thin (#4784). A heading, or a lead-in line, that would be left + No piece is thin. A heading, or a lead-in line, that would be left on its own is carried into the paragraph after it; a thin tail joins the piece before it.""" pieces: list[str] = [] current = "" # A hard cut never lands in the first quarter of the budget: the newline # it would find there is the one closing a heading or a lead-in, and - # cutting at it is how a heading became a chunk of its own. + # cutting at it would make a heading a chunk of its own. lo = budget // 4 for para in section.split("\n\n"): if current and _is_thin(current): @@ -589,15 +579,15 @@ def chunk_document(title: str | None, body: str | None) -> list[str]: The contract every retrieval surface builds on: - A record that fits the model window yields EXACTLY ONE chunk, identical - to the historical `title\\nbody` shape — snippets and reference notes, + to the single-vector `title\\nbody` shape — snippets and reference notes, the corpus's sharpest records, are byte-for-byte unaffected. - A longer record is split at markdown heading boundaries (fence-aware), small neighbouring sections merged, oversize sections split at paragraph boundaries, so every chunk fits the window. NOTHING is dropped: every line of the body lands in some chunk. - Every chunk is prefixed with the record's title — each vector carries - its own topical anchor, the property that makes snippets discriminative - (#2485). Pieces sub-split from one section also repeat that section's + its own topical anchor, the property that makes snippets + discriminative. Pieces sub-split from one section also repeat that section's heading line, so "which part of which topic" survives the split. - An empty record yields [] (callers gate on falsiness to skip embedding). @@ -615,7 +605,7 @@ def chunk_document(title: str | None, body: str | None) -> list[str]: # Merge small adjacent sections upward so tiny sections don't each spend a # vector, then split anything still over budget at paragraph boundaries. - # A THIN section takes the next one in whatever its size (#4784): a heading + # A THIN section takes the next one in whatever its size: a heading # with nothing under it ("## Findings" above its "### …" parts) or a lone # lead-in line would otherwise stand as a chunk, and the split below keeps # it attached to the first paragraph that follows. @@ -640,7 +630,7 @@ def chunk_document(title: str | None, body: str | None) -> list[str]: for i, piece in enumerate(pieces): # Repeat the heading a continuation piece sits under so each vector # still knows what topic it is part of. The LAST heading so far, - # not the section's first: a thin heading merged forward (#4784) + # not the section's first: a thin heading merged forward # means one section can hold several. if i > 0 and heading and not piece.startswith(heading): piece = f"{heading}\n{piece}" @@ -650,7 +640,7 @@ def chunk_document(title: str | None, body: str | None) -> list[str]: async def _claim_parent_row(session, id_column, row_id: int, label: str) -> bool: - """Lock the record a vector belongs to BEFORE rewriting that vector (#3262). + """Lock the record a vector belongs to BEFORE rewriting that vector. An embedding write and a cascading delete of the same record take the same two row locks in OPPOSITE orders. The embedder deletes the old chunk rows @@ -696,7 +686,7 @@ async def _claim_parent_row(session, id_column, row_id: int, label: str) -> bool async def _work_log_sections(note_id: int) -> list[tuple[object, str | None]]: - """A task's work logs, oldest first, for its embedded document (#4251). + """A task's work logs, oldest first, for its embedded document. Reads the table directly rather than through `task_logs.logs_for_task`, because that function asks a PERMISSION question — may this user read this @@ -707,8 +697,8 @@ async def _work_log_sections(note_id: int) -> list[tuple[object, str | None]]: That also settles what happens on a shared task: a collaborator's log is part of the task's document, so it becomes findable by everyone who can read the task and by nobody else. The same answer `logs_for_task` gives a - reader (#4241), which is the point — a log that can be read and not found - is the half-surface that issue was about. + reader, which is the point — a log that can be read but not found is half + a surface. Asked for every note, not only tasks. `upsert_note_embedding` is handed a note_id and no kind — and the three writers that call it would each have to @@ -797,14 +787,13 @@ async def upsert_note_embedding( # Kinds that belong to no single project, and are therefore reachable from a -# project-scoped search of a DIFFERENT project when a caller asks for them -# (milestone 385 step 3). +# project-scoped search of a DIFFERENT project when a caller asks for them. # # A lesson is the whole reason this exists. "A better way to think about this # problem" is not true only where it was learned, and a lesson confined to its # origin project would be unreachable exactly where it is most useful — on the -# next project, which is the case the kind was created for (#3727). -# `semantic_search_rules` has always had this property: it scopes by OWNERSHIP +# next project, which is the case the kind exists for. +# `semantic_search_rules` has the same property: it scopes by OWNERSHIP # rather than by what binds a given project, because "is there a rule about # this" is a question asked across a whole rulebook. A lesson asks the same # kind of question. @@ -816,11 +805,11 @@ GLOBAL_NOTE_TYPES: tuple[str, ...] = ("lesson",) # Both searches rank WITHOUT the threshold and apply it in Python, so the best -# rejected score stays observable (#3670). The qualifying set is provably -# unchanged: rows arrive ordered by distance ascending, so every above-bar row -# sorts ahead of every below-bar one, and an over-fetch that used to return N -# above-bar rows returns the same N plus some losers. What changes is only that -# the losers are now visible instead of discarded inside the query. +# rejected score stays observable. The qualifying set is the same as a +# filter inside the query would give: rows arrive ordered by distance +# ascending, so every above-bar row sorts ahead of every below-bar one, and an +# over-fetch returns the same N above-bar rows plus some losers. The losers +# are visible instead of discarded inside the query. # # That visibility is the entire point. A bar can only be judged from the calls # it TURNED AWAY — a 0.72 bar rejecting a stream of 0.71s is set too high by a @@ -836,15 +825,15 @@ def record_best_chunk(report: dict | None, chunks: dict[int, dict]) -> None: """Publish the winning chunk per record into `report["best_chunk"]`. Every semantic search here collapses several chunk rows to the best one per - record, which means each of them KNOWS which passage earned the hit — and - each of them used to drop it, leaving every caller to preview the head of - the document instead. The head is a different span, one the search has - already scored lower, and nothing in the result said so (#4243). + record, which means each of them KNOWS which passage earned the hit. + Dropping it would leave every caller to preview the head of the document + instead — a different span, one the search has already scored lower, with + nothing in the result saying so. It rides in `report` rather than in the return value because all three searches return `list[tuple[float, Record]]` and roughly thirty sites unpack that pair; widening it would be an interface change to every one of - them with nothing to catch a miss (lesson #4207). `report` is already the + them with nothing to catch a miss. `report` is already the side-channel these functions use for `searched` and `best_available_score`, so this adds a key to a channel callers already open. @@ -856,7 +845,7 @@ def record_best_chunk(report: dict | None, chunks: dict[int, dict]) -> None: report["best_chunk"] = chunks -# --- scope first, then rank (#4958, #4961) ----------------------------------- +# --- scope first, then rank ------------------------------------------------- # # Every semantic search here asks the same question of a shared table: the # nearest chunks AMONG THE ONES THIS CALLER MAY SEE. Ordered straight off a @@ -867,8 +856,8 @@ def record_best_chunk(report: dict | None, chunks: dict[int, dict]) -> None: # users' records are what fill those 40. The searches fail open, so nothing # reports it: the result is just shorter. # -# The fix is one shape, shared by every search so it cannot be fixed in one -# and missed in the next (which is how #4958 left three behind). The scope is +# The remedy is one shape, shared by every search so it cannot be fixed in one +# and missed in the next. The scope is # a MATERIALIZED CTE with the distance computed inside it; the index cannot # order a materialized CTE, so the ranking over it is exact. # @@ -876,7 +865,7 @@ def record_best_chunk(report: dict | None, chunks: dict[int, dict]) -> None: # Systems are hundreds of chunks. Notes are the big corpus, but the hook arms # that run on every prompt are project-scoped, so the pass is over one # project's chunks rather than the whole table; `retrieval_logs.duration_ms` -# on those arms is where that claim is checked after the deploy. If it ever +# on those arms is where that claim is checked. If it ever # stops holding, pgvector 0.8's `hnsw.iterative_scan` keeps the index and is # the next step — gated on the extension version, because an unknown `hnsw.*` # setting errors and these searches would read that error as "nothing found". @@ -938,14 +927,14 @@ async def semantic_search_notes( the function sets `report["best_available_score"]` to the highest score anything reached, or None when the corpus offered nothing at all. It is the only figure that survives a call returning nothing, and therefore the - only one a bar can be judged from (#3670). + only one a bar can be judged from. It also sets `report["searched"]`: False before anything can return, True only where a real result set exists. So an empty query, an unavailable embedder and a failed database query all leave it FALSE, and a caller can tell a search that found nothing from one that never ran. A caller logging telemetry must check it — recording a failed search as a zero-result call - reports a decline the ranker never made (#3765). ABSENT means no search + reports a decline the ranker never made. ABSENT means no search touched the dict at all, which is a stand-in in a test, not a real call. `note_type` narrows to a record kind, or several (e.g. "snippet", or @@ -981,15 +970,15 @@ async def semantic_search_notes( Ranking and the top-k cut happen in Postgres via pgvector's cosine-distance operator (`<=>`, exposed as ``Vector.cosine_distance``), EXACTLY and over - the in-scope chunks only (#4961, see `_rank_scoped`). It used to be an - indexed ``ORDER BY ... LIMIT k`` straight off the HNSW index from - migration 0067, which ranks before it filters and so could drop an - in-scope record that sat behind ~40 nearer ones the caller cannot see. + the in-scope chunks only (see `_rank_scoped`). An indexed + ``ORDER BY ... LIMIT k`` straight off the HNSW index would rank before it + filters, and so could drop an in-scope record that sat behind ~40 nearer + ones the caller cannot see. Cosine distance is ``1 - cosine_similarity``, so a similarity floor of *threshold* is a distance ceiling of ``1 - threshold`` and similarity is recovered as ``1 - distance``. - `demote_superseded` applies the supersession penalty (#278): a record a + `demote_superseded` applies the supersession penalty: a record a later note claims to have overtaken ranks below its equals. Callers asking "what is the current answer" want it; the near-duplicate gate does NOT, and passes False — a superseded record is still a duplicate of what you are @@ -999,7 +988,7 @@ async def semantic_search_notes( Returns an empty list if the embedder is unavailable or on any error. """ # Stamped FALSE before anything can return, flipped True only where a real - # result set exists (#3765). Every early return below leaves it false, so a + # result set exists. Every early return below leaves it false, so a # caller can tell a search that found nothing from one that never ran. It # has to be the first thing done to `report`: a return added above this # line would leave the key ABSENT, which reads as "no caller asked". @@ -1025,13 +1014,13 @@ async def semantic_search_notes( # and leave shared records unreachable by meaning. # # Every filter below is the SCOPE, built into the chunks the - # ranking draws from rather than applied to what it returns - # (#4961) — see `_rank_scoped`. + # ranking draws from rather than applied to what it returns — + # see `_rank_scoped`. stmt = ( # chunk_index/chunk_text ride along so the collapse below can # say WHICH passage matched. Without them the caller is left # previewing the head of the body — a span this query has - # already determined is not why the record ranked (#4243). + # already determined is not why the record ranked. _scoped_chunks(NoteEmbedding, NoteEmbedding.note_id, distance) .join(Note, NoteEmbedding.note_id == Note.id) .where( @@ -1047,7 +1036,7 @@ async def semantic_search_notes( in_project = or_( in_project, Note.note_type.in_(GLOBAL_NOTE_TYPES) ) - # Family canon (milestone 463 step 6) crosses the project + # Family canon crosses the project # line the same way a lesson does, but only to a project # on one of the idea's platforms: the canon ideas reaching # it, and their references in its languages. Membership @@ -1125,7 +1114,7 @@ async def semantic_search_notes( logger.warning("Failed to query note embeddings", exc_info=True) return [] - # Collapse chunk rows to BEST-CHUNK-PER-NOTE (#280): rows arrive ordered by + # Collapse chunk rows to BEST-CHUNK-PER-NOTE: rows arrive ordered by # distance, so the first appearance of a note is its best chunk and later # appearances are the same note matched less well. A note's relevance IS # its best section's relevance — a query about one topic of a long record @@ -1137,9 +1126,9 @@ async def semantic_search_notes( # one place that knows which passage earned the hit. Kept beside the score # rather than returned with it: the return type is list[tuple[float, Note]] # and ten callers unpack it at ~18 sites, so widening the tuple would be an - # interface change to every one of them with nothing to catch the misses - # (lesson #4207). `report` is the side-channel this function already uses - # for best_available_score. + # interface change to every one of them with nothing to catch the misses. + # `report` is the side-channel this function already uses for + # best_available_score. best_chunk: dict[int, dict] = {} for note, dist, chunk_index, chunk_text in rows: if int(note.id) in seen: @@ -1153,16 +1142,16 @@ async def semantic_search_notes( # The best score anything reached, bar or no bar. Recorded BEFORE the # filter because a call that returns nothing is exactly when it matters. if report is not None: - # `searched` is what stops a null score meaning four things (#3765). - # Every early return above — empty query, embedder down, and the broad - # `except` around the query itself — leaves this key ABSENT, so a - # caller can tell "I looked and there was nothing" from "I never - # looked" and from "the query failed". Set here, at the one point past - # which a real result set exists. + # `searched` is what stops a null score meaning four things. Every + # early return above — empty query, embedder down, and the broad + # `except` around the query itself — leaves it False, so a caller can + # tell "I looked and there was nothing" from "I never looked" and from + # "the query failed". Set here, at the one point past which a real + # result set exists. report["searched"] = True - # ONE unpack, so the score and the id cannot describe different records - # (#3807). Splitting these into two expressions is how a later edit - # pairs a score with its neighbour's id. + # ONE unpack, so the score and the id cannot describe different + # records. Split into two expressions, a later edit can pair a score + # with its neighbour's id. best = scored[0] if scored else None report["best_available_score"] = best[0] if best else None report["best_available_id"] = int(best[1].id) if best else None @@ -1181,18 +1170,18 @@ async def semantic_search_notes( return final -# --- backfill correctness: scan ids, read text at embed time (#4262) -------- +# --- backfill correctness: scan ids, read text at embed time --------------- # -# A backfill used to SELECT the text alongside the id and then iterate that -# snapshot, sleeping between records. On a corpus of any size the run takes -# minutes, and anything edited inside that window was re-embedded by its own -# write path and then OVERWRITTEN with the pre-edit text the scan had captured -# — stamped at the current version, so the next boot considered it current and -# never repaired it. A record left findable by what it used to say, silently, -# until someone happened to edit it again. +# A backfill that SELECTs the text alongside the id and then iterates that +# snapshot, sleeping between records, runs for minutes on a corpus of any +# size. Anything edited inside that window is re-embedded by its own write +# path and then OVERWRITTEN with the pre-edit text the scan captured — +# stamped at the current version, so the next boot considers it current and +# never repairs it. The record stays findable by what it used to say, +# silently, until someone edits it again. # -# The two functions below are the fix, and they are separate on purpose: one -# stops it happening, the other repairs what already happened. +# The two functions below are separate on purpose: one stops that happening, +# the other repairs rows it already damaged. async def _current_row(columns, id_column, row_id: int): @@ -1234,7 +1223,7 @@ async def _vectors_older_than_their_record( only signal that separates them, and it costs one grouped read at startup. A guard that refuses to write a bad value does not undo the bad values - already stored (#4202) — the rows are the thing that has to change, and + already stored — the rows are the thing that has to change, and this is what changes them. """ newest = ( @@ -1257,16 +1246,15 @@ async def _tasks_logged_since_embedding(session) -> set[int]: """Task ids whose newest work log is newer than their newest vector. The note corpus needs this on top of the plain timestamp comparison, and - the reason is a consequence of #4251 that is easy to miss: a task's + the reason is easy to miss: a task's embedded document carries its work logs, but a log lives in its own table, so writing one does NOT move `notes.updated_at`. A task re-embedded from stale text during a backfill therefore looks current by every other signal — the version is right and the note's own timestamp is older than the vectors — while the document those vectors encode is out of date. - These are the exact records the backfill race was most likely to hit, - because a session logging work is what "forward work during a long - backfill" MEANS. + These are the records a backfill race is most likely to hit, because a + session logging work is what "forward work during a long backfill" MEANS. """ newest_vector = ( select( @@ -1299,13 +1287,13 @@ async def backfill_note_embeddings() -> None: 1. no vectors at all; 2. vectors from an older chunker — version-awareness is what makes a - document-shape change deployable. Migration 0077 cleared the table - once; every later CHUNKER_VERSION bump re-embeds here instead; + document-shape change deployable: a CHUNKER_VERSION bump re-embeds + here rather than wiping the table; 3. vectors OLDER THAN THE TEXT THEY CLAIM TO ENCODE. A record embedded by the current chunker from text that has since changed is current by - every other signal and answers to what it used to say (#4262). + every other signal and answers to what it used to say. - Runs as a background task at startup, after the serving flag (#4181), with + Runs as a background task at startup, after the serving flag, with a small sleep between notes so a large backfill doesn't peg CPU. It is resumable by construction: progress IS the version stamp on each record's rows, so a restart re-queries and skips what finished — there is no @@ -1360,7 +1348,7 @@ async def backfill_note_embeddings() -> None: if row is None: continue # deleted between the scan and here user_id, title, body, note_type, data = row - # The EMBEDDED title, as the write path builds it (milestone 427). + # The EMBEDDED title, as the write path builds it. title = document_title(title, note_type, data, body) if not chunk_document(title, body): continue @@ -1371,7 +1359,7 @@ async def backfill_note_embeddings() -> None: logger.info("Embedding backfill complete: %d/%d notes embedded", success, len(notes_to_embed)) -# ── Rules (milestone 307, note 3026) ──────────────────────────────────── +# ── Task and rule documents ────────────────────────────────────────────── # The heading a work log gets inside its task's embedded document. A CONSTANT # because it is load-bearing twice over: `_split_sections` splits on it, so it @@ -1392,23 +1380,23 @@ def task_document( A synthesised embed-time shape, like `rule_document` and unlike a lesson: the stored record is the task row, and the logs live in their own table, so the document that should be searchable exists nowhere until it is built - here (#4251). + here. WHY THE LOGS BELONG IN THE TASK'S DOCUMENT rather than in rows of their own. "Has anyone tried this before?" is answered by a log and asked of a task — a hit on a bare log would have to be resolved back to its task to be worth anything, so the useful result is the task either way. The objection - to folding them in is that a long log drowns a short title, and that was - true before #280: one vector per record meant a 2,000-word log averaged the - task's own subject away, and everything past ~400 words was truncated - unread. Chunking removed both. Each log becomes its own section, each + to folding them in is that a long log drowns a short title: one vector + per record would let a 2,000-word log average the task's own subject + away, and truncate everything past ~400 words unread. Chunking removes + both. Each log becomes its own section, each section its own title-anchored vector, each scored separately — so a task is as findable as its best-matching log, and the task's own prose keeps the chunk it always had. - That the result is LEGIBLE is the other half, and it is this session's - other build: a search now hands back the chunk that won (#4243), so a hit - earned by a log shows that log's passage under the task's title. Without it + That the result is LEGIBLE is the other half: a search hands back the + chunk that won, so a hit earned by a log shows that log's passage under + the task's title. Without it the reader would get `body[:240]` of the task — the opening of a record whose relevance lives three hundred lines further down. @@ -1443,17 +1431,17 @@ def rule_document( ) -> tuple[str | None, str | None]: """The (title, body) a rule is EMBEDDED as — trigger first, `why` never. - Both halves of this are measured, not guessed (note 2485). That pass found - the snippet was the only sharp record in the corpus — a 0.153 top-to-second - gap against 0.010–0.023 for everything else — and that the cause was its + Both halves of this are measured, not guessed. Snippets are the sharp + records in the corpus — a ~0.15 top-to-second gap against 0.010–0.023 + for everything else — and the cause is their SHAPE: `{name} — {when_to_use}` as the title and `**When to use:** …` repeated in the body, so purpose appears twice in a short document and dominates the vector. This mirrors that exactly. - And it excludes `why` on the same evidence. `why` is dated incident - narrative — rule 46's runs to 4,300 characters of it — and long, - multi-topic prose is precisely what made sixteen dev-logs mutually - indistinguishable: the average lands on the centroid of "development", + And it excludes `why` on the same evidence. `why` is often incident + narrative running to thousands of characters, and long, multi-topic prose + is precisely what makes dev-logs mutually indistinguishable: the average + lands on the centroid of "development", which every one of them shares. Adding `why` would not give the vector more to work with; it would give every rule the same thing to work with. @@ -1545,17 +1533,17 @@ async def semantic_search_rules( the function sets `report["best_available_score"]` to the highest score anything reached, or None when the corpus offered nothing at all. It is the only figure that survives a call returning nothing, and therefore the - only one a bar can be judged from (#3670). + only one a bar can be judged from. It also sets `report["searched"]`: False before anything can return, True only where a real result set exists. So an empty query, an unavailable embedder and a failed database query all leave it FALSE, and a caller can tell a search that found nothing from one that never ran. A caller logging telemetry must check it — recording a failed search as a zero-result call - reports a decline the ranker never made (#3765). ABSENT means no search + reports a decline the ranker never made. ABSENT means no search touched the dict at all, which is a stand-in in a test, not a real call. - SCOPED, and the scope is a rule's home (milestone 414). A rule lives in a + SCOPED, and the scope is a rule's home. A rule lives in a rulebook topic — GLOBAL, it applies wherever its owner works — or on one project, where it applies to that project and nowhere else: @@ -1569,20 +1557,15 @@ async def semantic_search_rules( explicit whole-rulebook question — `search(content_type="rule")` with no project — where "is there a rule about this" is asked across everything. - This used to be scoped by OWNERSHIP alone, on the argument that "is there - a rule about this" wants the whole rulebook. That is still right for the - explicit ask. It was wrong for the hooks, which inject unasked: every - project's rules surfaced in every other project's sessions — one repo's - template conventions arriving while editing an unrelated one — and a - project rule meant nothing a session could feel. + Scoping by OWNERSHIP alone is right for the explicit ask and wrong for + the hooks, which inject unasked: every project's rules would surface in + every other project's sessions — one repo's template conventions arriving + while editing an unrelated one — and a project rule would mean nothing a + session could feel. - THERE IS NO TIER TO NARROW BY ANY MORE (milestone 394). This carried a - `tier` parameter, and the arms deliberately passed nothing: filtering on it - made a whole class of rules permanently ineligible for the one mechanism - that surfaces a rule AT the moment it applies. The tier is now gone - entirely, so every rule is eligible for every arm and relevance is the - threshold's job alone — see the block above RULEHINT_LIMIT in - services/plugin_context.py for what those scores are read against. + Every rule in scope is eligible for every arm, and relevance is the + threshold's job alone — see `_RULEHINT_BAND` in + services/retrieval_pipeline.py for what those scores are read against. `kind` narrows to `rule` or `preference`, and NONE is likewise the ordinary case: a caller asking "what governs this" wants both, because the reader @@ -1602,7 +1585,7 @@ async def semantic_search_rules( from scribe.models.rulebook import Rule from scribe.services.rule_scope import joined_to_homes, rule_home - # See the sibling search: stamped before anything can return (#3765). + # See the sibling search: stamped before anything can return. if report is not None: report["searched"] = False if not query or not query.strip(): @@ -1621,11 +1604,11 @@ async def semantic_search_rules( # function fails open. home = await rule_home(user_id, project_id, everywhere=everywhere) - # SCOPE FIRST, THEN RANK — exactly (#4958; the shape is - # `_rank_scoped`'s). The home filter is built into the chunks the - # ranking draws from: applied to what an HNSW walk returned, it ran - # after ~40 nearest chunks from every owner and project had been - # taken, and an in-scope rule ranked past them was silently gone. + # SCOPE FIRST, THEN RANK — exactly (the shape is `_rank_scoped`'s). + # The home filter is built into the chunks the ranking draws from: + # applied to what an HNSW walk returned, it would run after ~40 + # nearest chunks from every owner and project had been taken, and an + # in-scope rule ranked past them would be silently gone. scoped = ( joined_to_homes( _scoped_chunks(RuleEmbedding, RuleEmbedding.rule_id, distance) @@ -1654,7 +1637,7 @@ async def semantic_search_rules( best: dict[int, tuple[float, object]] = {} # Which chunk won, kept beside the score it won with — a rule's `why` and # `how_to_apply` are long, and a caller shown only the head cannot see the - # clause that actually matched (#4243). + # clause that actually matched. won: dict[int, dict] = {} for rule, dist, chunk_index, chunk_text in rows: score = 1.0 - float(dist) @@ -1669,9 +1652,9 @@ async def semantic_search_rules( int(r.id): won[int(r.id)] for _s, r in kept if int(r.id) in won }) if report is not None: - # See the sibling search: absent means the search never ran (#3765). + # See the sibling search: False means the search never ran. report["searched"] = True - # One unpack — see the sibling search (#3807). + # One unpack — see the sibling search. best = ranked[0] if ranked else None report["best_available_score"] = best[0] if best else None report["best_available_id"] = int(best[1].id) if best else None @@ -1688,7 +1671,7 @@ async def backfill_rule_embeddings() -> None: current = select(RuleEmbedding.rule_id).where( is_current_stamp(RuleEmbedding) ) - # IDS ONLY — the text is re-read per rule below (#4262). + # IDS ONLY — the text is re-read per rule below. by_version = { row[0] for row in (await session.execute( select(Rule.id) @@ -1720,7 +1703,7 @@ async def backfill_rule_embeddings() -> None: await upsert_rule_embedding(rule_id, *row) -# ── Milestones (milestone 415) ────────────────────────────────────────── +# ── Milestones ─────────────────────────────────────────────────────────── def milestone_document( title: str | None, description: str | None, body: str | None, @@ -1728,7 +1711,7 @@ def milestone_document( """The (title, body) a milestone is EMBEDDED as. Title and one-line description lead, the way a snippet's name and purpose - lead its document (note 2485): the question this search answers is "does + lead its document: the question this search answers is "does a plan for this already exist?", and a plan is recognised by what it is FOR. The plan body follows, chunked, so a milestone whose description is empty — most roadmap milestones written by hand — is still findable by the @@ -1749,7 +1732,7 @@ async def upsert_milestone_embedding( The rule twin's contract: one document definition shared by the write path and the backfill, and an atomic per-milestone replacement guarded by the - parent-row claim (#3262), so a milestone deleted mid-refresh wins. + parent-row claim, so a milestone deleted mid-refresh wins. """ from scribe.models.embedding import MilestoneEmbedding from scribe.models.milestone import Milestone @@ -1808,7 +1791,7 @@ async def semantic_search_milestones( before start_planning, and the one the planning gate asks for it. SCOPE. With `project_id`, that project's milestones, provided the caller - can read the project (access.can_read_project, rule 78) — a collaborator on + can read the project (access.can_read_project) — a collaborator on a shared project sees its plans. Without one, the milestones the caller owns across their projects. `status` narrows to "active" or "done". @@ -1835,7 +1818,7 @@ async def semantic_search_milestones( scope = Milestone.project_id == project_id else: scope = Milestone.user_id == user_id - # Scope first, then rank (#4961): see `_rank_scoped`. + # Scope first, then rank: see `_rank_scoped`. scoped = ( _scoped_chunks(MilestoneEmbedding, MilestoneEmbedding.milestone_id, distance) .join(Milestone, MilestoneEmbedding.milestone_id == Milestone.id) @@ -1874,7 +1857,7 @@ async def semantic_search_milestones( return kept -# --- Systems: the charter as an ANSWER, not a filter (#4251) ----------------- +# --- Systems: the charter as an ANSWER, not a filter ------------------------ def system_document( @@ -1909,7 +1892,7 @@ async def upsert_system_embedding( The third sibling's contract exactly: one document definition shared by the write path and the backfill, and an atomic per-System replacement guarded - by the parent-row claim (#3262), so a System deleted mid-refresh wins. + by the parent-row claim, so a System deleted mid-refresh wins. """ from scribe.models.embedding import SystemEmbedding from scribe.models.system import System @@ -1964,26 +1947,26 @@ async def semantic_search_systems( """Return up to *limit* (score, system) pairs most like *query*. Answers "where does this belong?" — the question asked before filing a - record or opening a file, and the one that had no tool: `list_systems` - enumerates and `system_id` filters, so a charter could narrow a search and - could never be the answer to one. + record or opening a file. `list_systems` enumerates and `system_id` + filters, so without this a charter could narrow a search and never be the + answer to one. - ITS OWN SEARCH rather than a note kind, for what note 3163 says about - milestones: a charter competing with the whole note corpus for one top-k + ITS OWN SEARCH rather than a note kind, for the reason milestones have + their own: a charter competing with the whole note corpus for one top-k would be outranked by the records filed under it, and the right answer would be crowded out by its own contents. The questions are different too — "where does this belong?" is not "what prior art is there?" — and a caller asking one should not have to read past answers to the other. SCOPE. With `project_id`, that project's Systems, provided the caller can - read the project (access.can_read_project, rule 78) — a collaborator on a + read the project (access.can_read_project) — a collaborator on a shared project sees its areas, which is the point of a charter. Without one, the Systems the caller owns across their projects. Archived Systems are excluded: an archived area is one the operator has said is no longer where things go, and that is exactly the question being asked. Collapses to best-chunk-per-System and publishes the winning chunk, like - the sibling searches — from the start rather than retrofitted (#4243). + the sibling searches. Returns an empty list if the embedder is unavailable, the project is not readable, or on any error: a recall aid must never break the call it serves. @@ -2007,7 +1990,7 @@ async def semantic_search_systems( scope = System.project_id == project_id else: scope = System.user_id == user_id - # Scope first, then rank (#4961): see `_rank_scoped`. + # Scope first, then rank: see `_rank_scoped`. scoped = ( _scoped_chunks(SystemEmbedding, SystemEmbedding.system_id, distance) .join(System, SystemEmbedding.system_id == System.id) @@ -2030,7 +2013,7 @@ async def semantic_search_systems( # A charter runs to several hundred words and a result shows its NAME — so # a match on the paragraph that actually decides where a record belongs # would be previewed by two words that cannot. The winning chunk is what - # the caller should see (#4243). + # the caller should see. won: dict[int, dict] = {} for system, dist, chunk_index, chunk_text in rows: score = 1.0 - float(dist) @@ -2051,8 +2034,8 @@ async def backfill_system_embeddings() -> None: """Embed Systems that have no current vectors. Runs at startup beside the note, rule and milestone backfills; a CHUNKER_VERSION bump re-embeds. - This is also the pass that makes every existing charter findable at all — - Systems got vectors in #4251, so before it runs there are none.""" + On an install whose Systems predate their vectors, this is also the pass + that makes every existing charter findable at all.""" from scribe.models.embedding import SystemEmbedding from scribe.models.system import System @@ -2061,7 +2044,7 @@ async def backfill_system_embeddings() -> None: current = select(SystemEmbedding.system_id).where( is_current_stamp(SystemEmbedding) ) - # IDS ONLY — the charter is re-read per System below (#4262). + # IDS ONLY — the charter is re-read per System below. by_version = { row[0] for row in (await session.execute( select(System.id) @@ -2104,7 +2087,7 @@ async def backfill_milestone_embeddings() -> None: current = select(MilestoneEmbedding.milestone_id).where( is_current_stamp(MilestoneEmbedding) ) - # IDS ONLY — the plan is re-read per milestone below (#4262). + # IDS ONLY — the plan is re-read per milestone below. by_version = { row[0] for row in (await session.execute( select(Milestone.id) diff --git a/src/scribe/services/plugin_context.py b/src/scribe/services/plugin_context.py index 1dfdab91..f8d850fe 100644 --- a/src/scribe/services/plugin_context.py +++ b/src/scribe/services/plugin_context.py @@ -1665,9 +1665,9 @@ async def build_write_path_hint( # resembles what is being written, noticed at the moment it is relevant # rather than by being resident in every session. # - # EVERY TIER — see the note on tiers near RULEHINT_LIMIT: a rule the - # session was told about an hour ago is not a rule in front of the reader - # now. + # EVERY RULE IS ELIGIBLE, including one already named this session: a + # rule the session was told about an hour ago is not a rule in front of + # the reader now. # # The arm itself is `retrieval_pipeline.run_rule_arm`: the band, the # fresh/repeat split, the unconditional call row and the fresh-only