Compare commits
2
Commits
57773c8b55
...
cdb49b2f9f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cdb49b2f9f | ||
|
|
0a310f326f |
@@ -16,9 +16,9 @@ Two signals, both scoped to the same owner + project + kind:
|
||||
1. Normalized-title exact match — cheap, always checked.
|
||||
2. Semantic similarity (cosine ≥ _SEMANTIC_THRESHOLD) — only when the incoming
|
||||
body is substantial. Short/title-only embeddings sit in a tight neighborhood
|
||||
and false-positive (the pre-pivot lesson: "Lore: Shell 0" vs
|
||||
"Lore: Reinitialization 0" matched at 0.91 with no body), so we gate it on
|
||||
a minimum body length.
|
||||
and false-positive (two title-only records like "Lore: Shell 0" and
|
||||
"Lore: Reinitialization 0" match at 0.91), so we gate it on a minimum
|
||||
body length.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -48,10 +48,10 @@ logger = logging.getLogger(__name__)
|
||||
_MIN_BODY_FOR_SEMANTIC = 200
|
||||
# Cosine threshold for "this is the same thing, reworded." Deliberately high to
|
||||
# keep false positives rare (a hard block with a force-override is unforgiving of
|
||||
# noise). Matches the 0.90 the pre-pivot dedup settled on.
|
||||
# noise).
|
||||
_SEMANTIC_THRESHOLD = 0.90
|
||||
|
||||
# SNIPPETS ARE MEASURED DIFFERENTLY, and #2518 is why. A snippet's embedded
|
||||
# SNIPPETS ARE MEASURED DIFFERENTLY. A snippet's embedded
|
||||
# document is mostly PROSE ABOUT the code — name, when-to-reach-for-it,
|
||||
# signature, the comments explaining the choice — with the artefact itself a
|
||||
# minority of the text. Two measurements on the same corpus:
|
||||
@@ -61,8 +61,8 @@ _SEMANTIC_THRESHOLD = 0.90
|
||||
# under a different name <0.90 a literal copy, ALLOWED THROUGH
|
||||
#
|
||||
# The second is the one that settles it. Identical code at an identical
|
||||
# repo·path·symbol sailed past the gate because the description differed, while
|
||||
# two deliberately-parallel variants were refused because theirs did not. The
|
||||
# repo·path·symbol passes a semantic gate because the description differs,
|
||||
# while two deliberately-parallel variants are refused because theirs do not. The
|
||||
# arm is not mis-tuned; it is reading the wrong field, and no threshold fixes
|
||||
# that — lowering it blocks more siblings, raising it allows more copies.
|
||||
#
|
||||
@@ -74,37 +74,36 @@ _SNIPPET_SEMANTIC_THRESHOLD = 0.96
|
||||
|
||||
# A LESSON is measured the same way, for the first of those reasons and not the
|
||||
# second. Its document is `{what} — {trigger}` over a body that opens by
|
||||
# restating the trigger (milestone 385 step 3) — the same prose-about-the-thing
|
||||
# restating the trigger — the same prose-about-the-thing
|
||||
# shape, so two GENUINELY DIFFERENT lessons about one area ("CI cannot see this
|
||||
# class of failure") land in the same sibling band that refused .btn-danger
|
||||
# against .btn-danger-outline at 0.92.
|
||||
# class of failure") land in the same sibling band as .btn-danger against
|
||||
# .btn-danger-outline at 0.92.
|
||||
#
|
||||
# Its own constant rather than reusing the snippet's, because the two are
|
||||
# separate facts that happen to coincide: this number is INHERITED from #2518's
|
||||
# measurement of a structurally analogous corpus, not measured on lessons —
|
||||
# there are none yet to measure. When there are, this moves without dragging
|
||||
# snippets with it.
|
||||
# separate facts that happen to coincide: this number is INHERITED from the
|
||||
# snippet measurement of a structurally analogous corpus, not measured on
|
||||
# lessons, and can move without dragging snippets with it.
|
||||
#
|
||||
# The trade-off differs and is worth naming. A snippet has structural signals
|
||||
# (code, repo·path·symbol) to catch the literal copy a high bar lets through; a
|
||||
# lesson has none, and is not in `_REPORT_KINDS` either, so the duplicate report
|
||||
# is not a backstop for it yet. What remains is the exact-title check, which
|
||||
# still fires. That is the right way round for a gate that BLOCKS: a false
|
||||
# lesson has none. What catches a lesson copy is the exact-title check at
|
||||
# write time and the duplicate report (lessons are in `_REPORT_KINDS`)
|
||||
# afterwards. That is the right way round for a gate that BLOCKS: a false
|
||||
# positive refuses a real lesson outright, while a false negative leaves two
|
||||
# records that can still be merged by hand.
|
||||
_LESSON_SEMANTIC_THRESHOLD = 0.96
|
||||
|
||||
# NOTES AND TASKS BLOCK ONLY A COPY, and surface the rest (#4306). Measured
|
||||
# 2026-09-22 with find_duplicate_records(note, 0.85): of the 74 note pairs at or
|
||||
# above the old 0.90 bar, almost all were DISTINCT siblings — consecutive
|
||||
# dev-logs (0.90–0.94), sub-notes of one design (0.90–0.94), research parts
|
||||
# (0.90–0.97), lore entries (0.95–0.98). The one clear copy sat at 0.997. A
|
||||
# block in that band refused the next dev-log and taught force=true, the same
|
||||
# finding #4134 made for rules. So only the copy band blocks; below it, a
|
||||
# near match is shown on the create reply for the session to judge.
|
||||
# NOTES AND TASKS BLOCK ONLY A COPY, and surface the rest. Measured with
|
||||
# find_duplicate_records(note, 0.85), almost every note pair at or above 0.90
|
||||
# is a DISTINCT sibling — consecutive dev-logs (0.90–0.94), sub-notes of one
|
||||
# design (0.90–0.94), research parts (0.90–0.97), lore entries (0.95–0.98) —
|
||||
# while a clear copy sits near 0.997. A block in that band refuses the next
|
||||
# dev-log and teaches force=true, as it does for rules. So only the copy band
|
||||
# blocks; below it, a near match is shown on the create reply for the session
|
||||
# to judge.
|
||||
_NOTE_COPY_THRESHOLD = 0.98
|
||||
# Where a near match starts being worth reading. The measured pair counts
|
||||
# climb steeply under 0.87 (36 pairs at 0.87 against 200 capped at 0.85), and
|
||||
# climb steeply under 0.87 (roughly five times as many at 0.85), and
|
||||
# the reply lists at most _NOTE_OVERLAP_LIMIT, so this is a cost floor — what
|
||||
# decides whether a match matters is the session reading it.
|
||||
_NOTE_OVERLAP_FLOOR = 0.87
|
||||
@@ -113,7 +112,7 @@ _NOTE_OVERLAP_LIMIT = 3
|
||||
# gate was not part of the measurement, so it keeps the general bar.
|
||||
_COPY_BAND_TYPES = {"note"}
|
||||
|
||||
# The constants above are the DEFAULTS; each is a setting (rule 25, #4385),
|
||||
# The constants above are the DEFAULTS; each is a setting,
|
||||
# because how alike two distinct records get depends on how uniform a corpus
|
||||
# is, which nobody can know from here.
|
||||
GATE_THRESHOLD_KEYS = {
|
||||
@@ -138,7 +137,7 @@ _GATE_MIN_BLOCK = 0.80
|
||||
# may go lower — but not so low that the three slots fill with noise.
|
||||
_GATE_MIN_OVERLAP = 0.70
|
||||
|
||||
# The gate queries per CHUNK of the candidate (#280) — this caps how many
|
||||
# The gate queries per CHUNK of the candidate — this caps how many
|
||||
# searches one save may cost. Eight chunks ≈ five thousand words of candidate;
|
||||
# a duplicate hiding past that is the duplicate report's job to find, not a
|
||||
# reason to stall the write path.
|
||||
@@ -210,7 +209,7 @@ async def _find_snippet_by_structure(
|
||||
"""Exact-identity duplicate of an incoming snippet, or None.
|
||||
|
||||
Two signals, both index-served off `notes.data` and both CERTAIN rather than
|
||||
probabilistic — which is what the semantic arm could not be (#2518):
|
||||
probabilistic — which is what the semantic arm cannot be:
|
||||
|
||||
location the same named thing in the same file. Requires BOTH path and
|
||||
symbol: a path alone is a directory of many artefacts, and
|
||||
@@ -341,15 +340,15 @@ async def find_duplicate_note(
|
||||
|
||||
`code` and `locations` are the snippet's structured fields. They are ignored
|
||||
for every other kind, and passing them is what lets the gate compare
|
||||
ARTEFACTS rather than descriptions of artefacts (#2518).
|
||||
ARTEFACTS rather than descriptions of artefacts.
|
||||
|
||||
`data` is the candidate's structured mirror. For a snippet or lesson it
|
||||
carries the trigger, which the TITLE no longer does (milestone 427): the
|
||||
carries the trigger, which the stored TITLE does not: the
|
||||
title check compares names, and the semantic check rebuilds the embedded
|
||||
document from `data`.
|
||||
|
||||
`overlaps`, when given, is filled with the near matches below the copy band
|
||||
(#4306) — from the SAME searches, so asking costs nothing extra. Only the
|
||||
`overlaps`, when given, is filled with the near matches below the copy
|
||||
band — from the SAME searches, so asking costs nothing extra. Only the
|
||||
kinds in _COPY_BAND_TYPES collect them. The caller creates the record and
|
||||
returns them with `note_overlap_response`.
|
||||
"""
|
||||
@@ -394,16 +393,15 @@ async def find_duplicate_note(
|
||||
|
||||
# --- Signal 3: semantic similarity (only with a substantial body) ---
|
||||
if body and len(body.strip()) >= _MIN_BODY_FOR_SEMANTIC:
|
||||
# Query with the SAME chunker the corpus was embedded with (#280). This
|
||||
# was the copy that mattered most and was easiest to miss: these are
|
||||
# Query with the SAME chunker the corpus was embedded with. These are
|
||||
# QUERY documents, compared against embedded ones — shaped differently
|
||||
# from the corpus, the gate degrades silently. Chunking also makes the
|
||||
# gate see what the whole-document query diluted: a long candidate that
|
||||
# duplicates an existing record IN ONE SECTION now matches on that
|
||||
# from the corpus, the gate degrades silently. Chunking also lets the
|
||||
# gate see what a whole-document query would dilute: a long candidate
|
||||
# that duplicates an existing record IN ONE SECTION matches on that
|
||||
# section. Capped so one pathological paste can't turn a save into
|
||||
# dozens of searches — a duplicate past the cap is the duplicate
|
||||
# report's job, not the gate's.
|
||||
# The EMBEDDED title (milestone 427): a snippet or lesson is stored
|
||||
# The EMBEDDED title: a snippet or lesson is stored
|
||||
# under its name and embedded under `name — trigger`, so the query
|
||||
# document is built the way the corpus was, from `data`.
|
||||
doc_title = embeddings_svc.document_title(title, note_type, data, body)
|
||||
@@ -427,7 +425,7 @@ async def find_duplicate_note(
|
||||
# else's record would refuse their write and point them at
|
||||
# something they may not be able to edit.
|
||||
scope="own",
|
||||
# NOT demoted by supersession (#278). A superseded record is
|
||||
# NOT demoted by supersession. A superseded record is
|
||||
# still a duplicate of what you are about to write — the claim
|
||||
# is that it is no longer CURRENT, not that it is gone. Demoting
|
||||
# it here would let the same note be recorded a second time, and
|
||||
@@ -456,7 +454,7 @@ async def find_duplicate_note(
|
||||
|
||||
def note_overlap_response(overlaps: list[NoteOverlap], kind: str) -> dict:
|
||||
"""The keys a note or task create adds to its reply when an existing record
|
||||
reads closely like the one just written (#4306). Empty when none.
|
||||
reads closely like the one just written. Empty when none.
|
||||
|
||||
The judgement is the session's: the embedding cannot tell a restatement
|
||||
from the next dev-log in a series, and a reader can in one look."""
|
||||
@@ -500,10 +498,11 @@ def batch_overlap_response(per_record: dict[int, list[NoteOverlap]]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# --- corpus-wide near-duplicate report (#2088) -------------------------------
|
||||
# --- corpus-wide near-duplicate report ---------------------------------------
|
||||
# The gate above PREVENTS a new duplicate; merge_snippets CURES one you point it
|
||||
# at. Neither FINDS the duplicates already sitting in the record — someone had to
|
||||
# notice them by hand, which is the exact failure the Drafter exists to remove.
|
||||
# at. Neither FINDS the duplicates already sitting in the record — without this,
|
||||
# someone has to notice them by hand, the exact failure the Drafter exists to
|
||||
# remove.
|
||||
#
|
||||
# WHY OWN SNIPPETS ONLY. merge_snippets requires every record to share the
|
||||
# target's owner (cross-owner merge is out of scope), so a report that surfaced
|
||||
@@ -511,21 +510,20 @@ def batch_overlap_response(per_record: dict[int, list[NoteOverlap]]) -> dict:
|
||||
# scope here is set by what the operator can actually act on, not by what they
|
||||
# can see.
|
||||
#
|
||||
# WHY THE FLOOR IS PER-KIND. Chunked embeddings (#280) changed what a pair
|
||||
# score MEANS for multi-chunk records: a note-pair's similarity is its closest
|
||||
# chunk pair, so any family of related long records — a dev-log run, a research
|
||||
# fan-out — clears a floor that whole-document vectors used to dilute below it.
|
||||
# Measured on the live corpus (2026-08-09): at 0.82 the note/task reports
|
||||
# saturate the pair cap with related-but-distinct families, while 0.93+ returns
|
||||
# the genuinely-alike records. Snippets are single-chunk (short by nature), so
|
||||
# their similarity scale never shifted and they keep the old floor.
|
||||
# WHY THE FLOOR IS PER-KIND. With chunked embeddings a note-pair's
|
||||
# similarity is its closest chunk pair, so any family of related long records
|
||||
# — a dev-log run, a research fan-out — clears a floor that whole-document
|
||||
# vectors would dilute below it. Measured: at 0.82 the note/task reports
|
||||
# saturate the pair cap with related-but-distinct families, while 0.93+
|
||||
# returns the genuinely-alike records. Snippets are single-chunk (short by
|
||||
# nature), so their scale is the whole-document one and their floor is lower.
|
||||
#
|
||||
# Snippets sit BELOW the 0.90 write gate — the report catches what the gate
|
||||
# lets through. Notes/tasks sit ABOVE it, and that is not a contradiction: the
|
||||
# gate compares a new record against best-matching chunks too, but it blocks a
|
||||
# WRITE and must stay forgiving, while the report proposes a REVIEW and at
|
||||
# chunk grain 0.90 would still drown it in families. Each is a setting rather
|
||||
# than a constant (rule #25) because the right value depends on how uniform a
|
||||
# than a constant because the right value depends on how uniform a
|
||||
# corpus is, and nobody can guess that from here.
|
||||
|
||||
DUPLICATE_THRESHOLD_KEYS = {
|
||||
@@ -622,13 +620,13 @@ def _drop_sibling_pairs(
|
||||
apart, and merging them would destroy a distinction someone made on purpose.
|
||||
|
||||
Without this, a design system's button family reports as a single merge set:
|
||||
eight recipes, every direct pair over the floor, top score 0.92 (#2518).
|
||||
eight recipes, every direct pair over the floor, top score 0.92.
|
||||
They resemble each other because variants of one component are SUPPOSED to —
|
||||
same selector prefix, same token families, deliberately parallel prose. The
|
||||
similarity is read correctly; it just does not mean "duplicate".
|
||||
|
||||
THE COST, stated plainly: a helper genuinely recorded twice under two names
|
||||
— `debounce` and `useDebouncedRef` — is no longer reported. That is real
|
||||
— `debounce` and `useDebouncedRef` — is not reported. That is real
|
||||
recall lost. It is the better trade because the report is a merge PROPOSAL:
|
||||
a missed pair costs a duplicate nobody was going to notice anyway, while a
|
||||
wrong set invites an operator to collapse a component family in one click.
|
||||
@@ -651,10 +649,10 @@ def _drop_sibling_pairs(
|
||||
|
||||
# What a duplicate group should become, per kind. Snippets merge losslessly —
|
||||
# one helper, every call site folded into the survivor. Notes and tasks do NOT
|
||||
# merge: folding two records destroys what each actually said, which is why
|
||||
# consolidated_at was dropped rather than built (#2483). The report can detect
|
||||
# the cluster and name the options; deciding which applies is the reader's job,
|
||||
# because it turns on what the records SAY, not how alike they score (#2547).
|
||||
# merge: folding two records destroys what each actually said. The report can
|
||||
# detect the cluster and name the options; deciding which applies is the
|
||||
# reader's job, because it turns on what the records SAY, not how alike they
|
||||
# score.
|
||||
_KIND_SUGGESTION = {
|
||||
"snippet": (
|
||||
"Same reusable thing recorded more than once → merge_snippets(target, "
|
||||
@@ -759,7 +757,7 @@ async def find_duplicate_records(
|
||||
left_note = aliased(Note, name="left_note")
|
||||
right_note = aliased(Note, name="right_note")
|
||||
distance = left.embedding.cosine_distance(right.embedding)
|
||||
# Chunk grain (#280): a note-pair's similarity is its closest CHUNK pair —
|
||||
# Chunk grain: a note-pair's similarity is its closest CHUNK pair —
|
||||
# two records duplicate each other where their most similar sections do,
|
||||
# which is the honest definition when one section of a long note restates
|
||||
# another record. GROUP BY collapses the chunk cross-product to one row
|
||||
@@ -929,11 +927,10 @@ async def find_duplicate_rule(
|
||||
project (a project rule) — the one signal certain enough to BLOCK on.
|
||||
Fail-open like find_duplicate_note.
|
||||
|
||||
This is not the only duplicate signal for rules. It said so until #4134 —
|
||||
"rules aren't a semantic-retrieval surface" — which stopped being true
|
||||
when rules were embedded (rule_document, semantic_search_rules), and a
|
||||
title is the field LEAST likely to collide when someone is deliberately
|
||||
writing a second record about the same moment. find_overlapping_rules is
|
||||
This is not the only duplicate signal for rules: rules are embedded
|
||||
(rule_document, semantic_search_rules), and a title is the field LEAST
|
||||
likely to collide when someone is deliberately writing a second record
|
||||
about the same moment. find_overlapping_rules is
|
||||
the meaning half; it surfaces rather than blocks, for the reason recorded
|
||||
above _RULE_OVERLAP_FLOOR."""
|
||||
norm = " ".join((title or "").split()).lower()
|
||||
@@ -958,7 +955,7 @@ async def find_duplicate_rule(
|
||||
|
||||
|
||||
|
||||
# --- rule / preference overlap (#4134) ----------------------------------------
|
||||
# --- rule / preference overlap -----------------------------------------------
|
||||
# Rules and preferences are one table and one ranking: every hook arm searches
|
||||
# them with no `kind` filter. So a preference that restates a rule is not a
|
||||
# harmless near-copy — when only the preference places, a session receives
|
||||
@@ -966,10 +963,9 @@ async def find_duplicate_rule(
|
||||
# gate above cannot see it: a second record about the same moment is exactly
|
||||
# the case where someone chose a different title.
|
||||
#
|
||||
# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured 2026-09-21 on bge-small-en-
|
||||
# v1.5, querying with the gate's own rule_document shape across 16 sampled
|
||||
# records (10 preferences, 6 rules) and reading the nearest OTHER record (#4134
|
||||
# has the ids):
|
||||
# WHY THIS SURFACES INSTEAD OF BLOCKING. Measured on bge-small-en-v1.5,
|
||||
# querying with the gate's own rule_document shape across 16 sampled records
|
||||
# (10 preferences, 6 rules) and reading the nearest OTHER record:
|
||||
#
|
||||
# a preference rewording an existing rule 0.850
|
||||
# nearest distinct neighbours, 16 samples 0.672 – 0.853
|
||||
@@ -1018,7 +1014,7 @@ async def find_overlapping_rules(
|
||||
|
||||
Queried with rule_document — the exact shape the corpus is embedded as —
|
||||
so the score compares like with like. Both kinds, because the harm is
|
||||
across them (#4134).
|
||||
across them.
|
||||
|
||||
Scope follows the new record's home. A project rule is compared with
|
||||
global rules plus that project's own, the set it will rank against. A
|
||||
@@ -1079,7 +1075,7 @@ def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# --- the plan gate (milestone 415) -------------------------------------------
|
||||
# --- the plan gate -----------------------------------------------------------
|
||||
# A session asked "what work is open?" that cannot see an existing plan makes a
|
||||
# second one: a new milestone beside the one that already covers the work, or
|
||||
# loose tasks beside it. Each copy then collects its own steps, and neither
|
||||
@@ -1095,14 +1091,14 @@ def overlap_response(overlaps: list[RuleOverlap], new_kind: str) -> dict:
|
||||
# project is exactly what adding steps to its existing plan needs, so a caller
|
||||
# who reaches this gate can act on whatever it returns.
|
||||
#
|
||||
# Its own threshold, as a setting (rule 25), and lower than the note gate's
|
||||
# Its own threshold, as a setting, and lower than the note gate's
|
||||
# 0.90 because plan documents are shaped differently: a milestone is embedded
|
||||
# as title, description and plan (embeddings.milestone_document), while the
|
||||
# candidate usually has no description and carries its steps instead, so even
|
||||
# a faithful rewording never scores like a copy.
|
||||
#
|
||||
# Measured after the first deploy (2026-09-15, bge-small-en-v1.5, #4079): three
|
||||
# plans reworded from existing active milestones scored 0.83, 0.85 and 0.87
|
||||
# Measured on bge-small-en-v1.5: three plans reworded from existing active
|
||||
# milestones scored 0.83, 0.85 and 0.87
|
||||
# against the milestone they restated; the nearest DIFFERENT plan for each, and
|
||||
# a distinct plan on a neighbouring topic, scored 0.72-0.77. At 0.90 the
|
||||
# semantic arm matched nothing, leaving only the title arm. 0.80 sits in the
|
||||
@@ -1133,7 +1129,7 @@ def plan_candidate_text(
|
||||
The steps belong in it: a plan passed with steps and no design is still
|
||||
recognisable by them, and what its steps say is most of what makes two
|
||||
plans the same plan. Each step is (title, body), joined by embedding_text
|
||||
like every other record that becomes embedded text (#2486).
|
||||
like every other record that becomes embedded text.
|
||||
"""
|
||||
parts = [(description or "").strip(), (body or "").strip()]
|
||||
parts += [embeddings_svc.embedding_text(t, b) for t, b in (steps or [])]
|
||||
@@ -1158,7 +1154,7 @@ async def find_matching_plan(
|
||||
|
||||
if not project_id:
|
||||
return None
|
||||
# Rule 78, before either arm: a match names a milestone, and a caller who
|
||||
# Access first, before either arm: a match names a milestone, and a caller who
|
||||
# cannot read the project must not learn its plans by guessing titles.
|
||||
try:
|
||||
if not await can_read_project(user_id, project_id):
|
||||
|
||||
@@ -267,8 +267,8 @@ async def _assert_project_owned(session, project_id: int, user_id: int) -> None:
|
||||
|
||||
|
||||
# The vocabularies migration 0088's CHECK constraints enforce. Named here so
|
||||
# a caller can be corrected before the database refuses it (rule 36 keeps the
|
||||
# two in step; this keeps the error readable).
|
||||
# a caller can be corrected before the database refuses it (a migration that
|
||||
# widens one widens the other; this keeps the error readable).
|
||||
RELATION_KINDS = ("co_surfaces", "overrides", "elaborates")
|
||||
# Migration 0098's CHECK. `rule` binds; `preference` is how the operator
|
||||
# wants work done — see the model comment for why both live on one table.
|
||||
@@ -303,23 +303,21 @@ def _valid_kind(kind: str) -> str:
|
||||
return kind if kind in KINDS else "rule"
|
||||
|
||||
|
||||
# Re-exported, not redefined. Notes gained the same trio in milestone 317 and
|
||||
# this reading of it is genuinely common, so it moved to services/verification
|
||||
# — the DRY win note 3163 names, as against sharing the QUERY, which the two
|
||||
# record types cannot (a rule scopes by rulebook ownership, a note by the note
|
||||
# ACL). Kept importable from here because callers already reach for it here.
|
||||
# Re-exported, not redefined. Notes carry the same verification trio and this
|
||||
# reading of it is common to both, so it lives in services/verification —
|
||||
# sharing the reading, not the QUERY, which the two record types cannot share
|
||||
# (a rule scopes by rulebook ownership, a note by the note ACL). Importable
|
||||
# from here because callers reach for it here.
|
||||
last_verified_label = _last_verified_label
|
||||
|
||||
|
||||
def rule_brief(rule: Rule, **extra) -> dict:
|
||||
"""The shape a rule takes when it is SURFACED rather than opened.
|
||||
|
||||
One builder for every payload that hands rules to an agent, because there
|
||||
were three copies of this dict and they had already diverged — two carried
|
||||
`topic_id`, one didn't, and none carried the timestamps the model has held
|
||||
all along. That omission is why a rule written before the capability it
|
||||
duplicates was indistinguishable, at read time, from one still doing work
|
||||
(the FabledCurator case, note 3026).
|
||||
One builder for every payload that hands rules to an agent, because
|
||||
copies of this dict diverge. It carries the timestamps: without them a
|
||||
rule written before the capability it duplicates is indistinguishable, at
|
||||
read time, from one still doing work.
|
||||
|
||||
`updated_at` is a DATE, not a stamp: the question it answers is "how old
|
||||
is this?", and a full ISO string across an always-on set is ~2k characters
|
||||
@@ -343,8 +341,8 @@ def rule_brief(rule: Rule, **extra) -> dict:
|
||||
"kind": rule.kind or "rule",
|
||||
"updated_at": rule.updated_at.date().isoformat() if rule.updated_at else None,
|
||||
}
|
||||
# Attached only when present (#2483: never a null key that reads as a
|
||||
# capability the record doesn't have).
|
||||
# Attached only when present: never a null key that reads as a
|
||||
# capability the record doesn't have.
|
||||
if rule.when_to_apply:
|
||||
out["when_to_apply"] = rule.when_to_apply
|
||||
if rule.arose_from_id:
|
||||
@@ -353,7 +351,7 @@ def rule_brief(rule: Rule, **extra) -> dict:
|
||||
# signal, and it says two things at once: this rule asserts a fact that
|
||||
# can go false, and here is how long ago anyone confirmed it. The check
|
||||
# text itself stays in get_rule; a listing needs to know WHICH rules can
|
||||
# rot, not how to test them. "never" rather than null, per #2483: a key
|
||||
# rot, not how to test them. "never" rather than null: a key
|
||||
# that reads as an unused capability is a different claim from a rule
|
||||
# nobody has ever verified.
|
||||
stamp = last_verified_label(rule)
|
||||
@@ -373,7 +371,7 @@ def _refresh_rule_embedding(rule: Rule) -> None:
|
||||
|
||||
Detaching also means this task races anything that deletes the rule out
|
||||
from under it. That is not handled here: `upsert_rule_embedding` claims
|
||||
the rule's row before touching its vectors, and loses if it can't (#3262).
|
||||
the rule's row before touching its vectors, and loses if it can't.
|
||||
"""
|
||||
try:
|
||||
import asyncio
|
||||
@@ -394,11 +392,10 @@ def _refresh_rule_embedding(rule: Rule) -> None:
|
||||
async def co_surfaced_partners(user_id: int, rule_ids: list[int]) -> list[Rule]:
|
||||
"""Rules that must arrive WITH the given ones, because they fail together.
|
||||
|
||||
This is the whole reason `co_surfaces` exists. Rule 144 was split off rule
|
||||
46 and folded back into it the same day, on the correct observation that
|
||||
"either rule could surface without the other and miss exposing a project to
|
||||
what the entire shape is intended to be." Merging was the only fix
|
||||
available; this is the fix that should have been available.
|
||||
This is the whole reason `co_surfaces` exists. Two rules that guard one
|
||||
shape between them fail apart: either can surface without the other and
|
||||
leave a project seeing half of what the shape is meant to be. Without
|
||||
this edge the only remedy is merging them into one rule.
|
||||
|
||||
Only rules the caller OWNS: an edge is not a back door into someone else's
|
||||
rulebook. Whether a partner can reach a given PROJECT is the caller's
|
||||
@@ -441,7 +438,7 @@ async def rule_detail(
|
||||
|
||||
ONE seam for both doors and every write path, so create, update and get
|
||||
cannot disagree about what a rule looks like coming back — the same
|
||||
reasoning as attach_relations for notes (#2859), and the same reasoning
|
||||
reasoning as attach_relations for notes, and the same reasoning
|
||||
rule_brief exists for one level down.
|
||||
|
||||
`system_ids=None` / `moments=None` mean "leave them alone"; a list
|
||||
@@ -457,7 +454,7 @@ async def rule_detail(
|
||||
systems = (await list_rule_systems([rule.id])).get(rule.id, [])
|
||||
mounted = (await list_rule_moments([rule.id])).get(rule.id, [])
|
||||
relations = (await list_rule_relations([rule.id])).get(rule.id, [])
|
||||
# Attached only when present (#2483): an empty key reads as a capability
|
||||
# Attached only when present: an empty key reads as a capability
|
||||
# the record has and isn't using, which is a different claim.
|
||||
if systems:
|
||||
data["systems"] = systems
|
||||
@@ -466,7 +463,7 @@ async def rule_detail(
|
||||
if relations:
|
||||
data["relations"] = relations
|
||||
# The concrete situations judged (or proposed) to be instances of this
|
||||
# rule — milestone 440. Same present-only convention as the two above.
|
||||
# rule. Same present-only convention as the two above.
|
||||
from scribe.services.lesson_rules import attach_rule_lessons
|
||||
|
||||
await attach_rule_lessons(user_id, data, rule.id)
|
||||
@@ -542,7 +539,7 @@ async def create_project_rule(
|
||||
"""Create a rule scoped to a single project (no rulebook ceremony).
|
||||
|
||||
Project-scoped rules apply only to the named project: retrieval surfaces
|
||||
them in that project's sessions and nowhere else (milestone 414), where a
|
||||
them in that project's sessions and nowhere else, where a
|
||||
rule in a rulebook topic is global. Topic_id is left NULL — the CHECK
|
||||
constraint enforces exactly-one of (topic_id, project_id).
|
||||
"""
|
||||
@@ -577,7 +574,7 @@ async def list_rules(
|
||||
) -> list[Rule]:
|
||||
"""List rules by rulebook, topic or project. Ownership-scoped.
|
||||
|
||||
A rule has one home (milestone 414): a rulebook topic, where it is global,
|
||||
A rule has one home: a rulebook topic, where it is global,
|
||||
or a project. So the filters name homes rather than reach:
|
||||
|
||||
- `project_id` lists that project's OWN rules. Global rules apply to every
|
||||
@@ -588,9 +585,6 @@ async def list_rules(
|
||||
- No filter lists every global rule. A user's project rules are left out:
|
||||
they belong to their projects, and mixing them into the rulebook listing
|
||||
would surprise its callers.
|
||||
|
||||
Before milestone 414, `project_id` returned the rules of every rulebook the
|
||||
project SUBSCRIBED to plus its own. Subscriptions are gone.
|
||||
"""
|
||||
from scribe.models.project import Project
|
||||
|
||||
@@ -759,15 +753,15 @@ async def move_rule(
|
||||
rule_id: int, user_id: int, *, topic_id: int = 0, project_id: int = 0,
|
||||
) -> Optional[Rule]:
|
||||
"""Give a rule a new home — into a rulebook topic (global) or onto a
|
||||
project — keeping its id, history, Systems and relations (milestone 414).
|
||||
project — keeping its id, history, Systems and relations.
|
||||
|
||||
A rule's home IS its reach: in a topic it applies to every project, on a
|
||||
project to that project alone. Recreating the rule in the other home and
|
||||
trashing the original would lose its id (and every record citing it), its
|
||||
edit history, its area tags and its typed edges, which is why this exists.
|
||||
|
||||
Exactly one of `topic_id` / `project_id`, matching the model's CHECK
|
||||
(migration 0059). Raises ValueError for: neither or both named, a target
|
||||
Exactly one of `topic_id` / `project_id`, matching the model's CHECK.
|
||||
Raises ValueError for: neither or both named, a target
|
||||
the caller does not own, the rule already living there, or a topic that
|
||||
already holds a live rule with this title (uq_rule_per_topic) — the message
|
||||
names that rule, rather than letting the constraint fail the commit.
|
||||
@@ -775,10 +769,10 @@ async def move_rule(
|
||||
|
||||
WHAT A MOVE DOES NOT DO, deliberately:
|
||||
|
||||
- No version. A rule's history records its TEXT (milestone 323, decision
|
||||
4); its place is not text, and folding it in would make "version" mean
|
||||
two things. The rule's `updated_at` moves; say why a rule moved where
|
||||
the decision is recorded.
|
||||
- No version. A rule's history records its TEXT; its place is not text,
|
||||
and folding it in would make "version" mean two things. The rule's
|
||||
`updated_at` moves; say why a rule moved where the decision is
|
||||
recorded.
|
||||
- No duplicate gate. Nothing new enters the corpus — the same rule changes
|
||||
home — so there is no second record to warn about.
|
||||
- No re-embed. The rule's document is its title, statement and trigger;
|
||||
@@ -820,7 +814,7 @@ async def move_rule(
|
||||
return rule
|
||||
|
||||
|
||||
# ── Edit history (milestone 323) ───────────────────────────────────────
|
||||
# ── Edit history ───────────────────────────────────────────────────────
|
||||
#
|
||||
# The ACL-scoped reads live HERE rather than in services/rule_versions.py,
|
||||
# and not by preference: rulebooks imports rule_versions for the write path,
|
||||
@@ -862,14 +856,14 @@ async def get_rule_version(rule_id: int, version_id: int, user_id: int):
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
# ── Drift: what Scribe changed about how it works with you (#3895) ─────
|
||||
# ── Drift: what Scribe changed about how it works with you ─────────────
|
||||
#
|
||||
# A preference is the one record kind the AGENT rewrites in the ordinary
|
||||
# course of working, which is the property that makes it useful and the
|
||||
# property that makes it dangerous. Milestone 399 named the risk up front:
|
||||
# an agent misreads one session, rewrites a preference, and follows the
|
||||
# rewritten version forever while the operator never sees the moment it
|
||||
# changed — a confident wrong answer wearing the operator's own authority.
|
||||
# property that makes it dangerous: an agent misreads one session, rewrites
|
||||
# a preference, and follows the rewritten version forever while the operator
|
||||
# never sees the moment it changed — a confident wrong answer wearing the
|
||||
# operator's own authority.
|
||||
#
|
||||
# `rule_versions` already records every rewrite. What it does not do is
|
||||
# ARRIVE: a history you have to open one rule at a time, having first
|
||||
@@ -921,8 +915,8 @@ async def recent_preference_drift(user_id: int, limit: int = 20) -> list[dict]:
|
||||
this get here" belongs.
|
||||
|
||||
Each row carries what it said before, what it says now, and `taught_by` —
|
||||
the record named by `arose_from_id`, which step 2 made required on every
|
||||
agent-written preference. That is the provenance: not who typed it (the
|
||||
the record named by `arose_from_id`, which every agent-written
|
||||
preference must carry. That is the provenance: not who typed it (the
|
||||
agent acts as the operator's own user, so the actor column cannot tell
|
||||
them apart) but what the change was learned from.
|
||||
|
||||
@@ -974,7 +968,7 @@ async def recent_preference_drift(user_id: int, limit: int = 20) -> list[dict]:
|
||||
row: dict = {
|
||||
# rule_brief, not a hand-written dict: the drift pane links
|
||||
# straight into the editor, so the row has to be the same shape
|
||||
# every other rule listing is (#3313's three-copies lesson).
|
||||
# every other rule listing is.
|
||||
"rule": rule_brief(rule),
|
||||
# What it said BEFORE this rewrite. `statement` and
|
||||
# `when_to_apply` only — those are the two fields that change how
|
||||
@@ -1037,8 +1031,7 @@ async def restore_rule_version(
|
||||
raise ValueError(
|
||||
"only a preference can be restored in one action. A rule is "
|
||||
"the operator's own decision and reverting it goes through an "
|
||||
"ordinary edit, so the reason for the rewrite stays visible "
|
||||
"(milestone 323)."
|
||||
"ordinary edit, so the reason for the rewrite stays visible."
|
||||
)
|
||||
version = (await session.execute(
|
||||
select(RuleVersion).where(
|
||||
@@ -1069,7 +1062,7 @@ async def restore_rule_version(
|
||||
)
|
||||
|
||||
|
||||
# ── Canon tags + typed edges (milestone 307) ───────────────────────────
|
||||
# ── Canon tags + typed edges ───────────────────────────────────────────
|
||||
|
||||
async def set_rule_systems(
|
||||
rule_id: int, user_id: int, canonical_ids: list[int],
|
||||
@@ -1114,7 +1107,7 @@ async def set_rule_moments(
|
||||
rule_id: int, user_id: int, moments: list[str], *,
|
||||
note: str = "", judged=(), verdict: str | None = None,
|
||||
) -> list[str] | None:
|
||||
"""Replace which MOMENTS a rule arrives at (milestone 458). None if not owned.
|
||||
"""Replace which MOMENTS a rule arrives at. None if not owned.
|
||||
|
||||
Set-semantics like set_rule_systems: the list given IS the state after.
|
||||
Every name is checked against the catalog first and the whole write is
|
||||
@@ -1122,7 +1115,7 @@ async def set_rule_moments(
|
||||
back as attached and never fire, the silent miss moments exist to end.
|
||||
|
||||
THE ONE WRITE PATH for mounts, so it is also where a change is recorded as
|
||||
a judgment (step 7): what was added is confirmed, what was removed is
|
||||
a judgment: what was added is confirmed, what was removed is
|
||||
rejected, in the same transaction as the mounts. `note`, `judged` and
|
||||
`verdict` come from `rule_moment_judgments.judge`, which says why.
|
||||
"""
|
||||
@@ -1178,7 +1171,7 @@ async def rules_on_moments(
|
||||
) -> list[tuple[Rule, str]]:
|
||||
"""The rules mounted on any of these moments that this caller may be shown.
|
||||
|
||||
The delivery read (milestone 458 step 4), and a LOOKUP: no score, no bar.
|
||||
The delivery read, and a LOOKUP: no score, no bar.
|
||||
Scoped by the same home clause the semantic search uses (rule_scope), so
|
||||
a mount never delivers another project's rule. Each rule appears once,
|
||||
under the first of `moments` it is mounted on — the caller passes them in
|
||||
@@ -1389,17 +1382,15 @@ async def get_applicable_rules(
|
||||
}
|
||||
|
||||
NOT WHAT A SESSION RECEIVES. Rules reach a session by retrieval, which
|
||||
reads a rule's home (milestone 414): global rules everywhere, a project's
|
||||
reads a rule's home: global rules everywhere, a project's
|
||||
own rules in that project. This is the listing a planning read carries so
|
||||
a reader can see which constraints are on the table, and it is narrower
|
||||
than retrieval on purpose — "every global rule" is not a list anyone reads.
|
||||
|
||||
`rules` is the global rules TAGGED to a canonical area this project works
|
||||
in (milestone 307, D7): a deterministic tag match, never a similarity
|
||||
score. Before milestone 414 this was every rule in a SUBSCRIBED rulebook,
|
||||
narrowed by area only where an author had tagged one. With subscriptions
|
||||
gone there is no opt-in left to scope the untagged ones, and an untagged
|
||||
global rule is general by construction — it arrives by retrieval when the
|
||||
in: a deterministic tag match, never a similarity score. There is no
|
||||
opt-in to scope the untagged ones, and an untagged global rule is general
|
||||
by construction — it arrives by retrieval when the
|
||||
work makes it relevant, like every other rule.
|
||||
|
||||
`project_rules` is the project's own, never filtered by area: a rule
|
||||
@@ -1467,7 +1458,7 @@ async def get_applicable_rules(
|
||||
)).all()
|
||||
project_rules = [rule_brief(rule) for (rule,) in proj_rule_rows]
|
||||
|
||||
# Edges travel with the rules they belong to (milestone 307).
|
||||
# Edges travel with the rules they belong to.
|
||||
#
|
||||
# A co_surfaces partner that was not otherwise selected is ADDED, because a
|
||||
# rule that arrives without the half it fails with is the failure the edge
|
||||
@@ -1510,9 +1501,8 @@ def rules_payload(
|
||||
source. Every one of those surfaces is a bulk delivery — the applicable set
|
||||
handed over whole, chosen by nobody — so this is the one place that has to
|
||||
emit for all of them. Doing it per-caller instead would be five sites to
|
||||
remember, and #3430 gap 2 is what that costs: the process→skill sync went
|
||||
un-emitted through an entire dedicated telemetry survey because nothing
|
||||
forced its surface to be accounted for.
|
||||
remember, and a site nobody is forced to account for goes un-emitted
|
||||
without anyone noticing.
|
||||
|
||||
`source` stays the CALLER's name rather than a constant, so the readout can
|
||||
still separate the session handshake from a mid-session milestone read;
|
||||
@@ -1527,12 +1517,11 @@ def rules_payload(
|
||||
EVERY FORM LISTS, NONE RESTATES. Rules reach a session in full by
|
||||
retrieval, so these payloads say which constraints exist — id and title,
|
||||
the topic a global rule sits in, `via` for a co_surfaces partner — and
|
||||
get_rule reads one. Planning reads carried the full rule_brief until a
|
||||
project's listing grew to every global rule tagged to its areas
|
||||
(milestone 414) and start_planning replied with 92k characters (#4081),
|
||||
the shape #4045 had just removed from the handshake.
|
||||
get_rule reads one. A full rule_brief per rule would scale with every
|
||||
global rule tagged to a project's areas, and a planning reply can run to
|
||||
tens of thousands of characters.
|
||||
|
||||
`brief` is the session handshake's form (#4045): the project's own rules
|
||||
`brief` is the session handshake's form: the project's own rules
|
||||
only. Only what is shown is recorded as surfaced.
|
||||
"""
|
||||
project_rules = [_rule_line(r) for r in applicable.get("project_rules", [])]
|
||||
@@ -1556,7 +1545,7 @@ def rules_payload(
|
||||
|
||||
def _rule_line(brief: dict) -> dict:
|
||||
"""One rule as a listing names it: enough to recognise it and fetch it.
|
||||
Keys a row does not carry are left out rather than sent empty (#2483)."""
|
||||
Keys a row does not carry are left out rather than sent empty."""
|
||||
line = {"id": brief["id"], "title": brief["title"]}
|
||||
for key in ("topic_title", "via"):
|
||||
if brief.get(key):
|
||||
@@ -1564,7 +1553,7 @@ def _rule_line(brief: dict) -> dict:
|
||||
return line
|
||||
|
||||
|
||||
# ── The staleness marker (milestone 323 step 5) ────────────────────────
|
||||
# ── The staleness marker ───────────────────────────────────────────────
|
||||
#
|
||||
# WHAT THIS CAN AND CANNOT SEE. An etag catches a rule that MOVED after a
|
||||
# session loaded it. It is not a general staleness check, and a reader who
|
||||
@@ -1585,7 +1574,7 @@ def _rule_line(brief: dict) -> dict:
|
||||
_ETAG_EMPTY = "empty|0"
|
||||
|
||||
|
||||
# ── The staleness sweep (milestone 312) ────────────────────────────────
|
||||
# ── The staleness sweep ────────────────────────────────────────────────
|
||||
|
||||
async def rules_due_for_verification(
|
||||
user_id: int,
|
||||
|
||||
Reference in New Issue
Block a user