feat(dedup): the create gate's similarity bars are settings (#4385, rule 25)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / TypeScript typecheck (push) Successful in 57s
CI & Build / integration (push) Successful in 56s
CI & Build / Python tests (push) Successful in 1m38s
CI & Build / Build & push image (push) Successful in 38s

gate_bars(user_id, note_type) resolves the block bar and, for notes and
tasks, the overlap floor from kb_gate_* settings, with the old constants
as defaults. Fail-open on an unreadable value; a block bar clamps at 0.80
and the overlap floor at 0.70 and never above the bar. Five fields in
Settings beside the duplicate-report floors.

CLAIM_LEASE stays a constant, with the reason written at it: a per-user
lease would make one shared task live to one reader and dead to another.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-24 07:11:10 -04:00
co-authored by Claude Opus 5.5
parent baf22179ef
commit 4502f0a1ae
5 changed files with 286 additions and 11 deletions
+63 -11
View File
@@ -113,6 +113,31 @@ _NOTE_OVERLAP_LIMIT = 3
# gate was not part of the measurement, so it keeps the general bar.
_COPY_BAND_TYPES = {"note"}
# The constants above are the DEFAULTS; each is a setting (rule 25, #4385),
# because how alike two distinct records get depends on how uniform a corpus
# is, which nobody can know from here.
GATE_THRESHOLD_KEYS = {
"general": "kb_gate_threshold",
SNIPPET_NOTE_TYPE: "kb_gate_threshold_snippet",
LESSON_NOTE_TYPE: "kb_gate_threshold_lesson",
"note_copy": "kb_gate_threshold_note_copy",
"note_overlap": "kb_gate_note_overlap_floor",
}
GATE_DEFAULT_THRESHOLDS = {
"general": _SEMANTIC_THRESHOLD,
SNIPPET_NOTE_TYPE: _SNIPPET_SEMANTIC_THRESHOLD,
LESSON_NOTE_TYPE: _LESSON_SEMANTIC_THRESHOLD,
"note_copy": _NOTE_COPY_THRESHOLD,
"note_overlap": _NOTE_OVERLAP_FLOOR,
}
# A BLOCK bar may not be set below this. The gate refuses the write, so a
# mistyped 0.1 would refuse every create that shared a topic with anything —
# the report's floors can go low because a report only proposes.
_GATE_MIN_BLOCK = 0.80
# The overlap floor only decides what is LISTED for the session to judge, so it
# may go lower — but not so low that the three slots fill with noise.
_GATE_MIN_OVERLAP = 0.70
# The gate queries per CHUNK of the candidate (#280) — this caps how many
# searches one save may cost. Eight chunks ≈ five thousand words of candidate;
# a duplicate hiding past that is the duplicate report's job to find, not a
@@ -248,16 +273,43 @@ async def _find_snippet_by_structure(
def _semantic_threshold(note_type: str) -> float:
"""The semantic bar for this kind — a lookup, so the kinds that need a
different one are named in a single place rather than in a conditional
that grows a branch per kind."""
if note_type == SNIPPET_NOTE_TYPE:
return _SNIPPET_SEMANTIC_THRESHOLD
if note_type == LESSON_NOTE_TYPE:
return _LESSON_SEMANTIC_THRESHOLD
"""The DEFAULT semantic bar for this kind — what `gate_bars` falls back on
when the user has not set one."""
return GATE_DEFAULT_THRESHOLDS[_gate_key(note_type)]
def _gate_key(note_type: str) -> str:
if note_type in (SNIPPET_NOTE_TYPE, LESSON_NOTE_TYPE):
return note_type
if note_type in _COPY_BAND_TYPES:
return _NOTE_COPY_THRESHOLD
return _SEMANTIC_THRESHOLD
return "note_copy"
return "general"
async def _gate_setting(user_id: int, key: str, lo: float) -> float:
from scribe.services.settings import get_setting
default = GATE_DEFAULT_THRESHOLDS[key]
try:
value = float(await get_setting(user_id, GATE_THRESHOLD_KEYS[key], str(default)))
except Exception:
# Fail-open like the rest of the gate: an unreadable setting falls back
# to the measured default rather than blocking or waving through.
value = default
return min(1.0, max(lo, value))
async def gate_bars(user_id: int, note_type: str) -> tuple[float, float]:
"""(block_at, overlap_floor) for `note_type` on this user's install.
The overlap floor is only read for the copy-band kinds, and never sits
above the block bar — a floor over the bar would list nothing.
"""
block_at = await _gate_setting(user_id, _gate_key(note_type), _GATE_MIN_BLOCK)
if note_type not in _COPY_BAND_TYPES:
return block_at, block_at
floor = await _gate_setting(user_id, "note_overlap", _GATE_MIN_OVERLAP)
return block_at, min(floor, block_at)
@dataclass
@@ -355,7 +407,7 @@ async def find_duplicate_note(
# under its name and embedded under `name — trigger`, so the query
# document is built the way the corpus was, from `data`.
doc_title = embeddings_svc.document_title(title, note_type, data, body)
block_at = _semantic_threshold(note_type)
block_at, overlap_floor = await gate_bars(user_id, note_type)
collect = overlaps is not None and note_type in _COPY_BAND_TYPES
near: dict[int, NoteOverlap] = {}
for query in embeddings_svc.chunk_document(doc_title, body)[:_GATE_MAX_CHUNKS]:
@@ -369,7 +421,7 @@ async def find_duplicate_note(
user_id, query, project_id=project_id, is_task=is_task,
orphan_only=(project_id is None),
limit=3,
threshold=_NOTE_OVERLAP_FLOOR if collect else block_at,
threshold=overlap_floor if collect else block_at,
# Owner-only, deliberately: this gate BLOCKS a create and tells
# the caller to update the match instead. Matching someone
# else's record would refuse their write and point them at
+6
View File
@@ -37,6 +37,12 @@ if TYPE_CHECKING:
# compaction, a resume or a long read does not kill it; short enough that a
# session gone overnight reads as gone. The cost either way is stated rather
# than hidden: readers show the age beside `live`, never the boolean alone.
#
# NOT A SETTING, deliberately (#4385 weighed it). Settings are per user, and a
# claim is read by everyone who can see the task: a per-user lease would make
# one shared task read as live to one collaborator and dead to another, and
# "is anyone on this?" only means something if every reader gets the same
# answer. It is also read synchronously in `to_dict`, which has no user to ask.
CLAIM_LEASE = timedelta(hours=2)