feat(dedup): the create gate's similarity bars are settings (#4385, rule 25)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / TypeScript typecheck (push) Successful in 57s
CI & Build / integration (push) Successful in 56s
CI & Build / Python tests (push) Successful in 1m38s
CI & Build / Build & push image (push) Successful in 38s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 10s
CI & Build / TypeScript typecheck (push) Successful in 57s
CI & Build / integration (push) Successful in 56s
CI & Build / Python tests (push) Successful in 1m38s
CI & Build / Build & push image (push) Successful in 38s
gate_bars(user_id, note_type) resolves the block bar and, for notes and tasks, the overlap floor from kb_gate_* settings, with the old constants as defaults. Fail-open on an unreadable value; a block bar clamps at 0.80 and the overlap floor at 0.70 and never above the bar. Five fields in Settings beside the duplicate-report floors. CLAIM_LEASE stays a constant, with the reason written at it: a per-user lease would make one shared task live to one reader and dead to another. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -113,6 +113,31 @@ _NOTE_OVERLAP_LIMIT = 3
|
||||
# gate was not part of the measurement, so it keeps the general bar.
|
||||
_COPY_BAND_TYPES = {"note"}
|
||||
|
||||
# The constants above are the DEFAULTS; each is a setting (rule 25, #4385),
|
||||
# because how alike two distinct records get depends on how uniform a corpus
|
||||
# is, which nobody can know from here.
|
||||
GATE_THRESHOLD_KEYS = {
|
||||
"general": "kb_gate_threshold",
|
||||
SNIPPET_NOTE_TYPE: "kb_gate_threshold_snippet",
|
||||
LESSON_NOTE_TYPE: "kb_gate_threshold_lesson",
|
||||
"note_copy": "kb_gate_threshold_note_copy",
|
||||
"note_overlap": "kb_gate_note_overlap_floor",
|
||||
}
|
||||
GATE_DEFAULT_THRESHOLDS = {
|
||||
"general": _SEMANTIC_THRESHOLD,
|
||||
SNIPPET_NOTE_TYPE: _SNIPPET_SEMANTIC_THRESHOLD,
|
||||
LESSON_NOTE_TYPE: _LESSON_SEMANTIC_THRESHOLD,
|
||||
"note_copy": _NOTE_COPY_THRESHOLD,
|
||||
"note_overlap": _NOTE_OVERLAP_FLOOR,
|
||||
}
|
||||
# A BLOCK bar may not be set below this. The gate refuses the write, so a
|
||||
# mistyped 0.1 would refuse every create that shared a topic with anything —
|
||||
# the report's floors can go low because a report only proposes.
|
||||
_GATE_MIN_BLOCK = 0.80
|
||||
# The overlap floor only decides what is LISTED for the session to judge, so it
|
||||
# may go lower — but not so low that the three slots fill with noise.
|
||||
_GATE_MIN_OVERLAP = 0.70
|
||||
|
||||
# The gate queries per CHUNK of the candidate (#280) — this caps how many
|
||||
# searches one save may cost. Eight chunks ≈ five thousand words of candidate;
|
||||
# a duplicate hiding past that is the duplicate report's job to find, not a
|
||||
@@ -248,16 +273,43 @@ async def _find_snippet_by_structure(
|
||||
|
||||
|
||||
def _semantic_threshold(note_type: str) -> float:
|
||||
"""The semantic bar for this kind — a lookup, so the kinds that need a
|
||||
different one are named in a single place rather than in a conditional
|
||||
that grows a branch per kind."""
|
||||
if note_type == SNIPPET_NOTE_TYPE:
|
||||
return _SNIPPET_SEMANTIC_THRESHOLD
|
||||
if note_type == LESSON_NOTE_TYPE:
|
||||
return _LESSON_SEMANTIC_THRESHOLD
|
||||
"""The DEFAULT semantic bar for this kind — what `gate_bars` falls back on
|
||||
when the user has not set one."""
|
||||
return GATE_DEFAULT_THRESHOLDS[_gate_key(note_type)]
|
||||
|
||||
|
||||
def _gate_key(note_type: str) -> str:
|
||||
if note_type in (SNIPPET_NOTE_TYPE, LESSON_NOTE_TYPE):
|
||||
return note_type
|
||||
if note_type in _COPY_BAND_TYPES:
|
||||
return _NOTE_COPY_THRESHOLD
|
||||
return _SEMANTIC_THRESHOLD
|
||||
return "note_copy"
|
||||
return "general"
|
||||
|
||||
|
||||
async def _gate_setting(user_id: int, key: str, lo: float) -> float:
|
||||
from scribe.services.settings import get_setting
|
||||
|
||||
default = GATE_DEFAULT_THRESHOLDS[key]
|
||||
try:
|
||||
value = float(await get_setting(user_id, GATE_THRESHOLD_KEYS[key], str(default)))
|
||||
except Exception:
|
||||
# Fail-open like the rest of the gate: an unreadable setting falls back
|
||||
# to the measured default rather than blocking or waving through.
|
||||
value = default
|
||||
return min(1.0, max(lo, value))
|
||||
|
||||
|
||||
async def gate_bars(user_id: int, note_type: str) -> tuple[float, float]:
|
||||
"""(block_at, overlap_floor) for `note_type` on this user's install.
|
||||
|
||||
The overlap floor is only read for the copy-band kinds, and never sits
|
||||
above the block bar — a floor over the bar would list nothing.
|
||||
"""
|
||||
block_at = await _gate_setting(user_id, _gate_key(note_type), _GATE_MIN_BLOCK)
|
||||
if note_type not in _COPY_BAND_TYPES:
|
||||
return block_at, block_at
|
||||
floor = await _gate_setting(user_id, "note_overlap", _GATE_MIN_OVERLAP)
|
||||
return block_at, min(floor, block_at)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -355,7 +407,7 @@ async def find_duplicate_note(
|
||||
# under its name and embedded under `name — trigger`, so the query
|
||||
# document is built the way the corpus was, from `data`.
|
||||
doc_title = embeddings_svc.document_title(title, note_type, data, body)
|
||||
block_at = _semantic_threshold(note_type)
|
||||
block_at, overlap_floor = await gate_bars(user_id, note_type)
|
||||
collect = overlaps is not None and note_type in _COPY_BAND_TYPES
|
||||
near: dict[int, NoteOverlap] = {}
|
||||
for query in embeddings_svc.chunk_document(doc_title, body)[:_GATE_MAX_CHUNKS]:
|
||||
@@ -369,7 +421,7 @@ async def find_duplicate_note(
|
||||
user_id, query, project_id=project_id, is_task=is_task,
|
||||
orphan_only=(project_id is None),
|
||||
limit=3,
|
||||
threshold=_NOTE_OVERLAP_FLOOR if collect else block_at,
|
||||
threshold=overlap_floor if collect else block_at,
|
||||
# Owner-only, deliberately: this gate BLOCKS a create and tells
|
||||
# the caller to update the match instead. Matching someone
|
||||
# else's record would refuse their write and point them at
|
||||
|
||||
Reference in New Issue
Block a user