Queue clear — shared ACL on lists, semantic q, full telemetry ledger, --color-* retired #103
@@ -32,10 +32,15 @@ async def list_notes(
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""List notes (non-task documents) stored in Scribe.
|
"""List notes (non-task documents) stored in Scribe.
|
||||||
|
|
||||||
Optionally filter by a single tag (plain string, no # prefix) or a keyword
|
Optionally filter by a single tag (plain string, no # prefix) or by
|
||||||
search against title and body. Results are ordered by last-updated descending.
|
`search_text`, which matches on MEANING — the same semantic match every
|
||||||
|
search surface uses (#2462; keyword ILIKE is only the fallback when the
|
||||||
|
embedder is down). With `search_text`, results order by relevance; without
|
||||||
|
it, by last-updated descending. Reaches your records plus shared-project
|
||||||
|
records, like every other list.
|
||||||
|
|
||||||
Use search for semantic/meaning-based lookup instead of exact keyword search.
|
Prefer `search` for a pure lookup — it returns scores and reaches
|
||||||
|
everything readable; this adds the lifecycle filters on top.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
project_id: Scope to one project. PASS THE ACTIVE PROJECT'S ID whenever a
|
project_id: Scope to one project. PASS THE ACTIVE PROJECT'S ID whenever a
|
||||||
|
|||||||
@@ -4,12 +4,14 @@ from quart import Blueprint, jsonify, request
|
|||||||
|
|
||||||
from scribe.auth import login_required, get_current_user_id
|
from scribe.auth import login_required, get_current_user_id
|
||||||
from scribe.services.access import owner_names_for
|
from scribe.services.access import owner_names_for
|
||||||
|
from scribe.services.embeddings import (
|
||||||
|
INTERACTIVE_SEARCH_THRESHOLD as _REST_SEARCH_THRESHOLD,
|
||||||
|
)
|
||||||
from scribe.services.embeddings import semantic_search_notes
|
from scribe.services.embeddings import semantic_search_notes
|
||||||
from scribe.services.retrieval_telemetry import record_retrieval
|
from scribe.services.retrieval_telemetry import record_retrieval
|
||||||
|
|
||||||
# This route searches with a looser floor than the MCP tool default — it powers
|
# The interactive floor lives in embeddings.py now, shared with Browse search
|
||||||
# an interactive feed where loosely-related hits still have value.
|
# and the list views' semantic `q` — one number, one rationale (#2463).
|
||||||
_REST_SEARCH_THRESHOLD = 0.3
|
|
||||||
|
|
||||||
search_bp = Blueprint("search", __name__, url_prefix="/api/search")
|
search_bp = Blueprint("search", __name__, url_prefix="/api/search")
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,15 @@ logger = logging.getLogger(__name__)
|
|||||||
# loosely-related results that pad the sidebar without adding real value.
|
# loosely-related results that pad the sidebar without adding real value.
|
||||||
_SIMILARITY_THRESHOLD = 0.45
|
_SIMILARITY_THRESHOLD = 0.45
|
||||||
|
|
||||||
|
# The floor for INTERACTIVE, human-facing feeds — REST /api/search, Browse
|
||||||
|
# search, and the list views' semantic `q`. Deliberately looser than the agent
|
||||||
|
# default above: a human scanning a result list gets value from loosely-related
|
||||||
|
# hits an agent would be misled by. One constant, because this number was
|
||||||
|
# written as `0.3` in two files with the reasoning attached to only one of
|
||||||
|
# them — the exact shape where a later tuner moves one and not the other
|
||||||
|
# (#2463 finding 3).
|
||||||
|
INTERACTIVE_SEARCH_THRESHOLD = 0.3
|
||||||
|
|
||||||
# Public alias so callers (and telemetry) can record the effective default
|
# Public alias so callers (and telemetry) can record the effective default
|
||||||
# threshold without reaching for the underscored name.
|
# threshold without reaching for the underscored name.
|
||||||
DEFAULT_SIMILARITY_THRESHOLD = _SIMILARITY_THRESHOLD
|
DEFAULT_SIMILARITY_THRESHOLD = _SIMILARITY_THRESHOLD
|
||||||
|
|||||||
@@ -396,14 +396,20 @@ async def _semantic_knowledge_search(
|
|||||||
# case a semantic search exists to serve.
|
# case a semantic search exists to serve.
|
||||||
semantic_notes: list[Note] = []
|
semantic_notes: list[Note] = []
|
||||||
try:
|
try:
|
||||||
from scribe.services.embeddings import semantic_search_notes
|
from scribe.services.embeddings import (
|
||||||
|
INTERACTIVE_SEARCH_THRESHOLD,
|
||||||
|
semantic_search_notes,
|
||||||
|
)
|
||||||
is_task_filter = True if note_type in ("task", "plan") else (False if note_type else None)
|
is_task_filter = True if note_type in ("task", "plan") else (False if note_type else None)
|
||||||
candidates = await semantic_search_notes(
|
candidates = await semantic_search_notes(
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
scope="read",
|
scope="read",
|
||||||
query=q,
|
query=q,
|
||||||
limit=min(200, limit * 4),
|
limit=min(200, limit * 4),
|
||||||
threshold=0.3,
|
# The shared interactive floor — this was a bare `0.3` while
|
||||||
|
# routes/search.py had the same number as a commented constant, the
|
||||||
|
# exact pair where one moves and the other doesn't (#2463).
|
||||||
|
threshold=INTERACTIVE_SEARCH_THRESHOLD,
|
||||||
is_task=is_task_filter,
|
is_task=is_task_filter,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -188,10 +188,32 @@ async def list_notes(
|
|||||||
limit: int = 50,
|
limit: int = 50,
|
||||||
offset: int = 0,
|
offset: int = 0,
|
||||||
) -> tuple[list[Note], int]:
|
) -> tuple[list[Note], int]:
|
||||||
|
"""Lifecycle-shaped listing. Two contracts worth knowing:
|
||||||
|
|
||||||
|
VISIBILITY is the shared browse clause (#47, #2462): the caller's own
|
||||||
|
records plus anything in a project they can reach — the same reach
|
||||||
|
query_knowledge has always had. Before this, list_notes was silently
|
||||||
|
owner-only, so a task in a shared project was invisible in list_tasks,
|
||||||
|
enter_project's open-task list and the web UI while the same project's
|
||||||
|
notes appeared. Browse, not read, per decision #2094: an ambient list must
|
||||||
|
never surface a record someone shared one-to-one — those stay search-only.
|
||||||
|
|
||||||
|
`q` is SEMANTIC (operator decision, 2026-08-06: "make it match") — the same
|
||||||
|
meaning-based match as Browse search, at the interactive floor. When `q` is
|
||||||
|
present, relevance ordering wins and `sort` is ignored; a query is a
|
||||||
|
relevance claim and sorting its results by date would shuffle the answer.
|
||||||
|
Falls back to ILIKE substring match only when the embedder is unavailable —
|
||||||
|
degraded but never empty. Superseded records are not demoted here: the
|
||||||
|
penalty reorders a top-k, and reordering a paginated, counted list would
|
||||||
|
make page boundaries lie. The search surfaces carry the demotion.
|
||||||
|
"""
|
||||||
|
from scribe.services.access import notes_visibility_clause
|
||||||
|
|
||||||
|
visible = notes_visibility_clause(user_id, "browse")
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
query = select(Note).where(Note.user_id == user_id, Note.deleted_at.is_(None))
|
query = select(Note).where(visible, Note.deleted_at.is_(None))
|
||||||
count_query = select(func.count(Note.id)).where(
|
count_query = select(func.count(Note.id)).where(
|
||||||
Note.user_id == user_id, Note.deleted_at.is_(None)
|
visible, Note.deleted_at.is_(None)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Filter by task vs note
|
# Filter by task vs note
|
||||||
@@ -202,7 +224,27 @@ async def list_notes(
|
|||||||
query = query.where(Note.status.is_(None))
|
query = query.where(Note.status.is_(None))
|
||||||
count_query = count_query.where(Note.status.is_(None))
|
count_query = count_query.where(Note.status.is_(None))
|
||||||
|
|
||||||
|
semantic_order = None
|
||||||
if q:
|
if q:
|
||||||
|
query_vec = None
|
||||||
|
try:
|
||||||
|
from scribe.services.embeddings import get_embedding
|
||||||
|
query_vec = await get_embedding(q)
|
||||||
|
except Exception:
|
||||||
|
query_vec = None # embedder down → keyword fallback below
|
||||||
|
if query_vec is not None:
|
||||||
|
from scribe.models.embedding import NoteEmbedding
|
||||||
|
from scribe.services.embeddings import INTERACTIVE_SEARCH_THRESHOLD
|
||||||
|
distance = NoteEmbedding.embedding.cosine_distance(query_vec)
|
||||||
|
sem_filter = distance <= (1.0 - INTERACTIVE_SEARCH_THRESHOLD)
|
||||||
|
query = query.join(
|
||||||
|
NoteEmbedding, NoteEmbedding.note_id == Note.id
|
||||||
|
).where(sem_filter)
|
||||||
|
count_query = count_query.join(
|
||||||
|
NoteEmbedding, NoteEmbedding.note_id == Note.id
|
||||||
|
).where(sem_filter)
|
||||||
|
semantic_order = distance.asc()
|
||||||
|
else:
|
||||||
terms = _strip_type_nouns(q)
|
terms = _strip_type_nouns(q)
|
||||||
for term in terms:
|
for term in terms:
|
||||||
escaped_term = term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
escaped_term = term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||||
@@ -275,6 +317,10 @@ async def list_notes(
|
|||||||
query = query.where(paused_filter)
|
query = query.where(paused_filter)
|
||||||
count_query = count_query.where(paused_filter)
|
count_query = count_query.where(paused_filter)
|
||||||
|
|
||||||
|
if semantic_order is not None:
|
||||||
|
# A query is a relevance claim — see the docstring.
|
||||||
|
query = query.order_by(semantic_order)
|
||||||
|
else:
|
||||||
sort_col = getattr(Note, sort, Note.updated_at)
|
sort_col = getattr(Note, sort, Note.updated_at)
|
||||||
if order == "asc":
|
if order == "asc":
|
||||||
query = query.order_by(sort_col.asc())
|
query = query.order_by(sort_col.asc())
|
||||||
|
|||||||
@@ -202,3 +202,46 @@ async def test_injected_menu_labels_the_record_kind():
|
|||||||
assert "body" not in out["context"]
|
assert "body" not in out["context"]
|
||||||
# The header can't claim they're all notes when the markers say otherwise.
|
# The header can't claim they're all notes when the markers say otherwise.
|
||||||
assert "Scribe records" in lines[0]
|
assert "Scribe records" in lines[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_shaped_services_use_the_shared_visibility_clause():
|
||||||
|
"""Every service that LISTS notes must declare its reach through
|
||||||
|
notes_visibility_clause / browsable_notes_clause — never a bare owner
|
||||||
|
filter (#47, #2462).
|
||||||
|
|
||||||
|
Source inspection, because this is the shape no behavioural test catches:
|
||||||
|
an owner-only list returns correct-looking rows and simply omits the shared
|
||||||
|
ones, which is how list_notes shipped owner-only for months while
|
||||||
|
query_knowledge beside it was deliberately browse-scoped. A task in a
|
||||||
|
shared project was invisible in list_tasks, enter_project and the web UI
|
||||||
|
while the same project's notes appeared.
|
||||||
|
"""
|
||||||
|
import ast
|
||||||
|
import inspect
|
||||||
|
import textwrap
|
||||||
|
|
||||||
|
from scribe.services import knowledge, notes
|
||||||
|
|
||||||
|
for fn, label in ((notes.list_notes, "notes.list_notes"),
|
||||||
|
(knowledge.query_knowledge, "knowledge.query_knowledge")):
|
||||||
|
source = textwrap.dedent(inspect.getsource(fn))
|
||||||
|
assert (
|
||||||
|
"notes_visibility_clause" in source
|
||||||
|
or "browsable_notes_clause" in source
|
||||||
|
), (
|
||||||
|
f"{label} does not reference a shared visibility clause — a bare "
|
||||||
|
f"owner filter silently hides shared-project records (#2462)"
|
||||||
|
)
|
||||||
|
# And it must not ALSO carry a bare owner filter on the main query,
|
||||||
|
# which would quietly re-narrow whatever the clause granted.
|
||||||
|
tree = ast.parse(source)
|
||||||
|
for node in ast.walk(tree):
|
||||||
|
if (isinstance(node, ast.Compare)
|
||||||
|
and isinstance(node.left, ast.Attribute)
|
||||||
|
and node.left.attr == "user_id"
|
||||||
|
and isinstance(node.left.value, ast.Name)
|
||||||
|
and node.left.value.id == "Note"):
|
||||||
|
raise AssertionError(
|
||||||
|
f"{label} compares Note.user_id directly — reach must come "
|
||||||
|
f"from the shared clause, not a bare owner filter"
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user