feat(search): milestones are searchable by meaning — "is there already a plan for this?" (#4078)
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 11s
CI & Build / TypeScript typecheck (push) Successful in 52s
CI & Build / integration (push) Successful in 56s
CI & Build / Python tests (push) Successful in 1m37s
CI & Build / Build & push image (push) Successful in 28s
CI & Build / Python lint (push) Successful in 3s
CI & Build / Plugin hooks (push) Successful in 11s
CI & Build / TypeScript typecheck (push) Successful in 52s
CI & Build / integration (push) Successful in 56s
CI & Build / Python tests (push) Successful in 1m37s
CI & Build / Build & push image (push) Successful in 28s
`search` covered notes, tasks and rules, and a milestone — the record a plan lives in — could not be found. A project whose roadmap was written as milestones had every later plan opened beside the one that already described it, because nothing could have told the session it existed. - milestone_embeddings (migration 0102): the third sibling of note_ and rule_embeddings, for note 3163's reason — the search is milestone-specific. The document is title — description, then description and the plan body, so a roadmap milestone with no description is still found by its design. - Written on create, on a title/description/body update, and for a plan made through start_planning / create_records, fire-and-forget with the parent-row claim (#3262); a startup backfill covers every existing milestone. Derived, so it joins _NOT_INCLUDED beside the other embeddings. - semantic_search_milestones: a project's milestones when the caller can read it (access.can_read_project), otherwise the caller's own; optional status. - search(content_type="milestone"): id, title, description, status, project and progress. Its own shape, and not part of "all", whose results are note-shaped. The docstring says what it is for: ask before start_planning. - Integration test on real Postgres: found in its project and not another, status narrows, an unreadable project returns nothing. Milestone 415 step 3. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy
This commit is contained in:
@@ -0,0 +1,59 @@
|
|||||||
|
"""milestone_embeddings — a plan becomes findable by meaning (milestone 415)
|
||||||
|
|
||||||
|
Revision ID: 0102
|
||||||
|
Revises: 0101
|
||||||
|
Create Date: 2026-09-15
|
||||||
|
|
||||||
|
`search` covered notes, tasks and rules, and a milestone — the record a plan
|
||||||
|
lives in — could not be found at all. So "is there already a plan for this?"
|
||||||
|
had no tool, and a project whose roadmap was written as milestones had every
|
||||||
|
later plan opened as a new milestone beside the one that already described it.
|
||||||
|
|
||||||
|
The sibling of rule_embeddings (0089), for the reasons its model docstring and
|
||||||
|
note 3163 give. The vectors are DERIVED: nothing is backfilled here, the startup
|
||||||
|
backfill writes them.
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0102"
|
||||||
|
down_revision = "0101"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
# Matches note_embeddings and rule_embeddings — bge-small-en-v1.5, 384-dim.
|
||||||
|
_EMBEDDING_DIM = 384
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"milestone_embeddings",
|
||||||
|
sa.Column(
|
||||||
|
"milestone_id", sa.Integer(),
|
||||||
|
sa.ForeignKey("milestones.id", ondelete="CASCADE"), primary_key=True,
|
||||||
|
),
|
||||||
|
sa.Column("chunk_index", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("chunk_text", sa.Text(), nullable=False),
|
||||||
|
sa.Column("chunker_version", sa.Integer(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# Raw DDL for the vector column, as 0067 and 0089 do: the type comes from
|
||||||
|
# the pgvector extension, not SQLAlchemy's type system.
|
||||||
|
op.execute(
|
||||||
|
f"ALTER TABLE milestone_embeddings ADD COLUMN embedding vector({_EMBEDDING_DIM}) NOT NULL"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
CREATE INDEX ix_milestone_embeddings_embedding_hnsw
|
||||||
|
ON milestone_embeddings
|
||||||
|
USING hnsw (embedding vector_cosine_ops)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("DROP INDEX IF EXISTS ix_milestone_embeddings_embedding_hnsw")
|
||||||
|
op.drop_table("milestone_embeddings")
|
||||||
+9
-1
@@ -161,7 +161,9 @@ def create_app() -> Quart:
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from scribe.services.auth import start_auth_token_retention_loop
|
from scribe.services.auth import start_auth_token_retention_loop
|
||||||
from scribe.services.embeddings import backfill_note_embeddings, backfill_rule_embeddings
|
from scribe.services.embeddings import (
|
||||||
|
backfill_milestone_embeddings, backfill_note_embeddings, backfill_rule_embeddings,
|
||||||
|
)
|
||||||
from scribe.services.logging import start_log_retention_loop
|
from scribe.services.logging import start_log_retention_loop
|
||||||
from scribe.services.notifications import start_notification_loop
|
from scribe.services.notifications import start_notification_loop
|
||||||
|
|
||||||
@@ -182,6 +184,12 @@ def create_app() -> Quart:
|
|||||||
await backfill_rule_embeddings()
|
await backfill_rule_embeddings()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Rule embedding backfill failed", exc_info=True)
|
logger.warning("Rule embedding backfill failed", exc_info=True)
|
||||||
|
# Milestones got vectors in milestone 415, so a plan written before
|
||||||
|
# it is findable only after this pass.
|
||||||
|
try:
|
||||||
|
await backfill_milestone_embeddings()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Milestone embedding backfill failed", exc_info=True)
|
||||||
# Snippets written before migration 0070 have no `notes.data` mirror,
|
# Snippets written before migration 0070 have no `notes.data` mirror,
|
||||||
# and the location reverse lookup queries that column — an unfilled
|
# and the location reverse lookup queries that column — an unfilled
|
||||||
# row would read as "no snippet here" rather than as a gap. Separate
|
# row would read as "no snippet here" rather than as a gap. Separate
|
||||||
|
|||||||
@@ -12,7 +12,8 @@ import time
|
|||||||
from scribe.mcp._context import current_user_id
|
from scribe.mcp._context import current_user_id
|
||||||
from scribe.services.access import owner_names_for
|
from scribe.services.access import owner_names_for
|
||||||
from scribe.services.embeddings import (
|
from scribe.services.embeddings import (
|
||||||
DEFAULT_SIMILARITY_THRESHOLD, semantic_search_notes, semantic_search_rules,
|
DEFAULT_SIMILARITY_THRESHOLD, semantic_search_milestones, semantic_search_notes,
|
||||||
|
semantic_search_rules,
|
||||||
)
|
)
|
||||||
from scribe.services import rulebooks as rulebooks_svc
|
from scribe.services import rulebooks as rulebooks_svc
|
||||||
from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary
|
from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary
|
||||||
@@ -67,6 +68,40 @@ async def _search_rules(uid: int, q: str, limit: int, project_id: int) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_milestones(uid: int, q: str, limit: int, project_id: int) -> dict:
|
||||||
|
"""Milestones by meaning — "is there already a plan for this?" (milestone 415).
|
||||||
|
|
||||||
|
Its own result shape, like rules: a milestone is a plan with progress, not
|
||||||
|
a note with a body. The plan itself is left out — get_milestone reads it —
|
||||||
|
because a search hit is for recognising a plan, and bodies run long.
|
||||||
|
Not part of content_type="all", whose results are note-shaped.
|
||||||
|
"""
|
||||||
|
raw = await semantic_search_milestones(uid, q, project_id=project_id or None, limit=limit)
|
||||||
|
progress: dict[int, dict] = {}
|
||||||
|
if raw:
|
||||||
|
from scribe.services import milestones as milestones_svc
|
||||||
|
|
||||||
|
for pid in {m.project_id for _s, m in raw}:
|
||||||
|
for row in await milestones_svc.get_project_milestone_summary(uid, pid):
|
||||||
|
progress[row["id"]] = row
|
||||||
|
return {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"id": m.id,
|
||||||
|
"title": m.title,
|
||||||
|
"description": m.description or "",
|
||||||
|
"status": m.status,
|
||||||
|
"project_id": m.project_id,
|
||||||
|
"total": progress.get(m.id, {}).get("total", 0),
|
||||||
|
"completed": progress.get(m.id, {}).get("completed", 0),
|
||||||
|
"similarity": float(score),
|
||||||
|
}
|
||||||
|
for score, m in raw
|
||||||
|
],
|
||||||
|
"total": len(raw),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async def search(
|
async def search(
|
||||||
q: str,
|
q: str,
|
||||||
content_type: str = "all",
|
content_type: str = "all",
|
||||||
@@ -93,7 +128,12 @@ async def search(
|
|||||||
tagging?". A hit carries the rule's `why` and `how_to_apply`,
|
tagging?". A hit carries the rule's `why` and `how_to_apply`,
|
||||||
which the session-start payload does not. With a project_id,
|
which the session-start payload does not. With a project_id,
|
||||||
rules come back as the global rules plus that project's own;
|
rules come back as the global rules plus that project's own;
|
||||||
with 0, every rule in the rulebook.
|
with 0, every rule in the rulebook. Or 'milestone' (PLANS):
|
||||||
|
reach for it before start_planning to ask whether a plan for
|
||||||
|
this work already exists — a match is where new steps go
|
||||||
|
(create_records(milestone_id=…)), not a reason to open a second
|
||||||
|
milestone. Hits carry title, description, status and progress;
|
||||||
|
get_milestone reads the plan. Not included in 'all'.
|
||||||
limit: maximum number of results (1-50).
|
limit: maximum number of results (1-50).
|
||||||
project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
|
project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
|
||||||
whenever a project is in scope (the one you entered with
|
whenever a project is in scope (the one you entered with
|
||||||
@@ -118,6 +158,8 @@ async def search(
|
|||||||
limit = max(1, min(limit, 50))
|
limit = max(1, min(limit, 50))
|
||||||
if content_type == "rule":
|
if content_type == "rule":
|
||||||
return await _search_rules(uid, q, limit, project_id)
|
return await _search_rules(uid, q, limit, project_id)
|
||||||
|
if content_type == "milestone":
|
||||||
|
return await _search_milestones(uid, q, limit, project_id)
|
||||||
is_task = {"note": False, "task": True}.get(content_type) # None => any
|
is_task = {"note": False, "task": True}.get(content_type) # None => any
|
||||||
t0 = time.perf_counter()
|
t0 = time.perf_counter()
|
||||||
report: dict = {}
|
report: dict = {}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from scribe.models.user import User # noqa: E402, F401
|
|||||||
from scribe.models.app_log import AppLog # noqa: E402, F401
|
from scribe.models.app_log import AppLog # noqa: E402, F401
|
||||||
from scribe.models.password_reset import PasswordResetToken # noqa: E402, F401
|
from scribe.models.password_reset import PasswordResetToken # noqa: E402, F401
|
||||||
from scribe.models.invitation import InvitationToken # noqa: E402, F401
|
from scribe.models.invitation import InvitationToken # noqa: E402, F401
|
||||||
from scribe.models.embedding import NoteEmbedding, RuleEmbedding # noqa: E402, F401
|
from scribe.models.embedding import MilestoneEmbedding, NoteEmbedding, RuleEmbedding # noqa: E402, F401
|
||||||
from scribe.models.retrieval_log import RetrievalLog # noqa: E402, F401
|
from scribe.models.retrieval_log import RetrievalLog # noqa: E402, F401
|
||||||
from scribe.models.note_usage import NoteUsageEvent # noqa: E402, F401
|
from scribe.models.note_usage import NoteUsageEvent # noqa: E402, F401
|
||||||
from scribe.models.rule_usage import RuleUsageEvent # noqa: E402, F401
|
from scribe.models.rule_usage import RuleUsageEvent # noqa: E402, F401
|
||||||
|
|||||||
@@ -95,3 +95,36 @@ class RuleEmbedding(Base):
|
|||||||
DateTime(timezone=True),
|
DateTime(timezone=True),
|
||||||
default=lambda: datetime.now(timezone.utc),
|
default=lambda: datetime.now(timezone.utc),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MilestoneEmbedding(Base):
|
||||||
|
"""One embedding vector per CHUNK of a milestone (milestone 415).
|
||||||
|
|
||||||
|
The third sibling, for note 3163's reason: the row could be shared, the
|
||||||
|
search cannot. A milestone is scoped by its project, has no share of its
|
||||||
|
own, and is searched to answer one question — "is there already a plan for
|
||||||
|
this?" — which no note or rule search can answer, because a plan is not a
|
||||||
|
note. Before this, a roadmap written as milestones was invisible to recall,
|
||||||
|
and every later plan was opened as a new milestone beside the one that
|
||||||
|
already described it.
|
||||||
|
|
||||||
|
The document is the title, the one-line description and the plan body, the
|
||||||
|
parts a reader uses to recognise a plan. Derived data: the startup backfill
|
||||||
|
regenerates it, which is also how a chunker-version bump is handled.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "milestone_embeddings"
|
||||||
|
|
||||||
|
milestone_id: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
ForeignKey("milestones.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
)
|
||||||
|
chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
|
||||||
|
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
default=lambda: datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
|||||||
@@ -115,7 +115,8 @@ _BACKED_UP = [
|
|||||||
# like coverage while naming nothing the schema could confirm.
|
# like coverage while naming nothing the schema could confirm.
|
||||||
_NOT_INCLUDED = [
|
_NOT_INCLUDED = [
|
||||||
"groups", "group_memberships", "project_shares", "note_shares",
|
"groups", "group_memberships", "project_shares", "note_shares",
|
||||||
"api_keys", "note_embeddings", "rule_embeddings", "app_logs", "notifications",
|
"api_keys", "note_embeddings", "rule_embeddings", "milestone_embeddings",
|
||||||
|
"app_logs", "notifications",
|
||||||
"invitation_tokens", "password_reset_tokens", "user_profiles",
|
"invitation_tokens", "password_reset_tokens", "user_profiles",
|
||||||
"retrieval_logs",
|
"retrieval_logs",
|
||||||
# Sensitive credentials, same reasoning as api_keys: a backup that carries
|
# Sensitive credentials, same reasoning as api_keys: a backup that carries
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ from scribe.models.embedding import NoteEmbedding, RuleEmbedding
|
|||||||
from scribe.models.note import Note
|
from scribe.models.note import Note
|
||||||
from scribe.services.access import can_read_project, notes_visibility_clause
|
from scribe.services.access import can_read_project, notes_visibility_clause
|
||||||
|
|
||||||
if TYPE_CHECKING: # resolves the Rule forward ref without importing at runtime
|
if TYPE_CHECKING: # resolves forward refs without importing at runtime
|
||||||
|
from scribe.models.milestone import Milestone
|
||||||
from scribe.models.rulebook import Rule
|
from scribe.models.rulebook import Rule
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -975,3 +976,169 @@ async def backfill_rule_embeddings() -> None:
|
|||||||
logger.info("Rule embedding backfill: embedding %d rule(s)", len(stale))
|
logger.info("Rule embedding backfill: embedding %d rule(s)", len(stale))
|
||||||
for rule_id, title, statement, when_to_apply in stale:
|
for rule_id, title, statement, when_to_apply in stale:
|
||||||
await upsert_rule_embedding(rule_id, title, statement, when_to_apply)
|
await upsert_rule_embedding(rule_id, title, statement, when_to_apply)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Milestones (milestone 415) ──────────────────────────────────────────
|
||||||
|
|
||||||
|
def milestone_document(
|
||||||
|
title: str | None, description: str | None, body: str | None,
|
||||||
|
) -> tuple[str | None, str | None]:
|
||||||
|
"""The (title, body) a milestone is EMBEDDED as.
|
||||||
|
|
||||||
|
Title and one-line description lead, the way a snippet's name and purpose
|
||||||
|
lead its document (note 2485): the question this search answers is "does
|
||||||
|
a plan for this already exist?", and a plan is recognised by what it is
|
||||||
|
FOR. The plan body follows, chunked, so a milestone whose description is
|
||||||
|
empty — most roadmap milestones written by hand — is still findable by the
|
||||||
|
words of its design.
|
||||||
|
"""
|
||||||
|
name = (title or "").strip()
|
||||||
|
purpose = (description or "").strip()
|
||||||
|
plan = (body or "").strip()
|
||||||
|
doc_title = f"{name} — {purpose}" if name and purpose else (name or purpose or None)
|
||||||
|
parts = [p for p in (purpose, plan) if p]
|
||||||
|
return doc_title, "\n\n".join(parts) or None
|
||||||
|
|
||||||
|
|
||||||
|
async def upsert_milestone_embedding(
|
||||||
|
milestone_id: int, title: str | None, description: str | None, body: str | None,
|
||||||
|
) -> None:
|
||||||
|
"""Chunk, embed and persist a milestone's vectors. Safe to fire-and-forget.
|
||||||
|
|
||||||
|
The rule twin's contract: one document definition shared by the write path
|
||||||
|
and the backfill, and an atomic per-milestone replacement guarded by the
|
||||||
|
parent-row claim (#3262), so a milestone deleted mid-refresh wins.
|
||||||
|
"""
|
||||||
|
from scribe.models.embedding import MilestoneEmbedding
|
||||||
|
from scribe.models.milestone import Milestone
|
||||||
|
|
||||||
|
doc_title, doc_body = milestone_document(title, description, body)
|
||||||
|
chunks = chunk_document(doc_title, doc_body)
|
||||||
|
try:
|
||||||
|
if not chunks:
|
||||||
|
async with async_session() as session:
|
||||||
|
await session.execute(
|
||||||
|
delete(MilestoneEmbedding).where(MilestoneEmbedding.milestone_id == milestone_id)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to clear embedding for milestone %d", milestone_id, exc_info=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
vectors = await get_embeddings(chunks)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Skipping embedding for milestone %d — embedder unavailable", milestone_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
if not await _claim_parent_row(session, Milestone.id, milestone_id, "milestone"):
|
||||||
|
return
|
||||||
|
await session.execute(
|
||||||
|
delete(MilestoneEmbedding).where(MilestoneEmbedding.milestone_id == milestone_id)
|
||||||
|
)
|
||||||
|
for index, (chunk, vector) in enumerate(zip(chunks, vectors)):
|
||||||
|
session.add(MilestoneEmbedding(
|
||||||
|
milestone_id=milestone_id, chunk_index=index, embedding=vector,
|
||||||
|
chunk_text=chunk, chunker_version=CHUNKER_VERSION,
|
||||||
|
))
|
||||||
|
await session.commit()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to persist embedding for milestone %d", milestone_id, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def semantic_search_milestones(
|
||||||
|
user_id: int,
|
||||||
|
query: str,
|
||||||
|
*,
|
||||||
|
project_id: int | None = None,
|
||||||
|
status: str | None = None,
|
||||||
|
limit: int = 5,
|
||||||
|
threshold: float = _SIMILARITY_THRESHOLD,
|
||||||
|
) -> list[tuple[float, "Milestone"]]:
|
||||||
|
"""Return up to *limit* (score, milestone) pairs most like *query*.
|
||||||
|
|
||||||
|
Answers "is there already a plan for this?" — the question a session asks
|
||||||
|
before start_planning, and the one the planning gate asks for it.
|
||||||
|
|
||||||
|
SCOPE. With `project_id`, that project's milestones, provided the caller
|
||||||
|
can read the project (access.can_read_project, rule 78) — a collaborator on
|
||||||
|
a shared project sees its plans. Without one, the milestones the caller
|
||||||
|
owns across their projects. `status` narrows to "active" or "done".
|
||||||
|
|
||||||
|
Collapses to best-chunk-per-milestone, like the sibling searches. Returns
|
||||||
|
an empty list if the embedder is unavailable, the project is not readable,
|
||||||
|
or on any error: a recall aid must never break the call it serves.
|
||||||
|
"""
|
||||||
|
from scribe.models.embedding import MilestoneEmbedding
|
||||||
|
from scribe.models.milestone import Milestone
|
||||||
|
|
||||||
|
if not query or not query.strip():
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
query_vec = await get_embedding(query)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Milestone search skipped — embedder unavailable")
|
||||||
|
return []
|
||||||
|
|
||||||
|
distance = MilestoneEmbedding.embedding.cosine_distance(query_vec)
|
||||||
|
try:
|
||||||
|
if project_id:
|
||||||
|
if not await can_read_project(user_id, project_id):
|
||||||
|
return []
|
||||||
|
scope = Milestone.project_id == project_id
|
||||||
|
else:
|
||||||
|
scope = Milestone.user_id == user_id
|
||||||
|
async with async_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(Milestone, distance.label("distance"))
|
||||||
|
.select_from(MilestoneEmbedding)
|
||||||
|
.join(Milestone, MilestoneEmbedding.milestone_id == Milestone.id)
|
||||||
|
.where(
|
||||||
|
scope,
|
||||||
|
Milestone.deleted_at.is_(None),
|
||||||
|
*([Milestone.status == status] if status else []),
|
||||||
|
)
|
||||||
|
.order_by(distance)
|
||||||
|
.limit(limit * _CHUNK_OVERFETCH)
|
||||||
|
)).all()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Milestone semantic search failed", exc_info=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
best: dict[int, tuple[float, object]] = {}
|
||||||
|
for milestone, dist in rows:
|
||||||
|
score = 1.0 - float(dist)
|
||||||
|
if milestone.id not in best or score > best[milestone.id][0]:
|
||||||
|
best[milestone.id] = (score, milestone)
|
||||||
|
ranked = sorted(best.values(), key=lambda pair: pair[0], reverse=True)
|
||||||
|
return [pair for pair in ranked if pair[0] >= threshold][:limit]
|
||||||
|
|
||||||
|
|
||||||
|
async def backfill_milestone_embeddings() -> None:
|
||||||
|
"""Embed milestones that have no current vectors. Runs at startup beside
|
||||||
|
the note and rule backfills; a CHUNKER_VERSION bump re-embeds."""
|
||||||
|
from scribe.models.embedding import MilestoneEmbedding
|
||||||
|
from scribe.models.milestone import Milestone
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
current = select(MilestoneEmbedding.milestone_id).where(
|
||||||
|
MilestoneEmbedding.chunker_version == CHUNKER_VERSION
|
||||||
|
)
|
||||||
|
stale = (await session.execute(
|
||||||
|
select(Milestone.id, Milestone.title, Milestone.description, Milestone.body)
|
||||||
|
.where(Milestone.deleted_at.is_(None), Milestone.id.notin_(current))
|
||||||
|
)).all()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Milestone embedding backfill: failed to query milestones", exc_info=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not stale:
|
||||||
|
logger.info("Milestone embedding backfill: all milestones current at chunker v%d", CHUNKER_VERSION)
|
||||||
|
return
|
||||||
|
logger.info("Milestone embedding backfill: embedding %d milestone(s)", len(stale))
|
||||||
|
for milestone_id, title, description, body in stale:
|
||||||
|
await upsert_milestone_embedding(milestone_id, title, description, body)
|
||||||
|
|||||||
@@ -11,6 +11,30 @@ from scribe.models.note import Note
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def embed_milestone(milestone: Milestone) -> None:
|
||||||
|
"""Refresh a milestone's vectors, fire-and-forget (milestone 415).
|
||||||
|
|
||||||
|
At the service, so every path that writes a milestone gets it — the lesson
|
||||||
|
embed_note records (#2056): a record written through a door that forgot the
|
||||||
|
call stays out of search until a restart. Exceptions are swallowed because
|
||||||
|
a milestone that saved must not fail on its index refresh; no running loop
|
||||||
|
(a script, a unit test) is ordinary. A delete racing the refresh wins: the
|
||||||
|
upsert claims the milestone's row first (#3262).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from scribe.services.embeddings import upsert_milestone_embedding
|
||||||
|
|
||||||
|
asyncio.create_task(upsert_milestone_embedding(
|
||||||
|
milestone.id, milestone.title, milestone.description, milestone.body,
|
||||||
|
))
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
except Exception: # noqa: BLE001 - never let indexing break a write
|
||||||
|
logger.exception("embedding refresh failed for milestone %s", milestone.id)
|
||||||
|
|
||||||
|
|
||||||
async def create_milestone(
|
async def create_milestone(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
project_id: int,
|
project_id: int,
|
||||||
@@ -33,6 +57,7 @@ async def create_milestone(
|
|||||||
session.add(milestone)
|
session.add(milestone)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(milestone)
|
await session.refresh(milestone)
|
||||||
|
embed_milestone(milestone)
|
||||||
return milestone
|
return milestone
|
||||||
|
|
||||||
|
|
||||||
@@ -125,6 +150,8 @@ async def update_milestone(user_id: int, milestone_id: int, **fields: object) ->
|
|||||||
milestone.updated_at = datetime.now(timezone.utc)
|
milestone.updated_at = datetime.now(timezone.utc)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(milestone)
|
await session.refresh(milestone)
|
||||||
|
if {"title", "description", "body"} & set(fields):
|
||||||
|
embed_milestone(milestone)
|
||||||
return milestone
|
return milestone
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from scribe.models import async_session
|
|||||||
from scribe.models.milestone import Milestone
|
from scribe.models.milestone import Milestone
|
||||||
from scribe.models.note import Note
|
from scribe.models.note import Note
|
||||||
from scribe.services import access as access_svc
|
from scribe.services import access as access_svc
|
||||||
|
from scribe.services import milestones as milestones_svc
|
||||||
from scribe.services import notes as notes_svc
|
from scribe.services import notes as notes_svc
|
||||||
from scribe.services import systems as systems_svc
|
from scribe.services import systems as systems_svc
|
||||||
from scribe.services.record_refs import placeholder_keys, resolve_placeholders
|
from scribe.services.record_refs import placeholder_keys, resolve_placeholders
|
||||||
@@ -183,6 +184,8 @@ async def create_batch(
|
|||||||
# After the commit, as a single create does: embedding and System tags are
|
# After the commit, as a single create does: embedding and System tags are
|
||||||
# enrichment on records that now exist, and a failure in either must not
|
# enrichment on records that now exist, and a failure in either must not
|
||||||
# un-create them.
|
# un-create them.
|
||||||
|
if new_ms is not None:
|
||||||
|
milestones_svc.embed_milestone(new_ms)
|
||||||
for note, item in zip(notes, items):
|
for note, item in zip(notes, items):
|
||||||
notes_svc.embed_note(note)
|
notes_svc.embed_note(note)
|
||||||
if item.system_ids:
|
if item.system_ids:
|
||||||
|
|||||||
+4
-1
@@ -80,7 +80,10 @@ def _no_embedding():
|
|||||||
"""
|
"""
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
with patch("scribe.services.notes.embed_note", MagicMock()):
|
# Milestones embed too since milestone 415; a plan created in a test would
|
||||||
|
# otherwise detach the same model-loading task.
|
||||||
|
with patch("scribe.services.notes.embed_note", MagicMock()), \
|
||||||
|
patch("scribe.services.milestones.embed_milestone", MagicMock()):
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""Real-Postgres tests for finding a plan by meaning (milestone 415, step 3).
|
||||||
|
|
||||||
|
A project's roadmap written as milestones was invisible to recall: `search`
|
||||||
|
covered notes, tasks and rules, so "is there already a plan for this?" had no
|
||||||
|
tool. What a mock cannot show is the join scoping the vectors to a project and
|
||||||
|
to what the caller may read, so these seed real milestones with hand-made
|
||||||
|
vectors and stub only the embedder.
|
||||||
|
"""
|
||||||
|
import uuid
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import pytest_asyncio
|
||||||
|
|
||||||
|
from scribe.models import async_session
|
||||||
|
from scribe.models.embedding import EMBEDDING_DIM, MilestoneEmbedding
|
||||||
|
from scribe.models.milestone import Milestone
|
||||||
|
from scribe.models.project import Project
|
||||||
|
from scribe.services.embeddings import CHUNKER_VERSION, semantic_search_milestones
|
||||||
|
from tests.helpers import ensure_user
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.integration, pytest.mark.usefixtures("_dispose_engine", "_no_embedding")]
|
||||||
|
|
||||||
|
NEAR = [1.0] + [0.0] * (EMBEDDING_DIM - 1)
|
||||||
|
FAR = [0.0, 1.0] + [0.0] * (EMBEDDING_DIM - 2)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest_asyncio.fixture
|
||||||
|
async def roadmap():
|
||||||
|
tag = uuid.uuid4().hex[:8]
|
||||||
|
async with async_session() as s:
|
||||||
|
owner = await ensure_user(s, f"ms_search_owner_{tag}")
|
||||||
|
stranger = await ensure_user(s, f"ms_search_stranger_{tag}")
|
||||||
|
mine = Project(user_id=owner.id, title="Librarian")
|
||||||
|
other = Project(user_id=owner.id, title="Elsewhere")
|
||||||
|
s.add_all([mine, other])
|
||||||
|
await s.flush()
|
||||||
|
m3 = Milestone(user_id=owner.id, project_id=mine.id, title="M3 — Metadata",
|
||||||
|
description="works, editions, providers, provenance", status="active")
|
||||||
|
done = Milestone(user_id=owner.id, project_id=mine.id, title="Covers",
|
||||||
|
description="cover art", status="done")
|
||||||
|
unrelated = Milestone(user_id=owner.id, project_id=mine.id, title="Android client",
|
||||||
|
description="native app", status="active")
|
||||||
|
foreign = Milestone(user_id=owner.id, project_id=other.id, title="Metadata elsewhere",
|
||||||
|
description="same words, other project", status="active")
|
||||||
|
s.add_all([m3, done, unrelated, foreign])
|
||||||
|
await s.flush()
|
||||||
|
for ms, vec in ((m3, NEAR), (done, NEAR), (unrelated, FAR), (foreign, NEAR)):
|
||||||
|
s.add(MilestoneEmbedding(milestone_id=ms.id, chunk_index=0, embedding=vec,
|
||||||
|
chunk_text=ms.title, chunker_version=CHUNKER_VERSION))
|
||||||
|
ids = {"owner": owner.id, "stranger": stranger.id, "mine": mine.id,
|
||||||
|
"m3": m3.id, "done": done.id, "unrelated": unrelated.id, "foreign": foreign.id}
|
||||||
|
await s.commit()
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
async def _found(user_id, **kw) -> list[int]:
|
||||||
|
with patch("scribe.services.embeddings.get_embedding", AsyncMock(return_value=NEAR)):
|
||||||
|
hits = await semantic_search_milestones(user_id, "book metadata and providers",
|
||||||
|
threshold=0.5, limit=10, **kw)
|
||||||
|
return [m.id for _s, m in hits]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_plan_is_found_in_its_project_and_not_in_another(roadmap):
|
||||||
|
found = await _found(roadmap["owner"], project_id=roadmap["mine"])
|
||||||
|
assert set(found) == {roadmap["m3"], roadmap["done"]}
|
||||||
|
assert roadmap["foreign"] not in found and roadmap["unrelated"] not in found
|
||||||
|
|
||||||
|
|
||||||
|
async def test_status_narrows_to_open_plans(roadmap):
|
||||||
|
found = await _found(roadmap["owner"], project_id=roadmap["mine"], status="active")
|
||||||
|
assert found == [roadmap["m3"]]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_without_a_project_it_searches_the_callers_own(roadmap):
|
||||||
|
found = await _found(roadmap["owner"])
|
||||||
|
assert {roadmap["m3"], roadmap["done"], roadmap["foreign"]} <= set(found)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_project_the_caller_cannot_read_returns_nothing(roadmap):
|
||||||
|
assert await _found(roadmap["stranger"], project_id=roadmap["mine"]) == []
|
||||||
|
assert await _found(roadmap["stranger"]) == []
|
||||||
@@ -106,3 +106,34 @@ async def test_rule_search_scopes_to_the_project_it_is_given(project_id, scope):
|
|||||||
kwargs = found.await_args.kwargs
|
kwargs = found.await_args.kwargs
|
||||||
assert {k: kwargs[k] for k in scope} == scope
|
assert {k: kwargs[k] for k in scope} == scope
|
||||||
assert set(kwargs) & {"project_id", "everywhere"} == set(scope)
|
assert set(kwargs) & {"project_id", "everywhere"} == set(scope)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_milestone_is_embedded_by_what_it_is_for_then_its_plan():
|
||||||
|
from scribe.services.embeddings import milestone_document
|
||||||
|
|
||||||
|
assert milestone_document("M3", "metadata providers", "## Goal\nx") == (
|
||||||
|
"M3 — metadata providers", "metadata providers\n\n## Goal\nx")
|
||||||
|
# A roadmap milestone written with no description is still findable by its plan.
|
||||||
|
assert milestone_document("M3", None, "the plan") == ("M3", "the plan")
|
||||||
|
assert milestone_document(None, None, None) == (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_milestone_search_is_its_own_shape_and_scopes_to_the_project():
|
||||||
|
"""milestone 415: 'is there already a plan for this?' has a tool."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
_user_id_ctx.set(7)
|
||||||
|
ms = MagicMock(id=339, title="M3 — Metadata", description="works, editions",
|
||||||
|
status="active", project_id=30)
|
||||||
|
found = AsyncMock(return_value=[(0.81, ms)])
|
||||||
|
summary = AsyncMock(return_value=[{"id": 339, "total": 0, "completed": 0}])
|
||||||
|
with patch("scribe.mcp.tools.search.semantic_search_milestones", found), \
|
||||||
|
patch("scribe.services.milestones.get_project_milestone_summary", summary):
|
||||||
|
out = await search(q="book metadata", content_type="milestone", project_id=30)
|
||||||
|
assert found.await_args.kwargs["project_id"] == 30
|
||||||
|
assert out["results"] == [{
|
||||||
|
"id": 339, "title": "M3 — Metadata", "description": "works, editions",
|
||||||
|
"status": "active", "project_id": 30, "total": 0, "completed": 0,
|
||||||
|
"similarity": 0.81,
|
||||||
|
}]
|
||||||
|
|||||||
Reference in New Issue
Block a user