feat(search): milestones are searchable by meaning — "is there already a plan for this?" (#4078)

`search` covered notes, tasks and rules, and a milestone — the record a plan
lives in — could not be found. A project whose roadmap was written as
milestones had every later plan opened beside the one that already described
it, because nothing could have told the session it existed.

- milestone_embeddings (migration 0102): the third sibling of note_ and
  rule_embeddings, for note 3163's reason — the search is milestone-specific.
  The document is title — description, then description and the plan body,
  so a roadmap milestone with no description is still found by its design.
- Written on create, on a title/description/body update, and for a plan made
  through start_planning / create_records, fire-and-forget with the parent-row
  claim (#3262); a startup backfill covers every existing milestone. Derived,
  so it joins _NOT_INCLUDED beside the other embeddings.
- semantic_search_milestones: a project's milestones when the caller can read
  it (access.can_read_project), otherwise the caller's own; optional status.
- search(content_type="milestone"): id, title, description, status, project
  and progress. Its own shape, and not part of "all", whose results are
  note-shaped. The docstring says what it is for: ask before start_planning.
- Integration test on real Postgres: found in its project and not another,
  status narrows, an unreadable project returns nothing.

Milestone 415 step 3.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01821k5B3Ysecp9fNYs92Kuy

src/scribe/app.py

@@ -161,7 +161,9 @@ def create_app() -> Quart:
         import asyncio
 
         from scribe.services.auth import start_auth_token_retention_loop
-        from scribe.services.embeddings import backfill_note_embeddings, backfill_rule_embeddings
+        from scribe.services.embeddings import (
+            backfill_milestone_embeddings, backfill_note_embeddings, backfill_rule_embeddings,
+        )
         from scribe.services.logging import start_log_retention_loop
         from scribe.services.notifications import start_notification_loop
 
@@ -182,6 +184,12 @@ def create_app() -> Quart:
                 await backfill_rule_embeddings()
             except Exception:
                 logger.warning("Rule embedding backfill failed", exc_info=True)
+            # Milestones got vectors in milestone 415, so a plan written before
+            # it is findable only after this pass.
+            try:
+                await backfill_milestone_embeddings()
+            except Exception:
+                logger.warning("Milestone embedding backfill failed", exc_info=True)
             # Snippets written before migration 0070 have no `notes.data` mirror,
             # and the location reverse lookup queries that column — an unfilled
             # row would read as "no snippet here" rather than as a gap. Separate

src/scribe/mcp/tools/search.py

@@ -12,7 +12,8 @@ import time
 from scribe.mcp._context import current_user_id
 from scribe.services.access import owner_names_for
 from scribe.services.embeddings import (
-    DEFAULT_SIMILARITY_THRESHOLD, semantic_search_notes, semantic_search_rules,
+    DEFAULT_SIMILARITY_THRESHOLD, semantic_search_milestones, semantic_search_notes,
+    semantic_search_rules,
 )
 from scribe.services import rulebooks as rulebooks_svc
 from scribe.services.retrieval_telemetry import record_retrieval, retrieval_summary
@@ -67,6 +68,40 @@ async def _search_rules(uid: int, q: str, limit: int, project_id: int) -> dict:
     }
 
 
+async def _search_milestones(uid: int, q: str, limit: int, project_id: int) -> dict:
+    """Milestones by meaning — "is there already a plan for this?" (milestone 415).
+
+    Its own result shape, like rules: a milestone is a plan with progress, not
+    a note with a body. The plan itself is left out — get_milestone reads it —
+    because a search hit is for recognising a plan, and bodies run long.
+    Not part of content_type="all", whose results are note-shaped.
+    """
+    raw = await semantic_search_milestones(uid, q, project_id=project_id or None, limit=limit)
+    progress: dict[int, dict] = {}
+    if raw:
+        from scribe.services import milestones as milestones_svc
+
+        for pid in {m.project_id for _s, m in raw}:
+            for row in await milestones_svc.get_project_milestone_summary(uid, pid):
+                progress[row["id"]] = row
+    return {
+        "results": [
+            {
+                "id": m.id,
+                "title": m.title,
+                "description": m.description or "",
+                "status": m.status,
+                "project_id": m.project_id,
+                "total": progress.get(m.id, {}).get("total", 0),
+                "completed": progress.get(m.id, {}).get("completed", 0),
+                "similarity": float(score),
+            }
+            for score, m in raw
+        ],
+        "total": len(raw),
+    }
+
+
 async def search(
     q: str,
     content_type: str = "all",
@@ -93,7 +128,12 @@ async def search(
             tagging?". A hit carries the rule's `why` and `how_to_apply`,
             which the session-start payload does not. With a project_id,
             rules come back as the global rules plus that project's own;
-            with 0, every rule in the rulebook.
+            with 0, every rule in the rulebook. Or 'milestone' (PLANS):
+            reach for it before start_planning to ask whether a plan for
+            this work already exists — a match is where new steps go
+            (create_records(milestone_id=…)), not a reason to open a second
+            milestone. Hits carry title, description, status and progress;
+            get_milestone reads the plan. Not included in 'all'.
         limit: maximum number of results (1-50).
         project_id: Scope results to one project. PASS THE ACTIVE PROJECT'S ID
             whenever a project is in scope (the one you entered with
@@ -118,6 +158,8 @@ async def search(
     limit = max(1, min(limit, 50))
     if content_type == "rule":
         return await _search_rules(uid, q, limit, project_id)
+    if content_type == "milestone":
+        return await _search_milestones(uid, q, limit, project_id)
     is_task = {"note": False, "task": True}.get(content_type)  # None => any
     t0 = time.perf_counter()
     report: dict = {}

src/scribe/models/__init__.py

@@ -25,7 +25,7 @@ from scribe.models.user import User  # noqa: E402, F401
 from scribe.models.app_log import AppLog  # noqa: E402, F401
 from scribe.models.password_reset import PasswordResetToken  # noqa: E402, F401
 from scribe.models.invitation import InvitationToken  # noqa: E402, F401
-from scribe.models.embedding import NoteEmbedding, RuleEmbedding  # noqa: E402, F401
+from scribe.models.embedding import MilestoneEmbedding, NoteEmbedding, RuleEmbedding  # noqa: E402, F401
 from scribe.models.retrieval_log import RetrievalLog  # noqa: E402, F401
 from scribe.models.note_usage import NoteUsageEvent  # noqa: E402, F401
 from scribe.models.rule_usage import RuleUsageEvent  # noqa: E402, F401

src/scribe/models/embedding.py

@@ -95,3 +95,36 @@ class RuleEmbedding(Base):
         DateTime(timezone=True),
         default=lambda: datetime.now(timezone.utc),
     )
+
+
+class MilestoneEmbedding(Base):
+    """One embedding vector per CHUNK of a milestone (milestone 415).
+
+    The third sibling, for note 3163's reason: the row could be shared, the
+    search cannot. A milestone is scoped by its project, has no share of its
+    own, and is searched to answer one question — "is there already a plan for
+    this?" — which no note or rule search can answer, because a plan is not a
+    note. Before this, a roadmap written as milestones was invisible to recall,
+    and every later plan was opened as a new milestone beside the one that
+    already described it.
+
+    The document is the title, the one-line description and the plan body, the
+    parts a reader uses to recognise a plan. Derived data: the startup backfill
+    regenerates it, which is also how a chunker-version bump is handled.
+    """
+
+    __tablename__ = "milestone_embeddings"
+
+    milestone_id: Mapped[int] = mapped_column(
+        Integer,
+        ForeignKey("milestones.id", ondelete="CASCADE"),
+        primary_key=True,
+    )
+    chunk_index: Mapped[int] = mapped_column(Integer, primary_key=True)
+    embedding: Mapped[list] = mapped_column(Vector(EMBEDDING_DIM), nullable=False)
+    chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
+    chunker_version: Mapped[int] = mapped_column(Integer, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        default=lambda: datetime.now(timezone.utc),
+    )

src/scribe/services/backup.py

@@ -115,7 +115,8 @@ _BACKED_UP = [
 # like coverage while naming nothing the schema could confirm.
 _NOT_INCLUDED = [
     "groups", "group_memberships", "project_shares", "note_shares",
-    "api_keys", "note_embeddings", "rule_embeddings", "app_logs", "notifications",
+    "api_keys", "note_embeddings", "rule_embeddings", "milestone_embeddings",
+    "app_logs", "notifications",
     "invitation_tokens", "password_reset_tokens", "user_profiles",
     "retrieval_logs",
     # Sensitive credentials, same reasoning as api_keys: a backup that carries

src/scribe/services/embeddings.py

@@ -25,7 +25,8 @@ from scribe.models.embedding import NoteEmbedding, RuleEmbedding
 from scribe.models.note import Note
 from scribe.services.access import can_read_project, notes_visibility_clause
 
-if TYPE_CHECKING:  # resolves the Rule forward ref without importing at runtime
+if TYPE_CHECKING:  # resolves forward refs without importing at runtime
+    from scribe.models.milestone import Milestone
     from scribe.models.rulebook import Rule
 
 logger = logging.getLogger(__name__)
@@ -975,3 +976,169 @@ async def backfill_rule_embeddings() -> None:
     logger.info("Rule embedding backfill: embedding %d rule(s)", len(stale))
     for rule_id, title, statement, when_to_apply in stale:
         await upsert_rule_embedding(rule_id, title, statement, when_to_apply)
+
+
+# ── Milestones (milestone 415) ──────────────────────────────────────────
+
+def milestone_document(
+    title: str | None, description: str | None, body: str | None,
+) -> tuple[str | None, str | None]:
+    """The (title, body) a milestone is EMBEDDED as.
+
+    Title and one-line description lead, the way a snippet's name and purpose
+    lead its document (note 2485): the question this search answers is "does
+    a plan for this already exist?", and a plan is recognised by what it is
+    FOR. The plan body follows, chunked, so a milestone whose description is
+    empty — most roadmap milestones written by hand — is still findable by the
+    words of its design.
+    """
+    name = (title or "").strip()
+    purpose = (description or "").strip()
+    plan = (body or "").strip()
+    doc_title = f"{name} — {purpose}" if name and purpose else (name or purpose or None)
+    parts = [p for p in (purpose, plan) if p]
+    return doc_title, "\n\n".join(parts) or None
+
+
+async def upsert_milestone_embedding(
+    milestone_id: int, title: str | None, description: str | None, body: str | None,
+) -> None:
+    """Chunk, embed and persist a milestone's vectors. Safe to fire-and-forget.
+
+    The rule twin's contract: one document definition shared by the write path
+    and the backfill, and an atomic per-milestone replacement guarded by the
+    parent-row claim (#3262), so a milestone deleted mid-refresh wins.
+    """
+    from scribe.models.embedding import MilestoneEmbedding
+    from scribe.models.milestone import Milestone
+
+    doc_title, doc_body = milestone_document(title, description, body)
+    chunks = chunk_document(doc_title, doc_body)
+    try:
+        if not chunks:
+            async with async_session() as session:
+                await session.execute(
+                    delete(MilestoneEmbedding).where(MilestoneEmbedding.milestone_id == milestone_id)
+                )
+                await session.commit()
+            return
+    except Exception:
+        logger.warning("Failed to clear embedding for milestone %d", milestone_id, exc_info=True)
+        return
+
+    try:
+        vectors = await get_embeddings(chunks)
+    except Exception:
+        logger.debug("Skipping embedding for milestone %d — embedder unavailable", milestone_id)
+        return
+
+    try:
+        async with async_session() as session:
+            if not await _claim_parent_row(session, Milestone.id, milestone_id, "milestone"):
+                return
+            await session.execute(
+                delete(MilestoneEmbedding).where(MilestoneEmbedding.milestone_id == milestone_id)
+            )
+            for index, (chunk, vector) in enumerate(zip(chunks, vectors)):
+                session.add(MilestoneEmbedding(
+                    milestone_id=milestone_id, chunk_index=index, embedding=vector,
+                    chunk_text=chunk, chunker_version=CHUNKER_VERSION,
+                ))
+            await session.commit()
+    except Exception:
+        logger.warning("Failed to persist embedding for milestone %d", milestone_id, exc_info=True)
+
+
+async def semantic_search_milestones(
+    user_id: int,
+    query: str,
+    *,
+    project_id: int | None = None,
+    status: str | None = None,
+    limit: int = 5,
+    threshold: float = _SIMILARITY_THRESHOLD,
+) -> list[tuple[float, "Milestone"]]:
+    """Return up to *limit* (score, milestone) pairs most like *query*.
+
+    Answers "is there already a plan for this?" — the question a session asks
+    before start_planning, and the one the planning gate asks for it.
+
+    SCOPE. With `project_id`, that project's milestones, provided the caller
+    can read the project (access.can_read_project, rule 78) — a collaborator on
+    a shared project sees its plans. Without one, the milestones the caller
+    owns across their projects. `status` narrows to "active" or "done".
+
+    Collapses to best-chunk-per-milestone, like the sibling searches. Returns
+    an empty list if the embedder is unavailable, the project is not readable,
+    or on any error: a recall aid must never break the call it serves.
+    """
+    from scribe.models.embedding import MilestoneEmbedding
+    from scribe.models.milestone import Milestone
+
+    if not query or not query.strip():
+        return []
+    try:
+        query_vec = await get_embedding(query)
+    except Exception:
+        logger.debug("Milestone search skipped — embedder unavailable")
+        return []
+
+    distance = MilestoneEmbedding.embedding.cosine_distance(query_vec)
+    try:
+        if project_id:
+            if not await can_read_project(user_id, project_id):
+                return []
+            scope = Milestone.project_id == project_id
+        else:
+            scope = Milestone.user_id == user_id
+        async with async_session() as session:
+            rows = (await session.execute(
+                select(Milestone, distance.label("distance"))
+                .select_from(MilestoneEmbedding)
+                .join(Milestone, MilestoneEmbedding.milestone_id == Milestone.id)
+                .where(
+                    scope,
+                    Milestone.deleted_at.is_(None),
+                    *([Milestone.status == status] if status else []),
+                )
+                .order_by(distance)
+                .limit(limit * _CHUNK_OVERFETCH)
+            )).all()
+    except Exception:
+        logger.warning("Milestone semantic search failed", exc_info=True)
+        return []
+
+    best: dict[int, tuple[float, object]] = {}
+    for milestone, dist in rows:
+        score = 1.0 - float(dist)
+        if milestone.id not in best or score > best[milestone.id][0]:
+            best[milestone.id] = (score, milestone)
+    ranked = sorted(best.values(), key=lambda pair: pair[0], reverse=True)
+    return [pair for pair in ranked if pair[0] >= threshold][:limit]
+
+
+async def backfill_milestone_embeddings() -> None:
+    """Embed milestones that have no current vectors. Runs at startup beside
+    the note and rule backfills; a CHUNKER_VERSION bump re-embeds."""
+    from scribe.models.embedding import MilestoneEmbedding
+    from scribe.models.milestone import Milestone
+
+    try:
+        async with async_session() as session:
+            current = select(MilestoneEmbedding.milestone_id).where(
+                MilestoneEmbedding.chunker_version == CHUNKER_VERSION
+            )
+            stale = (await session.execute(
+                select(Milestone.id, Milestone.title, Milestone.description, Milestone.body)
+                .where(Milestone.deleted_at.is_(None), Milestone.id.notin_(current))
+            )).all()
+    except Exception:
+        logger.warning("Milestone embedding backfill: failed to query milestones", exc_info=True)
+        return
+
+    if not stale:
+        logger.info("Milestone embedding backfill: all milestones current at chunker v%d", CHUNKER_VERSION)
+        return
+    logger.info("Milestone embedding backfill: embedding %d milestone(s)", len(stale))
+    for milestone_id, title, description, body in stale:
+        await upsert_milestone_embedding(milestone_id, title, description, body)

src/scribe/services/milestones.py

@@ -11,6 +11,30 @@ from scribe.models.note import Note
 logger = logging.getLogger(__name__)
 
 
+def embed_milestone(milestone: Milestone) -> None:
+    """Refresh a milestone's vectors, fire-and-forget (milestone 415).
+
+    At the service, so every path that writes a milestone gets it — the lesson
+    embed_note records (#2056): a record written through a door that forgot the
+    call stays out of search until a restart. Exceptions are swallowed because
+    a milestone that saved must not fail on its index refresh; no running loop
+    (a script, a unit test) is ordinary. A delete racing the refresh wins: the
+    upsert claims the milestone's row first (#3262).
+    """
+    try:
+        import asyncio
+
+        from scribe.services.embeddings import upsert_milestone_embedding
+
+        asyncio.create_task(upsert_milestone_embedding(
+            milestone.id, milestone.title, milestone.description, milestone.body,
+        ))
+    except RuntimeError:
+        pass
+    except Exception:  # noqa: BLE001 - never let indexing break a write
+        logger.exception("embedding refresh failed for milestone %s", milestone.id)
+
+
 async def create_milestone(
     user_id: int,
     project_id: int,
@@ -33,6 +57,7 @@ async def create_milestone(
         session.add(milestone)
         await session.commit()
         await session.refresh(milestone)
+        embed_milestone(milestone)
         return milestone
 
 
@@ -125,6 +150,8 @@ async def update_milestone(user_id: int, milestone_id: int, **fields: object) ->
         milestone.updated_at = datetime.now(timezone.utc)
         await session.commit()
         await session.refresh(milestone)
+        if {"title", "description", "body"} & set(fields):
+            embed_milestone(milestone)
         return milestone
 
 

src/scribe/services/record_batch.py

@@ -28,6 +28,7 @@ from scribe.models import async_session
 from scribe.models.milestone import Milestone
 from scribe.models.note import Note
 from scribe.services import access as access_svc
+from scribe.services import milestones as milestones_svc
 from scribe.services import notes as notes_svc
 from scribe.services import systems as systems_svc
 from scribe.services.record_refs import placeholder_keys, resolve_placeholders
@@ -183,6 +184,8 @@ async def create_batch(
     # After the commit, as a single create does: embedding and System tags are
     # enrichment on records that now exist, and a failure in either must not
     # un-create them.
+    if new_ms is not None:
+        milestones_svc.embed_milestone(new_ms)
     for note, item in zip(notes, items):
         notes_svc.embed_note(note)
         if item.system_ids: