feat(rag): RAG scoping and context isolation controls

- Migration 0030: add conversations.rag_project_id (NULL=orphan-only,
  -1=all notes, positive=project), projects.auto_summary and
  projects.summary_updated_at
- Three-value scope semantics thread from build_context() → semantic
  search and keyword fallback via orphan_only + effective_project_id
- Project summarization background job (generate_project_summary,
  backfill_project_summaries) called via Ollama; triggered on project
  update and note saves (debounced 1h); runs at startup
- New LLM tools: search_projects (SequenceMatcher scoring on
  title+description+auto_summary) and set_rag_scope (persists to DB,
  workspace-guarded, emits new_rag_scope in SSE done event)
- execute_tool() accepts conv_id + workspace_project_id; generation_task
  passes both and captures scope changes for SSE done enrichment
- Frontend: Conversation type gets rag_project_id; chat store adds
  ragProjectId computed + updateRagScope(); SSE done handler syncs scope
- ChatView: replace sidebar ProjectSelector with a scope chip pill above
  the input bar, animated dropdown, pulse on model-driven scope change

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/fabledassistant/app.py

@@ -245,6 +245,11 @@ def create_app() -> Quart:
                 await backfill_note_embeddings()
             except Exception:
                 logger.warning("Embedding backfill failed", exc_info=True)
+            try:
+                from fabledassistant.services.projects import backfill_project_summaries
+                await backfill_project_summaries()
+            except Exception:
+                logger.warning("Project summary backfill failed", exc_info=True)
 
         asyncio.create_task(_delayed_backfill())
 

src/fabledassistant/models/conversation.py

@@ -23,6 +23,8 @@ class Conversation(Base, TimestampMixin):
     conversation_type: Mapped[str] = mapped_column(Text, default="chat", server_default="chat")
     # For briefing conversations only: the calendar date this briefing covers.
     briefing_date: Mapped[datetime.date | None] = mapped_column(Date, nullable=True)
+    # NULL = orphan notes only; -1 = all notes; positive int = specific project
+    rag_project_id: Mapped[int | None] = mapped_column(Integer, nullable=True, default=None)
 
     messages: Mapped[list["Message"]] = relationship(
         back_populates="conversation",
@@ -47,6 +49,7 @@ class Conversation(Base, TimestampMixin):
             "model": self.model,
             "conversation_type": self.conversation_type,
             "briefing_date": self.briefing_date.isoformat() if self.briefing_date else None,
+            "rag_project_id": self.rag_project_id,
             "message_count": msg_count,
             "created_at": self.created_at.isoformat(),
             "updated_at": self.updated_at.isoformat(),

src/fabledassistant/models/project.py

@@ -1,5 +1,6 @@
 import enum
-from sqlalchemy import ForeignKey, Integer, Text
+from datetime import datetime
+from sqlalchemy import DateTime, ForeignKey, Integer, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from fabledassistant.models import Base
 from fabledassistant.models.base import TimestampMixin
@@ -20,6 +21,10 @@ class Project(Base, TimestampMixin):
     goal: Mapped[str] = mapped_column(Text, default="")
     status: Mapped[str] = mapped_column(Text, default="active")
     color: Mapped[str | None] = mapped_column(Text, nullable=True)  # hex color
+    auto_summary: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
+    summary_updated_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True, default=None
+    )
 
     def to_dict(self) -> dict:
         return {

src/fabledassistant/routes/chat.py

@@ -115,13 +115,15 @@ async def delete_conversation_route(conv_id: int):
 @chat_bp.route("/conversations/<int:conv_id>", methods=["PATCH"])
 @login_required
 async def update_conversation_route(conv_id: int):
+    from fabledassistant.services.chat import _UNSET
     uid = get_current_user_id()
     data = await request.get_json()
     title = data.get("title")
     model = data.get("model")
-    if title is None and model is None:
-        return jsonify({"error": "title or model is required"}), 400
-    conv = await update_conversation(uid, conv_id, title=title, model=model)
+    rag_project_id = data.get("rag_project_id", _UNSET)
+    if title is None and model is None and rag_project_id is _UNSET:
+        return jsonify({"error": "title, model, or rag_project_id is required"}), 400
+    conv = await update_conversation(uid, conv_id, title=title, model=model, rag_project_id=rag_project_id)
     if conv is None:
         return not_found("Conversation")
     return jsonify(conv.to_dict())

src/fabledassistant/services/chat.py

@@ -139,11 +139,15 @@ async def cleanup_old_conversations(user_id: int, days: int) -> int:
         return len(result.fetchall())
 
 
+_UNSET = object()
+
+
 async def update_conversation(
     user_id: int,
     conversation_id: int,
     title: str | None = None,
     model: str | None = None,
+    rag_project_id: object = _UNSET,
 ) -> Conversation | None:
     async with async_session() as session:
         result = await session.execute(
@@ -159,6 +163,8 @@ async def update_conversation(
             conv.title = title
         if model is not None:
             conv.model = model
+        if rag_project_id is not _UNSET:
+            conv.rag_project_id = rag_project_id  # type: ignore[assignment]
         conv.updated_at = datetime.now(timezone.utc)
         await session.commit()
         await session.refresh(conv)

src/fabledassistant/services/embeddings.py

@@ -81,6 +81,7 @@ async def semantic_search_notes(
     threshold: float = _SIMILARITY_THRESHOLD,
     project_id: int | None = None,
     is_task: bool | None = None,
+    orphan_only: bool = False,
 ) -> list[tuple[float, Note]]:
     """Return up to *limit* (score, note) pairs most relevant to *query*.
 
@@ -101,7 +102,9 @@ async def semantic_search_notes(
                 .join(Note, NoteEmbedding.note_id == Note.id)
                 .where(NoteEmbedding.user_id == user_id)
             )
-            if project_id is not None:
+            if orphan_only:
+                stmt = stmt.where(Note.project_id.is_(None))
+            elif project_id is not None:
                 stmt = stmt.where(Note.project_id == project_id)
             if is_task is True:
                 stmt = stmt.where(Note.status.isnot(None))

src/fabledassistant/services/generation_task.py

@@ -208,6 +208,8 @@ async def run_generation(
 
     last_flush = time.monotonic()
     all_tool_calls: list[dict] = []
+    new_rag_scope: object = False  # sentinel; set to int|None when scope changes
+    new_rag_scope_label: str | None = None
 
     try:
         cancelled = False
@@ -280,7 +282,16 @@ async def run_generation(
                                 buf.content_so_far += err_text
                             research_completed = True
                         else:
-                            result = await execute_tool(user_id, tool_name, arguments)
+                            result = await execute_tool(
+                                user_id, tool_name, arguments,
+                                conv_id=conv_id,
+                                workspace_project_id=workspace_project_id,
+                            )
+
+                        # Capture RAG scope change for SSE done event
+                        if result.get("type") == "rag_scope_set" and result.get("success"):
+                            new_rag_scope = arguments.get("project_id")
+                            new_rag_scope_label = result.get("scope_label")
 
                         timing["tools"].append({"name": tool_name, "ms": int((time.monotonic() - t_tool) * 1000)})
                         logger.info("Tool %s result: success=%s", tool_name, result.get("success"))
@@ -352,7 +363,11 @@ async def run_generation(
 
         buf.state = GenerationState.COMPLETED
         buf.finished_at = time.monotonic()
-        buf.append_event("done", {"done": True, "message_id": msg_id, "timing": timing})
+        done_payload: dict = {"done": True, "message_id": msg_id, "timing": timing}
+        if new_rag_scope is not False:
+            done_payload["new_rag_scope"] = new_rag_scope
+            done_payload["new_rag_scope_label"] = new_rag_scope_label
+        buf.append_event("done", done_payload)
 
         # Fire push notification when complete (non-critical, fire-and-forget)
         try:

src/fabledassistant/services/llm.py

@@ -536,12 +536,18 @@ async def build_context(
     # (score, note) pairs — score is float for semantic results, None for keyword fallback.
     found_scored: list[tuple[float | None, object]] = []
 
+    # Derive scope flags from rag_project_id three-value semantics:
+    # None → orphan notes only; -1 → all notes; positive int → that project
+    orphan_only = rag_project_id is None
+    effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
+
     # Try semantic search first; fall back to keyword search on failure / no results.
     try:
         from fabledassistant.services.embeddings import semantic_search_notes
         for score, note in await semantic_search_notes(
             user_id, user_message, exclude_ids=search_exclude or None, limit=8,
-            project_id=rag_project_id,
+            project_id=effective_project_id,
+            orphan_only=orphan_only,
         ):
             found_scored.append((score, note))
     except Exception:
@@ -553,7 +559,8 @@ async def build_context(
             try:
                 for note in await search_notes_for_context(
                     user_id, keywords, exclude_ids=search_exclude or None, limit=8,
-                    project_id=rag_project_id,
+                    project_id=effective_project_id,
+                    orphan_only=orphan_only,
                 ):
                     found_scored.append((None, note))
             except Exception:

src/fabledassistant/services/notes.py

@@ -22,6 +22,29 @@ def _normalize_tags(tags: list[str]) -> list[str]:
     return out
 
 
+async def _maybe_trigger_project_summary(user_id: int, project_id: int) -> None:
+    """Fire generate_project_summary() if the project summary is missing or >1h old."""
+    import asyncio
+    from datetime import timedelta
+    from fabledassistant.models.project import Project
+    from fabledassistant.services.projects import generate_project_summary
+    try:
+        async with async_session() as session:
+            project = (await session.execute(
+                select(Project).where(Project.id == project_id)
+            )).scalars().first()
+        if project is None:
+            return
+        stale = (
+            project.summary_updated_at is None
+            or (datetime.now(timezone.utc) - project.summary_updated_at) > timedelta(hours=1)
+        )
+        if stale:
+            asyncio.create_task(generate_project_summary(user_id, project_id))
+    except Exception:
+        logger.debug("_maybe_trigger_project_summary failed for project %d", project_id, exc_info=True)
+
+
 async def create_note(
     user_id: int,
     title: str = "",
@@ -61,7 +84,11 @@ async def create_note(
         session.add(note)
         await session.commit()
         await session.refresh(note)
-        return note
+
+    if project_id is not None:
+        await _maybe_trigger_project_summary(user_id, project_id)
+
+    return note
 
 
 async def get_note(user_id: int, note_id: int) -> Note | None:
@@ -222,6 +249,9 @@ async def update_note(user_id: int, note_id: int, **fields: object) -> Note | No
         from fabledassistant.services.note_versions import create_version
         await create_version(user_id, note_id, old_body, old_title, old_tags)
 
+    if note.project_id is not None:
+        await _maybe_trigger_project_summary(user_id, note.project_id)
+
     return note
 
 
@@ -303,6 +333,7 @@ async def search_notes_for_context(
     exclude_ids: set[int] | None = None,
     limit: int = 3,
     project_id: int | None = None,
+    orphan_only: bool = False,
 ) -> list[Note]:
     """Search notes by keywords with OR logic. Optimized for context building — no count query."""
     async with async_session() as session:
@@ -313,7 +344,9 @@ async def search_notes_for_context(
             keyword_filters.append(or_(Note.title.ilike(pattern), Note.body.ilike(pattern)))
 
         query = select(Note).where(Note.user_id == user_id, or_(*keyword_filters))
-        if project_id is not None:
+        if orphan_only:
+            query = query.where(Note.project_id.is_(None))
+        elif project_id is not None:
             query = query.where(Note.project_id == project_id)
         if exclude_ids:
             query = query.where(Note.id.notin_(exclude_ids))

src/fabledassistant/services/projects.py

@@ -70,6 +70,7 @@ async def list_projects(user_id: int, status: str | None = None) -> list[Project
 
 
 async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
+    import asyncio
     async with async_session() as session:
         result = await session.execute(
             select(Project).where(Project.id == project_id, Project.user_id == user_id)
@@ -83,7 +84,72 @@ async def update_project(user_id: int, project_id: int, **fields: object) -> Pro
         project.updated_at = datetime.now(timezone.utc)
         await session.commit()
         await session.refresh(project)
-        return project
+    asyncio.create_task(generate_project_summary(user_id, project_id))
+    return project
+
+
+async def generate_project_summary(user_id: int, project_id: int) -> None:
+    """Generate an LLM summary for a project and persist it. Fire-and-forget safe."""
+    try:
+        async with async_session() as session:
+            project = (await session.execute(
+                select(Project).where(Project.id == project_id, Project.user_id == user_id)
+            )).scalars().first()
+            if project is None:
+                return
+
+            note_rows = (await session.execute(
+                select(Note.title, Note.body)
+                .where(Note.project_id == project_id, Note.user_id == user_id)
+                .order_by(Note.updated_at.desc())
+                .limit(10)
+            )).all()
+
+        title = project.title or ""
+        description = project.description or ""
+        goal = project.goal or ""
+        note_snippets = "\n".join(
+            f"- {r.title}: {(r.body or '')[:200]}" for r in note_rows
+        )
+
+        prompt = (
+            f"Summarize this project in 3-4 sentences covering its purpose, themes, and content.\n"
+            f"Title: {title}\nDescription: {description}\nGoal: {goal}\n"
+            f"Recent notes:\n{note_snippets}"
+        )
+
+        from fabledassistant.services.llm import generate_completion
+        from fabledassistant.config import Config
+        messages = [{"role": "user", "content": prompt}]
+        summary = await generate_completion(messages, model=Config.OLLAMA_MODEL, max_tokens=400)
+        if not summary:
+            return
+
+        async with async_session() as session:
+            project = (await session.execute(
+                select(Project).where(Project.id == project_id)
+            )).scalars().first()
+            if project:
+                project.auto_summary = summary.strip()
+                project.summary_updated_at = datetime.now(timezone.utc)
+                await session.commit()
+        logger.debug("Generated summary for project %d", project_id)
+    except Exception:
+        logger.debug("Failed to generate summary for project %d", project_id, exc_info=True)
+
+
+async def backfill_project_summaries() -> None:
+    """Generate summaries for all projects missing auto_summary. Fire-and-forget."""
+    import asyncio
+    try:
+        async with async_session() as session:
+            rows = (await session.execute(
+                select(Project.id, Project.user_id).where(Project.auto_summary.is_(None))
+            )).all()
+        for row in rows:
+            asyncio.create_task(generate_project_summary(row.user_id, row.id))
+    except Exception:
+        logger.debug("backfill_project_summaries failed", exc_info=True)
 
 
 async def delete_project(user_id: int, project_id: int) -> bool:

src/fabledassistant/services/tools.py

@@ -838,9 +838,48 @@ _IMAGE_TOOLS = [
 ]
 
 
+_RAG_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_projects",
+            "description": "Search for projects by name, description, or theme. Use this to find which project a user is referring to and get its ID for set_rag_scope.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "Search query — project name, topic, or theme",
+                    }
+                },
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "set_rag_scope",
+            "description": "Change the RAG scope for this conversation. Use project_id=<int> to scope to a project, project_id=null to scope to orphan notes only, project_id=-1 for all notes.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "project_id": {
+                        "type": ["integer", "null"],
+                        "description": "Project ID to scope to, null for orphan-only, -1 for all notes",
+                    }
+                },
+                "required": ["project_id"],
+            },
+        },
+    },
+]
+
+
 async def get_tools_for_user(user_id: int) -> list[dict]:
     """Build the tool list for a user based on their configured integrations."""
     tools = list(_CORE_TOOLS)
+    tools.extend(_RAG_TOOLS)
     if await is_caldav_configured(user_id):
         tools.extend(_CALDAV_TOOLS)
     if Config.searxng_enabled():
@@ -883,7 +922,13 @@ def _fuzzy_title_match(title: str, candidates, threshold: float = 0.82):
 
 
 
-async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
+async def execute_tool(
+    user_id: int,
+    tool_name: str,
+    arguments: dict,
+    conv_id: int | None = None,
+    workspace_project_id: int | None = None,
+) -> dict:
     """Execute a tool call and return the result."""
     try:
         if tool_name == "create_task":
@@ -1735,6 +1780,52 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
             items = await get_recent_items(user_id, limit=limit)
             return {"data": {"items": items, "count": len(items)}}
 
+        elif tool_name == "search_projects":
+            from difflib import SequenceMatcher
+            query = str(arguments.get("query", "")).lower()
+            from fabledassistant.services.projects import list_projects
+            projects = await list_projects(user_id)
+            scored: list[tuple[float, object]] = []
+            for p in projects:
+                combined = f"{p.title} {p.description or ''} {p.auto_summary or ''}".lower()
+                base_score = SequenceMatcher(None, query, combined).ratio()
+                # Bonus for keyword overlap
+                query_words = set(query.split())
+                overlap = sum(1 for w in query_words if w in combined)
+                score = base_score + overlap * 0.05
+                scored.append((score, p))
+            scored.sort(key=lambda x: x[0], reverse=True)
+            results = []
+            for score, p in scored[:5]:
+                results.append({
+                    "id": p.id,
+                    "title": p.title,
+                    "summary_snippet": (p.auto_summary or p.description or "")[:200],
+                    "score": round(score, 3),
+                })
+            return {"type": "projects_list", "data": {"projects": results}}
+
+        elif tool_name == "set_rag_scope":
+            if workspace_project_id is not None:
+                return {"success": False, "error": "Cannot change RAG scope in workspace view"}
+            if conv_id is None:
+                return {"success": False, "error": "No conversation context available"}
+            project_id = arguments.get("project_id")  # None, -1, or positive int
+            # Validate positive project_id belongs to user
+            if project_id is not None and project_id != -1:
+                from fabledassistant.services.projects import get_project
+                proj = await get_project(user_id, int(project_id))
+                if proj is None:
+                    return {"success": False, "error": "Project not found"}
+                scope_label = proj.title
+            elif project_id == -1:
+                scope_label = "All notes"
+            else:
+                scope_label = "Orphan notes only"
+            from fabledassistant.services.chat import update_conversation
+            await update_conversation(user_id, conv_id, rag_project_id=project_id)
+            return {"success": True, "type": "rag_scope_set", "scope_label": scope_label}
+
         else:
             return {"success": False, "error": f"Unknown tool: {tool_name}"}