Project-aware assist, link suggestions, project-scoped RAG, semantic search tool, SSE race fix

- Writing assistant: inject project notes as context (definition-tagged first), wikilink suggestions
- Link suggestions: server-side endpoint finds unlinked term occurrences, NoteEditorView sidebar panel
- Project-scoped RAG: ChatView ProjectSelector filters semantic+keyword search to selected project
- Semantic search tool: LLM search_notes upgraded to hybrid semantic (0.40 threshold) + keyword merge
- SSE race condition fix: drain remaining events after stream loop exits in chat.py and notes.py
- RAG_AUTO_SNIPPET raised 800→4000; sidebar include uses full note body; MAX_BODY_CHARS 8000→24000
- Enter-to-submit on writing assistant instruction textareas (note and task editors)
- DiffView: equal-line collapsing with 3-line context around changes

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/fabledassistant/services/assist.py

@@ -1,18 +1,50 @@
-MAX_BODY_CHARS = 8000
+MAX_BODY_CHARS = 24000
+MAX_CONTEXT_NOTE_CHARS = 1500
+
+
+def _build_context_block(context_notes: list[dict]) -> str:
+    """Format project context notes into a prompt block."""
+    lines = [
+        "--- Project Context ---",
+        "The following notes are from the same project. Notes tagged 'definition' are "
+        "canonical term definitions — treat them as authoritative. Use them to maintain "
+        "consistency in terminology, tone, style, and [[wikilinks]] to other documents.",
+        "",
+    ]
+    for n in context_notes:
+        tags_str = f"  [tags: {', '.join(n['tags'])}]" if n.get("tags") else ""
+        body_preview = (n["body"] or "")[:MAX_CONTEXT_NOTE_CHARS]
+        if len(n["body"] or "") > MAX_CONTEXT_NOTE_CHARS:
+            body_preview += "…"
+        lines.append(f"**{n['title']}**{tags_str}")
+        if body_preview:
+            lines.append(body_preview)
+        lines.append("")
+    lines.append("--- End Project Context ---")
+    return "\n".join(lines)
 
 
 def build_assist_messages(
-    body: str, target_section: str, instruction: str, whole_doc: bool = False
+    body: str,
+    target_section: str,
+    instruction: str,
+    whole_doc: bool = False,
+    context_notes: list[dict] | None = None,
 ) -> list[dict]:
     """Build Ollama messages for writing assist.
 
     When whole_doc=True, the model revises the entire document.
     When whole_doc=False (section mode), the full body is provided as read-only
     context and the model outputs only the replacement for the target section.
+
+    context_notes: optional list of {title, tags, body} dicts from the same project,
+    injected so the model can maintain consistency with related documents.
     """
     truncated_body = body[:MAX_BODY_CHARS]
     if len(body) > MAX_BODY_CHARS:
-        truncated_body += "\n... (truncated)"
+        truncated_body += "\n… (truncated)"
+
+    context_block = (_build_context_block(context_notes) + "\n\n") if context_notes else ""
 
     if whole_doc:
         system_content = (
@@ -21,9 +53,12 @@ def build_assist_messages(
             "content not addressed by the instruction. "
             "Preserve all markdown formatting exactly — including bullet lists (- item), "
             "numbered lists (1. item), nested/indented sub-items (  - sub), bold (**text**), "
-            "italic (_text_), and code blocks. Never flatten nested lists into plain text."
+            "italic (_text_), and code blocks. Never flatten nested lists into plain text. "
+            "When referencing concepts defined in the Project Context, use [[wikilinks]] "
+            "to link to the relevant note by its exact title."
         )
         user_content = (
+            f"{context_block}"
             f"--- Document ---\n{truncated_body}\n--- End Document ---\n\n"
             f"Instruction: {instruction}"
         )
@@ -41,9 +76,12 @@ def build_assist_messages(
             "Match the document's existing tone and style. "
             "IMPORTANT: Preserve all markdown formatting exactly — including bullet lists (- item), "
             "numbered lists (1. item), nested/indented sub-items (  - sub), bold (**text**), "
-            "italic (_text_), and code blocks. Never flatten nested lists into plain text."
+            "italic (_text_), and code blocks. Never flatten nested lists into plain text. "
+            "When referencing concepts defined in the Project Context, use [[wikilinks]] "
+            "to link to the relevant note by its exact title."
         )
         user_content = (
+            f"{context_block}"
             f"--- Target Section ---\n{target_section}\n--- End Target Section ---\n\n"
             f"Instruction: {instruction}"
         )

src/fabledassistant/services/embeddings.py

@@ -79,6 +79,7 @@ async def semantic_search_notes(
     exclude_ids: set[int] | None = None,
     limit: int = 8,
     threshold: float = _SIMILARITY_THRESHOLD,
+    project_id: int | None = None,
 ) -> list[tuple[float, Note]]:
     """Return up to *limit* (score, note) pairs most relevant to *query*.
 
@@ -99,6 +100,8 @@ async def semantic_search_notes(
                 .join(Note, NoteEmbedding.note_id == Note.id)
                 .where(NoteEmbedding.user_id == user_id)
             )
+            if project_id is not None:
+                stmt = stmt.where(Note.project_id == project_id)
             if exclude_ids:
                 stmt = stmt.where(NoteEmbedding.note_id.notin_(exclude_ids))
             rows = list((await session.execute(stmt)).all())

src/fabledassistant/services/generation_task.py

@@ -45,7 +45,7 @@ _TOOL_LABELS: dict[str, str] = {
     "get_note": "Reading note",
     "list_notes": "Listing notes",
     "list_tasks": "Searching tasks",
-    "search_notes": "Searching notes",
+    "search_notes": "Searching notes (semantic)",
     "create_event": "Creating calendar event",
     "list_events": "Searching calendar",
     "search_events": "Searching calendar",
@@ -149,6 +149,7 @@ async def run_generation(
     include_note_ids: list[int] | None = None,
     excluded_note_ids: list[int] | None = None,
     think: bool = False,
+    rag_project_id: int | None = None,
 ) -> None:
     """Stream LLM response into buffer with periodic DB flushes."""
     MAX_TOOL_ROUNDS = 5
@@ -172,13 +173,14 @@ async def run_generation(
         history_to_use, history_summary = await summarize_history_for_context(history, model)
 
     # Phase 3: Build context and wait for model in parallel.
-    model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=90.0))
+    model_load_task = asyncio.create_task(wait_for_model_loaded(model, timeout=180.0))
 
     context_task = asyncio.create_task(build_context(
         user_id, history_to_use, context_note_id, user_content,
         history_summary=history_summary,
         include_note_ids=include_note_ids,
         excluded_note_ids=excluded_note_ids,
+        rag_project_id=rag_project_id,
     ))
 
     messages, context_meta = await context_task
@@ -192,7 +194,7 @@ async def run_generation(
         buf.append_event("status", {"status": "Loading model..."})
     loaded = await model_load_task
     if not loaded:
-        logger.warning("Model %s did not load within 90s — proceeding anyway", model)
+        logger.warning("Model %s did not load within 180s — proceeding anyway", model)
 
     t_start = time.monotonic()
     timing: dict = {
@@ -415,6 +417,9 @@ async def run_assist_generation(
     On each retry the accumulated content is reset so the done event
     always reflects only the successful generation.
     """
+    input_chars = sum(len(m.get("content", "")) for m in messages)
+    logger.info("Assist generation started: model=%s, input_chars=%d", model, input_chars)
+
     last_exc: BaseException | None = None
     for attempt in range(3):
         if attempt > 0:
@@ -429,9 +434,15 @@ async def run_assist_generation(
                 buf.content_so_far += chunk
                 buf.append_event("chunk", {"chunk": chunk})
 
+            output_chars = len(buf.content_so_far)
+            logger.info(
+                "Assist generation complete: output_chars=%d, events=%d",
+                output_chars, len(buf.events),
+            )
             buf.state = GenerationState.COMPLETED
             buf.finished_at = time.monotonic()
             buf.append_event("done", {"done": True, "full_text": buf.content_so_far})
+            logger.info("Assist done event appended (event index %d)", len(buf.events) - 1)
             return
 
         except httpx.HTTPStatusError as exc:

src/fabledassistant/services/llm.py

@@ -1,11 +1,14 @@
 import asyncio
+import ipaddress
 import json
 import logging
 import re
+import socket
 import time
 from collections.abc import AsyncGenerator
 from dataclasses import dataclass, field
 from typing import Literal
+from urllib.parse import urlparse
 
 import httpx
 
@@ -29,7 +32,7 @@ STOP_WORDS = frozenset({
 
 RAG_AUTO_THRESHOLD = 0.60
 RAG_AUTO_LIMIT = 3
-RAG_AUTO_SNIPPET = 800
+RAG_AUTO_SNIPPET = 4000
 
 
 async def get_installed_models() -> set[str]:
@@ -268,11 +271,32 @@ async def generate_completion(
     raise last_exc
 
 
+def _is_private_url(url: str) -> bool:
+    """Return True if the URL resolves to a private/loopback/link-local address (SSRF guard)."""
+    try:
+        host = urlparse(url).hostname
+        if not host:
+            return True
+        if host.lower() in ("localhost", "::1"):
+            return True
+        addr_info = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+        for entry in addr_info:
+            ip = ipaddress.ip_address(entry[4][0])
+            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast:
+                return True
+        return False
+    except Exception:
+        return True  # Block on resolution failure
+
+
 async def fetch_url_content(url: str) -> str:
     """Fetch a URL and return text content (HTML tags stripped)."""
+    if _is_private_url(url):
+        logger.warning("Blocked fetch of private/internal URL: %s", url)
+        return "[URL blocked: internal network access not permitted]"
     try:
         async with httpx.AsyncClient(
-            timeout=15.0, follow_redirects=True, headers={"User-Agent": "FabledAssistant/1.0"}
+            timeout=15.0, follow_redirects=False, headers={"User-Agent": "FabledAssistant/1.0"}
         ) as client:
             resp = await client.get(url)
             resp.raise_for_status()
@@ -426,6 +450,7 @@ async def build_context(
     history_summary: str | None = None,
     include_note_ids: list[int] | None = None,
     excluded_note_ids: list[int] | None = None,
+    rag_project_id: int | None = None,
 ) -> tuple[list[dict], dict]:
     """Build messages array for Ollama with system prompt and context.
 
@@ -465,6 +490,9 @@ async def build_context(
         "If a note was created earlier in the conversation and the user provides more content for it, use update_note. "
         "Use get_note to read the full content of a specific note. "
         "Use list_notes to browse notes by recency or tag. "
+        "Use search_notes for conceptual/semantic queries — e.g. 'what notes do I have about X' or "
+        "'find notes related to Y' — it uses semantic understanding to find thematically related content "
+        "even when exact words don't match. Pass project= to scope the search to a specific project. "
         "Use delete_note / delete_task only when explicitly asked to delete — these require confirmation."
     )
     tool_guidance = "\n".join(tool_lines)
@@ -511,7 +539,8 @@ async def build_context(
     try:
         from fabledassistant.services.embeddings import semantic_search_notes
         for score, note in await semantic_search_notes(
-            user_id, user_message, exclude_ids=search_exclude or None, limit=8
+            user_id, user_message, exclude_ids=search_exclude or None, limit=8,
+            project_id=rag_project_id,
         ):
             found_scored.append((score, note))
     except Exception:
@@ -522,7 +551,8 @@ async def build_context(
         if keywords:
             try:
                 for note in await search_notes_for_context(
-                    user_id, keywords, exclude_ids=search_exclude or None, limit=8
+                    user_id, keywords, exclude_ids=search_exclude or None, limit=8,
+                    project_id=rag_project_id,
                 ):
                     found_scored.append((None, note))
             except Exception:
@@ -587,7 +617,7 @@ async def build_context(
             try:
                 n = await _get_note(user_id, nid)
                 if n:
-                    body_preview = n.body[:2000] if n.body else ""
+                    body_preview = n.body or ""
                     included_snippets.append(f"- {n.title}: {body_preview}")
             except Exception:
                 logger.warning("Failed to load included note %d for context", nid, exc_info=True)

src/fabledassistant/services/note_versions.py

@@ -6,9 +6,11 @@ from fabledassistant.models.note_version import NoteVersion
 MAX_VERSIONS = 20
 
 
-async def create_version(user_id: int, note_id: int, body: str, title: str) -> NoteVersion:
+async def create_version(
+    user_id: int, note_id: int, body: str, title: str, tags: list[str] | None = None
+) -> NoteVersion:
     async with async_session() as session:
-        version = NoteVersion(note_id=note_id, user_id=user_id, body=body, title=title)
+        version = NoteVersion(note_id=note_id, user_id=user_id, body=body, title=title, tags=tags or [])
         session.add(version)
         await session.commit()
         await session.refresh(version)

src/fabledassistant/services/notes.py

@@ -197,6 +197,7 @@ async def update_note(user_id: int, note_id: int, **fields: object) -> Note | No
         # Snapshot before changes for version creation
         old_body = note.body
         old_title = note.title
+        old_tags = list(note.tags or [])
         for key, value in fields.items():
             if not hasattr(note, key):
                 continue
@@ -214,7 +215,7 @@ async def update_note(user_id: int, note_id: int, **fields: object) -> Note | No
     # Create a version snapshot when body actually changes
     if "body" in fields and fields["body"] != old_body:
         from fabledassistant.services.note_versions import create_version
-        await create_version(user_id, note_id, old_body, old_title)
+        await create_version(user_id, note_id, old_body, old_title, old_tags)
 
     return note
 
@@ -234,19 +235,24 @@ async def delete_note(user_id: int, note_id: int) -> bool:
 
 async def get_all_tags(user_id: int, q: str | None = None) -> list[str]:
     async with async_session() as session:
-        result = await session.execute(
-            text(
-                "SELECT DISTINCT unnest(tags) AS tag FROM notes"
-                " WHERE tags != '{}' AND user_id = :user_id"
-            ).bindparams(user_id=user_id)
-        )
-        all_tags = [row[0] for row in result.fetchall()]
-
         if q:
-            q_lower = q.lower()
-            all_tags = [t for t in all_tags if q_lower in t.lower()]
-
-        return sorted(all_tags)
+            result = await session.execute(
+                text(
+                    "SELECT DISTINCT tag FROM"
+                    " (SELECT unnest(tags) AS tag FROM notes"
+                    "  WHERE tags != '{}' AND user_id = :user_id) t"
+                    " WHERE tag ILIKE :q_pattern ORDER BY tag LIMIT 100"
+                ).bindparams(user_id=user_id, q_pattern=f"%{q}%")
+            )
+        else:
+            result = await session.execute(
+                text(
+                    "SELECT DISTINCT unnest(tags) AS tag FROM notes"
+                    " WHERE tags != '{}' AND user_id = :user_id"
+                    " ORDER BY tag LIMIT 100"
+                ).bindparams(user_id=user_id)
+            )
+        return [row[0] for row in result.fetchall()]
 
 
 async def convert_note_to_task(user_id: int, note_id: int) -> Note:
@@ -291,6 +297,7 @@ async def search_notes_for_context(
     keywords: list[str],
     exclude_ids: set[int] | None = None,
     limit: int = 3,
+    project_id: int | None = None,
 ) -> list[Note]:
     """Search notes by keywords with OR logic. Optimized for context building — no count query."""
     async with async_session() as session:
@@ -301,6 +308,8 @@ async def search_notes_for_context(
             keyword_filters.append(or_(Note.title.ilike(pattern), Note.body.ilike(pattern)))
 
         query = select(Note).where(Note.user_id == user_id, or_(*keyword_filters))
+        if project_id is not None:
+            query = query.where(Note.project_id == project_id)
         if exclude_ids:
             query = query.where(Note.id.notin_(exclude_ids))
         query = query.order_by(Note.updated_at.desc()).limit(limit)

src/fabledassistant/services/tools.py

@@ -182,19 +182,29 @@ _CORE_TOOLS = [
         "type": "function",
         "function": {
             "name": "search_notes",
-            "description": "Search the user's notes and tasks. Use this when the user asks about their existing notes, tasks, or wants to find something they've written.",
+            "description": (
+                "Search notes and tasks using semantic understanding. Finds content related to a concept, "
+                "theme, or topic — even when exact keywords don't appear in the title or body. "
+                "Use this when the user asks what notes they have about a topic, wants to explore related "
+                "content, or is looking for something they've written. Falls back to keyword search "
+                "automatically if semantic search is unavailable."
+            ),
             "parameters": {
                 "type": "object",
                 "properties": {
                     "query": {
                         "type": "string",
-                        "description": "Search query to find notes/tasks",
+                        "description": "A natural-language description of what you're looking for — concepts, themes, topics, or keywords",
                     },
                     "type": {
                         "type": "string",
                         "enum": ["note", "task"],
                         "description": "Restrict results to only notes or only tasks. Omit to search both.",
                     },
+                    "project": {
+                        "type": "string",
+                        "description": "Optional project name to restrict search to notes in that project",
+                    },
                 },
                 "required": ["query"],
             },
@@ -1070,27 +1080,79 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
         elif tool_name == "search_notes":
             query = arguments.get("query", "")
             type_filter = arguments.get("type")
+            project_name = arguments.get("project")
+
             is_task: bool | None = None
             if type_filter == "note":
                 is_task = False
             elif type_filter == "task":
                 is_task = True
-            notes, total = await list_notes(user_id=user_id, q=query, is_task=is_task, limit=5)
-            results = []
-            for n in notes:
-                results.append({
-                    "id": n.id,
-                    "title": n.title,
-                    "type": "task" if n.status is not None else "note",
-                    "status": n.status,
-                    "preview": (n.body[:200] + "...") if n.body and len(n.body) > 200 else (n.body or ""),
-                })
+
+            # Resolve project name → id
+            search_project_id: int | None = None
+            if project_name:
+                from fabledassistant.services.projects import get_project_by_title as _gpbt2, list_projects as _lp2
+                proj = await _gpbt2(user_id, project_name)
+                if proj is None:
+                    all_p = await _lp2(user_id)
+                    matches = [p for p in all_p if project_name.lower() in p.title.lower()]
+                    proj = matches[0] if matches else None
+                if proj:
+                    search_project_id = proj.id
+
+            results_map: dict[int, dict] = {}
+
+            # 1) Semantic search — broad threshold (0.40) since this is an intentional query
+            try:
+                from fabledassistant.services.embeddings import semantic_search_notes as _ssn
+                sem_results = await _ssn(
+                    user_id, query, limit=8, threshold=0.40,
+                    project_id=search_project_id,
+                )
+                for score, n in sem_results:
+                    n_is_task = n.status is not None
+                    if is_task is True and not n_is_task:
+                        continue
+                    if is_task is False and n_is_task:
+                        continue
+                    results_map[n.id] = {
+                        "id": n.id,
+                        "title": n.title,
+                        "type": "task" if n_is_task else "note",
+                        "status": n.status,
+                        "relevance": f"{round(score * 100)}%",
+                        "preview": (n.body[:300] + "…") if n.body and len(n.body) > 300 else (n.body or ""),
+                    }
+                logger.debug("search_notes semantic: %d results for %r", len(results_map), query)
+            except Exception:
+                logger.debug("Semantic search unavailable in search_notes, using keyword only", exc_info=True)
+
+            # 2) Keyword search — merge in anything not already found
+            try:
+                kw_notes, _ = await list_notes(
+                    user_id=user_id, q=query, is_task=is_task,
+                    project_id=search_project_id, limit=8,
+                )
+                for n in kw_notes:
+                    if n.id not in results_map:
+                        results_map[n.id] = {
+                            "id": n.id,
+                            "title": n.title,
+                            "type": "task" if n.status is not None else "note",
+                            "status": n.status,
+                            "relevance": "keyword",
+                            "preview": (n.body[:300] + "…") if n.body and len(n.body) > 300 else (n.body or ""),
+                        }
+            except Exception:
+                logger.warning("Keyword fallback in search_notes failed", exc_info=True)
+
+            results = list(results_map.values())[:8]
             return {
                 "success": True,
                 "type": "search",
                 "data": {
                     "query": query,
-                    "total": total,
+                    "count": len(results),
                     "results": results,
                 },
             }