feat(research): linked section notes + executive summary in index

Research pipeline now produces an index note with:
- Executive summary (2-3 paragraphs synthesized from sections)
- Clickable links to each section note (/notes/{id})
- Section notes have parent_id pointing to the index

Also improves outline resilience: lowered minimum sections from 3
to 2, retries once on failure before falling back to monolithic note.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/fabledassistant/services/research.py

@@ -9,7 +9,7 @@ import httpx
 
 from fabledassistant.config import Config
 from fabledassistant.services.llm import fetch_url_content, generate_completion, stream_chat
-from fabledassistant.services.notes import create_note
+from fabledassistant.services.notes import create_note, update_note
 from fabledassistant.models.note import Note
 
 logger = logging.getLogger(__name__)
@@ -35,8 +35,8 @@ def _build_sources_block(sources: list[dict]) -> str:
 async def _generate_outline(topic: str, sources: list[dict], model: str) -> list[dict]:
     """Generate a topic outline from fetched research sources.
 
-    Returns a list of {"title": str, "focus": str} dicts (3–8 entries).
-    Returns [] on failure — callers must fall back to single-note synthesis.
+    Returns a list of {"title": str, "focus": str} dicts (2–8 entries).
+    Retries once on failure before returning [] (callers fall back to single-note).
     """
     import json as _json
 
@@ -61,23 +61,27 @@ async def _generate_outline(topic: str, sources: list[dict], model: str) -> list
             "content": f"Topic: {topic}\n\nSources:\n{sources_block}",
         },
     ]
-    try:
-        raw = await generate_completion(messages, model, max_tokens=400)
-        raw = raw.strip()
-        raw = re.sub(r"^```(?:json)?\s*", "", raw)
-        raw = re.sub(r"\s*```$", "", raw)
-        idx = raw.find("[")
-        if idx >= 0:
-            parsed, _ = _json.JSONDecoder().raw_decode(raw[idx:])
-            if isinstance(parsed, list):
-                sections = [
-                    s for s in parsed
-                    if isinstance(s, dict) and s.get("title") and s.get("focus")
-                ]
-                if len(sections) >= 3:
-                    return sections[:8]
-    except Exception:
-        logger.warning("Outline generation failed for topic '%s'", topic, exc_info=True)
+    for attempt in range(2):
+        try:
+            raw = await generate_completion(messages, model, max_tokens=400)
+            raw = raw.strip()
+            raw = re.sub(r"^```(?:json)?\s*", "", raw)
+            raw = re.sub(r"\s*```$", "", raw)
+            idx = raw.find("[")
+            if idx >= 0:
+                parsed, _ = _json.JSONDecoder().raw_decode(raw[idx:])
+                if isinstance(parsed, list):
+                    sections = [
+                        s for s in parsed
+                        if isinstance(s, dict) and s.get("title") and s.get("focus")
+                    ]
+                    if len(sections) >= 2:
+                        return sections[:8]
+        except Exception:
+            logger.warning(
+                "Outline generation attempt %d failed for topic '%s'",
+                attempt + 1, topic, exc_info=True,
+            )
     return []
 
 
@@ -121,6 +125,49 @@ async def _synthesize_section(
     return section_title, raw.strip()
 
 
+async def _generate_executive_summary(
+    topic: str,
+    section_bodies: list[tuple[str, str]],
+    model: str,
+) -> str:
+    """Generate a 2-3 paragraph executive summary from completed section notes.
+
+    Args:
+        section_bodies: list of (title, body) pairs from the section notes.
+
+    Returns summary markdown (no heading — caller adds structure).
+    """
+    sections_block = "\n\n".join(
+        f"### {title}\n{body[:1500]}" for title, body in section_bodies
+    )
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are a research summarizer. Given several completed research sections on a topic, "
+                "write a concise executive summary.\n\n"
+                "Requirements:\n"
+                "- 2–3 paragraphs of substantive prose (150–300 words total)\n"
+                "- Cover the key findings and insights across all sections\n"
+                "- Highlight the most important or surprising takeaways\n"
+                "- Write so someone can decide which sections to read in detail\n"
+                "- Do NOT include headings, bullet points, or source citations\n"
+                "- Do NOT start with 'This research' or 'This document' — jump straight into the content"
+            ),
+        },
+        {
+            "role": "user",
+            "content": f"Topic: {topic}\n\nSections:\n{sections_block}",
+        },
+    ]
+    try:
+        raw = await generate_completion(messages, model, max_tokens=600)
+        return raw.strip()
+    except Exception:
+        logger.warning("Executive summary generation failed for '%s'", topic, exc_info=True)
+        return ""
+
+
 async def run_research_pipeline(
     topic: str,
     user_id: int,
@@ -220,28 +267,17 @@ async def run_research_pipeline(
         return_exceptions=True,
     )
 
-    # Step 5: Create section notes sequentially
-    _status(f"Saving {len(outline)} notes...")
-    section_note_pairs: list[tuple[dict, Note]] = []
+    # Collect successful results for index + summary generation
+    section_results: list[tuple[dict, str, str]] = []  # (outline_entry, title, body)
     for section, result in zip(outline, raw_results):
         if isinstance(result, Exception):
             logger.warning("Section synthesis failed for '%s': %s", section["title"], result)
             continue
         sec_title, sec_body = result
-        try:
-            note = await create_note(
-                user_id=user_id,
-                title=sec_title,
-                body=sec_body,
-                tags=["research"],
-                project_id=project_id,
-            )
-            section_note_pairs.append((section, note))
-        except Exception:
-            logger.warning("Failed to save section note '%s'", sec_title, exc_info=True)
+        section_results.append((section, sec_title, sec_body))
 
     # All sections failed — fall back to single note
-    if not section_note_pairs:
+    if not section_results:
         logger.warning("All section syntheses failed, falling back to single note for '%s'", topic)
         _status("Synthesizing report (fallback)...")
         title, body = await _synthesize_note(topic, synthesis_sources, model, buf=None)
@@ -250,20 +286,24 @@ async def run_research_pipeline(
         )
         return note
 
-    # Step 6: Create index note
+    # Step 5: Generate executive summary from section content
+    _status("Writing summary...")
+    executive_summary = await _generate_executive_summary(
+        topic, [(t, b) for _, t, b in section_results], model,
+    )
+
+    # Step 6: Create index note first (so section notes can reference it via parent_id)
     from datetime import date as _date
     index_lines = [
         f"Research overview for **{topic}** — {_date.today().isoformat()}",
         "",
-        f"Generated from {len(synthesis_sources)} web sources across {len(section_note_pairs)} sections.",
-        "",
-        "## Sections",
+        f"Generated from {len(synthesis_sources)} web sources across {len(section_results)} sections.",
         "",
     ]
-    for section, note in section_note_pairs:
-        index_lines.append(f"- **{note.title}** — {section['focus']}")
-    index_lines += ["", "*Search for any section title to read it.*"]
-
+    if executive_summary:
+        index_lines += ["## Summary", "", executive_summary, ""]
+    index_lines += ["## Sections", ""]
+    # Placeholder — will be updated with real links after section notes are created
     index_note = await create_note(
         user_id=user_id,
         title=f"Research: {topic}",
@@ -271,6 +311,34 @@ async def run_research_pipeline(
         tags=["research", "research-index"],
         project_id=project_id,
     )
+
+    # Step 7: Create section notes with parent_id pointing to index
+    _status(f"Saving {len(section_results)} notes...")
+    section_note_pairs: list[tuple[dict, Note]] = []
+    for section, sec_title, sec_body in section_results:
+        try:
+            note = await create_note(
+                user_id=user_id,
+                title=sec_title,
+                body=sec_body,
+                tags=["research"],
+                project_id=project_id,
+                parent_id=index_note.id,
+            )
+            section_note_pairs.append((section, note))
+        except Exception:
+            logger.warning("Failed to save section note '%s'", sec_title, exc_info=True)
+
+    # Step 8: Update index note body with real links to section notes
+    for section, note in section_note_pairs:
+        index_lines.append(f"- [{note.title}](/notes/{note.id}) — {section['focus']}")
+
+    await update_note(
+        user_id=user_id,
+        note_id=index_note.id,
+        body="\n".join(index_lines),
+    )
+
     logger.info(
         "Research: created %d section notes + index id=%d for topic '%s'",
         len(section_note_pairs), index_note.id, topic,

tests/test_research_pipeline.py

@@ -34,8 +34,8 @@ async def test_generate_outline_returns_empty_on_parse_failure():
 
 
 @pytest.mark.asyncio
-async def test_generate_outline_returns_empty_when_fewer_than_3_sections():
-    """_generate_outline returns [] when model returns fewer than 3 valid sections."""
+async def test_generate_outline_returns_empty_when_fewer_than_2_sections():
+    """_generate_outline returns [] when model returns fewer than 2 valid sections."""
     from fabledassistant.services.research import _generate_outline
 
     outline_json = json.dumps([{"title": "Only One", "focus": "Focus"}])
@@ -106,7 +106,7 @@ async def test_synthesize_section_strips_whitespace():
 
 @pytest.mark.asyncio
 async def test_pipeline_creates_section_notes_and_index():
-    """run_research_pipeline creates N section notes + 1 index note, returns index."""
+    """run_research_pipeline creates N section notes + 1 index note with links, returns index."""
     from unittest.mock import MagicMock
 
     outline = [
@@ -117,23 +117,31 @@ async def test_pipeline_creates_section_notes_and_index():
 
     note_id_counter = iter(range(10, 20))
 
-    def _make_note(user_id, title, body, tags, project_id=None):
+    def _make_note(user_id, title, body, tags, project_id=None, parent_id=None):
         n = MagicMock()
         n.id = next(note_id_counter)
         n.title = title
         return n
 
+    mock_update = AsyncMock()
+
     with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
          patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
          patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
          patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \
          patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \
-         patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note):
+         patch("fabledassistant.services.research._generate_executive_summary", new_callable=AsyncMock, return_value="Executive summary text."), \
+         patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note), \
+         patch("fabledassistant.services.research.update_note", new_callable=AsyncMock) as mock_update:
 
         from fabledassistant.services.research import run_research_pipeline
         result = await run_research_pipeline("test topic", user_id=1, model="test-model")
 
     assert result.title == "Research: test topic"
+    # Index note body should be updated with section links
+    mock_update.assert_called_once()
+    updated_body = mock_update.call_args.kwargs.get("body", "")
+    assert "/notes/" in updated_body
 
 
 @pytest.mark.asyncio