From 1d6905ccc8c6a3bac4c3e62544f1e114749dbe78 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 12 Apr 2026 12:18:31 -0400 Subject: [PATCH] feat(research): linked section notes + executive summary in index Research pipeline now produces an index note with: - Executive summary (2-3 paragraphs synthesized from sections) - Clickable links to each section note (/notes/{id}) - Section notes have parent_id pointing to the index Also improves outline resilience: lowered minimum sections from 3 to 2, retries once on failure before falling back to monolithic note. Co-Authored-By: Claude Opus 4.6 --- src/fabledassistant/services/research.py | 154 ++++++++++++++++------- tests/test_research_pipeline.py | 18 ++- 2 files changed, 124 insertions(+), 48 deletions(-) diff --git a/src/fabledassistant/services/research.py b/src/fabledassistant/services/research.py index 7293da8..8525e6e 100644 --- a/src/fabledassistant/services/research.py +++ b/src/fabledassistant/services/research.py @@ -9,7 +9,7 @@ import httpx from fabledassistant.config import Config from fabledassistant.services.llm import fetch_url_content, generate_completion, stream_chat -from fabledassistant.services.notes import create_note +from fabledassistant.services.notes import create_note, update_note from fabledassistant.models.note import Note logger = logging.getLogger(__name__) @@ -35,8 +35,8 @@ def _build_sources_block(sources: list[dict]) -> str: async def _generate_outline(topic: str, sources: list[dict], model: str) -> list[dict]: """Generate a topic outline from fetched research sources. - Returns a list of {"title": str, "focus": str} dicts (3–8 entries). - Returns [] on failure — callers must fall back to single-note synthesis. + Returns a list of {"title": str, "focus": str} dicts (2–8 entries). + Retries once on failure before returning [] (callers fall back to single-note). """ import json as _json @@ -61,23 +61,27 @@ async def _generate_outline(topic: str, sources: list[dict], model: str) -> list "content": f"Topic: {topic}\n\nSources:\n{sources_block}", }, ] - try: - raw = await generate_completion(messages, model, max_tokens=400) - raw = raw.strip() - raw = re.sub(r"^```(?:json)?\s*", "", raw) - raw = re.sub(r"\s*```$", "", raw) - idx = raw.find("[") - if idx >= 0: - parsed, _ = _json.JSONDecoder().raw_decode(raw[idx:]) - if isinstance(parsed, list): - sections = [ - s for s in parsed - if isinstance(s, dict) and s.get("title") and s.get("focus") - ] - if len(sections) >= 3: - return sections[:8] - except Exception: - logger.warning("Outline generation failed for topic '%s'", topic, exc_info=True) + for attempt in range(2): + try: + raw = await generate_completion(messages, model, max_tokens=400) + raw = raw.strip() + raw = re.sub(r"^```(?:json)?\s*", "", raw) + raw = re.sub(r"\s*```$", "", raw) + idx = raw.find("[") + if idx >= 0: + parsed, _ = _json.JSONDecoder().raw_decode(raw[idx:]) + if isinstance(parsed, list): + sections = [ + s for s in parsed + if isinstance(s, dict) and s.get("title") and s.get("focus") + ] + if len(sections) >= 2: + return sections[:8] + except Exception: + logger.warning( + "Outline generation attempt %d failed for topic '%s'", + attempt + 1, topic, exc_info=True, + ) return [] @@ -121,6 +125,49 @@ async def _synthesize_section( return section_title, raw.strip() +async def _generate_executive_summary( + topic: str, + section_bodies: list[tuple[str, str]], + model: str, +) -> str: + """Generate a 2-3 paragraph executive summary from completed section notes. + + Args: + section_bodies: list of (title, body) pairs from the section notes. + + Returns summary markdown (no heading — caller adds structure). + """ + sections_block = "\n\n".join( + f"### {title}\n{body[:1500]}" for title, body in section_bodies + ) + messages = [ + { + "role": "system", + "content": ( + "You are a research summarizer. Given several completed research sections on a topic, " + "write a concise executive summary.\n\n" + "Requirements:\n" + "- 2–3 paragraphs of substantive prose (150–300 words total)\n" + "- Cover the key findings and insights across all sections\n" + "- Highlight the most important or surprising takeaways\n" + "- Write so someone can decide which sections to read in detail\n" + "- Do NOT include headings, bullet points, or source citations\n" + "- Do NOT start with 'This research' or 'This document' — jump straight into the content" + ), + }, + { + "role": "user", + "content": f"Topic: {topic}\n\nSections:\n{sections_block}", + }, + ] + try: + raw = await generate_completion(messages, model, max_tokens=600) + return raw.strip() + except Exception: + logger.warning("Executive summary generation failed for '%s'", topic, exc_info=True) + return "" + + async def run_research_pipeline( topic: str, user_id: int, @@ -220,28 +267,17 @@ async def run_research_pipeline( return_exceptions=True, ) - # Step 5: Create section notes sequentially - _status(f"Saving {len(outline)} notes...") - section_note_pairs: list[tuple[dict, Note]] = [] + # Collect successful results for index + summary generation + section_results: list[tuple[dict, str, str]] = [] # (outline_entry, title, body) for section, result in zip(outline, raw_results): if isinstance(result, Exception): logger.warning("Section synthesis failed for '%s': %s", section["title"], result) continue sec_title, sec_body = result - try: - note = await create_note( - user_id=user_id, - title=sec_title, - body=sec_body, - tags=["research"], - project_id=project_id, - ) - section_note_pairs.append((section, note)) - except Exception: - logger.warning("Failed to save section note '%s'", sec_title, exc_info=True) + section_results.append((section, sec_title, sec_body)) # All sections failed — fall back to single note - if not section_note_pairs: + if not section_results: logger.warning("All section syntheses failed, falling back to single note for '%s'", topic) _status("Synthesizing report (fallback)...") title, body = await _synthesize_note(topic, synthesis_sources, model, buf=None) @@ -250,20 +286,24 @@ async def run_research_pipeline( ) return note - # Step 6: Create index note + # Step 5: Generate executive summary from section content + _status("Writing summary...") + executive_summary = await _generate_executive_summary( + topic, [(t, b) for _, t, b in section_results], model, + ) + + # Step 6: Create index note first (so section notes can reference it via parent_id) from datetime import date as _date index_lines = [ f"Research overview for **{topic}** — {_date.today().isoformat()}", "", - f"Generated from {len(synthesis_sources)} web sources across {len(section_note_pairs)} sections.", - "", - "## Sections", + f"Generated from {len(synthesis_sources)} web sources across {len(section_results)} sections.", "", ] - for section, note in section_note_pairs: - index_lines.append(f"- **{note.title}** — {section['focus']}") - index_lines += ["", "*Search for any section title to read it.*"] - + if executive_summary: + index_lines += ["## Summary", "", executive_summary, ""] + index_lines += ["## Sections", ""] + # Placeholder — will be updated with real links after section notes are created index_note = await create_note( user_id=user_id, title=f"Research: {topic}", @@ -271,6 +311,34 @@ async def run_research_pipeline( tags=["research", "research-index"], project_id=project_id, ) + + # Step 7: Create section notes with parent_id pointing to index + _status(f"Saving {len(section_results)} notes...") + section_note_pairs: list[tuple[dict, Note]] = [] + for section, sec_title, sec_body in section_results: + try: + note = await create_note( + user_id=user_id, + title=sec_title, + body=sec_body, + tags=["research"], + project_id=project_id, + parent_id=index_note.id, + ) + section_note_pairs.append((section, note)) + except Exception: + logger.warning("Failed to save section note '%s'", sec_title, exc_info=True) + + # Step 8: Update index note body with real links to section notes + for section, note in section_note_pairs: + index_lines.append(f"- [{note.title}](/notes/{note.id}) — {section['focus']}") + + await update_note( + user_id=user_id, + note_id=index_note.id, + body="\n".join(index_lines), + ) + logger.info( "Research: created %d section notes + index id=%d for topic '%s'", len(section_note_pairs), index_note.id, topic, diff --git a/tests/test_research_pipeline.py b/tests/test_research_pipeline.py index ae29516..9c61422 100644 --- a/tests/test_research_pipeline.py +++ b/tests/test_research_pipeline.py @@ -34,8 +34,8 @@ async def test_generate_outline_returns_empty_on_parse_failure(): @pytest.mark.asyncio -async def test_generate_outline_returns_empty_when_fewer_than_3_sections(): - """_generate_outline returns [] when model returns fewer than 3 valid sections.""" +async def test_generate_outline_returns_empty_when_fewer_than_2_sections(): + """_generate_outline returns [] when model returns fewer than 2 valid sections.""" from fabledassistant.services.research import _generate_outline outline_json = json.dumps([{"title": "Only One", "focus": "Focus"}]) @@ -106,7 +106,7 @@ async def test_synthesize_section_strips_whitespace(): @pytest.mark.asyncio async def test_pipeline_creates_section_notes_and_index(): - """run_research_pipeline creates N section notes + 1 index note, returns index.""" + """run_research_pipeline creates N section notes + 1 index note with links, returns index.""" from unittest.mock import MagicMock outline = [ @@ -117,23 +117,31 @@ async def test_pipeline_creates_section_notes_and_index(): note_id_counter = iter(range(10, 20)) - def _make_note(user_id, title, body, tags, project_id=None): + def _make_note(user_id, title, body, tags, project_id=None, parent_id=None): n = MagicMock() n.id = next(note_id_counter) n.title = title return n + mock_update = AsyncMock() + with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \ patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \ patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \ patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \ patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \ - patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note): + patch("fabledassistant.services.research._generate_executive_summary", new_callable=AsyncMock, return_value="Executive summary text."), \ + patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note), \ + patch("fabledassistant.services.research.update_note", new_callable=AsyncMock) as mock_update: from fabledassistant.services.research import run_research_pipeline result = await run_research_pipeline("test topic", user_id=1, model="test-model") assert result.title == "Research: test topic" + # Index note body should be updated with section links + mock_update.assert_called_once() + updated_body = mock_update.call_args.kwargs.get("body", "") + assert "/notes/" in updated_body @pytest.mark.asyncio