bvandeusen
eb92b2a976
Replaces the single monolithic research note with topic-driven section notes plus an index note. Two new LLM calls: _generate_outline (JSON outline, 3-8 sections) and _synthesize_section (300-600 word focused note per section, parallelised via asyncio.gather). Public signature of run_research_pipeline unchanged; falls back to single-note synthesis on outline failure or if all sections fail. Also extracts _build_sources_block helper and adds full test suite. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
187 lines
8.8 KiB
Python
187 lines
8.8 KiB
Python
"""Tests for the multi-note research pipeline."""
|
|
import json
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generate_outline_parses_valid_json():
|
|
"""_generate_outline returns parsed sections when model returns valid JSON."""
|
|
from fabledassistant.services.research import _generate_outline
|
|
|
|
outline_json = json.dumps([
|
|
{"title": "Section One", "focus": "Focus one"},
|
|
{"title": "Section Two", "focus": "Focus two"},
|
|
{"title": "Section Three", "focus": "Focus three"},
|
|
])
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
|
result = await _generate_outline("test topic", [], "test-model")
|
|
|
|
assert len(result) == 3
|
|
assert result[0]["title"] == "Section One"
|
|
assert result[0]["focus"] == "Focus one"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generate_outline_returns_empty_on_parse_failure():
|
|
"""_generate_outline returns [] when model output is not parseable JSON."""
|
|
from fabledassistant.services.research import _generate_outline
|
|
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="not json at all"):
|
|
result = await _generate_outline("test topic", [], "test-model")
|
|
|
|
assert result == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generate_outline_returns_empty_when_fewer_than_3_sections():
|
|
"""_generate_outline returns [] when model returns fewer than 3 valid sections."""
|
|
from fabledassistant.services.research import _generate_outline
|
|
|
|
outline_json = json.dumps([{"title": "Only One", "focus": "Focus"}])
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
|
result = await _generate_outline("test topic", [], "test-model")
|
|
|
|
assert result == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generate_outline_truncates_to_8():
|
|
"""_generate_outline truncates results to at most 8 sections."""
|
|
from fabledassistant.services.research import _generate_outline
|
|
|
|
sections = [{"title": f"Section {i}", "focus": f"Focus {i}"} for i in range(10)]
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=json.dumps(sections)):
|
|
result = await _generate_outline("test topic", [], "test-model")
|
|
|
|
assert len(result) == 8
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generate_outline_skips_malformed_entries():
|
|
"""_generate_outline filters out entries missing title or focus."""
|
|
from fabledassistant.services.research import _generate_outline
|
|
|
|
outline_json = json.dumps([
|
|
{"title": "Good One", "focus": "Good focus"},
|
|
{"title": "Missing focus"},
|
|
{"focus": "Missing title"},
|
|
{"title": "Good Two", "focus": "Good focus two"},
|
|
{"title": "Good Three", "focus": "Good focus three"},
|
|
])
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
|
result = await _generate_outline("test topic", [], "test-model")
|
|
|
|
assert len(result) == 3
|
|
assert all(s["title"] and s["focus"] for s in result)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_synthesize_section_returns_title_and_body():
|
|
"""_synthesize_section returns (section_title, body) from model output."""
|
|
from fabledassistant.services.research import _synthesize_section
|
|
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="The body of this section."):
|
|
title, body = await _synthesize_section(
|
|
section_title="Quantum Entanglement: Mechanisms",
|
|
section_focus="How entanglement works at the particle level",
|
|
sources=[],
|
|
model="test-model",
|
|
)
|
|
|
|
assert title == "Quantum Entanglement: Mechanisms"
|
|
assert body == "The body of this section."
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_synthesize_section_strips_whitespace():
|
|
"""_synthesize_section strips leading/trailing whitespace from body."""
|
|
from fabledassistant.services.research import _synthesize_section
|
|
|
|
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="\n\n body text \n\n"):
|
|
title, body = await _synthesize_section("Title", "Focus", [], "model")
|
|
|
|
assert body == "body text"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_creates_section_notes_and_index():
|
|
"""run_research_pipeline creates N section notes + 1 index note, returns index."""
|
|
from unittest.mock import MagicMock
|
|
|
|
outline = [
|
|
{"title": "Section A", "focus": "Focus A"},
|
|
{"title": "Section B", "focus": "Focus B"},
|
|
{"title": "Section C", "focus": "Focus C"},
|
|
]
|
|
|
|
note_id_counter = iter(range(10, 20))
|
|
|
|
def _make_note(user_id, title, body, tags, project_id=None):
|
|
n = MagicMock()
|
|
n.id = next(note_id_counter)
|
|
n.title = title
|
|
return n
|
|
|
|
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
|
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
|
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
|
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \
|
|
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \
|
|
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note):
|
|
|
|
from fabledassistant.services.research import run_research_pipeline
|
|
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
|
|
|
assert result.title == "Research: test topic"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_falls_back_to_single_note_when_outline_empty():
|
|
"""run_research_pipeline falls back to single-note synthesis when _generate_outline returns []."""
|
|
from unittest.mock import MagicMock
|
|
|
|
single_note = MagicMock()
|
|
single_note.id = 99
|
|
single_note.title = "Research: test topic"
|
|
|
|
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
|
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
|
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
|
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=[]), \
|
|
patch("fabledassistant.services.research._synthesize_note", new_callable=AsyncMock, return_value=("Research: test topic", "body")), \
|
|
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, return_value=single_note):
|
|
|
|
from fabledassistant.services.research import run_research_pipeline
|
|
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
|
|
|
assert result.id == 99
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_falls_back_when_all_sections_fail():
|
|
"""run_research_pipeline falls back to single note when all section syntheses raise."""
|
|
from unittest.mock import MagicMock
|
|
|
|
outline = [
|
|
{"title": "Section A", "focus": "Focus A"},
|
|
{"title": "Section B", "focus": "Focus B"},
|
|
{"title": "Section C", "focus": "Focus C"},
|
|
]
|
|
fallback_note = MagicMock()
|
|
fallback_note.id = 77
|
|
fallback_note.title = "Research: test topic"
|
|
|
|
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
|
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
|
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
|
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \
|
|
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=RuntimeError("synthesis failed")), \
|
|
patch("fabledassistant.services.research._synthesize_note", new_callable=AsyncMock, return_value=("Research: test topic", "body")), \
|
|
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, return_value=fallback_note):
|
|
|
|
from fabledassistant.services.research import run_research_pipeline
|
|
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
|
|
|
assert result.id == 77
|