From a551f52682b77845ee2c90a7b2d7761c2464bbe1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 12 May 2026 20:30:37 -0400 Subject: [PATCH] fix(tools): score_project_match strips 'project' filler + uses title for SequenceMatcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI surfaced three issues: - 'famous supply project' didn't substring-match 'Famous-Supply Work topics' because the trailing filler word 'project' blocked the substring tier. Strip {project, projects} from the query before the substring check. - SequenceMatcher fallback against `combined` (title + description + summary) diluted ratios to ~0.5 for plausible matches. Use title directly; the 0.70 tier already handles description/summary mentions. - Test patches used patch.object on a consumer module where list_projects is imported locally — patch the source module instead. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../services/tools/_helpers.py | 27 ++++++++++++++----- tests/test_tool_use_fixes.py | 6 +++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/fabledassistant/services/tools/_helpers.py b/src/fabledassistant/services/tools/_helpers.py index 2ec5b1b..218237b 100644 --- a/src/fabledassistant/services/tools/_helpers.py +++ b/src/fabledassistant/services/tools/_helpers.py @@ -22,21 +22,33 @@ def schedule_embedding(note_id: int, user_id: int, title: str, body: str) -> Non asyncio.create_task(upsert_note_embedding(note_id, user_id, text)) +_PROJECT_QUERY_NOISE = {"project", "projects"} + + def _normalize(s: str) -> str: """Lowercase and collapse non-alphanumerics to single spaces.""" return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip() +def _normalize_query(query: str) -> str: + """Normalize plus drop trailing type-nouns ('project' / 'projects') + that users add as filler when referring to a project by name.""" + tokens = [t for t in _normalize(query).split() if t not in _PROJECT_QUERY_NOISE] + return " ".join(tokens) + + def score_project_match(query: str, project) -> float: """Score how well `query` matches `project`. Range [0.0, 1.0]. Tiered: exact title → 1.0, substring either-way → 0.85, query found in - description/summary → 0.70, otherwise SequenceMatcher ratio. Substring - tiers exist because LLM-generated colloquial queries (e.g. "famous - supply project" for "Famous-Supply Work topics") would otherwise score - too low under pure SequenceMatcher and be treated as no match. + description/summary → 0.70, otherwise SequenceMatcher ratio against the + title. Substring tiers exist because LLM-generated colloquial queries + (e.g. "famous supply project" for "Famous-Supply Work topics") would + otherwise score too low under pure SequenceMatcher and be treated as + no match. Filler words like "project" are stripped from the query so + the substring check still fires. """ - q = _normalize(query) + q = _normalize_query(query) if not q: return 0.0 title = _normalize(project.title) @@ -50,7 +62,10 @@ def score_project_match(query: str, project) -> float: return 0.85 if q in combined: return 0.70 - return SequenceMatcher(None, q, combined).ratio() + # SequenceMatcher against the title — comparing against `combined` + # dilutes the ratio with long description/summary text and produces + # uniformly low scores even for plausible matches. + return SequenceMatcher(None, q, title).ratio() async def resolve_project(user_id: int, project_name: str): diff --git a/tests/test_tool_use_fixes.py b/tests/test_tool_use_fixes.py index 18b8278..c63e2da 100644 --- a/tests/test_tool_use_fixes.py +++ b/tests/test_tool_use_fixes.py @@ -20,7 +20,9 @@ async def test_search_projects_tool_returns_success_true(): fake_projects = [_project(5, "Famous-Supply Work topics", "AT&T fiber circuit")] - with patch.object(projects_tool, "list_projects", new=AsyncMock(return_value=fake_projects)): + # `list_projects` is imported locally inside search_projects_tool, so we + # patch the source module rather than the consumer. + with patch("fabledassistant.services.projects.list_projects", new=AsyncMock(return_value=fake_projects)): result = await projects_tool.search_projects_tool( user_id=1, arguments={"query": "famous supply"}, ) @@ -101,7 +103,7 @@ async def test_search_projects_tool_ranks_substring_match_above_others(): _project(13, "ImageRepo", auto_summary="self-hosted Flask app"), ] - with patch.object(projects_tool, "list_projects", new=AsyncMock(return_value=fake_projects)): + with patch("fabledassistant.services.projects.list_projects", new=AsyncMock(return_value=fake_projects)): result = await projects_tool.search_projects_tool( user_id=1, arguments={"query": "famous supply project"}, )