Files
FabledScribe/tests/test_record_moment_guards.py
T
bvandeusen 4f18023284 fix(journal): server-side guards on record_moment links + place names (#158)
Belt-and-suspenders to the prompt-layer changes in 6c309f1. Even when
the model emits bogus task or place links, the server now refuses to
persist them.

## Task auto-linking guard

Reproducer (2026-04-27): a moment about restaging Docker on the swarm
ended up with `task_ids: [2]` (Weston's ADHD Evaluation) — the only
task in that day's prep. The model picked it up as filler.

`_filter_task_ids_by_keyword_overlap` now runs after id resolution: it
fetches each linked task's title, tokenizes both content and title
through `_content_keywords` (lowercased, stopwords stripped, <3-char
tokens dropped), and drops any link whose title shares no meaningful
keyword with the moment content. The drop is logged at INFO so we can
observe how often it fires post-deploy.

The guard runs against the merged id list, so it covers both the
preferred `task_titles` resolution path and the discouraged explicit
`task_ids` path.

## Place placeholder guard

Reproducer (2026-04-27): `place_names=["work"]` got passed to
`record_moment`. "work" / "home" / "office" aren't places — they're
role-labels for already-known geocoded locations.

`_filter_placeholder_places` drops a small set of generic single-word
labels before name resolution. Real user-named places that happen to
be one word (e.g. "Akron") pass through.

## Tests

9 new unit tests in `tests/test_record_moment_guards.py` cover:
  - keyword tokenization & stopword stripping
  - placeholder place filtering (generic, case-insensitive, real-place
    pass-through)
  - keyword-overlap filtering (the exact 4/27 reproducer, the genuine-
    reference case, mixed/partial relevance, empty input)

13 tests pass; ruff clean.

Closes Fable task #158.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-29 09:25:16 -04:00

182 lines
6.2 KiB
Python

"""Tests for the record_moment server-side data-hygiene guards.
These guard against two failure modes observed in real journal usage:
1. The LLM emits ``task_titles`` referencing a task that's only in the
prep context — not actually mentioned by the user. Without a check,
every moment ends up linked to whatever's open in the user's queue.
2. The LLM emits generic placeholder ``place_names`` like ``"work"`` /
``"home"`` instead of real place notes. These role-labels aren't
places.
Both are exercised through the pure helpers; full-stack handler tests
would require a session-bound DB fixture this suite doesn't have yet.
"""
from unittest.mock import AsyncMock, patch
import pytest
def test_content_keywords_drops_stopwords_and_short_tokens():
from fabledassistant.services.tools.journal import _content_keywords
kw = _content_keywords(
"I went to the store to pick up milk and bread for the kids."
)
# Stopwords (the, to, for, and, i) gone; short tokens (kids? 4 chars - kept)
# filtered. Real content words kept.
assert "store" in kw
assert "milk" in kw
assert "bread" in kw
assert "kids" in kw
assert "the" not in kw
assert "to" not in kw
assert "for" not in kw
assert "i" not in kw
def test_content_keywords_extracts_named_entities():
"""Place names and project nouns survive tokenization. Short numeric
fragments (e.g. '14') get filtered alongside other <3-char tokens —
the surrounding alpha keywords carry the overlap weight in practice."""
from fabledassistant.services.tools.journal import _content_keywords
kw = _content_keywords("Migration prep at Branch 14 Bedford.")
assert "branch" in kw
assert "bedford" in kw
assert "migration" in kw
def test_filter_placeholder_places_drops_generic_role_labels():
from fabledassistant.services.tools.journal import _filter_placeholder_places
kept, dropped = _filter_placeholder_places(
["work", "Famous Supply", "home", "Branch 14 Bedford", "the office"]
)
assert "Famous Supply" in kept
assert "Branch 14 Bedford" in kept
assert "work" in dropped
assert "home" in dropped
assert "the office" in dropped
def test_filter_placeholder_places_is_case_insensitive():
from fabledassistant.services.tools.journal import _filter_placeholder_places
kept, dropped = _filter_placeholder_places(["WORK", "Home", "OFFICE"])
assert kept == []
assert set(dropped) == {"WORK", "Home", "OFFICE"}
def test_filter_placeholder_places_preserves_real_places_named_similarly():
"""A user-defined place that happens to be ONE word but isn't a generic
role-label (e.g. 'Akron') stays."""
from fabledassistant.services.tools.journal import _filter_placeholder_places
kept, dropped = _filter_placeholder_places(["Akron", "Cleveland"])
assert kept == ["Akron", "Cleveland"]
assert dropped == []
@pytest.mark.asyncio
async def test_keyword_overlap_drops_unrelated_task_link():
"""The reported failure mode: model attached Weston's ADHD task to a
Docker-swarm moment. With the keyword guard, the link gets dropped."""
from fabledassistant.services.tools.journal import (
_filter_task_ids_by_keyword_overlap,
)
# Mock the DB lookup to return the offending task title for id=2.
fake_session = AsyncMock()
fake_session.execute = AsyncMock()
fake_session.execute.return_value.all = lambda: [
(2, "Research Weston's ADHD Evaluation"),
]
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
with patch(
"fabledassistant.services.tools.journal.async_session",
return_value=fake_session,
):
kept = await _filter_task_ids_by_keyword_overlap(
user_id=1,
content="Working on restaging Docker in the swarm; encountered network breakage on a Windows node.",
task_ids=[2],
)
assert kept == []
@pytest.mark.asyncio
async def test_keyword_overlap_keeps_genuinely_referenced_task():
"""When the moment content shares a meaningful keyword with the task
title, the link is preserved."""
from fabledassistant.services.tools.journal import (
_filter_task_ids_by_keyword_overlap,
)
fake_session = AsyncMock()
fake_session.execute = AsyncMock()
fake_session.execute.return_value.all = lambda: [
(5, "Restage Docker on Fam-dockerwin04"),
]
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
with patch(
"fabledassistant.services.tools.journal.async_session",
return_value=fake_session,
):
kept = await _filter_task_ids_by_keyword_overlap(
user_id=1,
content="Reinstalled Microsoft container support; Docker restage now works.",
task_ids=[5],
)
# 'docker' appears in both → keep
assert kept == [5]
@pytest.mark.asyncio
async def test_keyword_overlap_handles_partial_relevance():
"""Mixed ids: keep the related one, drop the unrelated one."""
from fabledassistant.services.tools.journal import (
_filter_task_ids_by_keyword_overlap,
)
fake_session = AsyncMock()
fake_session.execute = AsyncMock()
fake_session.execute.return_value.all = lambda: [
(2, "Research Weston's ADHD Evaluation"),
(5, "Restage Docker on Fam-dockerwin04"),
]
fake_session.__aenter__ = AsyncMock(return_value=fake_session)
fake_session.__aexit__ = AsyncMock(return_value=None)
with patch(
"fabledassistant.services.tools.journal.async_session",
return_value=fake_session,
):
kept = await _filter_task_ids_by_keyword_overlap(
user_id=1,
content="Working on restaging Docker in the swarm.",
task_ids=[2, 5],
)
assert kept == [5]
@pytest.mark.asyncio
async def test_keyword_overlap_empty_task_ids_returns_empty():
from fabledassistant.services.tools.journal import (
_filter_task_ids_by_keyword_overlap,
)
kept = await _filter_task_ids_by_keyword_overlap(
user_id=1, content="anything", task_ids=[],
)
assert kept == []