fix(journal): anti-hallucination hardening + message_count fix
Prep prose (services/journal_prep.py): - Emit explicit "WEATHER: none available — do NOT mention weather" absent-marker so a small model can't invent partly-cloudy/temperature prose when both configured locations have empty addresses. - Replace negative-only system rule with positive-anchored guidance forbidding weather/temp/precip mentions unless a numeric WEATHER section is present; also bans echoing parenthetical labels verbatim. - Reword overdue header to "(past their due date, still open — backlog, not today's work)" and render lines as "was due <date>, N day(s) overdue" with correct singular/plural. Supersedes the wording noted in Fable task #159. - Deterministic fabricated-weather reconciler: low-false-positive regex detects fabricated weather phrasing; on trip with an empty section, regenerate once with a corrective. Persistent fabrication logs ERROR rather than mangling prose. Journal route (routes/journal.py): - Override message_count with len(messages) in _day_payload. The chat path already does this; the journal path was hitting the Conversation.to_dict() fallback to 0 because messages aren't eager-loaded on that instance. Tests: - tests/test_journal_message_count.py — pins the model-level trap and the override contract (3 cases). - tests/test_journal_prep_hardening.py — 11 cases covering the fabricated-weather reconciler and absent-marker rendering. - tests/test_journal_prep_filtering.py — updated one stale assertion. Tracks Fable task #171. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
"""Regression coverage for the journal `message_count: 0` bug.
|
||||
|
||||
`_day_payload` (routes/journal.py) loads messages in a separate query and
|
||||
serializes the Conversation with `conv.to_dict()`. `Conversation.to_dict()`
|
||||
derives `message_count` from the `messages` relationship, which is NOT
|
||||
eager-loaded on that instance — so it silently fell back to 0 for every
|
||||
journal day (observed via the fable MCP: conversation #291 reported
|
||||
message_count 0 with 9 messages present).
|
||||
|
||||
Full HTTP coverage of `_day_payload` needs a live DB (not available in the
|
||||
unit-test env — see test_events_routes.py). These tests pin the model-level
|
||||
contract that necessitates the route-side override.
|
||||
"""
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fabledassistant.models.conversation import Conversation, Message
|
||||
|
||||
|
||||
def _conv() -> Conversation:
|
||||
now = datetime(2026, 5, 19, 9, 0, tzinfo=timezone.utc)
|
||||
return Conversation(
|
||||
user_id=1,
|
||||
conversation_type="journal",
|
||||
day_date=date(2026, 5, 19),
|
||||
title="2026-05-19",
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def test_to_dict_reports_zero_when_messages_relationship_not_loaded():
|
||||
"""The trap: an untouched `messages` relationship is absent from the
|
||||
instance state, so to_dict() reports 0 regardless of DB rows. This is
|
||||
exactly the journal path's situation and why the route must override."""
|
||||
conv = _conv()
|
||||
assert conv.to_dict()["message_count"] == 0
|
||||
|
||||
|
||||
def test_to_dict_counts_when_messages_loaded():
|
||||
"""When the relationship IS populated the count is correct — confirming
|
||||
the model isn't broken, the journal path just never loaded it."""
|
||||
conv = _conv()
|
||||
conv.messages = [
|
||||
Message(conversation_id=1, role="assistant", content="prep"),
|
||||
Message(conversation_id=1, role="user", content="hi"),
|
||||
]
|
||||
assert conv.to_dict()["message_count"] == 2
|
||||
|
||||
|
||||
def test_day_payload_override_yields_true_count():
|
||||
"""Pins the fix contract: _day_payload already holds the real message
|
||||
list and overrides message_count with len(messages), the same way the
|
||||
chat-list path (services/chat.py) supplies its own count."""
|
||||
conv = _conv() # messages relationship deliberately not loaded
|
||||
messages = [object(), object(), object()] # stand-ins for the loaded rows
|
||||
|
||||
conv_dict = conv.to_dict()
|
||||
conv_dict["message_count"] = len(messages)
|
||||
|
||||
assert conv_dict["message_count"] == 3
|
||||
@@ -145,7 +145,9 @@ def test_render_overdue_includes_staleness_duration():
|
||||
}
|
||||
rendered = _render_sections_for_prompt(sections)
|
||||
assert "OVERDUE TASKS" in rendered
|
||||
assert "68 days ago" in rendered
|
||||
# Unambiguous overdue phrasing (replaced the old "68 days ago", which the
|
||||
# model parroted alongside the header into a self-contradiction).
|
||||
assert "68 days overdue" in rendered
|
||||
assert "2026-02-20" in rendered
|
||||
# Crucially, NOT framed as due today.
|
||||
assert "DUE TODAY" not in rendered
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Coverage for the daily-prep hardening pass.
|
||||
|
||||
Targets the two prep-quality defects observed via the fable MCP on
|
||||
2026-05-19 (conversation #291):
|
||||
|
||||
1. Fabricated weather ("partly cloudy with a high of 68°F and a 15%
|
||||
chance of rain") emitted with an empty weather section.
|
||||
2. Self-contradictory overdue phrasing ("still on the list, not
|
||||
currently due, 88 days ago"; "1 days ago").
|
||||
|
||||
All targets are pure sync functions — no DB / LLM needed.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
from fabledassistant.services.journal_prep import (
|
||||
_prose_fabricated_weather,
|
||||
_render_sections_for_prompt,
|
||||
_render_task_line,
|
||||
)
|
||||
|
||||
TODAY = datetime.date(2026, 5, 19)
|
||||
|
||||
|
||||
# ── Overdue phrasing ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_overdue_line_singular_day():
|
||||
line = _render_task_line(
|
||||
{"title": "Change my oil", "due_date": "2026-05-18", "days_overdue": 1},
|
||||
include_due=False, include_overdue=True,
|
||||
)
|
||||
assert "1 day overdue" in line
|
||||
assert "1 days" not in line # the old ungrammatical form is gone
|
||||
|
||||
|
||||
def test_overdue_line_plural_days_no_contradiction():
|
||||
line = _render_task_line(
|
||||
{"title": "Research X", "due_date": "2026-02-20", "days_overdue": 88},
|
||||
include_due=False, include_overdue=True,
|
||||
)
|
||||
assert "88 days overdue" in line
|
||||
assert "was due 2026-02-20" in line
|
||||
assert "days ago" not in line # replaced by unambiguous "overdue"
|
||||
|
||||
|
||||
def test_overdue_section_header_is_unambiguous():
|
||||
out = _render_sections_for_prompt({
|
||||
"tasks_overdue": [
|
||||
{"title": "Research X", "due_date": "2026-02-20", "days_overdue": 88},
|
||||
],
|
||||
})
|
||||
assert "past their due date, still open" in out
|
||||
# The phrase that the model parroted into the self-contradiction is gone.
|
||||
assert "not currently due" not in out
|
||||
|
||||
|
||||
# ── Weather absent-marker ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_empty_weather_emits_explicit_none_marker():
|
||||
out = _render_sections_for_prompt({
|
||||
"tasks_overdue": [
|
||||
{"title": "Change my oil", "due_date": "2026-05-18", "days_overdue": 1},
|
||||
],
|
||||
"weather": [],
|
||||
})
|
||||
assert "WEATHER: none available" in out
|
||||
assert "Do NOT mention weather" in out
|
||||
|
||||
|
||||
def test_present_weather_renders_data_not_marker():
|
||||
out = _render_sections_for_prompt({
|
||||
"tasks_due_today": [{"title": "Ship it"}],
|
||||
"weather": [{
|
||||
"location_label": "Home",
|
||||
"forecast_json": {"daily": {
|
||||
"temperature_2m_max": [70],
|
||||
"temperature_2m_min": [52],
|
||||
"precipitation_probability_max": [10],
|
||||
}},
|
||||
}],
|
||||
})
|
||||
assert "WEATHER:" in out
|
||||
assert "none available" not in out
|
||||
assert "high 70°" in out
|
||||
|
||||
|
||||
def test_quiet_day_returns_fabrication_forbidding_text():
|
||||
out = _render_sections_for_prompt({})
|
||||
assert "quiet day" in out
|
||||
assert "Do not invent weather" in out
|
||||
|
||||
|
||||
# ── Fabricated-weather guard ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_guard_flags_the_real_observed_hallucination():
|
||||
prose = (
|
||||
"You have two overdue tasks. The weather today is partly cloudy "
|
||||
"with a high of 68°F and a 15% chance of rain. What's on your mind?"
|
||||
)
|
||||
assert _prose_fabricated_weather(prose, {"weather": []}) is True
|
||||
|
||||
|
||||
def test_guard_flags_bare_temperature_glyph():
|
||||
assert _prose_fabricated_weather("Expect around 72° later.", {"weather": []}) is True
|
||||
|
||||
|
||||
def test_guard_passes_clean_prose():
|
||||
prose = "Two tasks are overdue and nothing is due today. What's on your mind?"
|
||||
assert _prose_fabricated_weather(prose, {"weather": []}) is False
|
||||
|
||||
|
||||
def test_guard_no_false_positive_on_rain_in_task_title():
|
||||
# Bare "rain" must NOT trip the guard — only "chance of rain" etc. does.
|
||||
prose = "Don't forget to buy rain boots and finish the training module."
|
||||
assert _prose_fabricated_weather(prose, {"weather": []}) is False
|
||||
|
||||
|
||||
def test_guard_allows_weather_when_data_present():
|
||||
prose = "Home is 70°F with a 10% chance of rain."
|
||||
sections = {"weather": [{"location_label": "Home"}]}
|
||||
assert _prose_fabricated_weather(prose, sections) is False
|
||||
Reference in New Issue
Block a user