fix(journal): anti-hallucination hardening + message_count fix

Prep prose (services/journal_prep.py):
- Emit explicit "WEATHER: none available — do NOT mention weather"
  absent-marker so a small model can't invent partly-cloudy/temperature
  prose when both configured locations have empty addresses.
- Replace negative-only system rule with positive-anchored guidance
  forbidding weather/temp/precip mentions unless a numeric WEATHER
  section is present; also bans echoing parenthetical labels verbatim.
- Reword overdue header to "(past their due date, still open — backlog,
  not today's work)" and render lines as "was due <date>, N day(s)
  overdue" with correct singular/plural. Supersedes the wording noted
  in Fable task #159.
- Deterministic fabricated-weather reconciler: low-false-positive regex
  detects fabricated weather phrasing; on trip with an empty section,
  regenerate once with a corrective. Persistent fabrication logs ERROR
  rather than mangling prose.

Journal route (routes/journal.py):
- Override message_count with len(messages) in _day_payload. The chat
  path already does this; the journal path was hitting the
  Conversation.to_dict() fallback to 0 because messages aren't
  eager-loaded on that instance.

Tests:
- tests/test_journal_message_count.py — pins the model-level trap and
  the override contract (3 cases).
- tests/test_journal_prep_hardening.py — 11 cases covering the
  fabricated-weather reconciler and absent-marker rendering.
- tests/test_journal_prep_filtering.py — updated one stale assertion.

Tracks Fable task #171.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-20 18:54:35 -04:00
parent e6f2ee2b94
commit 5d2d27c499
5 changed files with 306 additions and 28 deletions
+60
View File
@@ -0,0 +1,60 @@
"""Regression coverage for the journal `message_count: 0` bug.
`_day_payload` (routes/journal.py) loads messages in a separate query and
serializes the Conversation with `conv.to_dict()`. `Conversation.to_dict()`
derives `message_count` from the `messages` relationship, which is NOT
eager-loaded on that instance — so it silently fell back to 0 for every
journal day (observed via the fable MCP: conversation #291 reported
message_count 0 with 9 messages present).
Full HTTP coverage of `_day_payload` needs a live DB (not available in the
unit-test env — see test_events_routes.py). These tests pin the model-level
contract that necessitates the route-side override.
"""
from datetime import date, datetime, timezone
from fabledassistant.models.conversation import Conversation, Message
def _conv() -> Conversation:
now = datetime(2026, 5, 19, 9, 0, tzinfo=timezone.utc)
return Conversation(
user_id=1,
conversation_type="journal",
day_date=date(2026, 5, 19),
title="2026-05-19",
created_at=now,
updated_at=now,
)
def test_to_dict_reports_zero_when_messages_relationship_not_loaded():
"""The trap: an untouched `messages` relationship is absent from the
instance state, so to_dict() reports 0 regardless of DB rows. This is
exactly the journal path's situation and why the route must override."""
conv = _conv()
assert conv.to_dict()["message_count"] == 0
def test_to_dict_counts_when_messages_loaded():
"""When the relationship IS populated the count is correct — confirming
the model isn't broken, the journal path just never loaded it."""
conv = _conv()
conv.messages = [
Message(conversation_id=1, role="assistant", content="prep"),
Message(conversation_id=1, role="user", content="hi"),
]
assert conv.to_dict()["message_count"] == 2
def test_day_payload_override_yields_true_count():
"""Pins the fix contract: _day_payload already holds the real message
list and overrides message_count with len(messages), the same way the
chat-list path (services/chat.py) supplies its own count."""
conv = _conv() # messages relationship deliberately not loaded
messages = [object(), object(), object()] # stand-ins for the loaded rows
conv_dict = conv.to_dict()
conv_dict["message_count"] = len(messages)
assert conv_dict["message_count"] == 3
+3 -1
View File
@@ -145,7 +145,9 @@ def test_render_overdue_includes_staleness_duration():
}
rendered = _render_sections_for_prompt(sections)
assert "OVERDUE TASKS" in rendered
assert "68 days ago" in rendered
# Unambiguous overdue phrasing (replaced the old "68 days ago", which the
# model parroted alongside the header into a self-contradiction).
assert "68 days overdue" in rendered
assert "2026-02-20" in rendered
# Crucially, NOT framed as due today.
assert "DUE TODAY" not in rendered
+123
View File
@@ -0,0 +1,123 @@
"""Coverage for the daily-prep hardening pass.
Targets the two prep-quality defects observed via the fable MCP on
2026-05-19 (conversation #291):
1. Fabricated weather ("partly cloudy with a high of 68°F and a 15%
chance of rain") emitted with an empty weather section.
2. Self-contradictory overdue phrasing ("still on the list, not
currently due, 88 days ago"; "1 days ago").
All targets are pure sync functions — no DB / LLM needed.
"""
import datetime
from fabledassistant.services.journal_prep import (
_prose_fabricated_weather,
_render_sections_for_prompt,
_render_task_line,
)
TODAY = datetime.date(2026, 5, 19)
# ── Overdue phrasing ────────────────────────────────────────────────────────
def test_overdue_line_singular_day():
line = _render_task_line(
{"title": "Change my oil", "due_date": "2026-05-18", "days_overdue": 1},
include_due=False, include_overdue=True,
)
assert "1 day overdue" in line
assert "1 days" not in line # the old ungrammatical form is gone
def test_overdue_line_plural_days_no_contradiction():
line = _render_task_line(
{"title": "Research X", "due_date": "2026-02-20", "days_overdue": 88},
include_due=False, include_overdue=True,
)
assert "88 days overdue" in line
assert "was due 2026-02-20" in line
assert "days ago" not in line # replaced by unambiguous "overdue"
def test_overdue_section_header_is_unambiguous():
out = _render_sections_for_prompt({
"tasks_overdue": [
{"title": "Research X", "due_date": "2026-02-20", "days_overdue": 88},
],
})
assert "past their due date, still open" in out
# The phrase that the model parroted into the self-contradiction is gone.
assert "not currently due" not in out
# ── Weather absent-marker ───────────────────────────────────────────────────
def test_empty_weather_emits_explicit_none_marker():
out = _render_sections_for_prompt({
"tasks_overdue": [
{"title": "Change my oil", "due_date": "2026-05-18", "days_overdue": 1},
],
"weather": [],
})
assert "WEATHER: none available" in out
assert "Do NOT mention weather" in out
def test_present_weather_renders_data_not_marker():
out = _render_sections_for_prompt({
"tasks_due_today": [{"title": "Ship it"}],
"weather": [{
"location_label": "Home",
"forecast_json": {"daily": {
"temperature_2m_max": [70],
"temperature_2m_min": [52],
"precipitation_probability_max": [10],
}},
}],
})
assert "WEATHER:" in out
assert "none available" not in out
assert "high 70°" in out
def test_quiet_day_returns_fabrication_forbidding_text():
out = _render_sections_for_prompt({})
assert "quiet day" in out
assert "Do not invent weather" in out
# ── Fabricated-weather guard ────────────────────────────────────────────────
def test_guard_flags_the_real_observed_hallucination():
prose = (
"You have two overdue tasks. The weather today is partly cloudy "
"with a high of 68°F and a 15% chance of rain. What's on your mind?"
)
assert _prose_fabricated_weather(prose, {"weather": []}) is True
def test_guard_flags_bare_temperature_glyph():
assert _prose_fabricated_weather("Expect around 72° later.", {"weather": []}) is True
def test_guard_passes_clean_prose():
prose = "Two tasks are overdue and nothing is due today. What's on your mind?"
assert _prose_fabricated_weather(prose, {"weather": []}) is False
def test_guard_no_false_positive_on_rain_in_task_title():
# Bare "rain" must NOT trip the guard — only "chance of rain" etc. does.
prose = "Don't forget to buy rain boots and finish the training module."
assert _prose_fabricated_weather(prose, {"weather": []}) is False
def test_guard_allows_weather_when_data_present():
prose = "Home is 70°F with a 10% chance of rain."
sections = {"weather": [{"location_label": "Home"}]}
assert _prose_fabricated_weather(prose, sections) is False