fix(journal): anti-hallucination hardening + message_count fix
Prep prose (services/journal_prep.py): - Emit explicit "WEATHER: none available — do NOT mention weather" absent-marker so a small model can't invent partly-cloudy/temperature prose when both configured locations have empty addresses. - Replace negative-only system rule with positive-anchored guidance forbidding weather/temp/precip mentions unless a numeric WEATHER section is present; also bans echoing parenthetical labels verbatim. - Reword overdue header to "(past their due date, still open — backlog, not today's work)" and render lines as "was due <date>, N day(s) overdue" with correct singular/plural. Supersedes the wording noted in Fable task #159. - Deterministic fabricated-weather reconciler: low-false-positive regex detects fabricated weather phrasing; on trip with an empty section, regenerate once with a corrective. Persistent fabrication logs ERROR rather than mangling prose. Journal route (routes/journal.py): - Override message_count with len(messages) in _day_payload. The chat path already does this; the journal path was hitting the Conversation.to_dict() fallback to 0 because messages aren't eager-loaded on that instance. Tests: - tests/test_journal_message_count.py — pins the model-level trap and the override contract (3 cases). - tests/test_journal_prep_hardening.py — 11 cases covering the fabricated-weather reconciler and absent-marker rendering. - tests/test_journal_prep_filtering.py — updated one stale assertion. Tracks Fable task #171. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -407,8 +407,14 @@ async def _day_payload(*, user_id: int, day_date: datetime.date):
|
||||
.order_by(Message.created_at)
|
||||
)
|
||||
messages = (await session.execute(msgs_stmt)).scalars().all()
|
||||
# conv.to_dict() recomputes message_count from the `messages`
|
||||
# relationship, which isn't eager-loaded here, so it would report 0.
|
||||
# We already have the real list — override with the known count, same
|
||||
# convention the chat-list path uses (services/chat.py).
|
||||
conv_dict = conv.to_dict()
|
||||
conv_dict["message_count"] = len(messages)
|
||||
return jsonify({
|
||||
"day_date": day_date.isoformat(),
|
||||
"conversation": conv.to_dict(),
|
||||
"conversation": conv_dict,
|
||||
"messages": [m.to_dict() for m in messages],
|
||||
})
|
||||
|
||||
@@ -23,6 +23,7 @@ from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -249,7 +250,9 @@ async def _open_threads(*, user_id: int, day_date: datetime.date) -> list[dict]:
|
||||
def _render_task_line(t: dict, *, include_due: bool, include_overdue: bool) -> str:
|
||||
line = f" - {t.get('title', '?')}"
|
||||
if include_overdue and t.get("days_overdue"):
|
||||
line += f" (due {t['due_date']}, {t['days_overdue']} days ago)"
|
||||
n = t["days_overdue"]
|
||||
unit = "day" if n == 1 else "days"
|
||||
line += f" (was due {t['due_date']}, {n} {unit} overdue)"
|
||||
elif include_due and t.get("due_date"):
|
||||
line += f" (due {t['due_date']})"
|
||||
if t.get("priority") and t["priority"] not in (None, "none"):
|
||||
@@ -277,7 +280,7 @@ def _render_sections_for_prompt(sections: dict) -> str:
|
||||
lines.append(_render_task_line(t, include_due=True, include_overdue=False))
|
||||
lines.append("")
|
||||
if overdue:
|
||||
lines.append("OVERDUE TASKS (still on the list, not currently due):")
|
||||
lines.append("OVERDUE TASKS (past their due date, still open — backlog, not today's work):")
|
||||
for t in overdue[:8]:
|
||||
lines.append(_render_task_line(t, include_due=False, include_overdue=True))
|
||||
lines.append("")
|
||||
@@ -312,6 +315,17 @@ def _render_sections_for_prompt(sections: dict) -> str:
|
||||
bits.append(f"{precip}% chance of precipitation")
|
||||
lines.append(" - " + ", ".join(bits))
|
||||
lines.append("")
|
||||
else:
|
||||
# Explicit absent-marker. A silently-omitted weather block leaves a
|
||||
# small model an unanchored void it tends to fill with plausible
|
||||
# fabricated weather (observed: invented "68°F, 15% rain" with an
|
||||
# empty weather section). A concrete "none" line + directive holds
|
||||
# far better than relying on a negative system-prompt rule alone.
|
||||
lines.append(
|
||||
"WEATHER: none available — no weather, temperature, or precipitation "
|
||||
"data exists for today. Do NOT mention weather in any form."
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
projects = sections.get("projects") or []
|
||||
if projects:
|
||||
@@ -342,8 +356,14 @@ def _render_sections_for_prompt(sections: dict) -> str:
|
||||
lines.append(f" - [{day}] {content}")
|
||||
lines.append("")
|
||||
|
||||
if not lines:
|
||||
return "(No data for today — quiet morning.)"
|
||||
# The weather-none marker is always emitted, so `lines` is never empty;
|
||||
# a quiet day is one with no *substantive* sections beyond that marker.
|
||||
substantive = [ln for ln in lines if ln and not ln.startswith("WEATHER: none")]
|
||||
if not substantive:
|
||||
return (
|
||||
"(No tasks, events, or notable data for today — a quiet day. "
|
||||
"Do not invent weather or any other details.)"
|
||||
)
|
||||
return "\n".join(lines).rstrip()
|
||||
|
||||
|
||||
@@ -360,15 +380,19 @@ _PREP_SYSTEM_PROMPT = (
|
||||
"greetings unless the actual content warrants two clauses' worth.\n"
|
||||
"- TASK BUCKETS — three sections may appear: TASKS DUE TODAY, UPCOMING TASKS, "
|
||||
"OVERDUE TASKS. Lead with TASKS DUE TODAY when present. Do NOT call overdue items "
|
||||
"\"due today\" — they aren't. When OVERDUE TASKS appears, surface it with the "
|
||||
"staleness duration (\"still on the list 68 days\") and frame it as something to "
|
||||
"revisit, not as today's work. If the only data is overdue, lead with it but "
|
||||
"frame it as a backlog reminder.\n"
|
||||
"\"due today\" — they aren't. When OVERDUE TASKS appears, state the overdue "
|
||||
"duration exactly as given (e.g. \"3 days overdue\") and frame it as backlog to "
|
||||
"revisit, not as today's work. Do NOT echo the parenthetical section labels "
|
||||
"verbatim. If the only data is overdue, lead with it but frame it as a backlog "
|
||||
"reminder.\n"
|
||||
"- If RECENT JOURNAL MOMENTS or OPEN THREADS are present, mention one or two BRIEFLY "
|
||||
"at the end as context — not as the lead. Skip them if nothing notable.\n"
|
||||
"- Close with one short invitation to journal: \"What's on your mind?\", "
|
||||
"\"Anything to set down?\", \"How's the morning shaping up?\" — pick one, keep it under 8 words.\n"
|
||||
"- Don't fabricate. Skip categories with no data; don't acknowledge their absence.\n"
|
||||
"- Use ONLY the data below. A category marked 'none' (or absent) genuinely has no "
|
||||
"data — do not invent it and do not mention it. In particular, NEVER state weather, "
|
||||
"temperature, or precipitation unless an explicit WEATHER section with numbers "
|
||||
"appears below.\n"
|
||||
"- Voice is competent assistant briefing the user. Not a friend writing a letter."
|
||||
)
|
||||
|
||||
@@ -379,6 +403,40 @@ def _fallback_prep_text(day_date: datetime.date) -> str:
|
||||
return f"{weekday}, {day_date.isoformat()}. What's on your mind?"
|
||||
|
||||
|
||||
# Strong, low-false-positive weather signals. Deliberately NOT bare words like
|
||||
# "rain"/"sunny"/"weather" (those legitimately appear in task/event titles —
|
||||
# "buy rain boots"). Targets the concrete phrasings small models actually
|
||||
# emit when fabricating ("partly cloudy with a high of 68°F and a 15% chance
|
||||
# of rain"): temperature glyphs, "high/low of N", "chance of <precip>",
|
||||
# "(partly|mostly) (cloudy|sunny)", "overcast", "precipitation", "forecast".
|
||||
_WEATHER_SIGNAL_RE = re.compile(
|
||||
r"""
|
||||
\d{1,3}\s?°
|
||||
| \b\d{1,3}\s?°?\s?(?:degrees|fahrenheit|celsius)\b
|
||||
| \b(?:high|low)\s+of\s+\d
|
||||
| \bchance\s+of\s+(?:rain|showers?|precipitation|snow|sleet|storms?|thunder)
|
||||
| \b(?:partly|mostly)\s+(?:cloudy|sunny)\b
|
||||
| \bovercast\b
|
||||
| \bprecipitation\b
|
||||
| \bforecast\b
|
||||
""",
|
||||
re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
def _prose_fabricated_weather(prose: str, sections: dict) -> bool:
|
||||
"""True when the prose talks weather but no weather data was gathered.
|
||||
|
||||
The deterministic backstop for the system-prompt rule: an 8–14B model
|
||||
still invents weather on quiet days even when told not to. If the
|
||||
WEATHER section is genuinely empty and the prose trips a strong weather
|
||||
signal, that text is fabricated.
|
||||
"""
|
||||
if sections.get("weather"):
|
||||
return False
|
||||
return bool(_WEATHER_SIGNAL_RE.search(prose or ""))
|
||||
|
||||
|
||||
async def _generate_prep_prose(
|
||||
*,
|
||||
sections: dict,
|
||||
@@ -400,25 +458,54 @@ async def _generate_prep_prose(
|
||||
f"Write the opener for today's journal."
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": _PREP_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_trigger},
|
||||
]
|
||||
_WEATHER_CORRECTION = (
|
||||
"\n\nIMPORTANT: there is NO weather data for today. Do not mention "
|
||||
"weather, temperature, sky conditions, or precipitation in any form."
|
||||
)
|
||||
|
||||
try:
|
||||
prose = await generate_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
max_tokens=400,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Daily prep prose generation failed for day %s", day_date)
|
||||
return _fallback_prep_text(day_date)
|
||||
# Up to 2 attempts: if the first trips the fabricated-weather guard, retry
|
||||
# once with an explicit corrective appended to the user turn. A 14B model
|
||||
# almost always complies on the corrected pass; if it still doesn't we log
|
||||
# and accept (surgically excising a sentence risks breaking prose flow —
|
||||
# better a rare stray clause than mangled output, and the log lets us
|
||||
# measure whether a model bump is actually warranted).
|
||||
prose = ""
|
||||
for attempt in (1, 2):
|
||||
trigger = user_trigger
|
||||
if attempt == 2:
|
||||
trigger = user_trigger + _WEATHER_CORRECTION
|
||||
messages = [
|
||||
{"role": "system", "content": _PREP_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": trigger},
|
||||
]
|
||||
try:
|
||||
raw = await generate_completion(
|
||||
messages=messages,
|
||||
model=model,
|
||||
max_tokens=400,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Daily prep prose generation failed for day %s", day_date)
|
||||
return _fallback_prep_text(day_date)
|
||||
|
||||
prose = (prose or "").strip()
|
||||
if not prose:
|
||||
logger.warning("LLM returned empty prep prose for day %s — using fallback", day_date)
|
||||
return _fallback_prep_text(day_date)
|
||||
prose = (raw or "").strip()
|
||||
if not prose:
|
||||
logger.warning("LLM returned empty prep prose for day %s — using fallback", day_date)
|
||||
return _fallback_prep_text(day_date)
|
||||
|
||||
if not _prose_fabricated_weather(prose, sections):
|
||||
return prose
|
||||
|
||||
if attempt == 1:
|
||||
logger.warning(
|
||||
"daily_prep: fabricated weather detected for day %s (no weather "
|
||||
"data gathered) — regenerating with corrective", day_date,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"daily_prep: weather still fabricated after corrective retry "
|
||||
"for day %s — accepting prose as-is", day_date,
|
||||
)
|
||||
return prose
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user