From 9f8b451d15570494c1eef435cbded62e779e57f7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 29 Apr 2026 09:31:12 -0400 Subject: [PATCH] fix(journal-prep): bucket tasks + drop non-proximate events (#159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two filtering issues that made the daily prep noisy and trained the user to ignore it. ## Tasks: bucket into due-today / upcoming / overdue The prep was calling `list_notes(due_before=day_date)` and labeling the result as "tasks due today". That filter is strictly less-than, so it returned only OVERDUE tasks (a single 68-day-stale task in this user's case), while the prompt still framed them as fresh today's work. Each day of the prep treated the same overdue task as new — the user learned to ignore the line entirely. `gather_daily_sections` now runs three queries: - `tasks_due_today` — `due_after=day_date AND due_before=day_date+1` - `tasks_upcoming` — next 7 days, exclusive of today - `tasks_overdue` — strictly before today Overdue entries carry a `days_overdue` count. `_render_sections_for_prompt` emits three labeled headers ("TASKS DUE TODAY", "UPCOMING TASKS", "OVERDUE TASKS (still on the list, not currently due)"). The system prompt has a new TASK BUCKETS rule telling the model: don't call overdue items "due today"; surface them with their staleness duration ("still on the list 68 days") and frame as a backlog reminder rather than today's work. Backwards-compat: `sections["tasks"]` still exists, now as the union of all three buckets — strictly more useful than the prior overdue- only behavior any frontend consumer was getting before. ## Events: tz-aware window + proximity filter The user's "Birthday — 2026-09-29 (FREQ=YEARLY)" event was surfacing in every daily prep, 5 months out. Root cause: `gather_daily_sections` built `day_start`/`day_end` as NAIVE datetimes; `list_events` then called `rrulestr(...).between(naive_from, naive_to)` against an aware `dtstart`, which throws TypeError, hits the `except Exception` fallback, and appends the canonical event row — regardless of whether today is anywhere near a recurrence. Fix: 1. Construct the day window as TZ-aware in the user's local timezone and convert to UTC before the query. RRULE expansion now runs correctly. 2. Defense-in-depth `_filter_proximate_events` drops events whose start_dt is more than 7 days from `day_date` (in the user's local TZ — not UTC, so a Friday 23:00 NY event isn't misclassified as Saturday). If list_events ever leaks a far-future row again, the prep doesn't surface it. 10 new tests in `tests/test_journal_prep_filtering.py` cover task bucketing (overdue marker, due-today no-marker, no-due-date), the proximity filter (the 4/29 reproducer, in-window keeps, local-vs-UTC boundary, unparseable dates kept rather than suppressed), and the rendering (overdue staleness shown, due-today doesn't repeat the date, correct section ordering). 53 tests pass across journal_prep + journal_search + record_moment + calendar_tool + events. Ruff clean. Closes Fable task #159. Co-Authored-By: Claude Opus 4.7 --- src/fabledassistant/services/journal_prep.py | 180 +++++++++++++---- tests/test_journal_prep_filtering.py | 193 +++++++++++++++++++ 2 files changed, 340 insertions(+), 33 deletions(-) create mode 100644 tests/test_journal_prep_filtering.py diff --git a/src/fabledassistant/services/journal_prep.py b/src/fabledassistant/services/journal_prep.py index ec7966a..73734ac 100644 --- a/src/fabledassistant/services/journal_prep.py +++ b/src/fabledassistant/services/journal_prep.py @@ -23,6 +23,7 @@ from __future__ import annotations import datetime import logging +from zoneinfo import ZoneInfo from sqlalchemy import select @@ -38,6 +39,65 @@ from fabledassistant.services.weather import get_cached_weather_rows logger = logging.getLogger(__name__) +# How many days out from today an event needs to be before the prep treats +# it as too far-future to surface. Catches recurring-event canonical rows +# whose RRULE expansion missed (an `rrulestr` failure falls back to the +# canonical event in `list_events`, which leaks far-future occurrences +# into today's prep). +_EVENT_PROXIMITY_DAYS = 7 + + +def _task_to_prep_dict(task, today: datetime.date) -> dict: + """Render a Note row as a prep-payload task entry, tagging overdue + staleness when relevant so the prompt can frame it correctly.""" + d = { + "id": task.id, + "title": task.title, + "status": task.status, + "priority": task.priority, + "due_date": task.due_date.isoformat() if task.due_date else None, + } + if task.due_date and task.due_date < today: + d["days_overdue"] = (today - task.due_date).days + return d + + +def _filter_proximate_events( + events: list[dict], *, day_date: datetime.date, user_tz: ZoneInfo +) -> list[dict]: + """Drop events whose start_dt is more than ``_EVENT_PROXIMITY_DAYS`` + away from ``day_date`` in the user's local timezone. + + Belt-and-suspenders against `list_events` returning a canonical + far-future event (e.g. when RRULE expansion fails and the loop falls + back to the original event row, regardless of date). The user + observed "Birthday — 2026-09-29 (FREQ=YEARLY)" surfacing in every + daily prep 5 months out; this filter keeps the prep proximate. + """ + proximate: list[dict] = [] + for e in events: + raw = e.get("start_dt") or "" + try: + start_dt = datetime.datetime.fromisoformat( + raw.replace("Z", "+00:00") if isinstance(raw, str) else "" + ) + local_date = start_dt.astimezone(user_tz).date() + delta = abs((local_date - day_date).days) + except (ValueError, TypeError, AttributeError): + # Unparseable date — keep the event rather than suppress real data. + proximate.append(e) + continue + if delta <= _EVENT_PROXIMITY_DAYS: + proximate.append(e) + else: + logger.info( + "daily_prep: dropping non-proximate event %r start_local=%s " + "(%d days from day %s)", + e.get("title"), local_date.isoformat(), delta, day_date.isoformat(), + ) + return proximate + + async def gather_daily_sections( *, user_id: int, @@ -48,41 +108,71 @@ async def gather_daily_sections( Pure data fetching — no LLM call. Each section degrades to an empty list/dict on failure so the caller always gets a complete shape. + + Tasks are returned in three explicit buckets so the prompt can frame + overdue items correctly (instead of calling them "due today" — the + pre-2026-04-29 behavior, before this rewrite). """ sections: dict = {} + next_day = day_date + datetime.timedelta(days=1) + upcoming_end = day_date + datetime.timedelta(days=8) try: - tasks_today, _ = await list_notes( - user_id=user_id, - is_task=True, - status=["todo", "in_progress"], - due_before=day_date, - limit=20, - sort="due_date", - order="asc", + due_today_rows, _ = await list_notes( + user_id=user_id, is_task=True, status=["todo", "in_progress"], + due_after=day_date, due_before=next_day, + limit=20, sort="due_date", order="asc", + ) + upcoming_rows, _ = await list_notes( + user_id=user_id, is_task=True, status=["todo", "in_progress"], + due_after=next_day, due_before=upcoming_end, + limit=20, sort="due_date", order="asc", + ) + overdue_rows, _ = await list_notes( + user_id=user_id, is_task=True, status=["todo", "in_progress"], + due_before=day_date, + limit=20, sort="due_date", order="asc", + ) + sections["tasks_due_today"] = [_task_to_prep_dict(t, day_date) for t in due_today_rows] + sections["tasks_upcoming"] = [_task_to_prep_dict(t, day_date) for t in upcoming_rows] + sections["tasks_overdue"] = [_task_to_prep_dict(t, day_date) for t in overdue_rows] + # Backwards-compat alias for any consumers still reading sections["tasks"]. + # The combined view is more useful than the prior overdue-only behavior. + sections["tasks"] = ( + sections["tasks_due_today"] + + sections["tasks_upcoming"] + + sections["tasks_overdue"] ) - sections["tasks"] = [ - { - "id": t.id, - "title": t.title, - "status": t.status, - "priority": t.priority, - "due_date": t.due_date.isoformat() if t.due_date else None, - } - for t in tasks_today - ] except Exception: logger.exception("daily_prep tasks section failed for user %d", user_id) + sections["tasks_due_today"] = [] + sections["tasks_upcoming"] = [] + sections["tasks_overdue"] = [] sections["tasks"] = [] try: - day_start = datetime.datetime.combine(day_date, datetime.time.min) - day_end = datetime.datetime.combine(day_date, datetime.time.max) - sections["events"] = await list_events( + try: + user_tz = ZoneInfo(user_timezone) + except Exception: + logger.warning("daily_prep: invalid user_timezone %r — defaulting to UTC", user_timezone) + user_tz = ZoneInfo("UTC") + # Build the local-day window in the user's TZ, then convert to UTC + # for the DB / RRULE expansion. A naive datetime here previously + # caused rrule.between() to throw, falling back to the canonical + # event row regardless of date — the source of stale recurring + # events polluting every daily prep. + day_start_local = datetime.datetime.combine(day_date, datetime.time.min, tzinfo=user_tz) + day_end_local = datetime.datetime.combine(day_date, datetime.time.max, tzinfo=user_tz) + day_start = day_start_local.astimezone(datetime.timezone.utc) + day_end = day_end_local.astimezone(datetime.timezone.utc) + all_events = await list_events( user_id=user_id, date_from=day_start, date_to=day_end, ) + sections["events"] = _filter_proximate_events( + all_events, day_date=day_date, user_tz=user_tz, + ) except Exception: logger.exception("daily_prep events section failed for user %d", user_id) sections["events"] = [] @@ -148,22 +238,40 @@ async def _open_threads(*, user_id: int, day_date: datetime.date) -> list[dict]: ] +def _render_task_line(t: dict, *, include_due: bool, include_overdue: bool) -> str: + line = f" - {t.get('title', '?')}" + if include_overdue and t.get("days_overdue"): + line += f" (due {t['due_date']}, {t['days_overdue']} days ago)" + elif include_due and t.get("due_date"): + line += f" (due {t['due_date']})" + if t.get("priority") and t["priority"] not in (None, "none"): + line += f" [{t['priority']} priority]" + if t.get("status") == "in_progress": + line += " [in progress]" + return line + + def _render_sections_for_prompt(sections: dict) -> str: """Render the gathered sections as a structured plain-text block for the LLM.""" lines: list[str] = [] - tasks = sections.get("tasks") or [] - if tasks: - lines.append("TASKS (todo or in-progress):") - for t in tasks[:12]: - line = f" - {t.get('title', '?')}" - if t.get("due_date"): - line += f" (due {t['due_date']})" - if t.get("priority") and t["priority"] not in (None, "none"): - line += f" [{t['priority']} priority]" - if t.get("status") == "in_progress": - line += " [in progress]" - lines.append(line) + due_today = sections.get("tasks_due_today") or [] + upcoming = sections.get("tasks_upcoming") or [] + overdue = sections.get("tasks_overdue") or [] + if due_today: + lines.append("TASKS DUE TODAY:") + for t in due_today[:8]: + lines.append(_render_task_line(t, include_due=False, include_overdue=False)) + lines.append("") + if upcoming: + lines.append("UPCOMING TASKS (next 7 days):") + for t in upcoming[:8]: + lines.append(_render_task_line(t, include_due=True, include_overdue=False)) + lines.append("") + if overdue: + lines.append("OVERDUE TASKS (still on the list, not currently due):") + for t in overdue[:8]: + lines.append(_render_task_line(t, include_due=False, include_overdue=True)) lines.append("") events = sections.get("events") or [] @@ -242,6 +350,12 @@ _PREP_SYSTEM_PROMPT = ( "keep the prose factual and useful, not sentimental.\n" "- 4 to 7 sentences total. Tight. No padding, no flowery openings, no \"Good morning\" " "greetings unless the actual content warrants two clauses' worth.\n" + "- TASK BUCKETS — three sections may appear: TASKS DUE TODAY, UPCOMING TASKS, " + "OVERDUE TASKS. Lead with TASKS DUE TODAY when present. Do NOT call overdue items " + "\"due today\" — they aren't. When OVERDUE TASKS appears, surface it with the " + "staleness duration (\"still on the list 68 days\") and frame it as something to " + "revisit, not as today's work. If the only data is overdue, lead with it but " + "frame it as a backlog reminder.\n" "- If RECENT JOURNAL MOMENTS or OPEN THREADS are present, mention one or two BRIEFLY " "at the end as context — not as the lead. Skip them if nothing notable.\n" "- Close with one short invitation to journal: \"What's on your mind?\", " diff --git a/tests/test_journal_prep_filtering.py b/tests/test_journal_prep_filtering.py new file mode 100644 index 0000000..58f68fe --- /dev/null +++ b/tests/test_journal_prep_filtering.py @@ -0,0 +1,193 @@ +"""Tests for the journal prep filtering helpers added in #159.""" + +import datetime +from types import SimpleNamespace +from zoneinfo import ZoneInfo + +import pytest + + +# ── _task_to_prep_dict ──────────────────────────────────────────────────────── + + +def test_task_to_prep_dict_marks_overdue_with_days_count(): + from fabledassistant.services.journal_prep import _task_to_prep_dict + + today = datetime.date(2026, 4, 29) + task = SimpleNamespace( + id=2, title="Research Weston's ADHD Evaluation", + status="todo", priority="high", + due_date=datetime.date(2026, 2, 20), + ) + d = _task_to_prep_dict(task, today) + assert d["days_overdue"] == 68 + assert d["due_date"] == "2026-02-20" + + +def test_task_to_prep_dict_due_today_no_overdue_marker(): + from fabledassistant.services.journal_prep import _task_to_prep_dict + + today = datetime.date(2026, 4, 29) + task = SimpleNamespace( + id=10, title="Pick up dry cleaning", + status="todo", priority="none", + due_date=today, + ) + d = _task_to_prep_dict(task, today) + assert "days_overdue" not in d + + +def test_task_to_prep_dict_handles_no_due_date(): + from fabledassistant.services.journal_prep import _task_to_prep_dict + + task = SimpleNamespace( + id=11, title="Long-running side project", + status="in_progress", priority="medium", + due_date=None, + ) + d = _task_to_prep_dict(task, datetime.date(2026, 4, 29)) + assert d["due_date"] is None + assert "days_overdue" not in d + + +# ── _filter_proximate_events ────────────────────────────────────────────────── + + +def test_filter_proximate_drops_far_future_recurring_event(): + """The reported failure: a recurring Birthday event with start_dt + 2026-09-29 surfaced in every daily prep. The proximity filter drops it.""" + from fabledassistant.services.journal_prep import _filter_proximate_events + + events = [ + {"id": 13, "title": "Birthday", "start_dt": "2026-09-29T00:00:00+00:00"}, + ] + kept = _filter_proximate_events( + events, + day_date=datetime.date(2026, 4, 29), + user_tz=ZoneInfo("America/New_York"), + ) + assert kept == [] + + +def test_filter_proximate_keeps_events_within_window(): + """Events within ±7 days of the prep date stay.""" + from fabledassistant.services.journal_prep import _filter_proximate_events + + events = [ + {"id": 1, "title": "Today", "start_dt": "2026-04-29T13:00:00+00:00"}, + {"id": 2, "title": "Tomorrow", "start_dt": "2026-04-30T12:00:00+00:00"}, + {"id": 3, "title": "Next week", "start_dt": "2026-05-05T15:00:00+00:00"}, + {"id": 4, "title": "Two weeks out", "start_dt": "2026-05-15T15:00:00+00:00"}, + ] + kept = _filter_proximate_events( + events, + day_date=datetime.date(2026, 4, 29), + user_tz=ZoneInfo("America/New_York"), + ) + kept_ids = [e["id"] for e in kept] + assert 1 in kept_ids + assert 2 in kept_ids + assert 3 in kept_ids + assert 4 not in kept_ids + + +def test_filter_proximate_uses_local_date_not_utc(): + """An event at 04:30 UTC on 2026-04-30 = 00:30 local on 2026-04-30 in + NY (EDT, UTC-4). Local date is 2026-04-30, delta from 2026-04-29 = 1 + day. Must NOT cross-classify based on UTC date alone.""" + from fabledassistant.services.journal_prep import _filter_proximate_events + + events = [ + {"id": 99, "title": "Late-night dentist", "start_dt": "2026-04-30T04:30:00+00:00"}, + ] + kept = _filter_proximate_events( + events, + day_date=datetime.date(2026, 4, 29), + user_tz=ZoneInfo("America/New_York"), + ) + assert len(kept) == 1 + + +def test_filter_proximate_keeps_unparseable_dates(): + """Bad date strings are kept rather than silently suppressing real + events on a parser bug.""" + from fabledassistant.services.journal_prep import _filter_proximate_events + + events = [ + {"id": 50, "title": "Garbage", "start_dt": "not-a-date"}, + {"id": 51, "title": "Empty", "start_dt": ""}, + {"id": 52, "title": "Missing"}, + ] + kept = _filter_proximate_events( + events, + day_date=datetime.date(2026, 4, 29), + user_tz=ZoneInfo("America/New_York"), + ) + assert len(kept) == 3 + + +# ── _render_sections_for_prompt — overdue framing ──────────────────────────── + + +def test_render_overdue_includes_staleness_duration(): + """The rendered prompt block must surface days_overdue so the LLM + can frame stale tasks correctly.""" + from fabledassistant.services.journal_prep import _render_sections_for_prompt + + sections = { + "tasks_due_today": [], + "tasks_upcoming": [], + "tasks_overdue": [{ + "id": 2, "title": "Research Weston's ADHD Evaluation", + "status": "todo", "priority": "high", + "due_date": "2026-02-20", "days_overdue": 68, + }], + } + rendered = _render_sections_for_prompt(sections) + assert "OVERDUE TASKS" in rendered + assert "68 days ago" in rendered + assert "2026-02-20" in rendered + # Crucially, NOT framed as due today. + assert "DUE TODAY" not in rendered + + +def test_render_due_today_no_due_date_repetition(): + """Tasks in the DUE TODAY bucket don't need the (due 2026-04-29) + parenthetical — the section header already says 'today'.""" + from fabledassistant.services.journal_prep import _render_sections_for_prompt + + sections = { + "tasks_due_today": [{ + "id": 1, "title": "Pick up dry cleaning", + "status": "todo", "priority": "none", + "due_date": "2026-04-29", + }], + "tasks_upcoming": [], + "tasks_overdue": [], + } + rendered = _render_sections_for_prompt(sections) + assert "TASKS DUE TODAY" in rendered + assert "Pick up dry cleaning" in rendered + # The line itself shouldn't repeat the due date. + line = next( + (l for l in rendered.splitlines() if "Pick up dry cleaning" in l), + "", + ) + assert "2026-04-29" not in line + + +def test_render_three_buckets_in_correct_order(): + """When all three buckets have content, the prompt sees them in + DUE TODAY → UPCOMING → OVERDUE order so the LLM leads with today.""" + from fabledassistant.services.journal_prep import _render_sections_for_prompt + + sections = { + "tasks_due_today": [{"id": 1, "title": "Today task", "status": "todo", "priority": "none", "due_date": "2026-04-29"}], + "tasks_upcoming": [{"id": 2, "title": "Upcoming task", "status": "todo", "priority": "none", "due_date": "2026-05-02"}], + "tasks_overdue": [{"id": 3, "title": "Stale task", "status": "todo", "priority": "none", "due_date": "2026-02-20", "days_overdue": 68}], + } + rendered = _render_sections_for_prompt(sections) + today_idx = rendered.index("TASKS DUE TODAY") + upcoming_idx = rendered.index("UPCOMING TASKS") + overdue_idx = rendered.index("OVERDUE TASKS") + assert today_idx < upcoming_idx < overdue_idx