bvandeusen/FabledScribe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 50 Wiki Activity
Files
404698521fd10125b258da0a72dc26926ff74a13
FabledScribe/tests/test_calendar_tool_tz.py
T
bvandeusen 03d725ea3e fix(calendar-tool): anchor today's weekday in prompts + verify expected_weekday on create/update 
A user asked Fable to schedule "this Friday at 8am" on Wednesday 4/29
2026. The model picked 4/30 (Thursday) and confidently labeled it
"Friday." The TZ pipeline did everything correctly given the model's
date — the bug was upstream: the model was guessing weekdays from ISO
dates without an anchor, and the calendar tools had no way to verify.

Three layered fixes:

1. **System prompts now name the weekday alongside the ISO date.**
   Both the journal-conversation prompt and the general chat prompt
   used to say "Today is 2026-04-29 (America/New_York)." They now say
   "Today is Wednesday, 2026-04-29 (...)." LLMs are unreliable at
   deriving weekday names from ISO dates; supplying the name removes
   the guess.

2. **`expected_weekday` parameter on create_event / update_event.**
   When the model passes `expected_weekday="friday"`, the backend
   computes the resolved start_date's weekday in the user's local
   timezone and rejects mismatches with a self-correcting error
   ("Date 2026-04-30 falls on Thursday, not Friday. Recompute..."),
   without creating the event. The check is local-aware: a Friday
   23:00 event in Tokyo crosses midnight UTC but the local view
   stays Friday, and the validator respects that.

3. **Tool descriptions instruct echo-and-confirm.** create_event and
   update_event descriptions now tell the model: when the user names
   a weekday, state the resolved date in the reply BEFORE calling
   the tool, and pass `expected_weekday`. Costs nothing in code,
   reinforces the validator.

6 new tests — match success, mismatch rejection (with create/update
not invoked), omitted-param backcompat, invalid weekday name, local-
not-UTC weekday computation, and the update_event variant. All 18
calendar-tool tests + 33 event-related tests pass; ruff clean.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-29 08:43:32 -04:00

614 lines
21 KiB
Python
Raw Blame History

"""Regression tests: calendar tool respects the user's configured timezone."""
from datetime import timezone
from unittest.mock import AsyncMock, patch
import pytest
@pytest.mark.asyncio
async def test_create_event_all_day_bare_date_interprets_local_midnight():
"""Bare date like '2026-09-30' for a NY user = 2026-09-30 00:00 NY,
stored as 2026-09-30 04:00 UTC — NOT 2026-09-30 00:00 UTC (which would
be 2026-09-29 19:00 NY and land on the wrong day)."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1, **{k: v for k, v in kwargs.items() if not callable(v)}}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={"title": "Birthday", "start": "2026-09-30"},
)
assert result["success"] is True
start_dt = captured["start_dt"]
assert start_dt.tzinfo is not None
# Converted to UTC: 00:00 NY (EDT, UTC-4) == 04:00 UTC on the same day
utc = start_dt.astimezone(timezone.utc)
assert (utc.year, utc.month, utc.day) == (2026, 9, 30)
assert utc.hour == 4 # EDT offset; would be 5 in EST
assert captured["all_day"] is True
@pytest.mark.asyncio
async def test_create_event_naive_datetime_is_user_local():
"""'2026-03-15T14:30' for a NY user means 2:30pm NY, not 2:30pm UTC."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
await create_event_tool(
user_id=1,
arguments={"title": "Meeting", "start": "2026-03-15T14:30"},
)
utc = captured["start_dt"].astimezone(timezone.utc)
# 14:30 NY on 2026-03-15 (after DST start) = 18:30 UTC
assert (utc.year, utc.month, utc.day, utc.hour, utc.minute) == (2026, 3, 15, 18, 30)
@pytest.mark.asyncio
async def test_create_event_explicit_offset_is_respected():
"""Model-supplied timezone offsets must not be overridden by user TZ."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
await create_event_tool(
user_id=1,
arguments={"title": "Meeting", "start": "2026-03-15T14:30+00:00"},
)
utc = captured["start_dt"].astimezone(timezone.utc)
assert (utc.hour, utc.minute) == (14, 30)
@pytest.mark.asyncio
async def test_list_events_bare_date_range_covers_local_day():
"""date_from/date_to as bare dates should cover the full LOCAL day."""
from fabledassistant.services.tools.calendar import list_events_tool
captured = {}
async def fake_list_events(*, user_id, date_from, date_to):
captured["date_from"] = date_from
captured["date_to"] = date_to
return []
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_list_events",
side_effect=fake_list_events,
):
await list_events_tool(
user_id=1,
arguments={"date_from": "2026-09-30", "date_to": "2026-09-30"},
)
df = captured["date_from"].astimezone(timezone.utc)
dt = captured["date_to"].astimezone(timezone.utc)
# 2026-09-30 00:00 NY (EDT) = 04:00 UTC same day
assert (df.year, df.month, df.day, df.hour) == (2026, 9, 30, 4)
# 2026-09-30 23:59:59 NY (EDT) = 03:59:59 UTC next day
assert (dt.year, dt.month, dt.day, dt.hour) == (2026, 10, 1, 3)
# ── Split date/time field tests (durable shape) ──────────────────────────────
#
# These exercise the start_date + start_time path that the model is now
# steered toward. The split-field shape is structurally immune to the
# class of bugs where a model emits a TZ-tagged combined datetime that
# the parser correctly honors but lands on the wrong calendar day for
# negative-offset users.
@pytest.mark.asyncio
async def test_create_event_split_fields_friday_8am_no_drift_eastern():
"""The reported bug: 'next Friday at 8am' for a NY user must land on
2026-05-01 08:00 NY, never 04-30 19:00. With split fields the model
can't TZ-tag the date string, so the calendar day is fixed."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "Meeting",
"start_date": "2026-05-01",
"start_time": "08:00",
},
)
assert result["success"] is True
utc = captured["start_dt"].astimezone(timezone.utc)
# 08:00 NY (EDT, UTC-4) on 2026-05-01 = 12:00 UTC same day
assert (utc.year, utc.month, utc.day, utc.hour, utc.minute) == (2026, 5, 1, 12, 0)
assert captured["all_day"] is False
@pytest.mark.asyncio
async def test_create_event_split_fields_no_drift_pacific():
"""Same scenario for a UTC-8 user — the calendar day must be 5/1
regardless of how big the offset gets."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/Los_Angeles")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
await create_event_tool(
user_id=1,
arguments={
"title": "Standup",
"start_date": "2026-05-01",
"start_time": "08:00",
},
)
utc = captured["start_dt"].astimezone(timezone.utc)
# 08:00 LA (PDT, UTC-7) = 15:00 UTC same day
assert (utc.year, utc.month, utc.day, utc.hour) == (2026, 5, 1, 15)
@pytest.mark.asyncio
async def test_create_event_split_fields_no_drift_positive_offset():
"""A positive-offset user (Tokyo, UTC+9) — 08:00 Tokyo on 2026-05-01
is 23:00 UTC on 2026-04-30, but the local calendar day must still be
stored as 2026-05-01 from the user's perspective."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("Asia/Tokyo")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
await create_event_tool(
user_id=1,
arguments={
"title": "Sync",
"start_date": "2026-05-01",
"start_time": "08:00",
},
)
utc = captured["start_dt"].astimezone(timezone.utc)
# 08:00 Tokyo (UTC+9) = 23:00 UTC previous day; the round-trip back
# to Tokyo TZ recovers 2026-05-01 08:00.
assert (utc.year, utc.month, utc.day, utc.hour) == (2026, 4, 30, 23)
tokyo = captured["start_dt"].astimezone(__import__("zoneinfo").ZoneInfo("Asia/Tokyo"))
assert (tokyo.year, tokyo.month, tokyo.day, tokyo.hour) == (2026, 5, 1, 8)
@pytest.mark.asyncio
async def test_create_event_split_date_only_is_all_day():
"""Omitting start_time means the event is all-day; cache row uses
local midnight."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
await create_event_tool(
user_id=1,
arguments={"title": "Birthday", "start_date": "2026-09-30"},
)
assert captured["all_day"] is True
utc = captured["start_dt"].astimezone(timezone.utc)
assert (utc.year, utc.month, utc.day, utc.hour) == (2026, 9, 30, 4)
@pytest.mark.asyncio
async def test_create_event_split_rejects_tz_suffix_in_date():
"""The whole point of split fields: a TZ suffix on the date string
must be rejected, not silently honored."""
from fabledassistant.services.tools.calendar import create_event_tool
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
):
result = await create_event_tool(
user_id=1,
arguments={"title": "x", "start_date": "2026-05-01Z", "start_time": "08:00"},
)
assert result["success"] is False
assert "YYYY-MM-DD" in result["error"]
@pytest.mark.asyncio
async def test_create_event_split_rejects_tz_suffix_in_time():
"""A TZ suffix on the time string must also be rejected."""
from fabledassistant.services.tools.calendar import create_event_tool
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
):
result = await create_event_tool(
user_id=1,
arguments={"title": "x", "start_date": "2026-05-01", "start_time": "08:00 UTC"},
)
assert result["success"] is False
assert "HH:MM" in result["error"]
@pytest.mark.asyncio
async def test_create_event_legacy_combined_start_still_works():
"""Backcompat: saved tool-call payloads using the old `start` field
must still produce the same UTC datetime as before."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={"title": "Legacy", "start": "2026-05-01T08:00"},
)
assert result["success"] is True
utc = captured["start_dt"].astimezone(timezone.utc)
assert (utc.year, utc.month, utc.day, utc.hour) == (2026, 5, 1, 12)
@pytest.mark.asyncio
async def test_update_event_split_fields_reschedule_no_drift():
"""update_event with split fields must drift no calendar day either."""
from fabledassistant.services.tools.calendar import update_event_tool
captured = {}
async def fake_find(*, user_id, query):
ev = AsyncMock()
ev.id = 99
ev.title = "Coffee"
return [ev]
async def fake_update(*, user_id, event_id, **fields):
captured.update(fields)
ev = AsyncMock()
ev.to_dict.return_value = {"id": event_id, **fields}
return ev
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.find_events_by_query",
side_effect=fake_find,
), patch(
"fabledassistant.services.tools.calendar.events_update_event",
side_effect=fake_update,
):
result = await update_event_tool(
user_id=1,
arguments={
"query": "Coffee",
"start_date": "2026-05-01",
"start_time": "08:00",
},
)
assert result["success"] is True
utc = captured["start_dt"].astimezone(timezone.utc)
assert (utc.year, utc.month, utc.day, utc.hour) == (2026, 5, 1, 12)
# ── expected_weekday verification (catches "this Friday" → Thursday bugs) ────
@pytest.mark.asyncio
async def test_create_event_expected_weekday_match_succeeds():
"""When expected_weekday agrees with the resolved local date's
weekday, the event is created normally."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "Meeting",
"start_date": "2026-05-01", # is a Friday
"start_time": "08:00",
"expected_weekday": "friday",
},
)
assert result["success"] is True
assert captured["start_dt"] is not None
@pytest.mark.asyncio
async def test_create_event_expected_weekday_mismatch_rejects_and_names_actual():
"""The reported failure mode: model picks Thursday and calls it
Friday. With expected_weekday set, the create is rejected and the
error names the actual weekday so the model can self-correct."""
from fabledassistant.services.tools.calendar import create_event_tool
create_called = False
async def fake_create_event(**kwargs):
nonlocal create_called
create_called = True
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "Dentist",
"start_date": "2026-04-30", # Thursday
"start_time": "08:00",
"expected_weekday": "friday",
},
)
assert result["success"] is False
assert "Thursday" in result["error"]
assert "Friday" in result["error"]
# Critical: the event must NOT have been created when the check failed.
assert create_called is False
@pytest.mark.asyncio
async def test_create_event_expected_weekday_omitted_skips_check():
"""Backcompat: when expected_weekday isn't passed, no validation
runs — the existing create flow is unchanged."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "x",
"start_date": "2026-04-30",
"start_time": "08:00",
},
)
assert result["success"] is True
@pytest.mark.asyncio
async def test_create_event_expected_weekday_invalid_value_rejects():
"""Garbage in expected_weekday produces a clear validation error,
not a silent pass."""
from fabledassistant.services.tools.calendar import create_event_tool
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "x",
"start_date": "2026-05-01",
"start_time": "08:00",
"expected_weekday": "fri", # abbreviation not accepted
},
)
assert result["success"] is False
assert "weekday" in result["error"].lower()
@pytest.mark.asyncio
async def test_update_event_expected_weekday_mismatch_rejects():
"""update_event must enforce the same weekday check on reschedules."""
from fabledassistant.services.tools.calendar import update_event_tool
update_called = False
async def fake_find(*, user_id, query):
ev = AsyncMock()
ev.id = 99
ev.title = "Coffee"
return [ev]
async def fake_update(*, user_id, event_id, **fields):
nonlocal update_called
update_called = True
ev = AsyncMock()
ev.to_dict.return_value = {"id": event_id}
return ev
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("America/New_York")),
), patch(
"fabledassistant.services.tools.calendar.find_events_by_query",
side_effect=fake_find,
), patch(
"fabledassistant.services.tools.calendar.events_update_event",
side_effect=fake_update,
):
result = await update_event_tool(
user_id=1,
arguments={
"query": "Coffee",
"start_date": "2026-04-30", # Thursday
"start_time": "08:00",
"expected_weekday": "friday",
},
)
assert result["success"] is False
assert "Thursday" in result["error"]
assert update_called is False
@pytest.mark.asyncio
async def test_create_event_weekday_check_uses_local_not_utc():
"""The weekday check must use the LOCAL date, not the UTC date.
A late-evening Friday event in Tokyo (UTC+9) crosses midnight UTC,
so a UTC-day check would call it Saturday — but the user's calendar
says Friday. The check must respect the user's local view."""
from fabledassistant.services.tools.calendar import create_event_tool
captured = {}
async def fake_create_event(**kwargs):
captured.update(kwargs)
event = AsyncMock()
event.to_dict.return_value = {"id": 1}
return event
with patch(
"fabledassistant.services.tools.calendar.get_user_tz",
AsyncMock(return_value=__import__("zoneinfo").ZoneInfo("Asia/Tokyo")),
), patch(
"fabledassistant.services.tools.calendar.events_create_event",
side_effect=fake_create_event,
):
result = await create_event_tool(
user_id=1,
arguments={
"title": "Friday night",
"start_date": "2026-05-01", # Friday in Tokyo
"start_time": "23:00", # 14:00 UTC same day; safe
"expected_weekday": "friday",
},
)
assert result["success"] is True
Reference in New Issue View Git Blame Copy Permalink