diff --git a/src/fabledassistant/routes/chat.py b/src/fabledassistant/routes/chat.py index 55f6c2a..e0cdeb0 100644 --- a/src/fabledassistant/routes/chat.py +++ b/src/fabledassistant/routes/chat.py @@ -178,19 +178,58 @@ async def send_message_route(conv_id: int): model = await get_setting(uid, "default_model", Config.OLLAMA_MODEL) or Config.OLLAMA_MODEL - # Launch background generation task (context building happens inside the task) - asyncio.create_task(run_generation( - buf, history, model, - uid, conv_id, conv.title, content, - context_note_id=context_note_id, - include_note_ids=include_note_ids, - excluded_note_ids=excluded_note_ids, - think=think, - rag_project_id=effective_rag_project_id, - workspace_project_id=workspace_project_id, - user_timezone=user_timezone, - voice_mode=(conv.conversation_type == "voice"), - )) + # Launch background generation task (context building happens inside the task). + # + # Wrap in a top-level guard: `run_generation` is fire-and-forget via + # asyncio.create_task. Without a wrapper, an unhandled exception inside + # the coroutine is swallowed by the event loop, the buffer stays in + # GenerationState.RUNNING forever, and every subsequent POST to this + # conversation returns 409 — locking the user out of the chat surface + # with no log trail. Observed in production 2026-05-22 where the + # generation hung before any internal log line could fire. + async def _run_generation_guarded(): + try: + await run_generation( + buf, history, model, + uid, conv_id, conv.title, content, + context_note_id=context_note_id, + include_note_ids=include_note_ids, + excluded_note_ids=excluded_note_ids, + think=think, + rag_project_id=effective_rag_project_id, + workspace_project_id=workspace_project_id, + user_timezone=user_timezone, + voice_mode=(conv.conversation_type == "voice"), + ) + except Exception: + logger.exception( + "run_generation crashed for conv %d msg %d (model=%s); " + "transitioning buffer to FAILED so the route can be used again", + conv_id, assistant_msg.id, model, + ) + try: + buf.state = GenerationState.ERRORED + buf.append_event( + "done", + {"done": True, "error": "Generation crashed; see server logs"}, + ) + except Exception: + logger.exception("Failed to mark buffer ERRORED for conv %d", conv_id) + try: + from fabledassistant.services.generation_task import _update_message + await _update_message( + assistant_msg.id, + "", + "error", + tool_calls=None, + ) + except Exception: + logger.exception( + "Failed to mark assistant message %d as error after crash", + assistant_msg.id, + ) + + asyncio.create_task(_run_generation_guarded()) return jsonify({ "assistant_message_id": assistant_msg.id, diff --git a/src/fabledassistant/services/curator_scheduler.py b/src/fabledassistant/services/curator_scheduler.py index 798ce6d..7797d85 100644 --- a/src/fabledassistant/services/curator_scheduler.py +++ b/src/fabledassistant/services/curator_scheduler.py @@ -56,9 +56,15 @@ _loop: asyncio.AbstractEventLoop | None = None _running_lock = asyncio.Lock() # prevents overlapping sweeps if one runs long -async def _candidate_conversations() -> list[int]: - """Return conversation IDs that have user messages newer than the - last curator run. Limited to journal type, capped at the sweep limit. +async def _candidate_conversations() -> list[tuple[int, datetime.datetime | None]]: + """Return (conv_id, last_curator_run_at) for journal conversations that + have user messages newer than the last curator run. Capped at the + sweep limit. + + Returning the timestamp alongside the id lets `_sweep` pass it to the + curator as the `since` cutoff, so each sweep only sees messages added + since the previous successful pass — no re-extracting already-captured + beats. """ async with async_session() as session: # Per-conversation: newest USER message timestamp, vs last_curator_run_at. @@ -72,7 +78,7 @@ async def _candidate_conversations() -> list[int]: .subquery() ) stmt = ( - select(Conversation.id) + select(Conversation.id, Conversation.last_curator_run_at) .join(latest_user_msg, latest_user_msg.c.conv_id == Conversation.id) .where(Conversation.conversation_type == "journal") .where( @@ -83,7 +89,7 @@ async def _candidate_conversations() -> list[int]: .limit(_MAX_CONVERSATIONS_PER_SWEEP) ) rows = await session.execute(stmt) - return [row[0] for row in rows.all()] + return [(row[0], row[1]) for row in rows.all()] async def _stamp_last_run(conv_id: int, summary: str | None = None) -> None: @@ -129,9 +135,16 @@ async def _sweep() -> None: "Curator sweep: %d journal conversation(s) to process", len(candidates), ) - for conv_id in candidates: + for conv_id, last_run_at in candidates: try: - result = await run_curator_for_conversation(conv_id) + # Pass last_run_at as the `since` cutoff so the curator only + # examines messages added since the previous successful pass. + # Without this, every sweep re-loads the curator's default + # 24h window — and the LLM re-extracts beats from messages + # that were already captured, producing duplicate moments. + result = await run_curator_for_conversation( + conv_id, since=last_run_at, + ) if result.error: logger.warning( "Curator run errored for conv %d (will retry next sweep): %s",