diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 25b6eab..f488c5e 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -1,13 +1,16 @@ # CI runs first; build only proceeds if all checks pass. # -# Push to dev: typecheck + lint + test → build :dev + : -# Tag v* (release): typecheck + lint + test → build :latest + : + : -# Pull request: no CI run — code is validated on dev push before any PR is opened +# Push to any branch: typecheck + lint + test +# Push to dev: gates + build :dev + : +# Tag v* (release): gates + build :latest + : + : # # To cut a release: # Create a release via the Forgejo UI on main with a v* tag name. # The tag push triggers this workflow; build job pushes :latest + :. # +# PRs aren't triggered on purpose — this is a solo dev→main flow, so +# gating on branch push is already enough. +# # Required secrets (repo → Settings → Secrets → Actions): # REGISTRY_USER — your Forgejo username # REGISTRY_TOKEN — Forgejo PAT with write:packages scope @@ -15,7 +18,7 @@ name: CI & Build on: push: - branches: [dev] + branches: [dev, main] tags: ["v*"] paths: - "src/**" @@ -28,8 +31,18 @@ on: - "assets/**" - "fable-mcp/**" - ".forgejo/workflows/ci.yml" - # pull_request trigger intentionally omitted — all changes go through dev - # first, where CI already runs on push. PR runs would be redundant duplication. + +# Cancel older runs on the same branch when a newer push lands. Tag runs +# get their own group implicitly (refs/tags/v1.2.3 ≠ refs/heads/dev) and +# are never cancelled, so a release build can't kill itself mid-flight. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} + +# Least-privilege default. Jobs that need more (build pushes to the +# registry) upgrade explicitly. +permissions: + contents: read env: REGISTRY: git.fabledsword.com @@ -38,17 +51,23 @@ env: jobs: typecheck: name: TypeScript typecheck - runs-on: py3.12-node22 + runs-on: ci-runner steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 + # Cache node_modules directly (not the npm download cache). + # npm ci still has to extract + link every module even with a + # warm download cache, which is where the real time goes. Caching + # the output directory lets us skip npm ci entirely on hits. + - name: Cache node_modules + id: npm-cache + uses: actions/cache@v4 with: - node-version: "22" - cache: "npm" - cache-dependency-path: frontend/package-lock.json + path: frontend/node_modules + key: node-modules-${{ hashFiles('frontend/package-lock.json') }} - name: Install dependencies + if: steps.npm-cache.outputs.cache-hit != 'true' run: npm ci working-directory: frontend @@ -58,52 +77,56 @@ jobs: lint: name: Python lint - runs-on: py3.12-node22 + runs-on: ci-runner steps: - uses: actions/checkout@v6 - - name: Install ruff - run: pip install --break-system-packages ruff - + # ruff is pre-installed in the ci-runner base image — no install + # step needed, lint runs in ~2s. - name: Lint run: ruff check src/ test: name: Python tests - runs-on: py3.12-node22 + runs-on: ci-runner steps: - uses: actions/checkout@v6 - # Python 3.12 is pre-installed in the runner-base image (py3.12-node22). - - name: Cache pip wheels - uses: actions/cache@v3 + - name: Cache uv packages + uses: actions/cache@v4 with: - path: ~/.cache/pip - key: pip-${{ hashFiles('pyproject.toml') }} - restore-keys: pip- + path: ~/.cache/uv + key: uv-${{ hashFiles('pyproject.toml') }} + restore-keys: uv- - name: Create virtual environment - run: python3.12 -m venv /opt/venv + run: uv venv /opt/venv - name: Install package with dev deps run: | - /opt/venv/bin/pip install --upgrade pip setuptools wheel - /opt/venv/bin/pip install --no-build-isolation http-ece - /opt/venv/bin/pip install -e ".[dev]" + # http-ece doesn't declare setuptools as a build dep, and uv + # creates bare venvs without it. Install setuptools first so + # --no-build-isolation can find it. + uv pip install --python /opt/venv/bin/python setuptools wheel + uv pip install --python /opt/venv/bin/python --no-build-isolation http-ece + uv pip install --python /opt/venv/bin/python -e ".[dev]" - name: Run tests - run: /opt/venv/bin/python -m pytest tests/ -v + run: /opt/venv/bin/python -m pytest tests/ -q build: name: Build & push image needs: [typecheck, lint, test] # Build on dev branch pushes and version tag pushes only. - # main branch pushes run CI for safety but do not build — + # main branch pushes run the gates for safety but do not build — # the release tag (v*) is the sole trigger for a production image. if: | github.event_name == 'push' && (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) - runs-on: py3.12-node22 + runs-on: ci-runner + permissions: + contents: read + packages: write steps: - uses: actions/checkout@v6 @@ -122,11 +145,14 @@ jobs: echo "build_version=$BUILD_VERSION" >> $GITHUB_OUTPUT - name: Free disk space + # Self-hosted runner housekeeping. Two-step cleanup: + # 1. Prune dangling containers/images globally (stops the runner + # from accumulating cruft from past failed builds). + # 2. Trim the BuildKit layer cache to a 5GB ceiling so the pip + # mount cache survives but old intermediate layers don't + # accumulate indefinitely. run: | - # Remove all unused images (including old :SHA tags) and containers. docker system prune -af || true - # Keep the local BuildKit cache bounded so pip mount cache survives - # but stale intermediate layers don't accumulate indefinitely. docker builder prune --keep-storage 5g -f || true - name: Set up Docker Buildx @@ -147,3 +173,10 @@ jobs: provenance: false tags: ${{ steps.tags.outputs.value }} build-args: BUILD_VERSION=${{ steps.tags.outputs.build_version }} + # Registry-backed layer cache. Pull from :cache to prime + # BuildKit, push updated layers back to :cache so the next + # build starts warm even if the runner's local cache was + # pruned. `mode=max` exports all intermediate layers, not + # just the final image, which is what gives the ~80% speedup. + cache-from: type=registry,ref=${{ env.IMAGE }}:cache + cache-to: type=registry,ref=${{ env.IMAGE }}:cache,mode=max,ignore-error=true diff --git a/docs/2026-04-10-agentic-briefing-design.md b/docs/2026-04-10-agentic-briefing-design.md new file mode 100644 index 0000000..2614389 --- /dev/null +++ b/docs/2026-04-10-agentic-briefing-design.md @@ -0,0 +1,216 @@ +# Agentic Briefing — Design Spec + +**Date:** 2026-04-10 +**Status:** Proposed +**Author:** bvandeusen + Claude + +--- + +## Problem + +The current briefing pipeline hallucinates calendar events, tasks, and news items that do not exist in the user's actual data. Observed examples from production: + +- A morning briefing asserting "your dentist appointment is still in progress" when no such event existed +- An 8am check-in mentioning "a quick meeting at 10:30 AM" with no backing calendar entry +- A midday check-in inventing "a team huddle at 2:30 PM" and "the Q2 budget draft due by Friday" +- The model calling `search_images` in response to a clarifying question about a fabricated meeting + +These are not bugs in data retrieval — the data gathering code (`_gather_internal`, `_gather_external`) returns correct values. They are a structural consequence of how the briefing context is assembled. + +## Root cause — the "no receipt" problem + +The current `run_compilation` pipeline does this: + +1. Python code gathers data (tasks, events, weather, news) from the database +2. Python formats the data into a structured text blob as the user-role message: `TODAY'S EVENTS: ...`, `DUE TODAY: ...`, etc. +3. The LLM is called **once** with `[system_prompt, user_prompt]` and produces prose +4. Only the prose reply is written to the conversation. **The underlying data is never persisted in the conversation history.** + +When the user later chats in the briefing conversation, the chat endpoint loads the full conversation history — which contains the model's *prose* from earlier but not the data that prose was derived from. The model's own prior output becomes the only source of "truth" available for follow-up questions. If that prose asserted a fact (real or hallucinated), the model has no way to distinguish it from ground truth when generating the next reply, and it will double down. + +Compounding factors: + +- **Empty sections are silently omitted.** If `calendar_events` is empty, the user prompt contains no `TODAY'S EVENTS:` line at all. The model has no explicit "zero events" signal — combined with an imperative system prompt ("note calendar events and tasks"), it interprets the silence as "I should mention some" and fabricates. +- **Scheduled slot injections append synthetic turns.** `run_slot_injection` writes a fake `[Midday briefing update]` user message and the assistant reply into the persistent conversation. By evening, the chat history contains three separate briefings, each potentially with errors, all treated as equal-weight context on follow-up. +- **The `search_images` tool is available during briefing chat**, with a negative-instruction description ("Not for factual questions"). Small and mid-sized models frequently ignore negative guidance in tool descriptions and call the tool anyway. + +## Solution — agentic briefing (the receipt model) + +Replace the one-shot synthesis with a tool-call loop. The briefing is no longer "a text blob synthesized from pre-gathered data." It is **a scheduled agent run**: the LLM is given a system prompt, a curated set of read-only data tools, and a trigger ("generate the morning briefing"). The model must call tools to see what exists. Every tool call and tool result becomes part of the conversation history, where it lives as a permanent, structured receipt. + +### Why this fixes the hallucination + +When the model calls `list_events(today) → []`, that empty array is now a persistent message in the conversation. On a follow-up question like "what meeting?", the chat endpoint loads the conversation and the model sees its own tool result from the morning showing no events. Answering "you had a meeting at 10:30" would require the model to directly contradict a tool result sitting two messages back — something LLMs are much more reliable at avoiding than contradicting their own prior prose. + +In short: **the model cannot fabricate what has a visible receipt proving it does not exist.** + +### What the model sees — before vs after + +**Before** — the model sees one structured blob and produces prose. The blob is discarded. + +``` +[system] You are a personal assistant... weave together what matters... +[user] Date: 2026-04-10 + WEATHER: Home — partly cloudy, 8–16°C + DUE TODAY: Fix briefing hallucinations + OVERDUE (2 tasks): ... +[assistant] Morning! Looks like a quiet day with two overdue items... +``` + +**After** — the model is given tools and iterates to ground truth. + +``` +[system] You are the user's personal assistant giving their morning briefing. + Use tools to see what's relevant. Only mention things you learned from + tool calls. If a tool returns nothing, say so honestly. +[user] Generate the morning briefing. +[assistant] tool_call: list_tasks(filter="due_today") +[tool] [{"id": 81, "title": "Fix briefing hallucinations", ...}] +[assistant] tool_call: list_tasks(filter="overdue") +[tool] [{"id": 42, ...}, {"id": 43, ...}] +[assistant] tool_call: list_events(today) +[tool] [] +[assistant] tool_call: get_weather(location="home") +[tool] {"description": "partly cloudy", "temp_min": 8, "temp_max": 16} +[assistant] tool_call: get_rss_items(max=8) +[tool] [...] +[assistant] Morning! Partly cloudy today, 8 to 16 — nothing on the calendar + so it's a clean run at the desk. Two things to keep in mind... +``` + +The conversation now contains verifiable receipts: `list_events(today)` returned `[]`, and that result sits in context forever (until pruned). Follow-up questions operate against those receipts. + +## Architecture + +### Existing infrastructure (reused, not rebuilt) + +The codebase already has the agentic primitives — they're used for regular chat: + +- **`llm.py::stream_chat_with_tools`** — the streaming tool-use loop that talks to Ollama with a `tools` parameter and yields tool-call chunks +- **`generation_task.py::_stream_with_retry`** — wraps `stream_chat_with_tools` with retry-on-500 behavior for cold-model races +- **`tools.py`** — defines 40+ tool schemas and an execution dispatcher + +The briefing bypasses all of this and calls a one-shot `_llm_synthesise` helper. The refactor is mainly "route briefings through the same pipeline regular chat already uses." + +### New modules + +**`briefing_tools.py`** — a small wrapper exposing a curated read-only subset of `tools.py` for briefing runs. This is an **explicit allowlist**, not a blocklist, so newly-added tools must be opted in: + +| Tool | Purpose | +|---|---| +| `list_tasks` (with filter args) | See what's actionable today, overdue, high priority | +| `list_events` (today / upcoming) | Know what's on the calendar | +| `get_weather` | Current/forecast weather | +| `get_rss_items` | Pull news themes filtered by user preferences | +| `list_projects` | Understand project context | +| `search_projects` | Surface active project summaries | +| `list_notes` (recent) | Capture follow-ups from yesterday | + +**Explicitly omitted** from the briefing tool set: + +- `search_images`, `search_web`, `research_topic`, `read_article` — external search is not a briefing concern, and `search_images` is the source of the "Peter Kyle Science Secretary" image-search bug +- All `create_*`, `update_*`, `delete_*` — briefings are read-only; a scheduled background job must not decide to mutate the user's data on its own +- `set_rag_scope`, `calculate` — not relevant to briefing content + +### New briefing function + +**`briefing_pipeline.py::run_agentic_briefing(user_id, slot, model, conversation_id)`** replaces `run_compilation`'s body (and eventually `run_slot_injection`). Internally: + +1. Build a slot-specific system prompt (see below) +2. Load the curated briefing tools from `briefing_tools.py` +3. Seed messages with `[system, user]` where the user message is a simple trigger like `"Generate the morning briefing."` +4. Enter a tool-call loop (max 8 iterations): + - Call `stream_chat_with_tools` + - If the model returns `tool_calls`, execute them via the existing dispatcher, append tool results, continue + - If the model returns a final assistant message with no pending tool calls, break +5. Return `(final_prose, full_message_list, metadata)` + +The full message list is important: it's written to the conversation along with the final prose, so the tool-call receipts become part of the persistent record. + +### Slot-specific system prompts + +Compilation (full morning briefing): + +``` +You are the user's personal assistant giving their full morning briefing. +Use the tools available to see what's actually relevant today — tasks due, +overdue items, events on the calendar, weather, news themes, project state +— and weave it into a warm, natural-sounding summary. + +Rules: +- Call tools to see the data. Never assert facts you didn't learn from a tool. +- If a tool returns nothing (no events today, no overdue tasks), say so + honestly. Don't fabricate items to fill space. +- Write flowing prose. No markdown, no headers, no bullet points. +- Aim for 6–10 sentences. Skip topics that have nothing interesting. +- Close on one or two concrete, actionable suggestions. + +User profile (for tone and preferences): +{profile_body} +``` + +Check-ins (midday, afternoon): + +``` +You are the user's personal assistant giving a brief {slot} check-in. +Use tools to see what's changed since this morning. Focus on progress +and what's still unaddressed. + +Rules: +- Call tools to see current state. Never assert facts without tool results. +- If nothing meaningful has changed, say so briefly — don't invent progress. +- 3–5 sentences, natural prose, no markdown. +``` + +### Conversation hygiene — removing the fake user messages + +The current scheduler appends two fake messages on every slot injection: + +```python +await post_message(conv.id, "user", f"[{slot.title()} briefing update]") +await post_message(conv.id, "assistant", text) +``` + +Under the agentic model, we drop the fake `"user"` message entirely. Slot updates become plain assistant messages tagged with `metadata.briefing_slot = "midday"`. The chat endpoint's message loader is updated to filter these when building the LLM context on follow-ups, so a user chatting in the briefing conversation doesn't see three earlier briefings smashed into their history. The UI continues to show them as visible timeline entries. + +(This is the Option A filter from the earlier discussion — a small, surgical change compared to migrating briefings to a separate sidecar table. Sidecar storage remains a possible future step.) + +### Ollama setup compatibility + +The existing Ollama deployment uses non-parallel mode across two GPUs, with a concern about context duplication between cards. This is the correct setup for agentic briefings: + +- Each tool-call iteration shares the same KV cache on the same GPU, so appending tool results is cheap +- There is no race where a second iteration could land on a different card with a different cache state +- The trade-off is serialization: a briefing in progress will block a concurrent user chat request until it finishes, but scheduled briefings are rare enough (4× per day) that this is acceptable + +If GPU contention becomes a problem later, the right lever is pinning specific models to specific GPUs (e.g., background tasks on GPU 1, interactive models on GPU 0) — not enabling parallelism. + +## Cost & trade-offs + +- **Latency:** a briefing now makes 5–7 inference calls (one per tool-call decision plus the final prose) instead of 1. On a local Ollama with a 7B model, expect 15–40s per briefing vs the current 5–10s. Acceptable for a background job; if it becomes painful at the user-facing check-in slots, investigate letting the model batch independent tool calls in a single turn (Ollama supports this). +- **Model requirements:** the default model must be reliable at tool calling. `qwen2.5:7b`, `llama3.1:8b`, `mistral-small-3:24b`, and similar handle it well. Models in the ≤3B class typically fail — they either emit no tool calls and return empty prose, or hallucinate invalid tool arguments. If a user's `default_model` is too small, agentic mode should fall back to legacy mode with a log warning. +- **Context growth:** tool results bloat the conversation. At 8 tool calls per compilation × 4 slots per day, a daily briefing conversation can reach 20-30 KB of JSON-heavy history. Fine for a day; aged briefings should be archived/rolled up after 7 days. +- **Tool subset drift:** someone adds a new mutating tool to `tools.py` and forgets to update the briefing allowlist. Mitigated by the allowlist model — the default for new tools is "not exposed to briefings." +- **Infinite loop safety:** a buggy model could tool-call forever. Hard cap at 8 iterations, log a warning if hit, return whatever prose was last produced (or a fallback message). + +## Migration path + +Ship incrementally, each step independently reversible: + +**PR 1 — Agentic compilation behind a feature flag.** Add `briefing_tools.py`, add `run_agentic_briefing`, add a per-user setting `briefing_mode: "legacy" | "agentic"` (default legacy). Route only the 4am compilation through the new path when the flag is set. Keep slot-injection on the legacy path. Enable the flag on the author's account first, validate output quality over several days, then flip the default for all users. No DB migration required — the setting lives in the existing `settings` table. + +**PR 2 — Agentic slot injections + conversation hygiene.** Migrate midday/afternoon check-ins to the same pipeline. Remove the fake `[Midday briefing update]` user-role message; slot updates become plain assistant messages tagged with `metadata.briefing_slot`. Add a chat-message-loader filter that excludes slot-tagged messages from the LLM context on follow-ups (they remain visible in the UI). + +**PR 3 — UI polish.** Collapsed tool-call status row in the briefing card ("✓ checked calendar · ✓ looked at tasks · ✓ pulled weather"), expanding to show tool results on click. Tool-result cards (weather, news, task list) rendered inline where useful. + +**PR 4 (optional) — Sidecar storage for briefing snapshots.** If the chat-filter approach in PR 2 feels too hacky, migrate briefings out of the conversations table and into a dedicated `briefing_snapshots` table. Frontend renders them as pinned timeline cards separate from chat. Larger refactor; defer until PR 1–3 prove the approach works. + +## Secondary win — tightening the main chat system prompt + +The regular chat is already agentic (it uses `stream_chat_with_tools`), but its system prompt does not explicitly require the model to ground factual claims in tool results. The prompt discipline introduced for briefings — *"Never assert facts you didn't learn from a tool. If a tool returns nothing, say so honestly."* — is worth applying to the main chat system prompt in a follow-up PR. The mechanism already works; only the framing needs tightening. + +## Out of scope + +- The Android Flutter client's failure to render `search_images` tool-result cards. This is a separate rendering gap in the mobile app and does not affect the server-side fix. Tracked separately. +- Re-evaluating the Ollama parallelism setting. The current non-parallel config is correct for this work. +- Replacing the background model for title/summary/observation extraction. That model's role is unrelated to briefings. diff --git a/docs/architecture.md b/docs/architecture.md index 91ab316..9069ee9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -212,8 +212,8 @@ Permission resolution is centralised in `services/access.py`. `get_project_permi | `services/api_keys.py` | `generate_key()`, `create_api_key()`, `list_api_keys()`, `revoke_api_key()`, `lookup_key()` (SHA-256 hash lookup) | | `services/llm.py` | `build_context()`, RAG injection, history summarisation, `stream_chat_with_tools()`, URL fetching, SSRF guard | | `services/generation_task.py` | `run_generation()` — full chat pipeline: intent routing, tool loop, SSE fan-out, push notification; `run_assist_generation()` | -| `services/intent.py` | `classify_intent()` — fast non-streaming LLM call; intent skip heuristic; `_PRIOR_WORK_REFS` fast-path | -| `services/tools.py` | All LLM tool definitions + `execute_tool(user_id, tool_name, arguments, conv_id=None, workspace_project_id=None)` dispatcher; duplicate guards; `_resolve_project()` 4-step lookup; `search_projects` and `set_rag_scope` tools | +| ~~`services/intent.py`~~ | Removed — intent routing eliminated; the main model handles all tool routing directly | +| `services/tools/` | LLM tool package — decorator-based registry (`_registry.py`), shared helpers (`_helpers.py`), and one module per domain: `notes.py` (create/update/delete/search/list/read), `tasks.py` (list/log_work), `entities.py` (save_person/save_place/lists), `projects.py`, `calendar.py`, `web.py`, `rag.py`, `profile.py`, `rss.py`, `weather.py`, `utility.py` (calculate). `execute_tool()` and `get_tools_for_user()` are the public API. | | `services/projects.py` | Project CRUD + `generate_project_summary()` (Ollama, fire-and-forget) + `backfill_project_summaries()` (startup) | | `services/embeddings.py` | `upsert_note_embedding()`, `semantic_search_notes(orphan_only=False)` (pgvector cosine similarity) | | `services/generation_buffer.py` | In-memory SSE event buffer; `cancel_event`; 60s cleanup; supports both chat (int keys) and assist (string keys) | @@ -231,7 +231,7 @@ Permission resolution is centralised in `services/access.py`. `get_project_permi | `services/briefing_scheduler.py` | APScheduler `BackgroundScheduler`; slots with catch-up logic; async-safe via `asyncio.create_task` | | `services/briefing_conversations.py` | Briefing conversation persistence and history queries | | `services/briefing_profile.py` | Per-user profile note that the assistant updates over time | -| `services/research.py` | SearXNG research pipeline: 5 sub-queries → parallel fetch → synthesis; `search_images` for image category | +| `services/research.py` | SearXNG research pipeline: sub-queries → parallel fetch → outline → section synthesis → executive summary → index note with linked section notes | | `services/events.py` | Internal events CRUD: `list_events`, `create_event`, `update_event`, `delete_event`, `get_event`; source of truth for all event LLM tools | | `routes/events.py` | `/api/events` — event CRUD routes | | `services/caldav.py` | Optional CalDAV sync — user-configured external server; syncs to/from internal store via `caldav_uid` FK; `is_caldav_configured()` guards tool activation | @@ -291,7 +291,7 @@ Permission resolution is centralised in `services/access.py`. `get_project_permi | `services/access.py` | Permission resolution for all shared resources | | `services/llm.py` | `build_context()`, RAG injection, history summarisation | | `services/generation_task.py` | SSE streaming, tool-call loop, GenerationBuffer management | -| `services/tools.py` | All LLM tool implementations (`create_note`, `search_notes`, `get_weather`, …) | +| `services/tools/` | LLM tool implementations (38 tools across 11 modules); decorator-based registry | | `services/embeddings.py` | `upsert_note_embedding()`, `semantic_search_notes()` | | `services/briefing_pipeline.py` | Two-lane parallel gather → LLM synthesis → briefing output | | `services/briefing_scheduler.py` | APScheduler integration, catch-up logic for missed slots | @@ -311,26 +311,22 @@ See [sso-oauth.md](sso-oauth.md) for provider-specific setup instructions. ## LLM Pipeline Internals -### Intent Routing +### Tool Routing -Before the main model runs, a lightweight intent classifier (`services/intent.py`) runs concurrently with `build_context()`. It makes a fast non-streaming call using a smaller dedicated model (`OLLAMA_INTENT_MODEL`, default `qwen2.5:7b`) to determine if the message requires a tool call. - -**Skip heuristic** — Intent classification is skipped entirely for short messages (≤10 words) with no action/object keywords, saving 400–800ms on conversational replies. - -**Prior-work fast-path** — `_PRIOR_WORK_REFS` regex detects phrases like "research you did", "note you made", "using your research" and returns no-tool immediately, preventing `search_web` from firing when the user references existing notes. - -If a tool is detected, the intent's one-sentence `ack` field is streamed as the first chunk (TTFT), the tool executes, then the main model generates a follow-up with the tool result. For chat-only responses the main model streams directly. +No separate intent router — the main model handles all tool routing directly via Ollama's structured tool-calling output. The model receives the full tool schema list and decides whether to call a tool or respond conversationally. A thinking-mode heuristic (`_should_think()`) detects complex prompts and enables extended reasoning. ### Tool Loop -Multi-round tool loop (max 5 rounds). All implementations in `services/tools.py`; `execute_tool(user_id, tool_name, arguments, conv_id=None, workspace_project_id=None)` is the dispatcher. `conv_id` and `workspace_project_id` are threaded in from `run_generation()` so tools like `set_rag_scope` can write to the current conversation. +Multi-round tool loop (max 5 rounds). All implementations in `services/tools/` (decorator-based registry); `execute_tool(user_id, tool_name, arguments, conv_id=None, workspace_project_id=None)` is the dispatcher. `conv_id` and `workspace_project_id` are threaded in from `run_generation()` so tools like `set_rag_scope` can write to the current conversation. -**Duplicate protection on `create_note` / `create_task`:** +**Unified `create_note` tool** — creates both notes and tasks. Setting `status` (e.g. `"todo"`) creates a task; omitting it creates a knowledge note. All task fields (due_date, priority, milestone, parent_task, recurrence_rule) are available on the single tool. + +**Duplicate protection on `create_note`:** 1. Exact title match (case-insensitive) → hard block, redirect to `update_note` 2. Fuzzy title match (SequenceMatcher ≥ 82%; punctuation stripped before candidate search) → hard block -3. Semantic content similarity (threshold 0.90, body ≥ 200 chars) → soft block with `requires_confirmation: true` +3. Semantic content similarity (threshold 0.90, body ≥ 80 chars) → soft block with `requires_confirmation: true` -**Project resolution** (`_resolve_project`): 4-step lookup — (1) exact DB match, (2) `query in title` substring, (3) `title in query` reverse substring, (4) SequenceMatcher ≥ 0.55. +**Project resolution** (`_helpers.resolve_project`): 4-step lookup — (1) exact DB match, (2) `query in title` substring, (3) `title in query` reverse substring, (4) SequenceMatcher ≥ 0.55. ### Context Window and Summarisation @@ -344,8 +340,11 @@ History summarisation threshold: 30 messages. Keeps 8 recent messages. Summary m 1. Intent model generates 5 focused sub-queries 2. All 5 SearXNG queries run in parallel (200ms stagger to avoid rate limiter) 3. Up to 15 unique URLs fetched in parallel -4. Up to 12 sources passed to synthesis LLM -5. Result saved as a note with `tags=["research"]` +4. LLM generates an outline (2–8 sections with title + focus) +5. Each section synthesised in parallel from relevant sources +6. Executive summary generated from all section content +7. Index note created with executive summary + links to section notes; section notes linked back via `parent_id` +8. Falls back to single-note synthesis if outline generation fails SearXNG tip: add the app server IP to `botdetection.ip_lists.pass_ip` in SearXNG `settings.yml` to bypass the rate limiter for trusted backend requests. diff --git a/docs/features.md b/docs/features.md index 6c1ad9a..e9bbec4 100644 --- a/docs/features.md +++ b/docs/features.md @@ -68,7 +68,7 @@ Full conversation history with SSE streaming. Features: ## Web Research -The assistant can search the web (SearXNG) and fetch pages, synthesising findings into notes. A lightweight `search_web` tool answers quick questions inline without saving. Requires `SEARXNG_URL` to be configured. +The assistant can search the web (SearXNG) and fetch pages, synthesising findings into a structured multi-note research output: an index note with an executive summary and links to focused section notes. Each section covers a distinct aspect of the topic with cited sources. Falls back to a single note when outline generation fails. A lightweight `search_web` tool answers quick questions inline without saving. Requires `SEARXNG_URL` to be configured. ## Calendar diff --git a/fable-mcp/fable_mcp/server.py b/fable-mcp/fable_mcp/server.py index ca46da9..b7c0ff7 100644 --- a/fable-mcp/fable_mcp/server.py +++ b/fable-mcp/fable_mcp/server.py @@ -27,7 +27,7 @@ The hierarchy is: Project → Milestone → Task/Note. - **Tasks** belong to a project and optionally a milestone. They support sub-tasks via `parent_id`. - Status values: `todo`, `in_progress`, `done`, `cancelled` - - Priority values: `low`, `normal`, `high` + - Priority values: `none` (default), `low`, `medium`, `high` - **Notes** are free-form markdown documents. They can belong to a project or be standalone (orphan notes). Orphan notes are included in the default RAG scope for chat conversations. @@ -241,7 +241,7 @@ async def fable_create_task( title: Task title (required). body: Markdown description / notes for the task. status: Initial status — one of: todo (default), in_progress, done, cancelled. - priority: One of: low, normal, high. Omit for no priority. + priority: One of: low, medium, high. Omit for no priority (defaults to "none"). project_id: Associate with a project (0 = no project). milestone_id: Place within a project milestone (0 = no milestone). parent_id: Make this a sub-task of another task (0 = top-level). @@ -280,7 +280,7 @@ async def fable_update_task( title: New title, or omit to leave unchanged. body: New markdown body, or omit to leave unchanged. status: New status — one of: todo, in_progress, done, cancelled. - priority: New priority — one of: low, normal, high. + priority: New priority — one of: none, low, medium, high. project_id: New project (0 = remove from project). Omit to leave unchanged. milestone_id: New milestone (0 = remove from milestone). Omit to leave unchanged. """ @@ -628,6 +628,128 @@ async def fable_remove_rss_feed(feed_id: int) -> dict: return await briefing.remove_rss_feed(client, feed_id=feed_id) +# --------------------------------------------------------------------------- +# Briefing introspection & control +# --------------------------------------------------------------------------- + + +@mcp.tool() +async def fable_list_briefings() -> dict: + """List the user's briefing conversations, newest first. + + Each briefing has an associated date and conversation id. Use + ``fable_get_briefing_messages`` or ``fable_get_conversation`` with + the id to pull the actual briefing text and tool-call receipts. + + Returns a dict with a ``conversations`` list. + """ + async with FableClient() as client: + return await briefing.list_briefing_conversations(client) + + +@mcp.tool() +async def fable_get_today_briefing() -> dict: + """Fetch today's briefing conversation, creating it if needed. + + Returns the full conversation object including all messages — the + scheduled briefing assistant turns, any tool calls the agentic path + made, and any chat replies the user has sent in the briefing thread. + + Use this to inspect what a briefing actually said (and what tool + results grounded it) without having to query by id. + """ + async with FableClient() as client: + return await briefing.get_today_briefing(client) + + +@mcp.tool() +async def fable_get_briefing_messages(conversation_id: int) -> dict: + """Fetch all messages for a specific briefing conversation. + + Args: + conversation_id: The briefing conversation id from fable_list_briefings. + + Returns a dict with a ``messages`` list. Each message includes + role, content, tool_calls (with results), and metadata — the + metadata carries ``briefing_slot`` tags on agentic briefing turns. + """ + async with FableClient() as client: + return await briefing.get_briefing_messages( + client, conversation_id=conversation_id, + ) + + +@mcp.tool() +async def fable_trigger_briefing(slot: str = "compilation") -> dict: + """Manually run a briefing slot for the current user. + + Fires the same data refresh the scheduler does (RSS, weather), + runs the agentic briefing pipeline, and writes the result into + today's briefing conversation. Use this to test prompt changes + without waiting for the next scheduled slot. + + Args: + slot: One of ``compilation`` (full morning, default), ``morning``, + ``midday``, or ``afternoon``. + + Returns a dict with ``conversation_id``, ``message_id``, and ``slot``. + """ + async with FableClient() as client: + return await briefing.trigger_briefing(client, slot=slot) + + +@mcp.tool() +async def fable_reset_today_briefing(run_compilation: bool = True) -> dict: + """Wipe today's briefing and (optionally) regenerate from scratch. + + Deletes every message in today's briefing conversation — the + conversation row itself is kept so its id stays stable for any + open UI sessions. If ``run_compilation`` is True (the default), + immediately fires the compilation slot afterward so a fresh + briefing lands in place of the deleted content. + + Use this when iterating on briefing prompts or tools and you want + to start from a clean slate rather than append another slot update + on top of stale output. + + Args: + run_compilation: When True, fire ``POST /api/briefing/trigger`` + for the ``compilation`` slot immediately after wiping. + Set False to only wipe without regenerating. + + Returns a dict with ``reset`` (the delete result: deleted count + + conversation id) and ``triggered`` (the new message payload, or + null if regeneration was skipped). + """ + async with FableClient() as client: + return await briefing.reset_today_briefing( + client, run_compilation=run_compilation, + ) + + +# --------------------------------------------------------------------------- +# Generic conversation access +# --------------------------------------------------------------------------- + + +@mcp.tool() +async def fable_get_conversation(conversation_id: int) -> dict: + """Fetch any conversation (chat or briefing) with its full message list. + + Returns conversation metadata plus an ordered ``messages`` array. + Each message includes role, content, tool_calls (with results), + context_note_id, and msg_metadata. Tool calls are in the stored + flat format: ``[{"function": name, "arguments": {...}, "result": {...}}]``. + + Useful for debugging agentic briefings, inspecting chat history, + or verifying that a tool actually ran with the expected arguments. + """ + async with FableClient() as client: + return await briefing.get_conversation( + client, conversation_id=conversation_id, + ) + + # --------------------------------------------------------------------------- # Entry point # --------------------------------------------------------------------------- diff --git a/fable-mcp/fable_mcp/tools/briefing.py b/fable-mcp/fable_mcp/tools/briefing.py index 6c21c7e..f8d872a 100644 --- a/fable-mcp/fable_mcp/tools/briefing.py +++ b/fable-mcp/fable_mcp/tools/briefing.py @@ -1,4 +1,4 @@ -"""MCP tools for Fable RSS feed management.""" +"""MCP tools for Fable briefings and RSS feed management.""" from __future__ import annotations from typing import Any @@ -6,6 +6,8 @@ from typing import Any from fable_mcp.client import FableClient +# ── RSS feeds ──────────────────────────────────────────────────────────────── + async def list_rss_feeds(client: FableClient) -> dict[str, Any]: """List the user's RSS feeds.""" return await client.get("/api/briefing/feeds") @@ -30,3 +32,67 @@ async def add_rss_feed( async def remove_rss_feed(client: FableClient, *, feed_id: int) -> dict[str, Any]: """Remove an RSS feed by ID.""" return await client.delete(f"/api/briefing/feeds/{feed_id}") + + +# ── Briefings ──────────────────────────────────────────────────────────────── + +async def list_briefing_conversations(client: FableClient) -> dict[str, Any]: + """List the user's briefing conversations, newest first.""" + return await client.get("/api/briefing/conversations") + + +async def get_today_briefing(client: FableClient) -> dict[str, Any]: + """Fetch today's briefing conversation with all messages.""" + return await client.get("/api/briefing/conversations/today") + + +async def get_briefing_messages( + client: FableClient, + *, + conversation_id: int, +) -> dict[str, Any]: + """Fetch messages for a specific briefing conversation.""" + return await client.get(f"/api/briefing/conversations/{conversation_id}/messages") + + +async def trigger_briefing( + client: FableClient, + *, + slot: str = "compilation", +) -> dict[str, Any]: + """Manually trigger a briefing slot — fires data refresh and runs the pipeline. + + Slot is one of: compilation (full morning), morning, midday, afternoon. + """ + return await client.post("/api/briefing/trigger", json={"slot": slot}) + + +async def reset_today_briefing( + client: FableClient, + *, + run_compilation: bool = True, +) -> dict[str, Any]: + """Delete all messages in today's briefing and optionally regenerate. + + Wipes the messages in today's briefing conversation (keeping the + conversation row), then, if ``run_compilation`` is True, fires the + compilation slot so a fresh briefing lands in its place. + """ + reset = await client.post("/api/briefing/reset-today") + if not run_compilation: + return {"reset": reset, "triggered": None} + triggered = await client.post( + "/api/briefing/trigger", json={"slot": "compilation"} + ) + return {"reset": reset, "triggered": triggered} + + +# ── Generic conversations ─────────────────────────────────────────────────── + +async def get_conversation( + client: FableClient, + *, + conversation_id: int, +) -> dict[str, Any]: + """Fetch any conversation (chat or briefing) with all messages and tool calls.""" + return await client.get(f"/api/chat/conversations/{conversation_id}") diff --git a/fable-mcp/pyproject.toml b/fable-mcp/pyproject.toml index 0781762..ba6f5a3 100644 --- a/fable-mcp/pyproject.toml +++ b/fable-mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "fable-mcp" -version = "0.2.4" +version = "0.2.6" description = "MCP server for Fabled Assistant" requires-python = ">=3.12" dependencies = [ diff --git a/frontend/src/components/BriefingToolStatusRow.vue b/frontend/src/components/BriefingToolStatusRow.vue new file mode 100644 index 0000000..7ba64c4 --- /dev/null +++ b/frontend/src/components/BriefingToolStatusRow.vue @@ -0,0 +1,168 @@ + + + + + diff --git a/frontend/src/components/ChatMessage.vue b/frontend/src/components/ChatMessage.vue index 14dc4d1..ebb6f67 100644 --- a/frontend/src/components/ChatMessage.vue +++ b/frontend/src/components/ChatMessage.vue @@ -3,8 +3,16 @@ import { computed } from "vue"; import { renderMarkdown } from "@/utils/markdown"; import { useSettingsStore } from "@/stores/settings"; import ToolCallCard from "@/components/ToolCallCard.vue"; +import BriefingToolStatusRow from "@/components/BriefingToolStatusRow.vue"; import type { Message } from "@/types/chat"; +const SLOT_LABELS: Record = { + compilation: "Full Briefing", + morning: "Morning Update", + midday: "Midday Update", + afternoon: "Afternoon Update", +}; + const settingsStore = useSettingsStore(); const props = defineProps<{ @@ -41,6 +49,30 @@ function formatMs(ms: number | null | undefined): string { return `${(ms / 1000).toFixed(1)}s`; } +const metadata = computed(() => (props.message.metadata ?? {}) as Record); + +const isBriefingIntermediate = computed( + () => props.message.role === "assistant" && metadata.value.briefing_intermediate === true, +); + +const briefingSlot = computed(() => { + const slot = metadata.value.briefing_slot; + return typeof slot === "string" ? slot : null; +}); + +const slotLabel = computed(() => { + const slot = briefingSlot.value; + if (!slot) return null; + return SLOT_LABELS[slot] ?? slot; +}); + +const isSlotUpdate = computed( + () => + !isBriefingIntermediate.value && + briefingSlot.value != null && + briefingSlot.value !== "compilation", +); + const timingParts = computed((): string[] => { const t = props.message.timing; if (!t) return []; @@ -57,11 +89,16 @@ const timingParts = computed((): string[] => {