feat(rag): RAG scoping and context isolation controls
- Migration 0030: add conversations.rag_project_id (NULL=orphan-only, -1=all notes, positive=project), projects.auto_summary and projects.summary_updated_at - Three-value scope semantics thread from build_context() → semantic search and keyword fallback via orphan_only + effective_project_id - Project summarization background job (generate_project_summary, backfill_project_summaries) called via Ollama; triggered on project update and note saves (debounced 1h); runs at startup - New LLM tools: search_projects (SequenceMatcher scoring on title+description+auto_summary) and set_rag_scope (persists to DB, workspace-guarded, emits new_rag_scope in SSE done event) - execute_tool() accepts conv_id + workspace_project_id; generation_task passes both and captures scope changes for SSE done enrichment - Frontend: Conversation type gets rag_project_id; chat store adds ragProjectId computed + updateRagScope(); SSE done handler syncs scope - ChatView: replace sidebar ProjectSelector with a scope chip pill above the input bar, animated dropdown, pulse on model-driven scope change Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,23 @@
|
|||||||
|
"""Add rag_project_id to conversations; auto_summary columns to projects."""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0030"
|
||||||
|
down_revision = "0029"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute("""
|
||||||
|
ALTER TABLE conversations
|
||||||
|
ADD COLUMN IF NOT EXISTS rag_project_id INTEGER DEFAULT NULL
|
||||||
|
""")
|
||||||
|
op.execute("""
|
||||||
|
ALTER TABLE projects
|
||||||
|
ADD COLUMN IF NOT EXISTS auto_summary TEXT DEFAULT NULL,
|
||||||
|
ADD COLUMN IF NOT EXISTS summary_updated_at TIMESTAMPTZ DEFAULT NULL
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute("ALTER TABLE conversations DROP COLUMN IF EXISTS rag_project_id")
|
||||||
|
op.execute("ALTER TABLE projects DROP COLUMN IF EXISTS auto_summary, DROP COLUMN IF EXISTS summary_updated_at")
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -75,6 +75,17 @@ export const useChatStore = defineStore("chat", () => {
|
|||||||
const streamingPendingTool = computed(() => convStreams.value[currentConversation.value?.id ?? 0]?.pendingTool ?? null);
|
const streamingPendingTool = computed(() => convStreams.value[currentConversation.value?.id ?? 0]?.pendingTool ?? null);
|
||||||
const lastContextMeta = computed(() => convStreams.value[currentConversation.value?.id ?? 0]?.contextMeta ?? null);
|
const lastContextMeta = computed(() => convStreams.value[currentConversation.value?.id ?? 0]?.contextMeta ?? null);
|
||||||
|
|
||||||
|
const ragProjectId = computed<number | null>(
|
||||||
|
() => currentConversation.value?.rag_project_id ?? null
|
||||||
|
);
|
||||||
|
|
||||||
|
async function updateRagScope(convId: number, ragProjectId: number | null): Promise<void> {
|
||||||
|
await apiPatch(`/api/chat/conversations/${convId}`, { rag_project_id: ragProjectId });
|
||||||
|
if (currentConversation.value?.id === convId) {
|
||||||
|
currentConversation.value.rag_project_id = ragProjectId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function isStreamingConv(id: number): boolean {
|
function isStreamingConv(id: number): boolean {
|
||||||
return convStreams.value[id]?.streaming ?? false;
|
return convStreams.value[id]?.streaming ?? false;
|
||||||
}
|
}
|
||||||
@@ -387,6 +398,10 @@ export const useChatStore = defineStore("chat", () => {
|
|||||||
};
|
};
|
||||||
if (currentConversation.value?.id === convId) {
|
if (currentConversation.value?.id === convId) {
|
||||||
currentConversation.value.messages.push(assistantMsg);
|
currentConversation.value.messages.push(assistantMsg);
|
||||||
|
// Update RAG scope if the model changed it mid-conversation
|
||||||
|
if (event.data.new_rag_scope !== undefined) {
|
||||||
|
currentConversation.value.rag_project_id = event.data.new_rag_scope as number | null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Update updated_at only — message_count was already incremented at send time
|
// Update updated_at only — message_count was already incremented at send time
|
||||||
const idx = conversations.value.findIndex((c) => c.id === convId);
|
const idx = conversations.value.findIndex((c) => c.id === convId);
|
||||||
@@ -594,6 +609,8 @@ export const useChatStore = defineStore("chat", () => {
|
|||||||
streamingStatus,
|
streamingStatus,
|
||||||
streamingPendingTool,
|
streamingPendingTool,
|
||||||
lastContextMeta,
|
lastContextMeta,
|
||||||
|
ragProjectId,
|
||||||
|
updateRagScope,
|
||||||
ollamaStatus,
|
ollamaStatus,
|
||||||
modelStatus,
|
modelStatus,
|
||||||
defaultModel,
|
defaultModel,
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ export interface Conversation {
|
|||||||
title: string;
|
title: string;
|
||||||
model: string;
|
model: string;
|
||||||
message_count: number;
|
message_count: number;
|
||||||
|
rag_project_id: number | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|||||||
+132
-25
@@ -9,7 +9,6 @@ import ChatMessage from "@/components/ChatMessage.vue";
|
|||||||
import ToolCallCard from "@/components/ToolCallCard.vue";
|
import ToolCallCard from "@/components/ToolCallCard.vue";
|
||||||
import ToolConfirmCard from "@/components/ToolConfirmCard.vue";
|
import ToolConfirmCard from "@/components/ToolConfirmCard.vue";
|
||||||
import type { Note } from "@/types/note";
|
import type { Note } from "@/types/note";
|
||||||
import ProjectSelector from "@/components/ProjectSelector.vue";
|
|
||||||
|
|
||||||
const route = useRoute();
|
const route = useRoute();
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
@@ -47,8 +46,34 @@ const autoInjectedNotes = ref<{ id: number; title: string; score?: number | null
|
|||||||
// Note IDs excluded from auto-injection on next message
|
// Note IDs excluded from auto-injection on next message
|
||||||
const excludedNoteIds = ref<number[]>([]);
|
const excludedNoteIds = ref<number[]>([]);
|
||||||
|
|
||||||
// Project scope for RAG — when set, semantic & keyword search is restricted to this project
|
// Scope chip state
|
||||||
const ragProjectId = ref<number | null>(null);
|
const scopeDropdownOpen = ref(false);
|
||||||
|
const projects = ref<{ id: number; title: string }[]>([]);
|
||||||
|
const scopePulse = ref(false);
|
||||||
|
|
||||||
|
const scopeLabel = computed(() => {
|
||||||
|
const id = store.ragProjectId;
|
||||||
|
if (id === -1) return "All notes";
|
||||||
|
if (id === null) return "Orphan notes";
|
||||||
|
return projects.value.find((p) => p.id === id)?.title ?? `Project ${id}`;
|
||||||
|
});
|
||||||
|
|
||||||
|
async function loadProjects() {
|
||||||
|
try {
|
||||||
|
const data = await apiGet<{ projects: { id: number; title: string }[] }>("/api/projects?status=active");
|
||||||
|
projects.value = data.projects ?? [];
|
||||||
|
} catch {
|
||||||
|
projects.value = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onScopeSelect(value: number | null) {
|
||||||
|
scopeDropdownOpen.value = false;
|
||||||
|
if (!convId.value) return;
|
||||||
|
await store.updateRagScope(convId.value, value);
|
||||||
|
scopePulse.value = true;
|
||||||
|
setTimeout(() => { scopePulse.value = false; }, 600);
|
||||||
|
}
|
||||||
|
|
||||||
let prevConvId: number | null = null;
|
let prevConvId: number | null = null;
|
||||||
|
|
||||||
@@ -120,7 +145,7 @@ const inputPlaceholder = computed(() => {
|
|||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
document.addEventListener("keydown", onGlobalKeydown);
|
document.addEventListener("keydown", onGlobalKeydown);
|
||||||
await store.fetchConversations();
|
await Promise.all([store.fetchConversations(), loadProjects()]);
|
||||||
if (convId.value) {
|
if (convId.value) {
|
||||||
if (store.currentConversation?.id !== convId.value) {
|
if (store.currentConversation?.id !== convId.value) {
|
||||||
await store.fetchConversation(convId.value);
|
await store.fetchConversation(convId.value);
|
||||||
@@ -305,7 +330,7 @@ async function sendMessage() {
|
|||||||
true, // enable thinking in the full chat view
|
true, // enable thinking in the full chat view
|
||||||
undefined,
|
undefined,
|
||||||
excludedNoteIds.value.length ? excludedNoteIds.value : undefined,
|
excludedNoteIds.value.length ? excludedNoteIds.value : undefined,
|
||||||
ragProjectId.value,
|
store.ragProjectId,
|
||||||
);
|
);
|
||||||
sending.value = false;
|
sending.value = false;
|
||||||
|
|
||||||
@@ -520,13 +545,6 @@ onUnmounted(() => {
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- RAG project scope -->
|
|
||||||
<div class="rag-scope-section">
|
|
||||||
<label class="rag-scope-label">Scope notes to project</label>
|
|
||||||
<ProjectSelector v-model="ragProjectId" />
|
|
||||||
<p v-if="ragProjectId" class="rag-scope-hint">RAG search restricted to this project</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="conv-list">
|
<div class="conv-list">
|
||||||
<template v-for="group in groupedConversations" :key="group.label">
|
<template v-for="group in groupedConversations" :key="group.label">
|
||||||
<div class="conv-group-label">{{ group.label }}</div>
|
<div class="conv-group-label">{{ group.label }}</div>
|
||||||
@@ -726,6 +744,39 @@ onUnmounted(() => {
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="input-wrapper">
|
<div class="input-wrapper">
|
||||||
|
<!-- Scope chip above input -->
|
||||||
|
<div class="scope-chip-row">
|
||||||
|
<div class="scope-chip-wrapper">
|
||||||
|
<button
|
||||||
|
class="scope-chip"
|
||||||
|
:class="{ pulse: scopePulse }"
|
||||||
|
@click="scopeDropdownOpen = !scopeDropdownOpen"
|
||||||
|
title="Change RAG scope"
|
||||||
|
>
|
||||||
|
<span class="scope-dot">⊙</span> {{ scopeLabel }}
|
||||||
|
</button>
|
||||||
|
<div v-if="scopeDropdownOpen" class="scope-dropdown">
|
||||||
|
<button
|
||||||
|
class="scope-option"
|
||||||
|
:class="{ active: store.ragProjectId === null }"
|
||||||
|
@click="onScopeSelect(null)"
|
||||||
|
>Orphan notes only</button>
|
||||||
|
<button
|
||||||
|
v-for="p in projects"
|
||||||
|
:key="p.id"
|
||||||
|
class="scope-option"
|
||||||
|
:class="{ active: store.ragProjectId === p.id }"
|
||||||
|
@click="onScopeSelect(p.id)"
|
||||||
|
>{{ p.title }}</button>
|
||||||
|
<button
|
||||||
|
class="scope-option"
|
||||||
|
:class="{ active: store.ragProjectId === -1 }"
|
||||||
|
@click="onScopeSelect(-1)"
|
||||||
|
>All notes</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="input-area">
|
<div class="input-area">
|
||||||
<!-- Research button -->
|
<!-- Research button -->
|
||||||
<div class="research-wrapper">
|
<div class="research-wrapper">
|
||||||
@@ -950,26 +1001,82 @@ onUnmounted(() => {
|
|||||||
background: color-mix(in srgb, var(--color-primary) 8%, var(--color-bg-secondary));
|
background: color-mix(in srgb, var(--color-primary) 8%, var(--color-bg-secondary));
|
||||||
}
|
}
|
||||||
|
|
||||||
.rag-scope-section {
|
.scope-chip-row {
|
||||||
padding: 0.5rem 0.75rem 0.25rem;
|
padding: 0.35rem 0.75rem 0;
|
||||||
border-bottom: 1px solid var(--color-border);
|
display: flex;
|
||||||
}
|
}
|
||||||
|
|
||||||
.rag-scope-label {
|
.scope-chip-wrapper {
|
||||||
display: block;
|
position: relative;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-chip {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.3rem;
|
||||||
font-size: 0.72rem;
|
font-size: 0.72rem;
|
||||||
font-weight: 600;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.05em;
|
|
||||||
color: var(--color-text-muted);
|
color: var(--color-text-muted);
|
||||||
margin-bottom: 0.3rem;
|
background: var(--color-bg-secondary);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: 999px;
|
||||||
|
padding: 0.2rem 0.65rem;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: color 0.15s, border-color 0.15s;
|
||||||
}
|
}
|
||||||
|
|
||||||
.rag-scope-hint {
|
.scope-chip:hover {
|
||||||
margin: 0.3rem 0 0;
|
color: var(--color-text);
|
||||||
font-size: 0.7rem;
|
border-color: var(--color-primary);
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-dot {
|
||||||
color: var(--color-primary);
|
color: var(--color-primary);
|
||||||
font-style: italic;
|
font-size: 0.8rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes scope-pulse {
|
||||||
|
0% { box-shadow: 0 0 0 0 color-mix(in srgb, var(--color-primary) 40%, transparent); }
|
||||||
|
100% { box-shadow: 0 0 0 6px transparent; }
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-chip.pulse {
|
||||||
|
animation: scope-pulse 0.5s ease-out;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-dropdown {
|
||||||
|
position: absolute;
|
||||||
|
bottom: calc(100% + 4px);
|
||||||
|
left: 0;
|
||||||
|
z-index: 50;
|
||||||
|
background: var(--color-bg);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-md, 8px);
|
||||||
|
box-shadow: 0 4px 20px rgba(0,0,0,0.3);
|
||||||
|
min-width: 180px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-option {
|
||||||
|
display: block;
|
||||||
|
width: 100%;
|
||||||
|
text-align: left;
|
||||||
|
padding: 0.5rem 0.85rem;
|
||||||
|
font-size: 0.82rem;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
background: transparent;
|
||||||
|
border: none;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: background 0.1s, color 0.1s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-option:hover {
|
||||||
|
background: var(--color-bg-secondary);
|
||||||
|
color: var(--color-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.scope-option.active {
|
||||||
|
color: var(--color-primary);
|
||||||
|
font-weight: 600;
|
||||||
}
|
}
|
||||||
|
|
||||||
.conv-list {
|
.conv-list {
|
||||||
|
|||||||
@@ -245,6 +245,11 @@ def create_app() -> Quart:
|
|||||||
await backfill_note_embeddings()
|
await backfill_note_embeddings()
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Embedding backfill failed", exc_info=True)
|
logger.warning("Embedding backfill failed", exc_info=True)
|
||||||
|
try:
|
||||||
|
from fabledassistant.services.projects import backfill_project_summaries
|
||||||
|
await backfill_project_summaries()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Project summary backfill failed", exc_info=True)
|
||||||
|
|
||||||
asyncio.create_task(_delayed_backfill())
|
asyncio.create_task(_delayed_backfill())
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ class Conversation(Base, TimestampMixin):
|
|||||||
conversation_type: Mapped[str] = mapped_column(Text, default="chat", server_default="chat")
|
conversation_type: Mapped[str] = mapped_column(Text, default="chat", server_default="chat")
|
||||||
# For briefing conversations only: the calendar date this briefing covers.
|
# For briefing conversations only: the calendar date this briefing covers.
|
||||||
briefing_date: Mapped[datetime.date | None] = mapped_column(Date, nullable=True)
|
briefing_date: Mapped[datetime.date | None] = mapped_column(Date, nullable=True)
|
||||||
|
# NULL = orphan notes only; -1 = all notes; positive int = specific project
|
||||||
|
rag_project_id: Mapped[int | None] = mapped_column(Integer, nullable=True, default=None)
|
||||||
|
|
||||||
messages: Mapped[list["Message"]] = relationship(
|
messages: Mapped[list["Message"]] = relationship(
|
||||||
back_populates="conversation",
|
back_populates="conversation",
|
||||||
@@ -47,6 +49,7 @@ class Conversation(Base, TimestampMixin):
|
|||||||
"model": self.model,
|
"model": self.model,
|
||||||
"conversation_type": self.conversation_type,
|
"conversation_type": self.conversation_type,
|
||||||
"briefing_date": self.briefing_date.isoformat() if self.briefing_date else None,
|
"briefing_date": self.briefing_date.isoformat() if self.briefing_date else None,
|
||||||
|
"rag_project_id": self.rag_project_id,
|
||||||
"message_count": msg_count,
|
"message_count": msg_count,
|
||||||
"created_at": self.created_at.isoformat(),
|
"created_at": self.created_at.isoformat(),
|
||||||
"updated_at": self.updated_at.isoformat(),
|
"updated_at": self.updated_at.isoformat(),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import enum
|
import enum
|
||||||
from sqlalchemy import ForeignKey, Integer, Text
|
from datetime import datetime
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Integer, Text
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
from fabledassistant.models import Base
|
from fabledassistant.models import Base
|
||||||
from fabledassistant.models.base import TimestampMixin
|
from fabledassistant.models.base import TimestampMixin
|
||||||
@@ -20,6 +21,10 @@ class Project(Base, TimestampMixin):
|
|||||||
goal: Mapped[str] = mapped_column(Text, default="")
|
goal: Mapped[str] = mapped_column(Text, default="")
|
||||||
status: Mapped[str] = mapped_column(Text, default="active")
|
status: Mapped[str] = mapped_column(Text, default="active")
|
||||||
color: Mapped[str | None] = mapped_column(Text, nullable=True) # hex color
|
color: Mapped[str | None] = mapped_column(Text, nullable=True) # hex color
|
||||||
|
auto_summary: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
||||||
|
summary_updated_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True, default=None
|
||||||
|
)
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -115,13 +115,15 @@ async def delete_conversation_route(conv_id: int):
|
|||||||
@chat_bp.route("/conversations/<int:conv_id>", methods=["PATCH"])
|
@chat_bp.route("/conversations/<int:conv_id>", methods=["PATCH"])
|
||||||
@login_required
|
@login_required
|
||||||
async def update_conversation_route(conv_id: int):
|
async def update_conversation_route(conv_id: int):
|
||||||
|
from fabledassistant.services.chat import _UNSET
|
||||||
uid = get_current_user_id()
|
uid = get_current_user_id()
|
||||||
data = await request.get_json()
|
data = await request.get_json()
|
||||||
title = data.get("title")
|
title = data.get("title")
|
||||||
model = data.get("model")
|
model = data.get("model")
|
||||||
if title is None and model is None:
|
rag_project_id = data.get("rag_project_id", _UNSET)
|
||||||
return jsonify({"error": "title or model is required"}), 400
|
if title is None and model is None and rag_project_id is _UNSET:
|
||||||
conv = await update_conversation(uid, conv_id, title=title, model=model)
|
return jsonify({"error": "title, model, or rag_project_id is required"}), 400
|
||||||
|
conv = await update_conversation(uid, conv_id, title=title, model=model, rag_project_id=rag_project_id)
|
||||||
if conv is None:
|
if conv is None:
|
||||||
return not_found("Conversation")
|
return not_found("Conversation")
|
||||||
return jsonify(conv.to_dict())
|
return jsonify(conv.to_dict())
|
||||||
|
|||||||
@@ -139,11 +139,15 @@ async def cleanup_old_conversations(user_id: int, days: int) -> int:
|
|||||||
return len(result.fetchall())
|
return len(result.fetchall())
|
||||||
|
|
||||||
|
|
||||||
|
_UNSET = object()
|
||||||
|
|
||||||
|
|
||||||
async def update_conversation(
|
async def update_conversation(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
conversation_id: int,
|
conversation_id: int,
|
||||||
title: str | None = None,
|
title: str | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
|
rag_project_id: object = _UNSET,
|
||||||
) -> Conversation | None:
|
) -> Conversation | None:
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
@@ -159,6 +163,8 @@ async def update_conversation(
|
|||||||
conv.title = title
|
conv.title = title
|
||||||
if model is not None:
|
if model is not None:
|
||||||
conv.model = model
|
conv.model = model
|
||||||
|
if rag_project_id is not _UNSET:
|
||||||
|
conv.rag_project_id = rag_project_id # type: ignore[assignment]
|
||||||
conv.updated_at = datetime.now(timezone.utc)
|
conv.updated_at = datetime.now(timezone.utc)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(conv)
|
await session.refresh(conv)
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ async def semantic_search_notes(
|
|||||||
threshold: float = _SIMILARITY_THRESHOLD,
|
threshold: float = _SIMILARITY_THRESHOLD,
|
||||||
project_id: int | None = None,
|
project_id: int | None = None,
|
||||||
is_task: bool | None = None,
|
is_task: bool | None = None,
|
||||||
|
orphan_only: bool = False,
|
||||||
) -> list[tuple[float, Note]]:
|
) -> list[tuple[float, Note]]:
|
||||||
"""Return up to *limit* (score, note) pairs most relevant to *query*.
|
"""Return up to *limit* (score, note) pairs most relevant to *query*.
|
||||||
|
|
||||||
@@ -101,7 +102,9 @@ async def semantic_search_notes(
|
|||||||
.join(Note, NoteEmbedding.note_id == Note.id)
|
.join(Note, NoteEmbedding.note_id == Note.id)
|
||||||
.where(NoteEmbedding.user_id == user_id)
|
.where(NoteEmbedding.user_id == user_id)
|
||||||
)
|
)
|
||||||
if project_id is not None:
|
if orphan_only:
|
||||||
|
stmt = stmt.where(Note.project_id.is_(None))
|
||||||
|
elif project_id is not None:
|
||||||
stmt = stmt.where(Note.project_id == project_id)
|
stmt = stmt.where(Note.project_id == project_id)
|
||||||
if is_task is True:
|
if is_task is True:
|
||||||
stmt = stmt.where(Note.status.isnot(None))
|
stmt = stmt.where(Note.status.isnot(None))
|
||||||
|
|||||||
@@ -208,6 +208,8 @@ async def run_generation(
|
|||||||
|
|
||||||
last_flush = time.monotonic()
|
last_flush = time.monotonic()
|
||||||
all_tool_calls: list[dict] = []
|
all_tool_calls: list[dict] = []
|
||||||
|
new_rag_scope: object = False # sentinel; set to int|None when scope changes
|
||||||
|
new_rag_scope_label: str | None = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cancelled = False
|
cancelled = False
|
||||||
@@ -280,7 +282,16 @@ async def run_generation(
|
|||||||
buf.content_so_far += err_text
|
buf.content_so_far += err_text
|
||||||
research_completed = True
|
research_completed = True
|
||||||
else:
|
else:
|
||||||
result = await execute_tool(user_id, tool_name, arguments)
|
result = await execute_tool(
|
||||||
|
user_id, tool_name, arguments,
|
||||||
|
conv_id=conv_id,
|
||||||
|
workspace_project_id=workspace_project_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Capture RAG scope change for SSE done event
|
||||||
|
if result.get("type") == "rag_scope_set" and result.get("success"):
|
||||||
|
new_rag_scope = arguments.get("project_id")
|
||||||
|
new_rag_scope_label = result.get("scope_label")
|
||||||
|
|
||||||
timing["tools"].append({"name": tool_name, "ms": int((time.monotonic() - t_tool) * 1000)})
|
timing["tools"].append({"name": tool_name, "ms": int((time.monotonic() - t_tool) * 1000)})
|
||||||
logger.info("Tool %s result: success=%s", tool_name, result.get("success"))
|
logger.info("Tool %s result: success=%s", tool_name, result.get("success"))
|
||||||
@@ -352,7 +363,11 @@ async def run_generation(
|
|||||||
|
|
||||||
buf.state = GenerationState.COMPLETED
|
buf.state = GenerationState.COMPLETED
|
||||||
buf.finished_at = time.monotonic()
|
buf.finished_at = time.monotonic()
|
||||||
buf.append_event("done", {"done": True, "message_id": msg_id, "timing": timing})
|
done_payload: dict = {"done": True, "message_id": msg_id, "timing": timing}
|
||||||
|
if new_rag_scope is not False:
|
||||||
|
done_payload["new_rag_scope"] = new_rag_scope
|
||||||
|
done_payload["new_rag_scope_label"] = new_rag_scope_label
|
||||||
|
buf.append_event("done", done_payload)
|
||||||
|
|
||||||
# Fire push notification when complete (non-critical, fire-and-forget)
|
# Fire push notification when complete (non-critical, fire-and-forget)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -536,12 +536,18 @@ async def build_context(
|
|||||||
# (score, note) pairs — score is float for semantic results, None for keyword fallback.
|
# (score, note) pairs — score is float for semantic results, None for keyword fallback.
|
||||||
found_scored: list[tuple[float | None, object]] = []
|
found_scored: list[tuple[float | None, object]] = []
|
||||||
|
|
||||||
|
# Derive scope flags from rag_project_id three-value semantics:
|
||||||
|
# None → orphan notes only; -1 → all notes; positive int → that project
|
||||||
|
orphan_only = rag_project_id is None
|
||||||
|
effective_project_id = rag_project_id if (rag_project_id is not None and rag_project_id != -1) else None
|
||||||
|
|
||||||
# Try semantic search first; fall back to keyword search on failure / no results.
|
# Try semantic search first; fall back to keyword search on failure / no results.
|
||||||
try:
|
try:
|
||||||
from fabledassistant.services.embeddings import semantic_search_notes
|
from fabledassistant.services.embeddings import semantic_search_notes
|
||||||
for score, note in await semantic_search_notes(
|
for score, note in await semantic_search_notes(
|
||||||
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
user_id, user_message, exclude_ids=search_exclude or None, limit=8,
|
||||||
project_id=rag_project_id,
|
project_id=effective_project_id,
|
||||||
|
orphan_only=orphan_only,
|
||||||
):
|
):
|
||||||
found_scored.append((score, note))
|
found_scored.append((score, note))
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -553,7 +559,8 @@ async def build_context(
|
|||||||
try:
|
try:
|
||||||
for note in await search_notes_for_context(
|
for note in await search_notes_for_context(
|
||||||
user_id, keywords, exclude_ids=search_exclude or None, limit=8,
|
user_id, keywords, exclude_ids=search_exclude or None, limit=8,
|
||||||
project_id=rag_project_id,
|
project_id=effective_project_id,
|
||||||
|
orphan_only=orphan_only,
|
||||||
):
|
):
|
||||||
found_scored.append((None, note))
|
found_scored.append((None, note))
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -22,6 +22,29 @@ def _normalize_tags(tags: list[str]) -> list[str]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def _maybe_trigger_project_summary(user_id: int, project_id: int) -> None:
|
||||||
|
"""Fire generate_project_summary() if the project summary is missing or >1h old."""
|
||||||
|
import asyncio
|
||||||
|
from datetime import timedelta
|
||||||
|
from fabledassistant.models.project import Project
|
||||||
|
from fabledassistant.services.projects import generate_project_summary
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
project = (await session.execute(
|
||||||
|
select(Project).where(Project.id == project_id)
|
||||||
|
)).scalars().first()
|
||||||
|
if project is None:
|
||||||
|
return
|
||||||
|
stale = (
|
||||||
|
project.summary_updated_at is None
|
||||||
|
or (datetime.now(timezone.utc) - project.summary_updated_at) > timedelta(hours=1)
|
||||||
|
)
|
||||||
|
if stale:
|
||||||
|
asyncio.create_task(generate_project_summary(user_id, project_id))
|
||||||
|
except Exception:
|
||||||
|
logger.debug("_maybe_trigger_project_summary failed for project %d", project_id, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
async def create_note(
|
async def create_note(
|
||||||
user_id: int,
|
user_id: int,
|
||||||
title: str = "",
|
title: str = "",
|
||||||
@@ -61,7 +84,11 @@ async def create_note(
|
|||||||
session.add(note)
|
session.add(note)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(note)
|
await session.refresh(note)
|
||||||
return note
|
|
||||||
|
if project_id is not None:
|
||||||
|
await _maybe_trigger_project_summary(user_id, project_id)
|
||||||
|
|
||||||
|
return note
|
||||||
|
|
||||||
|
|
||||||
async def get_note(user_id: int, note_id: int) -> Note | None:
|
async def get_note(user_id: int, note_id: int) -> Note | None:
|
||||||
@@ -222,6 +249,9 @@ async def update_note(user_id: int, note_id: int, **fields: object) -> Note | No
|
|||||||
from fabledassistant.services.note_versions import create_version
|
from fabledassistant.services.note_versions import create_version
|
||||||
await create_version(user_id, note_id, old_body, old_title, old_tags)
|
await create_version(user_id, note_id, old_body, old_title, old_tags)
|
||||||
|
|
||||||
|
if note.project_id is not None:
|
||||||
|
await _maybe_trigger_project_summary(user_id, note.project_id)
|
||||||
|
|
||||||
return note
|
return note
|
||||||
|
|
||||||
|
|
||||||
@@ -303,6 +333,7 @@ async def search_notes_for_context(
|
|||||||
exclude_ids: set[int] | None = None,
|
exclude_ids: set[int] | None = None,
|
||||||
limit: int = 3,
|
limit: int = 3,
|
||||||
project_id: int | None = None,
|
project_id: int | None = None,
|
||||||
|
orphan_only: bool = False,
|
||||||
) -> list[Note]:
|
) -> list[Note]:
|
||||||
"""Search notes by keywords with OR logic. Optimized for context building — no count query."""
|
"""Search notes by keywords with OR logic. Optimized for context building — no count query."""
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
@@ -313,7 +344,9 @@ async def search_notes_for_context(
|
|||||||
keyword_filters.append(or_(Note.title.ilike(pattern), Note.body.ilike(pattern)))
|
keyword_filters.append(or_(Note.title.ilike(pattern), Note.body.ilike(pattern)))
|
||||||
|
|
||||||
query = select(Note).where(Note.user_id == user_id, or_(*keyword_filters))
|
query = select(Note).where(Note.user_id == user_id, or_(*keyword_filters))
|
||||||
if project_id is not None:
|
if orphan_only:
|
||||||
|
query = query.where(Note.project_id.is_(None))
|
||||||
|
elif project_id is not None:
|
||||||
query = query.where(Note.project_id == project_id)
|
query = query.where(Note.project_id == project_id)
|
||||||
if exclude_ids:
|
if exclude_ids:
|
||||||
query = query.where(Note.id.notin_(exclude_ids))
|
query = query.where(Note.id.notin_(exclude_ids))
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ async def list_projects(user_id: int, status: str | None = None) -> list[Project
|
|||||||
|
|
||||||
|
|
||||||
async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
|
async def update_project(user_id: int, project_id: int, **fields: object) -> Project | None:
|
||||||
|
import asyncio
|
||||||
async with async_session() as session:
|
async with async_session() as session:
|
||||||
result = await session.execute(
|
result = await session.execute(
|
||||||
select(Project).where(Project.id == project_id, Project.user_id == user_id)
|
select(Project).where(Project.id == project_id, Project.user_id == user_id)
|
||||||
@@ -83,7 +84,72 @@ async def update_project(user_id: int, project_id: int, **fields: object) -> Pro
|
|||||||
project.updated_at = datetime.now(timezone.utc)
|
project.updated_at = datetime.now(timezone.utc)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
await session.refresh(project)
|
await session.refresh(project)
|
||||||
return project
|
asyncio.create_task(generate_project_summary(user_id, project_id))
|
||||||
|
return project
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_project_summary(user_id: int, project_id: int) -> None:
|
||||||
|
"""Generate an LLM summary for a project and persist it. Fire-and-forget safe."""
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
project = (await session.execute(
|
||||||
|
select(Project).where(Project.id == project_id, Project.user_id == user_id)
|
||||||
|
)).scalars().first()
|
||||||
|
if project is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
note_rows = (await session.execute(
|
||||||
|
select(Note.title, Note.body)
|
||||||
|
.where(Note.project_id == project_id, Note.user_id == user_id)
|
||||||
|
.order_by(Note.updated_at.desc())
|
||||||
|
.limit(10)
|
||||||
|
)).all()
|
||||||
|
|
||||||
|
title = project.title or ""
|
||||||
|
description = project.description or ""
|
||||||
|
goal = project.goal or ""
|
||||||
|
note_snippets = "\n".join(
|
||||||
|
f"- {r.title}: {(r.body or '')[:200]}" for r in note_rows
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"Summarize this project in 3-4 sentences covering its purpose, themes, and content.\n"
|
||||||
|
f"Title: {title}\nDescription: {description}\nGoal: {goal}\n"
|
||||||
|
f"Recent notes:\n{note_snippets}"
|
||||||
|
)
|
||||||
|
|
||||||
|
from fabledassistant.services.llm import generate_completion
|
||||||
|
from fabledassistant.config import Config
|
||||||
|
messages = [{"role": "user", "content": prompt}]
|
||||||
|
summary = await generate_completion(messages, model=Config.OLLAMA_MODEL, max_tokens=400)
|
||||||
|
if not summary:
|
||||||
|
return
|
||||||
|
|
||||||
|
async with async_session() as session:
|
||||||
|
project = (await session.execute(
|
||||||
|
select(Project).where(Project.id == project_id)
|
||||||
|
)).scalars().first()
|
||||||
|
if project:
|
||||||
|
project.auto_summary = summary.strip()
|
||||||
|
project.summary_updated_at = datetime.now(timezone.utc)
|
||||||
|
await session.commit()
|
||||||
|
logger.debug("Generated summary for project %d", project_id)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("Failed to generate summary for project %d", project_id, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def backfill_project_summaries() -> None:
|
||||||
|
"""Generate summaries for all projects missing auto_summary. Fire-and-forget."""
|
||||||
|
import asyncio
|
||||||
|
try:
|
||||||
|
async with async_session() as session:
|
||||||
|
rows = (await session.execute(
|
||||||
|
select(Project.id, Project.user_id).where(Project.auto_summary.is_(None))
|
||||||
|
)).all()
|
||||||
|
for row in rows:
|
||||||
|
asyncio.create_task(generate_project_summary(row.user_id, row.id))
|
||||||
|
except Exception:
|
||||||
|
logger.debug("backfill_project_summaries failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
async def delete_project(user_id: int, project_id: int) -> bool:
|
async def delete_project(user_id: int, project_id: int) -> bool:
|
||||||
|
|||||||
@@ -838,9 +838,48 @@ _IMAGE_TOOLS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
_RAG_TOOLS = [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "search_projects",
|
||||||
|
"description": "Search for projects by name, description, or theme. Use this to find which project a user is referring to and get its ID for set_rag_scope.",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"query": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Search query — project name, topic, or theme",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["query"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "set_rag_scope",
|
||||||
|
"description": "Change the RAG scope for this conversation. Use project_id=<int> to scope to a project, project_id=null to scope to orphan notes only, project_id=-1 for all notes.",
|
||||||
|
"parameters": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"project_id": {
|
||||||
|
"type": ["integer", "null"],
|
||||||
|
"description": "Project ID to scope to, null for orphan-only, -1 for all notes",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["project_id"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
async def get_tools_for_user(user_id: int) -> list[dict]:
|
async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||||
"""Build the tool list for a user based on their configured integrations."""
|
"""Build the tool list for a user based on their configured integrations."""
|
||||||
tools = list(_CORE_TOOLS)
|
tools = list(_CORE_TOOLS)
|
||||||
|
tools.extend(_RAG_TOOLS)
|
||||||
if await is_caldav_configured(user_id):
|
if await is_caldav_configured(user_id):
|
||||||
tools.extend(_CALDAV_TOOLS)
|
tools.extend(_CALDAV_TOOLS)
|
||||||
if Config.searxng_enabled():
|
if Config.searxng_enabled():
|
||||||
@@ -883,7 +922,13 @@ def _fuzzy_title_match(title: str, candidates, threshold: float = 0.82):
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
|
async def execute_tool(
|
||||||
|
user_id: int,
|
||||||
|
tool_name: str,
|
||||||
|
arguments: dict,
|
||||||
|
conv_id: int | None = None,
|
||||||
|
workspace_project_id: int | None = None,
|
||||||
|
) -> dict:
|
||||||
"""Execute a tool call and return the result."""
|
"""Execute a tool call and return the result."""
|
||||||
try:
|
try:
|
||||||
if tool_name == "create_task":
|
if tool_name == "create_task":
|
||||||
@@ -1735,6 +1780,52 @@ async def execute_tool(user_id: int, tool_name: str, arguments: dict) -> dict:
|
|||||||
items = await get_recent_items(user_id, limit=limit)
|
items = await get_recent_items(user_id, limit=limit)
|
||||||
return {"data": {"items": items, "count": len(items)}}
|
return {"data": {"items": items, "count": len(items)}}
|
||||||
|
|
||||||
|
elif tool_name == "search_projects":
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
query = str(arguments.get("query", "")).lower()
|
||||||
|
from fabledassistant.services.projects import list_projects
|
||||||
|
projects = await list_projects(user_id)
|
||||||
|
scored: list[tuple[float, object]] = []
|
||||||
|
for p in projects:
|
||||||
|
combined = f"{p.title} {p.description or ''} {p.auto_summary or ''}".lower()
|
||||||
|
base_score = SequenceMatcher(None, query, combined).ratio()
|
||||||
|
# Bonus for keyword overlap
|
||||||
|
query_words = set(query.split())
|
||||||
|
overlap = sum(1 for w in query_words if w in combined)
|
||||||
|
score = base_score + overlap * 0.05
|
||||||
|
scored.append((score, p))
|
||||||
|
scored.sort(key=lambda x: x[0], reverse=True)
|
||||||
|
results = []
|
||||||
|
for score, p in scored[:5]:
|
||||||
|
results.append({
|
||||||
|
"id": p.id,
|
||||||
|
"title": p.title,
|
||||||
|
"summary_snippet": (p.auto_summary or p.description or "")[:200],
|
||||||
|
"score": round(score, 3),
|
||||||
|
})
|
||||||
|
return {"type": "projects_list", "data": {"projects": results}}
|
||||||
|
|
||||||
|
elif tool_name == "set_rag_scope":
|
||||||
|
if workspace_project_id is not None:
|
||||||
|
return {"success": False, "error": "Cannot change RAG scope in workspace view"}
|
||||||
|
if conv_id is None:
|
||||||
|
return {"success": False, "error": "No conversation context available"}
|
||||||
|
project_id = arguments.get("project_id") # None, -1, or positive int
|
||||||
|
# Validate positive project_id belongs to user
|
||||||
|
if project_id is not None and project_id != -1:
|
||||||
|
from fabledassistant.services.projects import get_project
|
||||||
|
proj = await get_project(user_id, int(project_id))
|
||||||
|
if proj is None:
|
||||||
|
return {"success": False, "error": "Project not found"}
|
||||||
|
scope_label = proj.title
|
||||||
|
elif project_id == -1:
|
||||||
|
scope_label = "All notes"
|
||||||
|
else:
|
||||||
|
scope_label = "Orphan notes only"
|
||||||
|
from fabledassistant.services.chat import update_conversation
|
||||||
|
await update_conversation(user_id, conv_id, rag_project_id=project_id)
|
||||||
|
return {"success": True, "type": "rag_scope_set", "scope_label": scope_label}
|
||||||
|
|
||||||
else:
|
else:
|
||||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user