Files
FabledScribe/src/fabledassistant/services/notes.py
T
bvandeusen ebc79b34f9 feat(rag): RAG scoping and context isolation controls
- Migration 0030: add conversations.rag_project_id (NULL=orphan-only,
  -1=all notes, positive=project), projects.auto_summary and
  projects.summary_updated_at
- Three-value scope semantics thread from build_context() → semantic
  search and keyword fallback via orphan_only + effective_project_id
- Project summarization background job (generate_project_summary,
  backfill_project_summaries) called via Ollama; triggered on project
  update and note saves (debounced 1h); runs at startup
- New LLM tools: search_projects (SequenceMatcher scoring on
  title+description+auto_summary) and set_rag_scope (persists to DB,
  workspace-guarded, emits new_rag_scope in SSE done event)
- execute_tool() accepts conv_id + workspace_project_id; generation_task
  passes both and captures scope changes for SSE done enrichment
- Frontend: Conversation type gets rag_project_id; chat store adds
  ragProjectId computed + updateRagScope(); SSE done handler syncs scope
- ChatView: replace sidebar ProjectSelector with a scope chip pill above
  the input bar, animated dropdown, pulse on model-driven scope change

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-25 17:44:39 -04:00

493 lines
18 KiB
Python

import logging
import re
from datetime import date, datetime, timezone
from sqlalchemy import func, or_, select, text
from fabledassistant.models import async_session
from fabledassistant.models.note import Note, TaskPriority, TaskStatus
logger = logging.getLogger(__name__)
def _normalize_tags(tags: list[str]) -> list[str]:
"""Lowercase, strip, deduplicate, and drop empty tags."""
seen: set[str] = set()
out: list[str] = []
for t in tags:
normalized = t.strip().lower()
if normalized and normalized not in seen:
seen.add(normalized)
out.append(normalized)
return out
async def _maybe_trigger_project_summary(user_id: int, project_id: int) -> None:
"""Fire generate_project_summary() if the project summary is missing or >1h old."""
import asyncio
from datetime import timedelta
from fabledassistant.models.project import Project
from fabledassistant.services.projects import generate_project_summary
try:
async with async_session() as session:
project = (await session.execute(
select(Project).where(Project.id == project_id)
)).scalars().first()
if project is None:
return
stale = (
project.summary_updated_at is None
or (datetime.now(timezone.utc) - project.summary_updated_at) > timedelta(hours=1)
)
if stale:
asyncio.create_task(generate_project_summary(user_id, project_id))
except Exception:
logger.debug("_maybe_trigger_project_summary failed for project %d", project_id, exc_info=True)
async def create_note(
user_id: int,
title: str = "",
body: str = "",
tags: list[str] | None = None,
parent_id: int | None = None,
project_id: int | None = None,
milestone_id: int | None = None,
status: str | None = None,
priority: str | None = None,
due_date: date | None = None,
) -> Note:
# Auto-populate project_id from milestone when not explicitly provided
if milestone_id is not None and project_id is None:
from fabledassistant.models.milestone import Milestone
async with async_session() as lookup:
result = await lookup.execute(
select(Milestone).where(Milestone.id == milestone_id, Milestone.user_id == user_id)
)
ms = result.scalars().first()
if ms is not None:
project_id = ms.project_id
async with async_session() as session:
note = Note(
user_id=user_id,
title=title,
body=body,
tags=_normalize_tags(tags or []),
parent_id=parent_id,
project_id=project_id,
milestone_id=milestone_id,
status=status,
priority=priority,
due_date=due_date,
)
session.add(note)
await session.commit()
await session.refresh(note)
if project_id is not None:
await _maybe_trigger_project_summary(user_id, project_id)
return note
async def get_note(user_id: int, note_id: int) -> Note | None:
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.id == note_id, Note.user_id == user_id)
)
return result.scalars().first()
async def list_notes(
user_id: int,
q: str | None = None,
tags: list[str] | None = None,
is_task: bool | None = None,
status: str | None = None,
priority: str | None = None,
due_before: date | None = None,
due_after: date | None = None,
project_id: int | None = None,
milestone_id: int | None = None,
milestone_ids: list[int] | None = None,
parent_id: int | None = None,
no_project: bool = False,
sort: str = "updated_at",
order: str = "desc",
limit: int = 50,
offset: int = 0,
) -> tuple[list[Note], int]:
async with async_session() as session:
query = select(Note).where(Note.user_id == user_id)
count_query = select(func.count(Note.id)).where(Note.user_id == user_id)
# Filter by task vs note
if is_task is True:
query = query.where(Note.status.isnot(None))
count_query = count_query.where(Note.status.isnot(None))
elif is_task is False:
query = query.where(Note.status.is_(None))
count_query = count_query.where(Note.status.is_(None))
if q:
terms = q.split()
for term in terms:
escaped_term = term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
pattern = f"%{escaped_term}%"
term_filter = or_(Note.title.ilike(pattern), Note.body.ilike(pattern))
query = query.where(term_filter)
count_query = count_query.where(term_filter)
if tags:
for i, tag in enumerate(tags):
param_tag = f"tag_{i}"
param_prefix = f"tag_prefix_{i}"
tag_filter = text(
f"EXISTS (SELECT 1 FROM unnest(notes.tags) AS t"
f" WHERE t = :{param_tag} OR t LIKE :{param_prefix})"
).bindparams(**{param_tag: tag, param_prefix: tag + "/%"})
query = query.where(tag_filter)
count_query = count_query.where(tag_filter)
if status:
query = query.where(Note.status == status)
count_query = count_query.where(Note.status == status)
if priority:
query = query.where(Note.priority == priority)
count_query = count_query.where(Note.priority == priority)
if due_before is not None:
query = query.where(Note.due_date < due_before)
count_query = count_query.where(Note.due_date < due_before)
if due_after is not None:
query = query.where(Note.due_date >= due_after)
count_query = count_query.where(Note.due_date >= due_after)
if project_id is not None:
if milestone_ids:
# OR: directly assigned to project, OR assigned to one of the project's milestones
project_filter = or_(Note.project_id == project_id, Note.milestone_id.in_(milestone_ids))
else:
project_filter = Note.project_id == project_id
query = query.where(project_filter)
count_query = count_query.where(project_filter)
if milestone_id is not None:
query = query.where(Note.milestone_id == milestone_id)
count_query = count_query.where(Note.milestone_id == milestone_id)
if parent_id is not None:
query = query.where(Note.parent_id == parent_id)
if no_project:
query = query.where(Note.project_id.is_(None))
count_query = count_query.where(Note.project_id.is_(None))
sort_col = getattr(Note, sort, Note.updated_at)
if order == "asc":
query = query.order_by(sort_col.asc())
else:
query = query.order_by(sort_col.desc())
query = query.limit(limit).offset(offset)
total = await session.scalar(count_query) or 0
result = await session.execute(query)
notes = list(result.scalars().all())
return notes, total
async def get_note_by_title(user_id: int, title: str) -> Note | None:
async with async_session() as session:
result = await session.execute(
select(Note).where(
Note.user_id == user_id,
func.lower(Note.title) == func.lower(title.strip()),
).limit(1)
)
return result.scalars().first()
async def get_or_create_note_by_title(user_id: int, title: str) -> Note:
title = title.strip()
note = await get_note_by_title(user_id, title)
if note:
return note
return await create_note(user_id, title=title)
async def update_note(user_id: int, note_id: int, **fields: object) -> Note | None:
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.id == note_id, Note.user_id == user_id)
)
note = result.scalars().first()
if note is None:
return None
# Snapshot before changes for version creation
old_body = note.body
old_title = note.title
old_tags = list(note.tags or [])
for key, value in fields.items():
if not hasattr(note, key):
continue
if key == "status" and isinstance(value, str):
value = TaskStatus(value).value
elif key == "priority" and isinstance(value, str):
value = TaskPriority(value).value
elif key == "tags" and isinstance(value, list):
value = _normalize_tags(value)
setattr(note, key, value)
note.updated_at = datetime.now(timezone.utc)
await session.commit()
await session.refresh(note)
# Create a version snapshot when body actually changes
if "body" in fields and fields["body"] != old_body:
from fabledassistant.services.note_versions import create_version
await create_version(user_id, note_id, old_body, old_title, old_tags)
if note.project_id is not None:
await _maybe_trigger_project_summary(user_id, note.project_id)
return note
async def delete_note(user_id: int, note_id: int) -> bool:
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.id == note_id, Note.user_id == user_id)
)
note = result.scalars().first()
if note is None:
return False
await session.delete(note)
await session.commit()
return True
async def get_all_tags(user_id: int, q: str | None = None) -> list[str]:
async with async_session() as session:
if q:
result = await session.execute(
text(
"SELECT DISTINCT tag FROM"
" (SELECT unnest(tags) AS tag FROM notes"
" WHERE tags != '{}' AND user_id = :user_id) t"
" WHERE tag ILIKE :q_pattern ORDER BY tag LIMIT 100"
).bindparams(user_id=user_id, q_pattern=f"%{q}%")
)
else:
result = await session.execute(
text(
"SELECT DISTINCT unnest(tags) AS tag FROM notes"
" WHERE tags != '{}' AND user_id = :user_id"
" ORDER BY tag LIMIT 100"
).bindparams(user_id=user_id)
)
return [row[0] for row in result.fetchall()]
async def convert_note_to_task(user_id: int, note_id: int) -> Note:
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.id == note_id, Note.user_id == user_id)
)
note = result.scalars().first()
if note is None:
logger.warning("convert_note_to_task: note %d not found", note_id)
raise ValueError("Note not found")
note.status = TaskStatus.todo.value
note.priority = TaskPriority.none.value
note.updated_at = datetime.now(timezone.utc)
await session.commit()
await session.refresh(note)
logger.info("Converted note %d to task", note_id)
return note
async def convert_task_to_note(user_id: int, note_id: int) -> Note:
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.id == note_id, Note.user_id == user_id)
)
note = result.scalars().first()
if note is None:
logger.warning("convert_task_to_note: note %d not found", note_id)
raise ValueError("Note not found")
note.status = None
note.priority = None
note.due_date = None
note.updated_at = datetime.now(timezone.utc)
await session.commit()
await session.refresh(note)
logger.info("Converted task %d to note", note_id)
return note
async def search_notes_for_context(
user_id: int,
keywords: list[str],
exclude_ids: set[int] | None = None,
limit: int = 3,
project_id: int | None = None,
orphan_only: bool = False,
) -> list[Note]:
"""Search notes by keywords with OR logic. Optimized for context building — no count query."""
async with async_session() as session:
keyword_filters = []
for kw in keywords:
escaped = kw.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
pattern = f"%{escaped}%"
keyword_filters.append(or_(Note.title.ilike(pattern), Note.body.ilike(pattern)))
query = select(Note).where(Note.user_id == user_id, or_(*keyword_filters))
if orphan_only:
query = query.where(Note.project_id.is_(None))
elif project_id is not None:
query = query.where(Note.project_id == project_id)
if exclude_ids:
query = query.where(Note.id.notin_(exclude_ids))
query = query.order_by(Note.updated_at.desc()).limit(limit)
result = await session.execute(query)
return list(result.scalars().all())
async def get_notes_by_ids(user_id: int, note_ids: list[int]) -> dict[int, Note]:
"""Batch fetch notes by ID list. Returns {note_id: Note}."""
if not note_ids:
return {}
async with async_session() as session:
result = await session.execute(
select(Note).where(Note.user_id == user_id, Note.id.in_(note_ids))
)
return {n.id: n for n in result.scalars().all()}
async def get_backlinks(user_id: int, note_id: int) -> list[dict]:
note = await get_note(user_id, note_id)
if note is None:
return []
title = note.title
if not title:
return []
async with async_session() as session:
escaped = title.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
pattern = f"%[[{escaped}]]%"
pattern_alias = f"%[[{escaped}|%"
results = await session.execute(
select(Note.id, Note.title, Note.status).where(
Note.user_id == user_id,
Note.id != note_id,
or_(Note.body.like(pattern), Note.body.like(pattern_alias)),
)
)
backlinks: list[dict] = []
for row in results.fetchall():
link_type = "task" if row[2] is not None else "note"
backlinks.append({"type": link_type, "id": row[0], "title": row[1]})
return backlinks
_WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
async def build_note_graph(
user_id: int,
project_id: int | None = None,
include_shared_tags: bool = False,
) -> dict:
notes, _ = await list_notes(user_id, project_id=project_id, limit=1000)
if not notes:
return {"nodes": [], "edges": []}
# Fetch project colours
project_ids = {n.project_id for n in notes if n.project_id is not None}
project_colors: dict[int, str] = {}
if project_ids:
from fabledassistant.models.project import Project
async with async_session() as session:
result = await session.execute(
select(Project.id, Project.color).where(Project.id.in_(project_ids))
)
for pid, color in result.fetchall():
project_colors[pid] = color or "#888888"
# Build title lookup (lowercase → id)
title_map: dict[str, int] = {}
for n in notes:
if n.title:
title_map[n.title.lower()] = n.id
# Build nodes
nodes = []
for n in notes:
nodes.append({
"id": n.id,
"title": n.title or "(untitled)",
"type": "task" if n.is_task else "note",
"tags": n.tags or [],
"project_id": n.project_id,
"project_color": project_colors.get(n.project_id) if n.project_id else None,
})
# Build wikilink edges
edge_set: set[tuple[int, int]] = set()
edges = []
for n in notes:
if not n.body:
continue
for match in _WIKILINK_RE.findall(n.body):
target_id = title_map.get(match.strip().lower())
if target_id is not None and target_id != n.id:
pair = (n.id, target_id)
if pair not in edge_set:
edge_set.add(pair)
edges.append({"source": n.id, "target": target_id, "type": "wikilink"})
# Build tag nodes + note→tag edges
if include_shared_tags:
tag_to_ids: dict[str, list[int]] = {}
for n in notes:
for tag in (n.tags or []):
tag_to_ids.setdefault(tag, []).append(n.id)
for tag, note_ids in tag_to_ids.items():
tag_node_id = f"tag:{tag}"
nodes.append({
"id": tag_node_id,
"title": tag,
"type": "tag",
"tags": [],
"project_id": None,
"project_color": None,
})
for nid in note_ids:
edges.append({"source": nid, "target": tag_node_id, "type": "tag"})
return {"nodes": nodes, "edges": edges}
# ---------------------------------------------------------------------------
# Shared-access variant
# ---------------------------------------------------------------------------
async def get_note_for_user(
accessing_user_id: int, note_id: int
) -> tuple["Note", str] | None:
"""Returns (note, permission) if user has any access, else None."""
from fabledassistant.services.access import get_note_permission
perm = await get_note_permission(accessing_user_id, note_id)
if perm is None:
return None
async with async_session() as session:
note = await session.get(Note, note_id)
return (note, perm) if note else None