FabledScribe/src/scribe/services/knowledge.py

"""Knowledge service — unified query across notes, tasks, plans, and processes.

ACL (rules #47/#78, decision note 2094): these queries were owner-only until
2026-07-25, which meant a record shared with you could be opened by id but never
*found*. They now honour shares — at two different widths:

  - **searching** (a `q` the caller typed) uses the full read scope, so a record
    shared directly with you is findable when you go looking for it. BOTH halves
    of the hybrid search — keyword and semantic — see equally, or a record would
    be findable by wording and invisible by meaning;
  - **browsing** (no `q`) and the facet counts beside it use the narrower browse
    scope: your own records plus anything in a project you can reach.

The asymmetry is the point. A record that appears unasked reads as one you
endorsed, so a one-off direct share has to be searched for rather than arriving
in your ambient lists.
"""
import logging

from sqlalchemy import func, select

from scribe.models import async_session
from scribe.models.note import Note
from scribe.services.access import browsable_notes_clause, readable_notes_clause

logger = logging.getLogger(__name__)

_SNIPPET_LEN = 200


def _note_to_item(note: Note) -> dict:
    item: dict = {
        "id": note.id,
        "note_type": note.note_type or "note",
        "title": note.title,
        "snippet": (note.body or "")[:_SNIPPET_LEN],
        "tags": note.tags or [],
        "project_id": note.project_id,
        # These lists now include records shared with the caller, so the client
        # needs the owner to tell "mine" from "someone else's" in a mixed list.
        "user_id": note.user_id,
        "created_at": note.created_at.isoformat(),
        "updated_at": note.updated_at.isoformat(),
    }
    # Task fields — override note_type and add status/priority/due_date
    if note.is_task:
        item["note_type"] = "task"
        item["task_kind"] = note.task_kind
        item["status"] = note.status
        item["priority"] = note.priority
        item["due_date"] = note.due_date.isoformat() if note.due_date else None

    return item


def _apply_type_filter(stmt, note_type: str | None):
    """Apply the type facet to a Note select.

    'task' = any task (status not null); 'plan' = a task with task_kind='plan';
    any other non-empty type = a non-task note of that note_type; None = all.

    Trashed rows (deleted_at set) are always excluded.
    """
    stmt = stmt.where(Note.deleted_at.is_(None))
    if note_type == "task":
        return stmt.where(Note.status.isnot(None))
    if note_type == "plan":
        return stmt.where(Note.status.isnot(None)).where(Note.task_kind == "plan")
    if note_type:
        return stmt.where(Note.note_type == note_type).where(Note.status.is_(None))
    return stmt


async def query_knowledge(
    user_id: int,
    note_type: str | None,
    tags: list[str],
    sort: str,
    q: str | None,
    limit: int,
    offset: int,
    project_id: int | None = None,
) -> tuple[list[dict], int]:
    """Query knowledge objects (non-task notes) with filters.

    `project_id` narrows to one project (None = every project).

    Returns (items, total_count).
    """
    # Semantic search path — scores take priority over sort
    if q:
        return await _semantic_knowledge_search(
            user_id, q, note_type=note_type, tags=tags, limit=limit,
            offset=offset, project_id=project_id,
        )

    # No query = browsing. Narrower scope: a record shared directly with the
    # caller is search-only and must not appear in an ambient list.
    visible = browsable_notes_clause(user_id)
    async with async_session() as session:
        base = select(Note).where(visible)

        base = _apply_type_filter(base, note_type)
        if project_id is not None:
            base = base.where(Note.project_id == project_id)

        for tag in tags:
            base = base.where(Note.tags.contains([tag]))

        # Count before pagination
        count_stmt = select(func.count()).select_from(base.subquery())
        total: int = (await session.execute(count_stmt)).scalar_one()

        # Apply sort
        if sort == "created":
            base = base.order_by(Note.created_at.desc())
        elif sort == "alpha":
            base = base.order_by(Note.title.asc())
        elif sort == "type":
            base = base.order_by(Note.note_type.asc(), Note.updated_at.desc())
        else:  # modified (default)
            base = base.order_by(Note.updated_at.desc())

        rows = list((await session.execute(base.limit(limit).offset(offset))).scalars().all())

    return [_note_to_item(n) for n in rows], total


async def _semantic_knowledge_search(
    user_id: int,
    q: str,
    note_type: str | None,
    tags: list[str],
    limit: int,
    offset: int,
    project_id: int | None = None,
) -> tuple[list[dict], int]:
    """Hybrid search: keyword matches first (title/body ILIKE), then semantic results.

    Exact keyword matches always rank above semantic-only matches so that
    searching for a name like "Weston" surfaces the note with that title
    before conceptually related notes.

    BEST-EFFORT TOP-N, not exhaustive pagination: the ranked candidate set is
    capped (keyword limit*2 + up to ~200 semantic), so `total` is the size of
    that window, NOT the true match count, and matches beyond the cap are not
    reachable by paging. Each page also recomputes the full merge (O(corpus)
    per page). Acceptable for an interactive "best results" feed; a cached
    ranked-id list or pgvector ORDER BY/LIMIT is the fix if exhaustive,
    cheap pagination is ever needed.
    """
    # 1. Keyword search — title and body ILIKE
    keyword_notes: list[Note] = []
    try:
        # A typed query is an explicit act, so it reaches the caller's full read
        # scope — including records shared directly with them.
        visible = readable_notes_clause(user_id)
        async with async_session() as session:
            pattern = f"%{q}%"
            base = (
                select(Note)
                .where(visible)
                .where(Note.title.ilike(pattern) | Note.body.ilike(pattern))
            )
            base = _apply_type_filter(base, note_type)
            if project_id is not None:
                base = base.where(Note.project_id == project_id)
            for tag in tags:
                base = base.where(Note.tags.contains([tag]))
            # Title matches first, then body-only matches, newest first within each
            base = base.order_by(
                Note.title.ilike(pattern).desc(),
                Note.updated_at.desc(),
            ).limit(limit * 2)
            keyword_notes = list((await session.execute(base)).scalars().all())
    except Exception:
        logger.warning("Keyword search failed", exc_info=True)

    # 2. Semantic search — conceptual similarity, at the SAME scope as the
    # keyword half above. Both halves of one search must see equally, or a shared
    # record would be findable by wording and invisible by meaning — which is the
    # case a semantic search exists to serve.
    semantic_notes: list[Note] = []
    try:
        from scribe.services.embeddings import semantic_search_notes
        is_task_filter = True if note_type in ("task", "plan") else (False if note_type else None)
        candidates = await semantic_search_notes(
            user_id=user_id,
            scope="read",
            query=q,
            limit=min(200, limit * 4),
            threshold=0.3,
            is_task=is_task_filter,
            project_id=project_id,
        )
        for _score, note in candidates:
            if note.deleted_at is not None:
                continue
            if note_type == "task" and not note.is_task:
                continue
            elif note_type == "plan" and (not note.is_task or note.task_kind != "plan"):
                continue
            elif note_type and note_type not in ("task", "plan") and note.note_type != note_type:
                continue
            if tags and not all(t in (note.tags or []) for t in tags):
                continue
            semantic_notes.append(note)
    except Exception:
        logger.warning("Semantic search unavailable, using keyword results only", exc_info=True)

    # 3. Merge — keyword matches first, then semantic (deduplicated)
    seen_ids: set[int] = set()
    merged: list[Note] = []
    for note in keyword_notes:
        if note.id not in seen_ids:
            seen_ids.add(note.id)
            merged.append(note)
    for note in semantic_notes:
        if note.id not in seen_ids:
            seen_ids.add(note.id)
            merged.append(note)

    total = len(merged)
    page_items = merged[offset: offset + limit]
    return [_note_to_item(n) for n in page_items], total


async def get_knowledge_tags(user_id: int, note_type: str | None = None) -> list[str]:
    """Distinct tags across what this user can BROWSE.

    Follows the browse list rather than the read scope: a facet is itself a
    passive surface, and offering a tag that only a search-only record carries
    would filter the visible list down to nothing."""
    visible = browsable_notes_clause(user_id)
    async with async_session() as session:
        base = (
            select(func.unnest(Note.tags).label("tag"))
            .where(visible)
        )
        base = _apply_type_filter(base, note_type)
        stmt = base.distinct().order_by("tag")
        rows = list((await session.execute(stmt)).scalars().all())
    return [r for r in rows if r]


async def get_knowledge_counts(user_id: int, tags: list[str] | None = None) -> dict[str, int]:
    """Per-type counts for the sidebar, over what this user can BROWSE — so the
    numbers match the list they sit beside rather than promising rows that only a
    search would surface."""
    visible = browsable_notes_clause(user_id)
    async with async_session() as session:
        # Count non-task types
        stmt = (
            select(Note.note_type, func.count(Note.id))
            .where(visible)
            .where(Note.status.is_(None))
            .where(Note.deleted_at.is_(None))
            .where(Note.note_type.in_(["note", "process"]))
            .group_by(Note.note_type)
        )
        if tags:
            for tag in tags:
                stmt = stmt.where(Note.tags.contains([tag]))
        rows = list((await session.execute(stmt)).all())
        counts = {row[0]: row[1] for row in rows}

        # Count tasks separately (is_task = status IS NOT NULL)
        task_stmt = (
            select(func.count(Note.id))
            .where(visible)
            .where(Note.status.isnot(None))
            .where(Note.deleted_at.is_(None))
        )
        if tags:
            for tag in tags:
                task_stmt = task_stmt.where(Note.tags.contains([tag]))
        task_count: int = (await session.execute(task_stmt)).scalar_one()
        counts["task"] = task_count

        # Plans are a subset of tasks (task_kind='plan'); counted for the facet
        # but NOT added to total to avoid double-counting against "task".
        plan_stmt = (
            select(func.count(Note.id))
            .where(visible)
            .where(Note.status.isnot(None))
            .where(Note.task_kind == "plan")
            .where(Note.deleted_at.is_(None))
        )
        if tags:
            for tag in tags:
                plan_stmt = plan_stmt.where(Note.tags.contains([tag]))
        counts["plan"] = (await session.execute(plan_stmt)).scalar_one()

    for t in ("note", "task", "plan", "process"):
        counts.setdefault(t, 0)
    counts["total"] = sum(counts[t] for t in ("note", "task", "process"))
    return counts


async def query_knowledge_ids(
    user_id: int,
    note_type: str | None,
    tags: list[str],
    sort: str,
    q: str | None,
    limit: int = 100,
    offset: int = 0,
) -> tuple[list[int], int]:
    """Return note IDs only — cheap query for the two-tier pagination feed."""
    if q:
        # Re-use semantic search, extract IDs in rank order
        items, total = await _semantic_knowledge_search(
            user_id, q, note_type=note_type, tags=tags,
            limit=limit, offset=offset,
        )
        return [item["id"] for item in items], total

    # Browsing (see query_knowledge) — narrower scope.
    visible = browsable_notes_clause(user_id)
    async with async_session() as session:
        base = select(Note.id).where(visible)

        base = _apply_type_filter(base, note_type)
        for tag in tags:
            base = base.where(Note.tags.contains([tag]))

        count_stmt = select(func.count()).select_from(base.subquery())
        total: int = (await session.execute(count_stmt)).scalar_one()

        if sort == "created":
            base = base.order_by(Note.created_at.desc())
        elif sort == "alpha":
            base = base.order_by(Note.title.asc())
        elif sort == "type":
            base = base.order_by(Note.note_type.asc(), Note.updated_at.desc())
        else:
            base = base.order_by(Note.updated_at.desc())

        ids = list((await session.execute(base.limit(limit).offset(offset))).scalars().all())

    return ids, total


async def get_knowledge_by_ids(user_id: int, ids: list[int]) -> list[dict]:
    """Fetch full items for the given IDs, preserving the requested order."""
    if not ids:
        return []
    # Fetching specific ids is explicit, so this takes the full read scope — the
    # ids came from either a browse or a search, and both must resolve.
    visible = readable_notes_clause(user_id)
    async with async_session() as session:
        stmt = (
            select(Note)
            .where(visible)
            .where(Note.id.in_(ids))
            .where(Note.deleted_at.is_(None))
        )
        rows = list((await session.execute(stmt)).scalars().all())
    by_id = {n.id: n for n in rows}
    return [_note_to_item(by_id[i]) for i in ids if i in by_id]