feat(journal): backend services (moments, search, prep, scheduler, pipeline)

- services/moments.py — CRUD with embedding sync + entity-link helpers - services/journal_search.py — three-mode search (temporal / entity / semantic) with notes-RAG isolation; cosine-similarity helper unit-tested - services/journal_prep.py — gathers tasks/events/weather/news/projects/ recent-moments/open-threads into a structured prep block - services/journal_scheduler.py — per-user APScheduler cron for daily prep, follows the BackgroundScheduler + threadsafe-async pattern from event_scheduler - services/journal_pipeline.py — system prompt (persona + calibration rules) with last-48h ambient moments injection - app.py: wire journal scheduler start/stop hooks - routes/settings.py: re-add live-reschedule on timezone change (now via journal_scheduler.update_user_schedule) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-25 22:37:39 -04:00
parent 7602bf2293
commit d9ab538ef4
7 changed files with 863 additions and 3 deletions
@@ -0,0 +1,204 @@
+"""Search across Moments and (optionally) journal transcripts.
+
+Three modes, expressed through one tool surface:
+1. Pure temporal: no query, just date range -> ordered by occurred_at DESC.
+2. Pure entity: person_id or place_id filter -> junction lookup.
+3. Semantic: query string -> embedding similarity, optionally constrained.
+
+This module ONLY queries Moments and (optionally) journal-conversation
+messages. It MUST NOT touch notes or note_embeddings. The notes-RAG and
+journal-RAG isolation is a hard invariant of the journal design.
+"""
+from __future__ import annotations
+
+import datetime
+import math
+from typing import Sequence
+
+from sqlalchemy import select
+
+from fabledassistant.models import (
+    Conversation,
+    Message,
+    Moment,
+    MomentEmbedding,
+    async_session,
+    moment_people,
+    moment_places,
+)
+from fabledassistant.services.embeddings import get_embedding
+
+DEFAULT_LIMIT = 10
+DEFAULT_THRESHOLD = 0.55
+
+
+def _cosine(a: Sequence[float], b: Sequence[float]) -> float:
+    if not a or not b:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(y * y for y in b))
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+async def search_journal(
+    *,
+    user_id: int,
+    query: str | None = None,
+    person_id: int | None = None,
+    place_id: int | None = None,
+    tag: str | None = None,
+    date_from: datetime.date | None = None,
+    date_to: datetime.date | None = None,
+    include_transcripts: bool = False,
+    limit: int = DEFAULT_LIMIT,
+    threshold: float = DEFAULT_THRESHOLD,
+) -> list[dict]:
+    """Return Moments (and optional transcript snippets) matching the filters.
+
+    Result rows: dicts from Moment.to_dict() plus an optional `score` when
+    `query` is set. Transcript snippets carry `kind='transcript'` and `id=None`
+    to distinguish them.
+    """
+    moment_rows = await _search_moments(
+        user_id=user_id,
+        query=query,
+        person_id=person_id,
+        place_id=place_id,
+        tag=tag,
+        date_from=date_from,
+        date_to=date_to,
+        limit=limit,
+        threshold=threshold,
+    )
+
+    if not include_transcripts:
+        return moment_rows
+
+    transcript_rows = await _search_transcripts(
+        user_id=user_id,
+        query=query,
+        date_from=date_from,
+        date_to=date_to,
+        limit=limit,
+    )
+    return moment_rows + transcript_rows
+
+
+async def _search_moments(
+    *,
+    user_id: int,
+    query: str | None,
+    person_id: int | None,
+    place_id: int | None,
+    tag: str | None,
+    date_from: datetime.date | None,
+    date_to: datetime.date | None,
+    limit: int,
+    threshold: float,
+) -> list[dict]:
+    async with async_session() as session:
+        stmt = select(Moment).where(Moment.user_id == user_id)
+
+        if person_id is not None:
+            stmt = stmt.join(moment_people, moment_people.c.moment_id == Moment.id).where(
+                moment_people.c.person_id == person_id
+            )
+        if place_id is not None:
+            stmt = stmt.join(moment_places, moment_places.c.moment_id == Moment.id).where(
+                moment_places.c.place_id == place_id
+            )
+        if tag is not None:
+            stmt = stmt.where(Moment.tags.any(tag))
+        if date_from is not None:
+            stmt = stmt.where(Moment.day_date >= date_from)
+        if date_to is not None:
+            stmt = stmt.where(Moment.day_date <= date_to)
+
+        if query is None:
+            stmt = stmt.order_by(Moment.occurred_at.desc()).limit(limit)
+            moments = (await session.execute(stmt)).scalars().all()
+            return [m.to_dict() for m in moments]
+
+        # Semantic mode. Pull a wider candidate set, then rank in Python.
+        candidate_stmt = stmt.order_by(Moment.occurred_at.desc()).limit(limit * 5)
+        candidates = (await session.execute(candidate_stmt)).scalars().all()
+        if not candidates:
+            return []
+
+        candidate_ids = [m.id for m in candidates]
+        emb_stmt = select(MomentEmbedding).where(
+            MomentEmbedding.moment_id.in_(candidate_ids)
+        )
+        embeddings = {
+            e.moment_id: e.embedding
+            for e in (await session.execute(emb_stmt)).scalars()
+        }
+
+        query_vec = await get_embedding(query)
+        scored = []
+        for m in candidates:
+            vec = embeddings.get(m.id)
+            if vec is None:
+                continue
+            score = _cosine(query_vec, vec)
+            if score >= threshold:
+                row = m.to_dict()
+                row["score"] = score
+                scored.append(row)
+
+        scored.sort(key=lambda r: r["score"], reverse=True)
+        return scored[:limit]
+
+
+async def _search_transcripts(
+    *,
+    user_id: int,
+    query: str | None,
+    date_from: datetime.date | None,
+    date_to: datetime.date | None,
+    limit: int,
+) -> list[dict]:
+    """Fallback substring search over raw journal Messages.
+
+    Substring is intentional: transcripts catch what the LLM didn't extract
+    as Moments. Semantic search over messages would require a third embedding
+    index, which we deliberately don't maintain.
+    """
+    if query is None:
+        return []
+    async with async_session() as session:
+        stmt = (
+            select(Message, Conversation.day_date)
+            .join(Conversation, Message.conversation_id == Conversation.id)
+            .where(
+                Conversation.user_id == user_id,
+                Conversation.conversation_type == "journal",
+                Message.content.ilike(f"%{query}%"),
+            )
+            .order_by(Message.created_at.desc())
+            .limit(limit)
+        )
+        if date_from is not None:
+            stmt = stmt.where(Conversation.day_date >= date_from)
+        if date_to is not None:
+            stmt = stmt.where(Conversation.day_date <= date_to)
+        rows = (await session.execute(stmt)).all()
+        return [
+            {
+                "id": None,
+                "kind": "transcript",
+                "message_id": msg.id,
+                "conversation_id": msg.conversation_id,
+                "day_date": day_date.isoformat() if day_date else None,
+                "occurred_at": msg.created_at.isoformat(),
+                "content": msg.content[:400],
+                "raw_excerpt": None,
+                "tags": [],
+                "people": [],
+                "places": [],
+            }
+            for msg, day_date in rows
+        ]