feat(knowledge): two-tier pagination — ID pre-fetch + content batch loading

Backend:
- GET /api/knowledge/ids: returns up to 100 note IDs cheaply (no body
  parsing), supports same filters as /api/knowledge, includes has_more
- GET /api/knowledge/batch?ids=...: fetches full items for given IDs in
  order; used by frontend to load content in controlled batches

Frontend (KnowledgeView):
- Fetch 100 IDs upfront, load first 50 as content on mount
- IntersectionObserver sentinel (root: null) triggers 24-item content
  batches as user scrolls
- Proactive ID refill when queue drops below 48 unloaded IDs
- fetchGen counter invalidates stale in-flight responses on filter reset
- IDs claimed before async fetch to prevent double-loading
- sentinelVisible ref drives post-load re-check when content doesn't
  push sentinel off screen

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

frontend/src/views/KnowledgeView.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { ref, watch, onMounted, onUnmounted } from "vue";
+import { ref, computed, watch, onMounted, onUnmounted, nextTick } from "vue";
 import { useRouter } from "vue-router";
 import { apiGet, apiPatch } from "@/api/client";
 import { fmtCompact } from "@/utils/dateFormat";
@@ -50,40 +50,94 @@ const sortMode = ref<"modified" | "created" | "alpha" | "type">("modified");
 const searchQuery = ref("");
 let searchDebounce: ReturnType<typeof setTimeout> | null = null;
 
-// ─── Items ────────────────────────────────────────────────────────────────────
+// ─── Two-tier pagination ──────────────────────────────────────────────────────
+
+const ID_BATCH = 100;         // IDs fetched per server round-trip
+const INITIAL_CONTENT = 50;   // items loaded on first render
+const CONTENT_PAGE = 24;      // items loaded per sentinel trigger
+const REFILL_THRESHOLD = 48;  // fetch more IDs when queue drops below this
 
 const items = ref<KnowledgeItem[]>([]);
-const total = ref(0);
-const page = ref(1);
-const perPage = 24;
-const loading = ref(false);
 const allTags = ref<string[]>([]);
+const idQueue = ref<number[]>([]);   // unloaded IDs ready to be content-fetched
+const idOffset = ref(0);             // next offset for ID batch requests
+const totalKnowledge = ref(0);       // total matching items on server
+const allIdsFetched = ref(false);    // no more ID pages to fetch
+const idsFetching = ref(false);      // ID batch request in-flight
+const contentFetching = ref(false);  // content batch request in-flight
+const sentinelVisible = ref(false);  // updated by IntersectionObserver
 
-async function fetchItems(reset = false) {
-  if (reset) page.value = 1;
-  loading.value = true;
+const loading = computed(() => idsFetching.value || contentFetching.value);
+
+let fetchGen = 0;  // incremented on each reset to invalidate stale responses
+
+function buildFilterParams(): URLSearchParams {
+  const p = new URLSearchParams();
+  if (activeType.value) p.set("type", activeType.value);
+  if (activeTag.value) p.set("tags", activeTag.value);
+  p.set("sort", sortMode.value);
+  if (searchQuery.value.trim()) p.set("q", searchQuery.value.trim());
+  return p;
+}
+
+async function fetchIdBatch(gen: number) {
+  if (idsFetching.value || allIdsFetched.value) return;
+  idsFetching.value = true;
   try {
-    const params = new URLSearchParams();
-    if (activeType.value) params.set("type", activeType.value);
-    if (activeTag.value) params.set("tags", activeTag.value);
-    params.set("sort", sortMode.value);
-    params.set("page", String(page.value));
-    params.set("per_page", String(perPage));
-    if (searchQuery.value.trim()) params.set("q", searchQuery.value.trim());
-    const data = await apiGet<{ items: KnowledgeItem[]; total: number }>(`/api/knowledge?${params}`);
-    if (reset || page.value === 1) {
-      items.value = data.items;
-    } else {
-      items.value = [...items.value, ...data.items];
+    const p = buildFilterParams();
+    p.set("limit", String(ID_BATCH));
+    p.set("offset", String(idOffset.value));
+    const data = await apiGet<{ ids: number[]; total: number; has_more: boolean }>(
+      `/api/knowledge/ids?${p}`
+    );
+    if (gen !== fetchGen) return;
+    idQueue.value.push(...data.ids);
+    idOffset.value += data.ids.length;
+    totalKnowledge.value = data.total;
+    if (!data.has_more) allIdsFetched.value = true;
+  } catch { /* silent */ }
+  finally { if (gen === fetchGen) idsFetching.value = false; }
+}
+
+async function loadNextContent(gen: number, count: number) {
+  if (contentFetching.value || idQueue.value.length === 0) return;
+  const toLoad = idQueue.value.splice(0, count);  // claim IDs immediately
+  contentFetching.value = true;
+  try {
+    const data = await apiGet<{ items: KnowledgeItem[] }>(
+      `/api/knowledge/batch?ids=${toLoad.join(",")}`
+    );
+    if (gen !== fetchGen) return;
+    items.value.push(...data.items);
+    // Proactively refill ID queue before it runs out
+    if (idQueue.value.length < REFILL_THRESHOLD && !allIdsFetched.value && !idsFetching.value) {
+      fetchIdBatch(gen);
+    }
+  } catch { /* silent */ }
+  finally {
+    if (gen !== fetchGen) return;
+    contentFetching.value = false;
+    // Sentinel may still be visible if new items didn't push it off screen
+    await nextTick();
+    if (sentinelVisible.value && idQueue.value.length > 0 && gen === fetchGen) {
+      loadNextContent(gen, CONTENT_PAGE);
     }
-    total.value = data.total;
-  } catch {
-    /* silent */
-  } finally {
-    loading.value = false;
   }
 }
 
+async function reset() {
+  fetchGen++;
+  const gen = fetchGen;
+  items.value = [];
+  idQueue.value = [];
+  idOffset.value = 0;
+  totalKnowledge.value = 0;
+  allIdsFetched.value = false;
+  await fetchIdBatch(gen);
+  if (gen !== fetchGen) return;
+  await loadNextContent(gen, INITIAL_CONTENT);
+}
+
 async function fetchTags() {
   try {
     const data = await apiGet<{ tags: string[] }>("/api/knowledge/tags");
@@ -91,12 +145,18 @@ async function fetchTags() {
   } catch { /* silent */ }
 }
 
-function onSearchInput() {
-  if (searchDebounce) clearTimeout(searchDebounce);
-  searchDebounce = setTimeout(() => fetchItems(true), 380);
+async function resetAndReobserve() {
+  await reset();
+  await nextTick();
+  setupObserver();
 }
 
-watch([activeType, activeTag, sortMode], () => fetchItems(true));
+function onSearchInput() {
+  if (searchDebounce) clearTimeout(searchDebounce);
+  searchDebounce = setTimeout(() => resetAndReobserve(), 380);
+}
+
+watch([activeType, activeTag, sortMode], () => resetAndReobserve());
 
 // ─── Today bar ────────────────────────────────────────────────────────────────
 
@@ -171,7 +231,7 @@ watch(
         ['create_note', 'update_note', 'create_task', 'update_task'].includes(tc.function) &&
         tc.status === 'success'
       ) {
-        fetchItems(true);
+        reset();
         fetchTags();
         break;
       }
@@ -220,7 +280,7 @@ async function toggleListItem(item: KnowledgeItem, index: number) {
   try {
     await apiPatch(`/api/notes/${item.id}`, { body: newBody });
   } catch {
-    fetchItems(true); // revert on error
+    reset(); // revert on error
   }
 }
 
@@ -242,37 +302,37 @@ function formatDate(iso: string): string {
   return d.toLocaleDateString(undefined, { month: "short", day: "numeric", year: "numeric" });
 }
 
-// ─── Infinite scroll ──────────────────────────────────────────────────────────
+// ─── Sentinel observer ────────────────────────────────────────────────────────
 
-const cardGridEl = ref<HTMLElement | null>(null);
-let scrollRafId: number | null = null;
+const sentinelEl = ref<HTMLElement | null>(null);
+let observer: IntersectionObserver | null = null;
 
-function onGridScroll() {
-  // Debounce via rAF — coalesces rapid scroll events into one check per frame
-  if (scrollRafId !== null) return;
-  scrollRafId = requestAnimationFrame(() => {
-    scrollRafId = null;
-    const el = cardGridEl.value;
-    if (!el || loading.value || items.value.length >= total.value) return;
-    const nearBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 300;
-    if (nearBottom) {
-      page.value++;
-      fetchItems();
-    }
-  });
+function setupObserver() {
+  observer?.disconnect();
+  if (!sentinelEl.value) return;
+  observer = new IntersectionObserver(
+    ([entry]) => {
+      sentinelVisible.value = entry.isIntersecting;
+      if (entry.isIntersecting) loadNextContent(fetchGen, CONTENT_PAGE);
+    },
+    { root: null, rootMargin: "200px" }
+  );
+  observer.observe(sentinelEl.value);
 }
 
 // ─── Lifecycle ────────────────────────────────────────────────────────────────
 
-onMounted(() => {
-  fetchItems(true);
+onMounted(async () => {
+  await reset();
   fetchTags();
   fetchTodayBar();
+  await nextTick();
+  setupObserver();
 });
 
 onUnmounted(() => {
   if (searchDebounce) clearTimeout(searchDebounce);
-  if (scrollRafId !== null) cancelAnimationFrame(scrollRafId);
+  observer?.disconnect();
 });
 </script>
 
@@ -375,7 +435,7 @@ onUnmounted(() => {
         </div>
 
         <!-- Card grid -->
-        <div v-else ref="cardGridEl" class="card-grid" @scroll.passive="onGridScroll">
+        <div v-else class="card-grid">
           <div
             v-for="item in items"
             :key="item.id"
@@ -440,8 +500,10 @@ onUnmounted(() => {
             </div>
           </div>
 
-          <!-- Loading indicator when fetching additional pages -->
-          <div v-if="loading && items.length > 0" class="load-more-indicator">Loading…</div>
+          <!-- Sentinel — IntersectionObserver triggers next content batch -->
+          <div ref="sentinelEl" class="scroll-sentinel">
+            <span v-if="contentFetching" class="sentinel-loading">Loading…</span>
+          </div>
         </div>
       </div>
 
@@ -869,11 +931,15 @@ onUnmounted(() => {
 }
 .empty-hint { font-size: 0.85rem; opacity: 0.7; }
 
-/* ── Load more indicator ─────────────────────────────────── */
-.load-more-indicator {
+/* ── Sentinel ────────────────────────────────────────────── */
+.scroll-sentinel {
   grid-column: 1 / -1;
-  padding: 16px;
-  text-align: center;
+  height: 40px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+.sentinel-loading {
   font-size: 0.8rem;
   color: var(--color-muted);
 }

src/fabledassistant/routes/knowledge.py

@@ -62,6 +62,63 @@ async def list_knowledge():
     })
 
 
+@knowledge_bp.route("/ids", methods=["GET"])
+@login_required
+async def list_knowledge_ids():
+    """Return note IDs only (cheap) for the two-tier pagination feed.
+
+    Same filter params as GET /api/knowledge.
+    Additional params: limit (default 100, max 200), offset (default 0).
+    Returns {ids, total, has_more}.
+    """
+    uid = get_current_user_id()
+    note_type = request.args.get("type", "").strip().lower() or None
+    tags_raw = request.args.get("tags", "").strip()
+    tags = [t.strip() for t in tags_raw.split(",") if t.strip()] if tags_raw else []
+    sort = request.args.get("sort", "modified").strip().lower()
+    q = request.args.get("q", "").strip() or None
+    if sort not in _VALID_SORTS:
+        sort = "modified"
+    try:
+        limit = min(int(request.args.get("limit", 100)), 200)
+        offset = max(0, int(request.args.get("offset", 0)))
+    except ValueError:
+        return jsonify({"error": "Invalid limit or offset"}), 400
+
+    if note_type and note_type not in _VALID_TYPES:
+        return jsonify({"error": "Invalid type"}), 400
+
+    from fabledassistant.services.knowledge import query_knowledge_ids
+    ids, total = await query_knowledge_ids(
+        user_id=uid, note_type=note_type, tags=tags,
+        sort=sort, q=q, limit=limit, offset=offset,
+    )
+    return jsonify({"ids": ids, "total": total, "has_more": (offset + len(ids)) < total})
+
+
+@knowledge_bp.route("/batch", methods=["GET"])
+@login_required
+async def get_knowledge_batch():
+    """Fetch full items for a comma-separated list of IDs (max 100).
+
+    Returns {items: [...]} in the order of the requested IDs.
+    """
+    uid = get_current_user_id()
+    ids_raw = request.args.get("ids", "").strip()
+    if not ids_raw:
+        return jsonify({"items": []})
+    try:
+        ids = [int(x) for x in ids_raw.split(",") if x.strip()]
+    except ValueError:
+        return jsonify({"error": "Invalid IDs"}), 400
+    if len(ids) > 100:
+        return jsonify({"error": "Too many IDs (max 100)"}), 400
+
+    from fabledassistant.services.knowledge import get_knowledge_by_ids
+    items = await get_knowledge_by_ids(uid, ids)
+    return jsonify({"items": items})
+
+
 @knowledge_bp.route("/tags", methods=["GET"])
 @login_required
 async def list_knowledge_tags():

src/fabledassistant/services/knowledge.py

@@ -167,6 +167,69 @@ async def get_knowledge_tags(user_id: int, note_type: str | None = None) -> list
     return [r for r in rows if r]
 
 
+async def query_knowledge_ids(
+    user_id: int,
+    note_type: str | None,
+    tags: list[str],
+    sort: str,
+    q: str | None,
+    limit: int = 100,
+    offset: int = 0,
+) -> tuple[list[int], int]:
+    """Return note IDs only — cheap query for the two-tier pagination feed."""
+    if q:
+        # Re-use semantic search, extract IDs in rank order
+        items, total = await _semantic_knowledge_search(
+            user_id, q, note_type=note_type, tags=tags,
+            limit=limit, offset=offset,
+        )
+        return [item["id"] for item in items], total
+
+    async with async_session() as session:
+        base = (
+            select(Note.id)
+            .where(Note.user_id == user_id)
+            .where(Note.status.is_(None))
+        )
+        if note_type:
+            base = base.where(Note.note_type == note_type)
+        else:
+            base = base.where(Note.note_type.in_(["note", "person", "place", "list"]))
+        for tag in tags:
+            base = base.where(Note.tags.contains([tag]))
+
+        count_stmt = select(func.count()).select_from(base.subquery())
+        total: int = (await session.execute(count_stmt)).scalar_one()
+
+        if sort == "created":
+            base = base.order_by(Note.created_at.desc())
+        elif sort == "alpha":
+            base = base.order_by(Note.title.asc())
+        elif sort == "type":
+            base = base.order_by(Note.note_type.asc(), Note.updated_at.desc())
+        else:
+            base = base.order_by(Note.updated_at.desc())
+
+        ids = list((await session.execute(base.limit(limit).offset(offset))).scalars().all())
+
+    return ids, total
+
+
+async def get_knowledge_by_ids(user_id: int, ids: list[int]) -> list[dict]:
+    """Fetch full items for the given IDs, preserving the requested order."""
+    if not ids:
+        return []
+    async with async_session() as session:
+        stmt = (
+            select(Note)
+            .where(Note.user_id == user_id)
+            .where(Note.id.in_(ids))
+        )
+        rows = list((await session.execute(stmt)).scalars().all())
+    by_id = {n.id: n for n in rows}
+    return [_note_to_item(by_id[i]) for i in ids if i in by_id]
+
+
 async def get_people_and_places_context(user_id: int) -> str:
     """Return a compact summary of known people and places for LLM system prompt injection."""
     async with async_session() as session: