diff --git a/docs/superpowers/plans/2026-04-03-streaming-tts.md b/docs/superpowers/plans/2026-04-03-streaming-tts.md new file mode 100644 index 0000000..0dc7b9c --- /dev/null +++ b/docs/superpowers/plans/2026-04-03-streaming-tts.md @@ -0,0 +1,594 @@ +# Streaming TTS Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Start playing TTS audio during LLM generation by splitting responses into sentences and synthesizing each sentence as it completes, rather than waiting for the full response. + +**Architecture:** A new `useStreamingTts` composable watches `streamingContent` for sentence boundaries, fires per-sentence `synthesiseSpeech` requests concurrently, and plays audio in strict insertion order using `useVoiceAudio`. ChatView, BriefingView, and WorkspaceView all use this composable, replacing their current post-stream speak logic. + +**Tech Stack:** Vue 3 Composition API, TypeScript, `useVoiceAudio` (existing), `synthesiseSpeech` from `api/client.ts` (existing), no backend changes. + +--- + +## File Map + +| Action | File | Responsibility | +|--------|------|----------------| +| **Create** | `frontend/src/composables/useStreamingTts.ts` | All streaming TTS logic: sentence splitting, TTS queuing, ordered playback | +| **Modify** | `frontend/src/views/ChatView.vue` | Replace `speakLastAssistantMessage` + old watch with `useStreamingTts` | +| **Modify** | `frontend/src/views/BriefingView.vue` | Replace `speakText` + `listenToLatest` + old watch with `useStreamingTts` | +| **Modify** | `frontend/src/views/WorkspaceView.vue` | Add listen mode toggle button + `useStreamingTts` | + +--- + +## Task 1: Create `useStreamingTts` composable + +**Files:** +- Create: `frontend/src/composables/useStreamingTts.ts` + +- [ ] **Step 1: Create the composable** + +Create `frontend/src/composables/useStreamingTts.ts` with the full implementation: + +```typescript +import { ref, watch, computed } from 'vue' +import type { Ref, ComputedRef } from 'vue' +import { synthesiseSpeech } from '@/api/client' +import { useVoiceAudio } from '@/composables/useVoiceAudio' + +/** Minimum stripped character count to bother synthesizing. */ +const MIN_CHARS = 3 + +/** Matches sentence-terminal punctuation followed by whitespace or end-of-string. */ +const SENTENCE_BOUNDARY = /[.!?]+(?=\s|$)/ + +function stripMarkdown(text: string): string { + return text + .replace(/```[\s\S]*?```/g, '') + .replace(/`[^`]+`/g, (m) => m.slice(1, -1)) + .replace(/#{1,6}\s+/g, '') + .replace(/\*\*([^*]+)\*\*/g, '$1') + .replace(/\*([^*]+)\*/g, '$1') + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') + .replace(/^\s*[-*+]\s+/gm, '') + .replace(/\n{2,}/g, ' ') + .trim() +} + +/** + * Extract completed sentences from `text` using SENTENCE_BOUNDARY. + * Returns the sentences found and the unconsumed remainder. + */ +function extractSentences(text: string): { sentences: string[]; remainder: string } { + const sentences: string[] = [] + let remaining = text + let match: RegExpExecArray | null + + while ((match = SENTENCE_BOUNDARY.exec(remaining)) !== null) { + const boundary = match.index + match[0].length + const sentence = remaining.slice(0, boundary).trim() + if (sentence) sentences.push(sentence) + remaining = remaining.slice(boundary) + } + + return { sentences, remainder: remaining } +} + +export interface UseStreamingTtsOptions { + streamingContent: Ref | ComputedRef + streaming: Ref | ComputedRef + enabled: Ref | ComputedRef +} + +export interface UseStreamingTtsReturn { + /** True while any synthesis request is in-flight or audio is playing. */ + speaking: ComputedRef + /** Cancel all in-flight synthesis/playback and clear the queue. */ + stop: () => void +} + +export function useStreamingTts(options: UseStreamingTtsOptions): UseStreamingTtsReturn { + const { streamingContent, streaming, enabled } = options + const audio = useVoiceAudio() + + let sentenceBuffer = '' + let lastSeenLength = 0 + let abortId = 0 + let playQueue: Promise = Promise.resolve() + const pendingCount = ref(0) + + const speaking = computed(() => pendingCount.value > 0 || audio.playing.value) + + function stop(): void { + abortId++ + sentenceBuffer = '' + lastSeenLength = 0 + playQueue = Promise.resolve() + audio.stop() + pendingCount.value = 0 + } + + async function enqueueSentence(sentence: string, myAbortId: number): Promise { + const stripped = stripMarkdown(sentence) + if (stripped.length < MIN_CHARS) return + + pendingCount.value++ + let blob: Blob | null = null + try { + blob = await synthesiseSpeech(stripped) + } catch (e) { + console.warn('[StreamingTTS] Synthesis failed, retrying sentence', { sentence: stripped, error: e }) + try { + blob = await synthesiseSpeech(stripped) + } catch (e2) { + console.warn('[StreamingTTS] Retry also failed, skipping sentence', { sentence: stripped, error: e2 }) + } + } finally { + pendingCount.value-- + } + + if (!blob) return + + // Capture blob for the closure — TS can't narrow after async gap + const resolvedBlob = blob + playQueue = playQueue.then(async () => { + if (abortId !== myAbortId) return + await audio.play(resolvedBlob) + }) + } + + function dispatchBuffer(flush: boolean): void { + if (!enabled.value) return + const myAbortId = abortId + const { sentences, remainder } = extractSentences(sentenceBuffer) + sentenceBuffer = flush ? '' : remainder + for (const sentence of sentences) { + enqueueSentence(sentence, myAbortId) + } + if (flush && remainder.trim().length >= MIN_CHARS) { + enqueueSentence(remainder.trim(), myAbortId) + } + } + + // Watch accumulating content — extract new characters since last check + watch(streamingContent, (newContent) => { + if (!enabled.value) return + const delta = newContent.slice(lastSeenLength) + lastSeenLength = newContent.length + sentenceBuffer += delta + dispatchBuffer(false) + }) + + // Watch streaming flag — stop on new message start, flush on end + watch(streaming, (isStreaming) => { + if (!enabled.value) return + if (isStreaming) { + // New message starting — cancel previous response's audio + stop() + } else { + // Stream ended — flush any remaining fragment + dispatchBuffer(true) + lastSeenLength = 0 + } + }) + + return { speaking, stop } +} +``` + +- [ ] **Step 2: TypeScript check** + +```bash +cd /home/bvandeusen/Nextcloud/Projects/fabledassistant/frontend +npx vue-tsc --noEmit 2>&1 | head -40 +``` + +Expected: no errors mentioning `useStreamingTts.ts`. + +- [ ] **Step 3: Commit** + +```bash +git add frontend/src/composables/useStreamingTts.ts +git commit -m "feat(tts): add useStreamingTts composable for sentence-level streaming" +``` + +--- + +## Task 2: Update ChatView + +**Files:** +- Modify: `frontend/src/views/ChatView.vue` + +Current TTS code to remove (lines ~35–66): + +```typescript +// REMOVE these: +const synthesising = ref(false); + +async function speakLastAssistantMessage() { ... } // entire function + +watch(() => store.streaming, async (streaming) => { + if (!streaming && listenMode.value && voiceTtsEnabled.value) { + await new Promise((r) => setTimeout(r, 200)); + await speakLastAssistantMessage(); + } +}); +``` + +Also remove the `synthesiseSpeech` import from `@/api/client` (it is no longer called directly in this file). + +- [ ] **Step 1: Add import and replace TTS logic** + +In `frontend/src/views/ChatView.vue`: + +1. Add to imports at the top of `