From 9b69e38aff9efd92f49534dc1e7c3054a8cee69a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 6 Apr 2026 19:40:56 -0400 Subject: [PATCH] docs: add web voice overlay polish implementation plan --- .../2026-04-06-web-voice-overlay-polish.md | 479 ++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-06-web-voice-overlay-polish.md diff --git a/docs/superpowers/plans/2026-04-06-web-voice-overlay-polish.md b/docs/superpowers/plans/2026-04-06-web-voice-overlay-polish.md new file mode 100644 index 0000000..95d04d4 --- /dev/null +++ b/docs/superpowers/plans/2026-04-06-web-voice-overlay-polish.md @@ -0,0 +1,479 @@ +# Web Voice Overlay Polish — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the dormant `VoiceOverlay` component by mounting it in `App.vue`, wiring the Space bar shortcut, and replacing push-to-talk with click-to-toggle silence detection backed by a new `useSilenceDetector` composable. + +**Architecture:** A new `useSilenceDetector` composable uses `AudioContext` + `AnalyserNode` to monitor amplitude from a live `MediaStream` and fires a callback after sustained silence. `VoiceOverlay` coordinates recording and silence detection, switching from hold-to-record to click-to-toggle. `App.vue` mounts the overlay and adds a Space bar handler that dispatches the existing `voice:ptt-toggle` custom event. + +**Tech Stack:** Vue 3 Composition API, TypeScript, Web Audio API (`AudioContext`, `AnalyserNode`), existing `useVoiceRecorder` / `useVoiceAudio` composables. + +--- + +## File Map + +| Action | Path | +|--------|------| +| Create | `frontend/src/composables/useSilenceDetector.ts` | +| Modify | `frontend/src/composables/useVoiceRecorder.ts` | +| Modify | `frontend/src/components/VoiceOverlay.vue` | +| Modify | `frontend/src/App.vue` | + +--- + +### Task 1: `useSilenceDetector` composable + +**Files:** +- Create: `frontend/src/composables/useSilenceDetector.ts` + +**Context:** The Web Audio API lets us pipe a `MediaStream` into an `AnalyserNode` and read frequency data as a byte array every 100 ms. RMS amplitude of that array gives a 0–1 loudness value; converting to dB lets us use the same `-40 dB` threshold as the Android app. The composable must be safe to call `stop()` on multiple times and must reset amplitude to 0 after stopping so the animated bars collapse. + +- [ ] **Step 1: Create the file with full implementation** + +`frontend/src/composables/useSilenceDetector.ts`: + +```ts +import { ref, readonly } from 'vue' + +export interface SilenceDetectorOptions { + thresholdDb?: number // default -40 + silenceDurationMs?: number // default 1500 + minRecordingMs?: number // default 500 +} + +export function useSilenceDetector(options: SilenceDetectorOptions = {}) { + const { + thresholdDb = -40, + silenceDurationMs = 1500, + minRecordingMs = 500, + } = options + + const amplitude = ref(0) + let audioCtx: AudioContext | null = null + let intervalId: ReturnType | null = null + let silenceMs = 0 + let startedAt = 0 + + function start(stream: MediaStream, onSilence: () => void): void { + stop() + audioCtx = new AudioContext() + const source = audioCtx.createMediaStreamSource(stream) + const analyser = audioCtx.createAnalyser() + analyser.fftSize = 256 + source.connect(analyser) + + const data = new Uint8Array(analyser.frequencyBinCount) + silenceMs = 0 + startedAt = Date.now() + + intervalId = setInterval(() => { + analyser.getByteFrequencyData(data) + const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255 + amplitude.value = rms + + const db = rms > 0 ? 20 * Math.log10(rms) : -100 + if (db < thresholdDb) { + silenceMs += 100 + if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) { + stop() + onSilence() + } + } else { + silenceMs = 0 + } + }, 100) + } + + function stop(): void { + if (intervalId !== null) { + clearInterval(intervalId) + intervalId = null + } + if (audioCtx) { + audioCtx.close().catch(() => {}) + audioCtx = null + } + amplitude.value = 0 + silenceMs = 0 + } + + return { amplitude: readonly(amplitude), start, stop } +} +``` + +- [ ] **Step 2: Verify TypeScript compiles** + +```bash +cd /path/to/fabledassistant/frontend +npx tsc --noEmit +``` + +Expected: no errors. + +- [ ] **Step 3: Commit** + +```bash +git add frontend/src/composables/useSilenceDetector.ts +git commit -m "feat: add useSilenceDetector composable with Web Audio API amplitude monitoring" +``` + +--- + +### Task 2: Expose `stream` ref from `useVoiceRecorder` + +**Files:** +- Modify: `frontend/src/composables/useVoiceRecorder.ts` + +**Context:** Currently `stream` is a plain `let` variable inside the closure. `VoiceOverlay` needs to pass the live `MediaStream` to `useSilenceDetector.start()` after recording begins. Exposing it as a readonly `Ref` is the minimal change — no other callers are broken because they don't currently read `stream` from the return value. + +The current file is at `frontend/src/composables/useVoiceRecorder.ts`. Read it before editing — the key lines to change are: + +1. Top of function body: `let stream: MediaStream | null = null` → `const streamRef = ref(null)` +2. In `startRecording()`: `stream = await navigator.mediaDevices.getUserMedia({ audio: true })` → `streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })` +3. In `startRecording()` catch block: `stream = null` if present — replace with `streamRef.value = null` (if the catch sets stream to null; if not, skip) +4. In `mediaRecorder.onstop`: `stream?.getTracks().forEach((t) => t.stop())` → `streamRef.value?.getTracks().forEach((t) => t.stop())` then `streamRef.value = null` +5. Return object: add `stream: readonly(streamRef)` + +- [ ] **Step 1: Add the `ref` import if not already present** + +The file already imports `{ ref, readonly }` from `'vue'` — confirm this. If `ref` is missing from the import, add it. + +- [ ] **Step 2: Replace the `stream` variable declaration** + +Find: +```ts +let stream: MediaStream | null = null +``` +Replace with: +```ts +const streamRef = ref(null) +``` + +- [ ] **Step 3: Update all usages of `stream` in `startRecording`** + +Find: +```ts + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) +``` +Replace with: +```ts + streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true }) +``` + +- [ ] **Step 4: Update `onstop` handler** + +Find: +```ts + stream?.getTracks().forEach((t) => t.stop()) + stream = null +``` +Replace with: +```ts + streamRef.value?.getTracks().forEach((t) => t.stop()) + streamRef.value = null +``` + +- [ ] **Step 5: Add `stream` to the return object** + +Find the return statement and add `stream: readonly(streamRef)`: +```ts + return { + recording: readonly(recording), + error: readonly(error), + isSupported, + startRecording, + stopRecording, + stream: readonly(streamRef), + } +``` + +- [ ] **Step 6: Verify TypeScript compiles** + +```bash +npx tsc --noEmit +``` + +Expected: no errors. + +- [ ] **Step 7: Commit** + +```bash +git add frontend/src/composables/useVoiceRecorder.ts +git commit -m "feat: expose live stream ref from useVoiceRecorder" +``` + +--- + +### Task 3: Update `VoiceOverlay` — silence detection, click-to-toggle, amplitude bars + +**Files:** +- Modify: `frontend/src/components/VoiceOverlay.vue` + +**Context:** `VoiceOverlay.vue` is a complete floating voice UI that was never mounted. It currently uses `@mousedown`/`@mouseup` for push-to-talk. This task switches it to click-to-toggle with automatic silence detection and adds animated amplitude bars during recording. Read the full file before making changes — the existing structure and style blocks must be preserved. + +#### Script changes + +- [ ] **Step 1: Import `useSilenceDetector`** + +At the top of `