Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 58e5c6bc60 | |||
| d3170e5545 | |||
| 814f44c3fb | |||
| 1d0cf4828b | |||
| 56f1c44b8e | |||
| a5e35f7c72 | |||
| 853dc810ff | |||
| ac7dde472f | |||
| 84926d4ba2 | |||
| 9b69e38aff | |||
| 1b68559bfe | |||
| edf0a9063e |
@@ -0,0 +1,479 @@
|
|||||||
|
# Web Voice Overlay Polish — Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Ship the dormant `VoiceOverlay` component by mounting it in `App.vue`, wiring the Space bar shortcut, and replacing push-to-talk with click-to-toggle silence detection backed by a new `useSilenceDetector` composable.
|
||||||
|
|
||||||
|
**Architecture:** A new `useSilenceDetector` composable uses `AudioContext` + `AnalyserNode` to monitor amplitude from a live `MediaStream` and fires a callback after sustained silence. `VoiceOverlay` coordinates recording and silence detection, switching from hold-to-record to click-to-toggle. `App.vue` mounts the overlay and adds a Space bar handler that dispatches the existing `voice:ptt-toggle` custom event.
|
||||||
|
|
||||||
|
**Tech Stack:** Vue 3 Composition API, TypeScript, Web Audio API (`AudioContext`, `AnalyserNode`), existing `useVoiceRecorder` / `useVoiceAudio` composables.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Map
|
||||||
|
|
||||||
|
| Action | Path |
|
||||||
|
|--------|------|
|
||||||
|
| Create | `frontend/src/composables/useSilenceDetector.ts` |
|
||||||
|
| Modify | `frontend/src/composables/useVoiceRecorder.ts` |
|
||||||
|
| Modify | `frontend/src/components/VoiceOverlay.vue` |
|
||||||
|
| Modify | `frontend/src/App.vue` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: `useSilenceDetector` composable
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `frontend/src/composables/useSilenceDetector.ts`
|
||||||
|
|
||||||
|
**Context:** The Web Audio API lets us pipe a `MediaStream` into an `AnalyserNode` and read frequency data as a byte array every 100 ms. RMS amplitude of that array gives a 0–1 loudness value; converting to dB lets us use the same `-40 dB` threshold as the Android app. The composable must be safe to call `stop()` on multiple times and must reset amplitude to 0 after stopping so the animated bars collapse.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Create the file with full implementation**
|
||||||
|
|
||||||
|
`frontend/src/composables/useSilenceDetector.ts`:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { ref, readonly } from 'vue'
|
||||||
|
|
||||||
|
export interface SilenceDetectorOptions {
|
||||||
|
thresholdDb?: number // default -40
|
||||||
|
silenceDurationMs?: number // default 1500
|
||||||
|
minRecordingMs?: number // default 500
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||||
|
const {
|
||||||
|
thresholdDb = -40,
|
||||||
|
silenceDurationMs = 1500,
|
||||||
|
minRecordingMs = 500,
|
||||||
|
} = options
|
||||||
|
|
||||||
|
const amplitude = ref(0)
|
||||||
|
let audioCtx: AudioContext | null = null
|
||||||
|
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||||
|
let silenceMs = 0
|
||||||
|
let startedAt = 0
|
||||||
|
|
||||||
|
function start(stream: MediaStream, onSilence: () => void): void {
|
||||||
|
stop()
|
||||||
|
audioCtx = new AudioContext()
|
||||||
|
const source = audioCtx.createMediaStreamSource(stream)
|
||||||
|
const analyser = audioCtx.createAnalyser()
|
||||||
|
analyser.fftSize = 256
|
||||||
|
source.connect(analyser)
|
||||||
|
|
||||||
|
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||||
|
silenceMs = 0
|
||||||
|
startedAt = Date.now()
|
||||||
|
|
||||||
|
intervalId = setInterval(() => {
|
||||||
|
analyser.getByteFrequencyData(data)
|
||||||
|
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||||
|
amplitude.value = rms
|
||||||
|
|
||||||
|
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||||
|
if (db < thresholdDb) {
|
||||||
|
silenceMs += 100
|
||||||
|
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||||
|
stop()
|
||||||
|
onSilence()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
}, 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop(): void {
|
||||||
|
if (intervalId !== null) {
|
||||||
|
clearInterval(intervalId)
|
||||||
|
intervalId = null
|
||||||
|
}
|
||||||
|
if (audioCtx) {
|
||||||
|
audioCtx.close().catch(() => {})
|
||||||
|
audioCtx = null
|
||||||
|
}
|
||||||
|
amplitude.value = 0
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return { amplitude: readonly(amplitude), start, stop }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Verify TypeScript compiles**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /path/to/fabledassistant/frontend
|
||||||
|
npx tsc --noEmit
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add frontend/src/composables/useSilenceDetector.ts
|
||||||
|
git commit -m "feat: add useSilenceDetector composable with Web Audio API amplitude monitoring"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Expose `stream` ref from `useVoiceRecorder`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/composables/useVoiceRecorder.ts`
|
||||||
|
|
||||||
|
**Context:** Currently `stream` is a plain `let` variable inside the closure. `VoiceOverlay` needs to pass the live `MediaStream` to `useSilenceDetector.start()` after recording begins. Exposing it as a readonly `Ref<MediaStream | null>` is the minimal change — no other callers are broken because they don't currently read `stream` from the return value.
|
||||||
|
|
||||||
|
The current file is at `frontend/src/composables/useVoiceRecorder.ts`. Read it before editing — the key lines to change are:
|
||||||
|
|
||||||
|
1. Top of function body: `let stream: MediaStream | null = null` → `const streamRef = ref<MediaStream | null>(null)`
|
||||||
|
2. In `startRecording()`: `stream = await navigator.mediaDevices.getUserMedia({ audio: true })` → `streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })`
|
||||||
|
3. In `startRecording()` catch block: `stream = null` if present — replace with `streamRef.value = null` (if the catch sets stream to null; if not, skip)
|
||||||
|
4. In `mediaRecorder.onstop`: `stream?.getTracks().forEach((t) => t.stop())` → `streamRef.value?.getTracks().forEach((t) => t.stop())` then `streamRef.value = null`
|
||||||
|
5. Return object: add `stream: readonly(streamRef)`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add the `ref` import if not already present**
|
||||||
|
|
||||||
|
The file already imports `{ ref, readonly }` from `'vue'` — confirm this. If `ref` is missing from the import, add it.
|
||||||
|
|
||||||
|
- [ ] **Step 2: Replace the `stream` variable declaration**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```ts
|
||||||
|
let stream: MediaStream | null = null
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```ts
|
||||||
|
const streamRef = ref<MediaStream | null>(null)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Update all usages of `stream` in `startRecording`**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```ts
|
||||||
|
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```ts
|
||||||
|
streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Update `onstop` handler**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```ts
|
||||||
|
stream?.getTracks().forEach((t) => t.stop())
|
||||||
|
stream = null
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```ts
|
||||||
|
streamRef.value?.getTracks().forEach((t) => t.stop())
|
||||||
|
streamRef.value = null
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Add `stream` to the return object**
|
||||||
|
|
||||||
|
Find the return statement and add `stream: readonly(streamRef)`:
|
||||||
|
```ts
|
||||||
|
return {
|
||||||
|
recording: readonly(recording),
|
||||||
|
error: readonly(error),
|
||||||
|
isSupported,
|
||||||
|
startRecording,
|
||||||
|
stopRecording,
|
||||||
|
stream: readonly(streamRef),
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 6: Verify TypeScript compiles**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx tsc --noEmit
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add frontend/src/composables/useVoiceRecorder.ts
|
||||||
|
git commit -m "feat: expose live stream ref from useVoiceRecorder"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: Update `VoiceOverlay` — silence detection, click-to-toggle, amplitude bars
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/components/VoiceOverlay.vue`
|
||||||
|
|
||||||
|
**Context:** `VoiceOverlay.vue` is a complete floating voice UI that was never mounted. It currently uses `@mousedown`/`@mouseup` for push-to-talk. This task switches it to click-to-toggle with automatic silence detection and adds animated amplitude bars during recording. Read the full file before making changes — the existing structure and style blocks must be preserved.
|
||||||
|
|
||||||
|
#### Script changes
|
||||||
|
|
||||||
|
- [ ] **Step 1: Import `useSilenceDetector`**
|
||||||
|
|
||||||
|
At the top of `<script setup>`, after the existing imports, add:
|
||||||
|
```ts
|
||||||
|
import { useSilenceDetector } from '@/composables/useSilenceDetector'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Instantiate the composable**
|
||||||
|
|
||||||
|
After `const audio = useVoiceAudio()`, add:
|
||||||
|
```ts
|
||||||
|
const silenceDetector = useSilenceDetector()
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3: Update `startPtt` to start silence detection**
|
||||||
|
|
||||||
|
Find the `startPtt` function. After `phase.value = 'recording'`, add:
|
||||||
|
```ts
|
||||||
|
if (recorder.stream.value) {
|
||||||
|
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The complete `startPtt` after the change:
|
||||||
|
```ts
|
||||||
|
async function startPtt() {
|
||||||
|
if (!voiceEnabled.value || isBusy.value) return
|
||||||
|
audio.stop()
|
||||||
|
errorMsg.value = ''
|
||||||
|
open.value = true
|
||||||
|
await recorder.startRecording()
|
||||||
|
if (recorder.error.value) {
|
||||||
|
phase.value = 'error'
|
||||||
|
errorMsg.value = recorder.error.value
|
||||||
|
return
|
||||||
|
}
|
||||||
|
phase.value = 'recording'
|
||||||
|
if (recorder.stream.value) {
|
||||||
|
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Update `stopPtt` to stop silence detection**
|
||||||
|
|
||||||
|
Add `silenceDetector.stop()` as the very first line of `stopPtt`:
|
||||||
|
```ts
|
||||||
|
async function stopPtt() {
|
||||||
|
silenceDetector.stop()
|
||||||
|
if (phase.value !== 'recording') return
|
||||||
|
// ... rest unchanged
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Update `cancelAll` to stop silence detection**
|
||||||
|
|
||||||
|
Add `silenceDetector.stop()` after `recorder.stopRecording().catch(() => {})`:
|
||||||
|
```ts
|
||||||
|
function cancelAll() {
|
||||||
|
silenceDetector.stop()
|
||||||
|
recorder.stopRecording().catch(() => {})
|
||||||
|
audio.stop()
|
||||||
|
phase.value = 'idle'
|
||||||
|
streamContent.value = ''
|
||||||
|
errorMsg.value = ''
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 6: Add `onBtnClick` function**
|
||||||
|
|
||||||
|
Add this function after `cancelAll`:
|
||||||
|
```ts
|
||||||
|
function onBtnClick() {
|
||||||
|
if (phase.value === 'error') { phase.value = 'idle'; return }
|
||||||
|
if (phase.value === 'recording') { stopPtt(); return }
|
||||||
|
if (phase.value === 'idle') { startPtt() }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Template changes
|
||||||
|
|
||||||
|
- [ ] **Step 7: Replace PTT mouse/touch handlers with `@click`**
|
||||||
|
|
||||||
|
On `.voice-ptt-btn`, replace:
|
||||||
|
```html
|
||||||
|
@mousedown.prevent="startPtt"
|
||||||
|
@mouseup.prevent="stopPtt"
|
||||||
|
@touchstart.prevent="startPtt"
|
||||||
|
@touchend.prevent="stopPtt"
|
||||||
|
@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"
|
||||||
|
```
|
||||||
|
with:
|
||||||
|
```html
|
||||||
|
@click.prevent="onBtnClick"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 8: Update aria-label and title on the button**
|
||||||
|
|
||||||
|
Replace:
|
||||||
|
```html
|
||||||
|
:aria-label="phase === 'recording' ? 'Release to send' : 'Hold to speak'"
|
||||||
|
:title="phase === 'recording' ? 'Release to send' : 'Hold Space or tap to speak'"
|
||||||
|
```
|
||||||
|
with:
|
||||||
|
```html
|
||||||
|
:aria-label="phase === 'recording' ? 'Click to stop' : 'Click to speak'"
|
||||||
|
:title="phase === 'recording' ? 'Click to stop or wait for silence' : 'Click or press Space to speak'"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 9: Replace the static recording icon with amplitude bars**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```html
|
||||||
|
<!-- Recording: waveform / stop icon -->
|
||||||
|
<svg v-else-if="phase === 'recording'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||||
|
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"/>
|
||||||
|
</svg>
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```html
|
||||||
|
<!-- Recording: amplitude bars -->
|
||||||
|
<span v-else-if="phase === 'recording'" class="voice-amp-bars">
|
||||||
|
<span
|
||||||
|
v-for="n in 3"
|
||||||
|
:key="n"
|
||||||
|
class="voice-amp-bar"
|
||||||
|
:style="{ transform: `scaleY(${0.3 + silenceDetector.amplitude.value * (0.4 + n * 0.15)})` }"
|
||||||
|
></span>
|
||||||
|
</span>
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 10: Update the idle hint label**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```html
|
||||||
|
Hold <kbd>Space</kbd> or tap
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```html
|
||||||
|
Tap or press <kbd>Space</kbd>
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Style changes
|
||||||
|
|
||||||
|
- [ ] **Step 11: Add amplitude bar styles to `<style scoped>`**
|
||||||
|
|
||||||
|
Append inside the `<style scoped>` block:
|
||||||
|
```css
|
||||||
|
/* ─── Amplitude bars (recording state) ──────────────────────────────────── */
|
||||||
|
.voice-amp-bars {
|
||||||
|
display: flex;
|
||||||
|
gap: 3px;
|
||||||
|
align-items: center;
|
||||||
|
height: 22px;
|
||||||
|
}
|
||||||
|
.voice-amp-bar {
|
||||||
|
width: 4px;
|
||||||
|
height: 18px;
|
||||||
|
background: #fff;
|
||||||
|
border-radius: 2px;
|
||||||
|
transform-origin: center;
|
||||||
|
transition: transform 0.08s ease;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 12: Verify TypeScript compiles**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx tsc --noEmit
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 13: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add frontend/src/components/VoiceOverlay.vue
|
||||||
|
git commit -m "feat: click-to-toggle silence detection and amplitude bars in VoiceOverlay"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 4: Mount `VoiceOverlay` and wire Space bar in `App.vue`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/App.vue`
|
||||||
|
|
||||||
|
**Context:** `App.vue` has a full `onGlobalKeydown` handler and a shortcuts overlay. The Space bar is already documented there as "Hold to speak (voice, when enabled)" but the handler was never added to `onGlobalKeydown`. `VoiceOverlay` uses `Teleport to="body"` so it renders at the document root regardless of where it's placed in the template — just needs to be inside the authenticated block.
|
||||||
|
|
||||||
|
#### Script changes
|
||||||
|
|
||||||
|
- [ ] **Step 1: Add `VoiceOverlay` import**
|
||||||
|
|
||||||
|
In `<script setup>`, after the existing component imports (after `ToastNotification`), add:
|
||||||
|
```ts
|
||||||
|
import VoiceOverlay from '@/components/VoiceOverlay.vue'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Add Space bar case to `onGlobalKeydown`**
|
||||||
|
|
||||||
|
The existing handler has a `switch (e.key)` block. The guard `if (isInputActive() || e.ctrlKey || e.metaKey || e.altKey) return` already runs before the switch, so the Space case only fires when the user isn't typing.
|
||||||
|
|
||||||
|
Inside the `switch (e.key)` block, add this case after the existing `'c'` case:
|
||||||
|
```ts
|
||||||
|
case ' ':
|
||||||
|
e.preventDefault()
|
||||||
|
document.dispatchEvent(new CustomEvent('voice:ptt-toggle'))
|
||||||
|
break
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Template changes
|
||||||
|
|
||||||
|
- [ ] **Step 3: Mount `VoiceOverlay` in the authenticated template**
|
||||||
|
|
||||||
|
Find `<ToastNotification />` near the bottom of the authenticated template block and add `<VoiceOverlay />` directly above it:
|
||||||
|
```html
|
||||||
|
<VoiceOverlay />
|
||||||
|
<ToastNotification />
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Update Space bar description in shortcuts panel**
|
||||||
|
|
||||||
|
Find:
|
||||||
|
```html
|
||||||
|
<span class="shortcut-desc">Hold to speak (voice, when enabled)</span>
|
||||||
|
```
|
||||||
|
Replace with:
|
||||||
|
```html
|
||||||
|
<span class="shortcut-desc">Tap to speak (voice, when enabled)</span>
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 5: Verify TypeScript compiles**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npx tsc --noEmit
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 6: Verify full build succeeds**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: build completes with no errors.
|
||||||
|
|
||||||
|
- [ ] **Step 7: Manual smoke test**
|
||||||
|
|
||||||
|
1. Start the dev server: `npm run dev`
|
||||||
|
2. Log in — confirm the floating mic button appears in the bottom-right corner
|
||||||
|
3. Ensure voice is enabled in Settings → Voice
|
||||||
|
4. Click the mic button — confirm it turns red with animated amplitude bars
|
||||||
|
5. Speak — bars should animate with your voice
|
||||||
|
6. Stop speaking — after ~1.5 s of silence, the button should switch to purple (transcribing), then green (speaking) as it plays back the response
|
||||||
|
7. Click the mic while recording — confirm it stops immediately
|
||||||
|
8. Press Space (not in an input field) — confirm it starts/stops recording
|
||||||
|
9. Press Space in the chat input — confirm it does NOT trigger voice
|
||||||
|
|
||||||
|
- [ ] **Step 8: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add frontend/src/App.vue
|
||||||
|
git commit -m "feat: mount VoiceOverlay and wire Space bar shortcut in App.vue"
|
||||||
|
```
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
# Web Voice Overlay Polish — Implementation Spec
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Ship the dormant `VoiceOverlay` component by mounting it, wiring the Space bar shortcut, and replacing push-to-talk with click-to-toggle silence detection.
|
||||||
|
|
||||||
|
**Architecture:** A new `useSilenceDetector` composable wraps the Web Audio API `AnalyserNode` and fires a callback when sustained silence is detected. `VoiceOverlay` coordinates `useVoiceRecorder` and `useSilenceDetector`, switching from hold-to-record to click-to-toggle. `App.vue` mounts the overlay and adds the Space bar handler.
|
||||||
|
|
||||||
|
**Tech Stack:** Vue 3 Composition API, Web Audio API (`AnalyserNode`), existing `useVoiceRecorder` / `useVoiceAudio` composables, TypeScript.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Map
|
||||||
|
|
||||||
|
| Action | Path |
|
||||||
|
|--------|------|
|
||||||
|
| Create | `frontend/src/composables/useSilenceDetector.ts` |
|
||||||
|
| Modify | `frontend/src/composables/useVoiceRecorder.ts` |
|
||||||
|
| Modify | `frontend/src/components/VoiceOverlay.vue` |
|
||||||
|
| Modify | `frontend/src/App.vue` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 1: `useSilenceDetector` composable
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `frontend/src/composables/useSilenceDetector.ts`
|
||||||
|
|
||||||
|
### Interface
|
||||||
|
|
||||||
|
```ts
|
||||||
|
export interface SilenceDetectorOptions {
|
||||||
|
thresholdDb?: number // default -40
|
||||||
|
silenceDurationMs?: number // default 1500
|
||||||
|
minRecordingMs?: number // default 500
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useSilenceDetector(options?: SilenceDetectorOptions): {
|
||||||
|
amplitude: Readonly<Ref<number>> // 0–1, for visualization
|
||||||
|
start(stream: MediaStream, onSilence: () => void): void
|
||||||
|
stop(): void
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Behaviour
|
||||||
|
|
||||||
|
- `start(stream, onSilence)`:
|
||||||
|
1. Creates `AudioContext`
|
||||||
|
2. `createMediaStreamSource(stream)` → connects to `AnalyserNode` (fftSize 256)
|
||||||
|
3. Records `startedAt = Date.now()`
|
||||||
|
4. Starts a `setInterval` at 100ms that:
|
||||||
|
- Calls `analyser.getByteFrequencyData(dataArray)`
|
||||||
|
- Computes RMS amplitude → maps to 0–1 range for `amplitude.value`
|
||||||
|
- Converts to approximate dB: `db = 20 * log10(rms)` (clamp to -100 when rms === 0)
|
||||||
|
- If `db < thresholdDb`: increments `silenceMs += 100`; else resets `silenceMs = 0`
|
||||||
|
- If `silenceMs >= silenceDurationMs` AND `Date.now() - startedAt >= minRecordingMs`: clears interval, fires `onSilence()`
|
||||||
|
- `stop()`: clears interval, closes `AudioContext`, resets `amplitude.value = 0`
|
||||||
|
- Safe to call `stop()` multiple times (guard with null check)
|
||||||
|
- `amplitude` resets to 0 after `stop()`
|
||||||
|
|
||||||
|
### Full implementation
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { ref, readonly } from 'vue'
|
||||||
|
|
||||||
|
export interface SilenceDetectorOptions {
|
||||||
|
thresholdDb?: number
|
||||||
|
silenceDurationMs?: number
|
||||||
|
minRecordingMs?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||||
|
const {
|
||||||
|
thresholdDb = -40,
|
||||||
|
silenceDurationMs = 1500,
|
||||||
|
minRecordingMs = 500,
|
||||||
|
} = options
|
||||||
|
|
||||||
|
const amplitude = ref(0)
|
||||||
|
let audioCtx: AudioContext | null = null
|
||||||
|
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||||
|
let silenceMs = 0
|
||||||
|
let startedAt = 0
|
||||||
|
|
||||||
|
function start(stream: MediaStream, onSilence: () => void) {
|
||||||
|
stop()
|
||||||
|
audioCtx = new AudioContext()
|
||||||
|
const source = audioCtx.createMediaStreamSource(stream)
|
||||||
|
const analyser = audioCtx.createAnalyser()
|
||||||
|
analyser.fftSize = 256
|
||||||
|
source.connect(analyser)
|
||||||
|
|
||||||
|
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||||
|
silenceMs = 0
|
||||||
|
startedAt = Date.now()
|
||||||
|
|
||||||
|
intervalId = setInterval(() => {
|
||||||
|
analyser.getByteFrequencyData(data)
|
||||||
|
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||||
|
amplitude.value = rms
|
||||||
|
|
||||||
|
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||||
|
if (db < thresholdDb) {
|
||||||
|
silenceMs += 100
|
||||||
|
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||||
|
stop()
|
||||||
|
onSilence()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
}, 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop() {
|
||||||
|
if (intervalId !== null) {
|
||||||
|
clearInterval(intervalId)
|
||||||
|
intervalId = null
|
||||||
|
}
|
||||||
|
if (audioCtx) {
|
||||||
|
audioCtx.close().catch(() => {})
|
||||||
|
audioCtx = null
|
||||||
|
}
|
||||||
|
amplitude.value = 0
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return { amplitude: readonly(amplitude), start, stop }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Write the file exactly as above
|
||||||
|
- [ ] Verify TypeScript compiles: `cd frontend && npx tsc --noEmit`
|
||||||
|
- [ ] Commit: `git add frontend/src/composables/useSilenceDetector.ts && git commit -m "feat: add useSilenceDetector composable"`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 2: Expose `stream` from `useVoiceRecorder`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/composables/useVoiceRecorder.ts`
|
||||||
|
|
||||||
|
Change the `stream` local variable to a `Ref<MediaStream | null>` and export it as readonly.
|
||||||
|
|
||||||
|
- [ ] Change `let stream: MediaStream | null = null` to `const streamRef = ref<MediaStream | null>(null)`
|
||||||
|
- [ ] Replace all `stream` assignments with `streamRef.value`:
|
||||||
|
- `stream = await navigator.mediaDevices.getUserMedia(...)` → `streamRef.value = await ...`
|
||||||
|
- `stream?.getTracks().forEach(...)` → `streamRef.value?.getTracks().forEach(...)`
|
||||||
|
- `stream = null` → `streamRef.value = null`
|
||||||
|
- [ ] Add `stream: readonly(streamRef)` to the return object
|
||||||
|
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||||
|
- [ ] Commit: `git add frontend/src/composables/useVoiceRecorder.ts && git commit -m "feat: expose stream ref from useVoiceRecorder"`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 3: Wire `VoiceOverlay` — silence detection + click-to-toggle
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/components/VoiceOverlay.vue`
|
||||||
|
|
||||||
|
### Script changes
|
||||||
|
|
||||||
|
- [ ] Import `useSilenceDetector` at the top of `<script setup>`
|
||||||
|
- [ ] Add `const silenceDetector = useSilenceDetector()` after the existing composable instantiations
|
||||||
|
- [ ] In `startPtt()`: after `phase.value = 'recording'`, add:
|
||||||
|
```ts
|
||||||
|
if (recorder.stream.value) {
|
||||||
|
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- [ ] In `stopPtt()`: add `silenceDetector.stop()` as the first line (before the guard check)
|
||||||
|
- [ ] In `cancelAll()`: add `silenceDetector.stop()` after `recorder.stopRecording().catch(() => {})`
|
||||||
|
|
||||||
|
### Button: click-to-toggle
|
||||||
|
|
||||||
|
Replace the PTT mouse/touch handlers on `.voice-ptt-btn` with click-to-toggle logic:
|
||||||
|
|
||||||
|
- [ ] Remove `@mousedown.prevent="startPtt"` and `@mouseup.prevent="stopPtt"`
|
||||||
|
- [ ] Remove `@touchstart.prevent="startPtt"` and `@touchend.prevent="stopPtt"`
|
||||||
|
- [ ] Replace `@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"` with:
|
||||||
|
```html
|
||||||
|
@click.prevent="onBtnClick"
|
||||||
|
```
|
||||||
|
- [ ] Add `onBtnClick` function in script:
|
||||||
|
```ts
|
||||||
|
function onBtnClick() {
|
||||||
|
if (phase.value === 'error') { phase.value = 'idle'; return }
|
||||||
|
if (phase.value === 'recording') { stopPtt(); return }
|
||||||
|
if (phase.value === 'idle') { startPtt() }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- [ ] Update `aria-label` and `title` on the button:
|
||||||
|
- `aria-label`: `phase === 'recording' ? 'Click to stop' : 'Click to speak'`
|
||||||
|
- `title`: `phase === 'recording' ? 'Click to stop or wait for silence' : 'Click or press Space to speak'`
|
||||||
|
|
||||||
|
### Amplitude visualization during recording
|
||||||
|
|
||||||
|
Inside the button, when `phase === 'recording'`, replace the static stop icon with animated amplitude bars:
|
||||||
|
|
||||||
|
- [ ] Replace the recording SVG block:
|
||||||
|
```html
|
||||||
|
<svg v-else-if="phase === 'recording'" ...>...</svg>
|
||||||
|
```
|
||||||
|
with:
|
||||||
|
```html
|
||||||
|
<span v-else-if="phase === 'recording'" class="voice-amp-bars">
|
||||||
|
<span
|
||||||
|
v-for="n in 3"
|
||||||
|
:key="n"
|
||||||
|
class="voice-amp-bar"
|
||||||
|
:style="{ transform: `scaleY(${0.3 + silenceDetector.amplitude.value * (0.4 + n * 0.15)})` }"
|
||||||
|
></span>
|
||||||
|
</span>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Hint label
|
||||||
|
|
||||||
|
- [ ] Change the idle hint from `Hold <kbd>Space</kbd> or tap` to `Tap or press <kbd>Space</kbd>`
|
||||||
|
|
||||||
|
### CSS for amplitude bars
|
||||||
|
|
||||||
|
- [ ] Add to `<style scoped>`:
|
||||||
|
```css
|
||||||
|
.voice-amp-bars {
|
||||||
|
display: flex;
|
||||||
|
gap: 3px;
|
||||||
|
align-items: center;
|
||||||
|
height: 22px;
|
||||||
|
}
|
||||||
|
.voice-amp-bar {
|
||||||
|
width: 4px;
|
||||||
|
height: 18px;
|
||||||
|
background: #fff;
|
||||||
|
border-radius: 2px;
|
||||||
|
transform-origin: center;
|
||||||
|
transition: transform 0.08s ease;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||||
|
- [ ] Commit: `git add frontend/src/components/VoiceOverlay.vue && git commit -m "feat: click-to-toggle silence detection in VoiceOverlay"`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Task 4: Mount overlay and wire Space bar in `App.vue`
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `frontend/src/App.vue`
|
||||||
|
|
||||||
|
### Mount VoiceOverlay
|
||||||
|
|
||||||
|
- [ ] Add import at top of `<script setup>`:
|
||||||
|
```ts
|
||||||
|
import VoiceOverlay from '@/components/VoiceOverlay.vue'
|
||||||
|
```
|
||||||
|
- [ ] Add `<VoiceOverlay />` inside the `<template v-if="authStore.isAuthenticated">` block, just before `<ToastNotification />`:
|
||||||
|
```html
|
||||||
|
<VoiceOverlay />
|
||||||
|
<ToastNotification />
|
||||||
|
```
|
||||||
|
|
||||||
|
### Space bar handler
|
||||||
|
|
||||||
|
- [ ] In `onGlobalKeydown`, add a `Space` case inside the `switch (e.key)` block (after the existing cases), only fires when `!isInputActive()`:
|
||||||
|
```ts
|
||||||
|
case ' ':
|
||||||
|
e.preventDefault()
|
||||||
|
document.dispatchEvent(new CustomEvent('voice:ptt-toggle'))
|
||||||
|
break
|
||||||
|
```
|
||||||
|
|
||||||
|
### Shortcuts panel label
|
||||||
|
|
||||||
|
- [ ] Update the Space shortcut description from `Hold to speak (voice, when enabled)` to `Tap to speak (voice, when enabled)`
|
||||||
|
|
||||||
|
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||||
|
- [ ] Verify full build: `npm run build`
|
||||||
|
- [ ] Commit: `git add frontend/src/App.vue && git commit -m "feat: mount VoiceOverlay and wire Space bar shortcut"`
|
||||||
@@ -3,6 +3,7 @@ import { onMounted, onUnmounted, ref, watch } from "vue";
|
|||||||
import { useRouter } from "vue-router";
|
import { useRouter } from "vue-router";
|
||||||
import AppHeader from "@/components/AppHeader.vue";
|
import AppHeader from "@/components/AppHeader.vue";
|
||||||
import ToastNotification from "@/components/ToastNotification.vue";
|
import ToastNotification from "@/components/ToastNotification.vue";
|
||||||
|
import VoiceOverlay from "@/components/VoiceOverlay.vue";
|
||||||
import { useTheme } from "@/composables/useTheme";
|
import { useTheme } from "@/composables/useTheme";
|
||||||
import { useShortcuts } from "@/composables/useShortcuts";
|
import { useShortcuts } from "@/composables/useShortcuts";
|
||||||
import { useAuthStore } from "@/stores/auth";
|
import { useAuthStore } from "@/stores/auth";
|
||||||
@@ -120,6 +121,10 @@ function onGlobalKeydown(e: KeyboardEvent) {
|
|||||||
router.push("/chat");
|
router.push("/chat");
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case " ":
|
||||||
|
e.preventDefault();
|
||||||
|
document.dispatchEvent(new CustomEvent("voice:ptt-toggle"));
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,7 +276,7 @@ onUnmounted(() => {
|
|||||||
</div>
|
</div>
|
||||||
<div class="shortcut-row">
|
<div class="shortcut-row">
|
||||||
<kbd class="shortcut-key">Space</kbd>
|
<kbd class="shortcut-key">Space</kbd>
|
||||||
<span class="shortcut-desc">Hold to speak (voice, when enabled)</span>
|
<span class="shortcut-desc">Tap to speak (voice, when enabled)</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -282,6 +287,7 @@ onUnmounted(() => {
|
|||||||
<template v-else>
|
<template v-else>
|
||||||
<router-view />
|
<router-view />
|
||||||
</template>
|
</template>
|
||||||
|
<VoiceOverlay />
|
||||||
<ToastNotification />
|
<ToastNotification />
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
import { ref, onMounted, onUnmounted, computed } from 'vue'
|
import { ref, onMounted, onUnmounted, computed } from 'vue'
|
||||||
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
||||||
import { useVoiceAudio } from '@/composables/useVoiceAudio'
|
import { useVoiceAudio } from '@/composables/useVoiceAudio'
|
||||||
|
import { useSilenceDetector } from '@/composables/useSilenceDetector'
|
||||||
import { apiPost, apiSSEStream, getVoiceStatus, transcribeAudio, synthesiseSpeech } from '@/api/client'
|
import { apiPost, apiSSEStream, getVoiceStatus, transcribeAudio, synthesiseSpeech } from '@/api/client'
|
||||||
|
|
||||||
// ─── Voice service availability ──────────────────────────────────────────────
|
// ─── Voice service availability ──────────────────────────────────────────────
|
||||||
@@ -64,6 +65,7 @@ const isBusy = computed(() => phase.value !== 'idle' && phase.value !== 'error')
|
|||||||
// ─── Composables ─────────────────────────────────────────────────────────────
|
// ─── Composables ─────────────────────────────────────────────────────────────
|
||||||
const recorder = useVoiceRecorder()
|
const recorder = useVoiceRecorder()
|
||||||
const audio = useVoiceAudio()
|
const audio = useVoiceAudio()
|
||||||
|
const silenceDetector = useSilenceDetector()
|
||||||
|
|
||||||
// ─── Core PTT flow ────────────────────────────────────────────────────────────
|
// ─── Core PTT flow ────────────────────────────────────────────────────────────
|
||||||
async function startPtt() {
|
async function startPtt() {
|
||||||
@@ -79,9 +81,13 @@ async function startPtt() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
phase.value = 'recording'
|
phase.value = 'recording'
|
||||||
|
if (recorder.stream.value) {
|
||||||
|
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function stopPtt() {
|
async function stopPtt() {
|
||||||
|
silenceDetector.stop()
|
||||||
if (phase.value !== 'recording') return
|
if (phase.value !== 'recording') return
|
||||||
|
|
||||||
phase.value = 'transcribing'
|
phase.value = 'transcribing'
|
||||||
@@ -187,7 +193,14 @@ async function stopPtt() {
|
|||||||
assistantMessageId // consumed; suppress lint
|
assistantMessageId // consumed; suppress lint
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function onBtnClick() {
|
||||||
|
if (phase.value === 'error') { phase.value = 'idle'; return }
|
||||||
|
if (phase.value === 'recording') { stopPtt(); return }
|
||||||
|
if (phase.value === 'idle') { startPtt() }
|
||||||
|
}
|
||||||
|
|
||||||
function cancelAll() {
|
function cancelAll() {
|
||||||
|
silenceDetector.stop()
|
||||||
recorder.stopRecording().catch(() => {})
|
recorder.stopRecording().catch(() => {})
|
||||||
audio.stop()
|
audio.stop()
|
||||||
phase.value = 'idle'
|
phase.value = 'idle'
|
||||||
@@ -263,7 +276,7 @@ onUnmounted(() => {
|
|||||||
<span v-else-if="phase === 'error'" class="voice-error-label">{{ errorMsg || 'Error' }}</span>
|
<span v-else-if="phase === 'error'" class="voice-error-label">{{ errorMsg || 'Error' }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div v-else-if="!open || !messages.length" class="voice-status-label voice-hint">
|
<div v-else-if="!open || !messages.length" class="voice-status-label voice-hint">
|
||||||
Hold <kbd>Space</kbd> or tap
|
Tap or press <kbd>Space</kbd>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Cancel button (shown while busy or speaking) -->
|
<!-- Cancel button (shown while busy or speaking) -->
|
||||||
@@ -288,23 +301,24 @@ onUnmounted(() => {
|
|||||||
'voice-ptt--speaking': phase === 'speaking' || audio.playing.value,
|
'voice-ptt--speaking': phase === 'speaking' || audio.playing.value,
|
||||||
'voice-ptt--error': phase === 'error',
|
'voice-ptt--error': phase === 'error',
|
||||||
}"
|
}"
|
||||||
@mousedown.prevent="startPtt"
|
@click.prevent="onBtnClick"
|
||||||
@mouseup.prevent="stopPtt"
|
|
||||||
@touchstart.prevent="startPtt"
|
|
||||||
@touchend.prevent="stopPtt"
|
|
||||||
@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"
|
|
||||||
:disabled="phase === 'transcribing' || phase === 'generating'"
|
:disabled="phase === 'transcribing' || phase === 'generating'"
|
||||||
:aria-label="phase === 'recording' ? 'Release to send' : 'Hold to speak'"
|
:aria-label="phase === 'recording' ? 'Click to stop' : 'Click to speak'"
|
||||||
:title="phase === 'recording' ? 'Release to send' : 'Hold Space or tap to speak'"
|
:title="phase === 'recording' ? 'Click to stop or wait for silence' : 'Click or press Space to speak'"
|
||||||
>
|
>
|
||||||
<!-- Idle: mic icon -->
|
<!-- Idle: mic icon -->
|
||||||
<svg v-if="phase === 'idle' || phase === 'error'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
<svg v-if="phase === 'idle' || phase === 'error'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||||
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
||||||
</svg>
|
</svg>
|
||||||
<!-- Recording: waveform / stop icon -->
|
<!-- Recording: amplitude bars -->
|
||||||
<svg v-else-if="phase === 'recording'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
<span v-else-if="phase === 'recording'" class="voice-amp-bars">
|
||||||
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"/>
|
<span
|
||||||
</svg>
|
v-for="n in 3"
|
||||||
|
:key="n"
|
||||||
|
class="voice-amp-bar"
|
||||||
|
:style="{ transform: `scaleY(${0.3 + silenceDetector.amplitude.value * (0.4 + n * 0.15)})` }"
|
||||||
|
></span>
|
||||||
|
</span>
|
||||||
<!-- Busy: spinner dots -->
|
<!-- Busy: spinner dots -->
|
||||||
<span v-else-if="phase === 'transcribing' || phase === 'generating'" class="voice-spinner">
|
<span v-else-if="phase === 'transcribing' || phase === 'generating'" class="voice-spinner">
|
||||||
<span></span><span></span><span></span>
|
<span></span><span></span><span></span>
|
||||||
@@ -525,7 +539,23 @@ onUnmounted(() => {
|
|||||||
40% { transform: scale(1); opacity: 1; }
|
40% { transform: scale(1); opacity: 1; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ─── Transition ─────────────────────────────────────────────────────────── */
|
/* ─── Amplitude bars (recording state) ──────────────────────────���───────── */
|
||||||
|
.voice-amp-bars {
|
||||||
|
display: flex;
|
||||||
|
gap: 3px;
|
||||||
|
align-items: center;
|
||||||
|
height: 22px;
|
||||||
|
}
|
||||||
|
.voice-amp-bar {
|
||||||
|
width: 4px;
|
||||||
|
height: 18px;
|
||||||
|
background: #fff;
|
||||||
|
border-radius: 2px;
|
||||||
|
transform-origin: center;
|
||||||
|
transition: transform 0.08s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ─── Transition ───────────────────────���──────────────────────���──────────── */
|
||||||
.panel-slide-enter-active,
|
.panel-slide-enter-active,
|
||||||
.panel-slide-leave-active {
|
.panel-slide-leave-active {
|
||||||
transition: opacity 0.2s ease, transform 0.2s ease;
|
transition: opacity 0.2s ease, transform 0.2s ease;
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
import { ref, readonly } from 'vue'
|
||||||
|
|
||||||
|
export interface SilenceDetectorOptions {
|
||||||
|
thresholdDb?: number // default -40
|
||||||
|
silenceDurationMs?: number // default 1500
|
||||||
|
minRecordingMs?: number // default 500
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||||
|
const {
|
||||||
|
thresholdDb = -40,
|
||||||
|
silenceDurationMs = 1500,
|
||||||
|
minRecordingMs = 500,
|
||||||
|
} = options
|
||||||
|
|
||||||
|
const amplitude = ref(0)
|
||||||
|
let audioCtx: AudioContext | null = null
|
||||||
|
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||||
|
let silenceMs = 0
|
||||||
|
let startedAt = 0
|
||||||
|
|
||||||
|
function start(stream: MediaStream, onSilence: () => void): void {
|
||||||
|
stop()
|
||||||
|
audioCtx = new AudioContext()
|
||||||
|
const source = audioCtx.createMediaStreamSource(stream)
|
||||||
|
const analyser = audioCtx.createAnalyser()
|
||||||
|
analyser.fftSize = 256
|
||||||
|
source.connect(analyser)
|
||||||
|
|
||||||
|
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||||
|
silenceMs = 0
|
||||||
|
startedAt = Date.now()
|
||||||
|
|
||||||
|
intervalId = setInterval(() => {
|
||||||
|
analyser.getByteFrequencyData(data)
|
||||||
|
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||||
|
amplitude.value = rms
|
||||||
|
|
||||||
|
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||||
|
if (db < thresholdDb) {
|
||||||
|
silenceMs += 100
|
||||||
|
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||||
|
stop()
|
||||||
|
onSilence()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
}, 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop(): void {
|
||||||
|
if (intervalId !== null) {
|
||||||
|
clearInterval(intervalId)
|
||||||
|
intervalId = null
|
||||||
|
}
|
||||||
|
if (audioCtx) {
|
||||||
|
audioCtx.close().catch(() => {})
|
||||||
|
audioCtx = null
|
||||||
|
}
|
||||||
|
amplitude.value = 0
|
||||||
|
silenceMs = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return { amplitude: readonly(amplitude), start, stop }
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { ref, readonly } from 'vue'
|
import { ref, readonly, type Ref } from 'vue'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push-to-talk recorder wrapping the browser MediaRecorder API.
|
* Push-to-talk recorder wrapping the browser MediaRecorder API.
|
||||||
@@ -15,7 +15,7 @@ export function useVoiceRecorder() {
|
|||||||
|
|
||||||
let mediaRecorder: MediaRecorder | null = null
|
let mediaRecorder: MediaRecorder | null = null
|
||||||
let chunks: Blob[] = []
|
let chunks: Blob[] = []
|
||||||
let stream: MediaStream | null = null
|
const streamRef = ref<MediaStream | null>(null)
|
||||||
let resolveStop: ((blob: Blob) => void) | null = null
|
let resolveStop: ((blob: Blob) => void) | null = null
|
||||||
let rejectStop: ((err: Error) => void) | null = null
|
let rejectStop: ((err: Error) => void) | null = null
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ export function useVoiceRecorder() {
|
|||||||
if (recording.value) return
|
if (recording.value) return
|
||||||
|
|
||||||
try {
|
try {
|
||||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
error.value = 'Microphone access denied'
|
error.value = 'Microphone access denied'
|
||||||
return
|
return
|
||||||
@@ -41,7 +41,7 @@ export function useVoiceRecorder() {
|
|||||||
? 'audio/webm'
|
? 'audio/webm'
|
||||||
: ''
|
: ''
|
||||||
|
|
||||||
mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream)
|
mediaRecorder = mimeType ? new MediaRecorder(streamRef.value!, { mimeType }) : new MediaRecorder(streamRef.value!)
|
||||||
|
|
||||||
mediaRecorder.ondataavailable = (e) => {
|
mediaRecorder.ondataavailable = (e) => {
|
||||||
if (e.data.size > 0) chunks.push(e.data)
|
if (e.data.size > 0) chunks.push(e.data)
|
||||||
@@ -50,8 +50,8 @@ export function useVoiceRecorder() {
|
|||||||
mediaRecorder.onstop = () => {
|
mediaRecorder.onstop = () => {
|
||||||
const blob = new Blob(chunks, { type: mediaRecorder?.mimeType ?? 'audio/webm' })
|
const blob = new Blob(chunks, { type: mediaRecorder?.mimeType ?? 'audio/webm' })
|
||||||
chunks = []
|
chunks = []
|
||||||
stream?.getTracks().forEach((t) => t.stop())
|
streamRef.value?.getTracks().forEach((t) => t.stop())
|
||||||
stream = null
|
streamRef.value = null
|
||||||
recording.value = false
|
recording.value = false
|
||||||
resolveStop?.(blob)
|
resolveStop?.(blob)
|
||||||
resolveStop = null
|
resolveStop = null
|
||||||
@@ -88,5 +88,6 @@ export function useVoiceRecorder() {
|
|||||||
isSupported,
|
isSupported,
|
||||||
startRecording,
|
startRecording,
|
||||||
stopRecording,
|
stopRecording,
|
||||||
|
stream: readonly(streamRef) as Readonly<Ref<MediaStream | null>>,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -428,6 +428,28 @@ const notifySecurityAlerts = ref(true);
|
|||||||
const savingNotifications = ref(false);
|
const savingNotifications = ref(false);
|
||||||
const notificationsSaved = ref(false);
|
const notificationsSaved = ref(false);
|
||||||
|
|
||||||
|
// VAPID key reset (admin)
|
||||||
|
const vapidResetting = ref(false);
|
||||||
|
const vapidResetMsg = ref("");
|
||||||
|
const vapidResetError = ref(false);
|
||||||
|
|
||||||
|
async function resetVapidKeys() {
|
||||||
|
if (!confirm("This will regenerate VAPID keys and clear all push subscriptions. You will need to re-enable notifications afterwards. Continue?")) return;
|
||||||
|
vapidResetting.value = true;
|
||||||
|
vapidResetMsg.value = "";
|
||||||
|
vapidResetError.value = false;
|
||||||
|
try {
|
||||||
|
await apiPost("/api/push/reset-vapid", {});
|
||||||
|
vapidResetMsg.value = "Keys regenerated. Please re-enable notifications in this browser.";
|
||||||
|
pushStore.checkSubscription();
|
||||||
|
} catch {
|
||||||
|
vapidResetError.value = true;
|
||||||
|
vapidResetMsg.value = "Reset failed — check server logs.";
|
||||||
|
} finally {
|
||||||
|
vapidResetting.value = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// CalDAV settings (per-user)
|
// CalDAV settings (per-user)
|
||||||
const caldav = ref({
|
const caldav = ref({
|
||||||
caldav_url: "",
|
caldav_url: "",
|
||||||
@@ -1882,6 +1904,22 @@ function formatUserDate(iso: string): string {
|
|||||||
Notifications are blocked. Allow them in your browser site settings to re-enable.
|
Notifications are blocked. Allow them in your browser site settings to re-enable.
|
||||||
</p>
|
</p>
|
||||||
</template>
|
</template>
|
||||||
|
<template v-if="authStore.isAdmin">
|
||||||
|
<div class="field-divider" style="margin: 1rem 0;" />
|
||||||
|
<p class="section-desc">
|
||||||
|
If push notifications fail due to a corrupted or misformatted VAPID key, regenerate
|
||||||
|
the key pair here. All existing subscriptions will be cleared — you will need to
|
||||||
|
re-enable notifications in this browser afterwards.
|
||||||
|
</p>
|
||||||
|
<div class="actions" style="margin-top: 0.5rem;">
|
||||||
|
<button class="btn-danger" :disabled="vapidResetting" @click="resetVapidKeys">
|
||||||
|
{{ vapidResetting ? 'Regenerating…' : 'Regenerate VAPID Keys' }}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<p v-if="vapidResetMsg" :class="vapidResetError ? 'field-error' : 'field-hint'" style="margin-top: 0.5rem;">
|
||||||
|
{{ vapidResetMsg }}
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section class="settings-section">
|
<section class="settings-section">
|
||||||
|
|||||||
@@ -453,7 +453,8 @@ async def discuss_article(item_id: int):
|
|||||||
if get_buffer(conv_id) is not None:
|
if get_buffer(conv_id) is not None:
|
||||||
return jsonify({"error": "Generation already in progress"}), 409
|
return jsonify({"error": "Generation already in progress"}), 409
|
||||||
|
|
||||||
article_content = item.content or ""
|
from fabledassistant.services.rss import _fetch_full_article
|
||||||
|
article_content = await _fetch_full_article(item.url) or item.content or ""
|
||||||
|
|
||||||
# Store synthetic assistant message with read_article tool result
|
# Store synthetic assistant message with read_article tool result
|
||||||
synthetic_tool_calls = [{
|
synthetic_tool_calls = [{
|
||||||
|
|||||||
@@ -533,8 +533,9 @@ async def create_conversation_from_article(item_id: int):
|
|||||||
conv_title = (item.title or "Article discussion")[:80]
|
conv_title = (item.title or "Article discussion")[:80]
|
||||||
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
||||||
|
|
||||||
|
from fabledassistant.services.rss import _fetch_full_article
|
||||||
source = feed_title or "News"
|
source = feed_title or "News"
|
||||||
content_body = (item.content or "").strip()
|
content_body = (await _fetch_full_article(item.url) if item.url else None) or (item.content or "").strip()
|
||||||
seeded_text = f"**{source}**\n\n**{item.title}**"
|
seeded_text = f"**{source}**\n\n**{item.title}**"
|
||||||
if content_body:
|
if content_body:
|
||||||
seeded_text += f"\n\n{content_body}"
|
seeded_text += f"\n\n{content_body}"
|
||||||
|
|||||||
@@ -3,9 +3,9 @@ import logging
|
|||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
|
|
||||||
from fabledassistant.auth import login_required, get_current_user_id
|
from fabledassistant.auth import login_required, get_current_user_id, admin_required
|
||||||
from fabledassistant.config import Config
|
from fabledassistant.config import Config
|
||||||
from fabledassistant.services.push import delete_subscription, save_subscription, vapid_enabled
|
from fabledassistant.services.push import delete_subscription, regenerate_vapid_keys, save_subscription, vapid_enabled
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -42,3 +42,13 @@ async def unsubscribe():
|
|||||||
return jsonify({"error": "endpoint is required"}), 400
|
return jsonify({"error": "endpoint is required"}), 400
|
||||||
await delete_subscription(uid, endpoint)
|
await delete_subscription(uid, endpoint)
|
||||||
return "", 204
|
return "", 204
|
||||||
|
|
||||||
|
|
||||||
|
@push_bp.route("/reset-vapid", methods=["POST"])
|
||||||
|
@admin_required
|
||||||
|
async def reset_vapid():
|
||||||
|
"""Regenerate VAPID keys and clear all push subscriptions."""
|
||||||
|
ok = await regenerate_vapid_keys()
|
||||||
|
if ok:
|
||||||
|
return jsonify({"publicKey": Config.VAPID_PUBLIC_KEY}), 200
|
||||||
|
return jsonify({"error": "Key regeneration failed"}), 500
|
||||||
|
|||||||
@@ -62,11 +62,14 @@ def ensure_vapid_keys() -> None:
|
|||||||
v = Vapid01()
|
v = Vapid01()
|
||||||
v.generate_keys()
|
v.generate_keys()
|
||||||
|
|
||||||
private_pem = v.private_key.private_bytes(
|
# pywebpush expects the private key as a base64url-encoded DER blob
|
||||||
serialization.Encoding.PEM,
|
# (passed to Vapid.from_string → from_der), NOT a PEM string.
|
||||||
|
private_der = v.private_key.private_bytes(
|
||||||
|
serialization.Encoding.DER,
|
||||||
serialization.PrivateFormat.TraditionalOpenSSL,
|
serialization.PrivateFormat.TraditionalOpenSSL,
|
||||||
serialization.NoEncryption(),
|
serialization.NoEncryption(),
|
||||||
).decode()
|
)
|
||||||
|
private_b64 = base64.urlsafe_b64encode(private_der).rstrip(b"=").decode()
|
||||||
|
|
||||||
pub_bytes = v.public_key.public_bytes(
|
pub_bytes = v.public_key.public_bytes(
|
||||||
serialization.Encoding.X962,
|
serialization.Encoding.X962,
|
||||||
@@ -76,15 +79,40 @@ def ensure_vapid_keys() -> None:
|
|||||||
|
|
||||||
# Persist so they survive container restarts.
|
# Persist so they survive container restarts.
|
||||||
_VAPID_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
_VAPID_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
_VAPID_KEYS_FILE.write_text(json.dumps({"private_key": private_pem, "public_key": public_b64}))
|
_VAPID_KEYS_FILE.write_text(json.dumps({"private_key": private_b64, "public_key": public_b64}))
|
||||||
|
|
||||||
Config.VAPID_PRIVATE_KEY = private_pem
|
Config.VAPID_PRIVATE_KEY = private_b64
|
||||||
Config.VAPID_PUBLIC_KEY = public_b64
|
Config.VAPID_PUBLIC_KEY = public_b64
|
||||||
logger.info("Generated new VAPID keys and saved to %s", _VAPID_KEYS_FILE)
|
logger.info("Generated new VAPID keys and saved to %s", _VAPID_KEYS_FILE)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.warning("Failed to generate VAPID keys — push notifications will be unavailable", exc_info=True)
|
logger.warning("Failed to generate VAPID keys — push notifications will be unavailable", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def regenerate_vapid_keys() -> bool:
|
||||||
|
"""Delete existing VAPID keys, clear all push subscriptions, and generate a fresh pair.
|
||||||
|
|
||||||
|
All existing browser subscriptions are invalidated when keys rotate, so they
|
||||||
|
must be cleared — users will need to re-enable notifications.
|
||||||
|
"""
|
||||||
|
if _VAPID_KEYS_FILE.exists():
|
||||||
|
_VAPID_KEYS_FILE.unlink()
|
||||||
|
Config.VAPID_PRIVATE_KEY = ""
|
||||||
|
Config.VAPID_PUBLIC_KEY = ""
|
||||||
|
|
||||||
|
# Clear all push subscriptions — they are bound to the old public key.
|
||||||
|
async with async_session() as session:
|
||||||
|
await session.execute(delete(PushSubscription))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
ensure_vapid_keys()
|
||||||
|
enabled = vapid_enabled()
|
||||||
|
if enabled:
|
||||||
|
logger.info("VAPID keys regenerated successfully")
|
||||||
|
else:
|
||||||
|
logger.error("VAPID key regeneration failed")
|
||||||
|
return enabled
|
||||||
|
|
||||||
|
|
||||||
async def save_subscription(user_id: int, subscription_json: dict, user_agent: str | None = None) -> PushSubscription:
|
async def save_subscription(user_id: int, subscription_json: dict, user_agent: str | None = None) -> PushSubscription:
|
||||||
"""Upsert a push subscription by endpoint."""
|
"""Upsert a push subscription by endpoint."""
|
||||||
endpoint = subscription_json.get("endpoint", "")
|
endpoint = subscription_json.get("endpoint", "")
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ _CORE_TOOLS = [
|
|||||||
},
|
},
|
||||||
"project": {
|
"project": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Optional project name to assign this task to",
|
"description": "Optional project name to assign this task to. Only set this if the user explicitly named a project. Do NOT infer a project from the task content or context.",
|
||||||
},
|
},
|
||||||
"parent_task": {
|
"parent_task": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@@ -147,7 +147,7 @@ _CORE_TOOLS = [
|
|||||||
},
|
},
|
||||||
"project": {
|
"project": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "Optional project name to assign this note to",
|
"description": "Optional project name to assign this note to. Only set this if the user explicitly named a project. Do NOT infer a project from the note content or context.",
|
||||||
},
|
},
|
||||||
"confirmed": {
|
"confirmed": {
|
||||||
"type": "boolean",
|
"type": "boolean",
|
||||||
|
|||||||
Reference in New Issue
Block a user