Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e3de91a6f | |||
| 6e0c528126 | |||
| dd3b59ed36 | |||
| 56b687c9f4 | |||
| 6d57840dc7 | |||
| 7f37cee49f | |||
| 39e554d938 | |||
| b3cf42863a | |||
| d290bebad2 | |||
| 58e5c6bc60 | |||
| d3170e5545 | |||
| 814f44c3fb | |||
| 1d0cf4828b | |||
| 56f1c44b8e | |||
| a5e35f7c72 | |||
| 853dc810ff | |||
| ac7dde472f | |||
| 84926d4ba2 | |||
| 9b69e38aff | |||
| 1b68559bfe | |||
| edf0a9063e | |||
| f2c2117b25 | |||
| b9d0716b01 | |||
| 9af8ab8f70 | |||
| a171210224 | |||
| eb92b2a976 | |||
| be805073a7 | |||
| e4c812a603 |
@@ -0,0 +1,479 @@
|
||||
# Web Voice Overlay Polish — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Ship the dormant `VoiceOverlay` component by mounting it in `App.vue`, wiring the Space bar shortcut, and replacing push-to-talk with click-to-toggle silence detection backed by a new `useSilenceDetector` composable.
|
||||
|
||||
**Architecture:** A new `useSilenceDetector` composable uses `AudioContext` + `AnalyserNode` to monitor amplitude from a live `MediaStream` and fires a callback after sustained silence. `VoiceOverlay` coordinates recording and silence detection, switching from hold-to-record to click-to-toggle. `App.vue` mounts the overlay and adds a Space bar handler that dispatches the existing `voice:ptt-toggle` custom event.
|
||||
|
||||
**Tech Stack:** Vue 3 Composition API, TypeScript, Web Audio API (`AudioContext`, `AnalyserNode`), existing `useVoiceRecorder` / `useVoiceAudio` composables.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
| Action | Path |
|
||||
|--------|------|
|
||||
| Create | `frontend/src/composables/useSilenceDetector.ts` |
|
||||
| Modify | `frontend/src/composables/useVoiceRecorder.ts` |
|
||||
| Modify | `frontend/src/components/VoiceOverlay.vue` |
|
||||
| Modify | `frontend/src/App.vue` |
|
||||
|
||||
---
|
||||
|
||||
### Task 1: `useSilenceDetector` composable
|
||||
|
||||
**Files:**
|
||||
- Create: `frontend/src/composables/useSilenceDetector.ts`
|
||||
|
||||
**Context:** The Web Audio API lets us pipe a `MediaStream` into an `AnalyserNode` and read frequency data as a byte array every 100 ms. RMS amplitude of that array gives a 0–1 loudness value; converting to dB lets us use the same `-40 dB` threshold as the Android app. The composable must be safe to call `stop()` on multiple times and must reset amplitude to 0 after stopping so the animated bars collapse.
|
||||
|
||||
- [ ] **Step 1: Create the file with full implementation**
|
||||
|
||||
`frontend/src/composables/useSilenceDetector.ts`:
|
||||
|
||||
```ts
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
export interface SilenceDetectorOptions {
|
||||
thresholdDb?: number // default -40
|
||||
silenceDurationMs?: number // default 1500
|
||||
minRecordingMs?: number // default 500
|
||||
}
|
||||
|
||||
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||
const {
|
||||
thresholdDb = -40,
|
||||
silenceDurationMs = 1500,
|
||||
minRecordingMs = 500,
|
||||
} = options
|
||||
|
||||
const amplitude = ref(0)
|
||||
let audioCtx: AudioContext | null = null
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
let silenceMs = 0
|
||||
let startedAt = 0
|
||||
|
||||
function start(stream: MediaStream, onSilence: () => void): void {
|
||||
stop()
|
||||
audioCtx = new AudioContext()
|
||||
const source = audioCtx.createMediaStreamSource(stream)
|
||||
const analyser = audioCtx.createAnalyser()
|
||||
analyser.fftSize = 256
|
||||
source.connect(analyser)
|
||||
|
||||
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||
silenceMs = 0
|
||||
startedAt = Date.now()
|
||||
|
||||
intervalId = setInterval(() => {
|
||||
analyser.getByteFrequencyData(data)
|
||||
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||
amplitude.value = rms
|
||||
|
||||
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||
if (db < thresholdDb) {
|
||||
silenceMs += 100
|
||||
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||
stop()
|
||||
onSilence()
|
||||
}
|
||||
} else {
|
||||
silenceMs = 0
|
||||
}
|
||||
}, 100)
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (intervalId !== null) {
|
||||
clearInterval(intervalId)
|
||||
intervalId = null
|
||||
}
|
||||
if (audioCtx) {
|
||||
audioCtx.close().catch(() => {})
|
||||
audioCtx = null
|
||||
}
|
||||
amplitude.value = 0
|
||||
silenceMs = 0
|
||||
}
|
||||
|
||||
return { amplitude: readonly(amplitude), start, stop }
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify TypeScript compiles**
|
||||
|
||||
```bash
|
||||
cd /path/to/fabledassistant/frontend
|
||||
npx tsc --noEmit
|
||||
```
|
||||
|
||||
Expected: no errors.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add frontend/src/composables/useSilenceDetector.ts
|
||||
git commit -m "feat: add useSilenceDetector composable with Web Audio API amplitude monitoring"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Expose `stream` ref from `useVoiceRecorder`
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/composables/useVoiceRecorder.ts`
|
||||
|
||||
**Context:** Currently `stream` is a plain `let` variable inside the closure. `VoiceOverlay` needs to pass the live `MediaStream` to `useSilenceDetector.start()` after recording begins. Exposing it as a readonly `Ref<MediaStream | null>` is the minimal change — no other callers are broken because they don't currently read `stream` from the return value.
|
||||
|
||||
The current file is at `frontend/src/composables/useVoiceRecorder.ts`. Read it before editing — the key lines to change are:
|
||||
|
||||
1. Top of function body: `let stream: MediaStream | null = null` → `const streamRef = ref<MediaStream | null>(null)`
|
||||
2. In `startRecording()`: `stream = await navigator.mediaDevices.getUserMedia({ audio: true })` → `streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })`
|
||||
3. In `startRecording()` catch block: `stream = null` if present — replace with `streamRef.value = null` (if the catch sets stream to null; if not, skip)
|
||||
4. In `mediaRecorder.onstop`: `stream?.getTracks().forEach((t) => t.stop())` → `streamRef.value?.getTracks().forEach((t) => t.stop())` then `streamRef.value = null`
|
||||
5. Return object: add `stream: readonly(streamRef)`
|
||||
|
||||
- [ ] **Step 1: Add the `ref` import if not already present**
|
||||
|
||||
The file already imports `{ ref, readonly }` from `'vue'` — confirm this. If `ref` is missing from the import, add it.
|
||||
|
||||
- [ ] **Step 2: Replace the `stream` variable declaration**
|
||||
|
||||
Find:
|
||||
```ts
|
||||
let stream: MediaStream | null = null
|
||||
```
|
||||
Replace with:
|
||||
```ts
|
||||
const streamRef = ref<MediaStream | null>(null)
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update all usages of `stream` in `startRecording`**
|
||||
|
||||
Find:
|
||||
```ts
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
```
|
||||
Replace with:
|
||||
```ts
|
||||
streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update `onstop` handler**
|
||||
|
||||
Find:
|
||||
```ts
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
stream = null
|
||||
```
|
||||
Replace with:
|
||||
```ts
|
||||
streamRef.value?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.value = null
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Add `stream` to the return object**
|
||||
|
||||
Find the return statement and add `stream: readonly(streamRef)`:
|
||||
```ts
|
||||
return {
|
||||
recording: readonly(recording),
|
||||
error: readonly(error),
|
||||
isSupported,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
stream: readonly(streamRef),
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Verify TypeScript compiles**
|
||||
|
||||
```bash
|
||||
npx tsc --noEmit
|
||||
```
|
||||
|
||||
Expected: no errors.
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add frontend/src/composables/useVoiceRecorder.ts
|
||||
git commit -m "feat: expose live stream ref from useVoiceRecorder"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Update `VoiceOverlay` — silence detection, click-to-toggle, amplitude bars
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/components/VoiceOverlay.vue`
|
||||
|
||||
**Context:** `VoiceOverlay.vue` is a complete floating voice UI that was never mounted. It currently uses `@mousedown`/`@mouseup` for push-to-talk. This task switches it to click-to-toggle with automatic silence detection and adds animated amplitude bars during recording. Read the full file before making changes — the existing structure and style blocks must be preserved.
|
||||
|
||||
#### Script changes
|
||||
|
||||
- [ ] **Step 1: Import `useSilenceDetector`**
|
||||
|
||||
At the top of `<script setup>`, after the existing imports, add:
|
||||
```ts
|
||||
import { useSilenceDetector } from '@/composables/useSilenceDetector'
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Instantiate the composable**
|
||||
|
||||
After `const audio = useVoiceAudio()`, add:
|
||||
```ts
|
||||
const silenceDetector = useSilenceDetector()
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Update `startPtt` to start silence detection**
|
||||
|
||||
Find the `startPtt` function. After `phase.value = 'recording'`, add:
|
||||
```ts
|
||||
if (recorder.stream.value) {
|
||||
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||
}
|
||||
```
|
||||
|
||||
The complete `startPtt` after the change:
|
||||
```ts
|
||||
async function startPtt() {
|
||||
if (!voiceEnabled.value || isBusy.value) return
|
||||
audio.stop()
|
||||
errorMsg.value = ''
|
||||
open.value = true
|
||||
await recorder.startRecording()
|
||||
if (recorder.error.value) {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = recorder.error.value
|
||||
return
|
||||
}
|
||||
phase.value = 'recording'
|
||||
if (recorder.stream.value) {
|
||||
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update `stopPtt` to stop silence detection**
|
||||
|
||||
Add `silenceDetector.stop()` as the very first line of `stopPtt`:
|
||||
```ts
|
||||
async function stopPtt() {
|
||||
silenceDetector.stop()
|
||||
if (phase.value !== 'recording') return
|
||||
// ... rest unchanged
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Update `cancelAll` to stop silence detection**
|
||||
|
||||
Add `silenceDetector.stop()` after `recorder.stopRecording().catch(() => {})`:
|
||||
```ts
|
||||
function cancelAll() {
|
||||
silenceDetector.stop()
|
||||
recorder.stopRecording().catch(() => {})
|
||||
audio.stop()
|
||||
phase.value = 'idle'
|
||||
streamContent.value = ''
|
||||
errorMsg.value = ''
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Add `onBtnClick` function**
|
||||
|
||||
Add this function after `cancelAll`:
|
||||
```ts
|
||||
function onBtnClick() {
|
||||
if (phase.value === 'error') { phase.value = 'idle'; return }
|
||||
if (phase.value === 'recording') { stopPtt(); return }
|
||||
if (phase.value === 'idle') { startPtt() }
|
||||
}
|
||||
```
|
||||
|
||||
#### Template changes
|
||||
|
||||
- [ ] **Step 7: Replace PTT mouse/touch handlers with `@click`**
|
||||
|
||||
On `.voice-ptt-btn`, replace:
|
||||
```html
|
||||
@mousedown.prevent="startPtt"
|
||||
@mouseup.prevent="stopPtt"
|
||||
@touchstart.prevent="startPtt"
|
||||
@touchend.prevent="stopPtt"
|
||||
@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"
|
||||
```
|
||||
with:
|
||||
```html
|
||||
@click.prevent="onBtnClick"
|
||||
```
|
||||
|
||||
- [ ] **Step 8: Update aria-label and title on the button**
|
||||
|
||||
Replace:
|
||||
```html
|
||||
:aria-label="phase === 'recording' ? 'Release to send' : 'Hold to speak'"
|
||||
:title="phase === 'recording' ? 'Release to send' : 'Hold Space or tap to speak'"
|
||||
```
|
||||
with:
|
||||
```html
|
||||
:aria-label="phase === 'recording' ? 'Click to stop' : 'Click to speak'"
|
||||
:title="phase === 'recording' ? 'Click to stop or wait for silence' : 'Click or press Space to speak'"
|
||||
```
|
||||
|
||||
- [ ] **Step 9: Replace the static recording icon with amplitude bars**
|
||||
|
||||
Find:
|
||||
```html
|
||||
<!-- Recording: waveform / stop icon -->
|
||||
<svg v-else-if="phase === 'recording'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"/>
|
||||
</svg>
|
||||
```
|
||||
Replace with:
|
||||
```html
|
||||
<!-- Recording: amplitude bars -->
|
||||
<span v-else-if="phase === 'recording'" class="voice-amp-bars">
|
||||
<span
|
||||
v-for="n in 3"
|
||||
:key="n"
|
||||
class="voice-amp-bar"
|
||||
:style="{ transform: `scaleY(${0.3 + silenceDetector.amplitude.value * (0.4 + n * 0.15)})` }"
|
||||
></span>
|
||||
</span>
|
||||
```
|
||||
|
||||
- [ ] **Step 10: Update the idle hint label**
|
||||
|
||||
Find:
|
||||
```html
|
||||
Hold <kbd>Space</kbd> or tap
|
||||
```
|
||||
Replace with:
|
||||
```html
|
||||
Tap or press <kbd>Space</kbd>
|
||||
```
|
||||
|
||||
#### Style changes
|
||||
|
||||
- [ ] **Step 11: Add amplitude bar styles to `<style scoped>`**
|
||||
|
||||
Append inside the `<style scoped>` block:
|
||||
```css
|
||||
/* ─── Amplitude bars (recording state) ──────────────────────────────────── */
|
||||
.voice-amp-bars {
|
||||
display: flex;
|
||||
gap: 3px;
|
||||
align-items: center;
|
||||
height: 22px;
|
||||
}
|
||||
.voice-amp-bar {
|
||||
width: 4px;
|
||||
height: 18px;
|
||||
background: #fff;
|
||||
border-radius: 2px;
|
||||
transform-origin: center;
|
||||
transition: transform 0.08s ease;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 12: Verify TypeScript compiles**
|
||||
|
||||
```bash
|
||||
npx tsc --noEmit
|
||||
```
|
||||
|
||||
Expected: no errors.
|
||||
|
||||
- [ ] **Step 13: Commit**
|
||||
|
||||
```bash
|
||||
git add frontend/src/components/VoiceOverlay.vue
|
||||
git commit -m "feat: click-to-toggle silence detection and amplitude bars in VoiceOverlay"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Mount `VoiceOverlay` and wire Space bar in `App.vue`
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/App.vue`
|
||||
|
||||
**Context:** `App.vue` has a full `onGlobalKeydown` handler and a shortcuts overlay. The Space bar is already documented there as "Hold to speak (voice, when enabled)" but the handler was never added to `onGlobalKeydown`. `VoiceOverlay` uses `Teleport to="body"` so it renders at the document root regardless of where it's placed in the template — just needs to be inside the authenticated block.
|
||||
|
||||
#### Script changes
|
||||
|
||||
- [ ] **Step 1: Add `VoiceOverlay` import**
|
||||
|
||||
In `<script setup>`, after the existing component imports (after `ToastNotification`), add:
|
||||
```ts
|
||||
import VoiceOverlay from '@/components/VoiceOverlay.vue'
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add Space bar case to `onGlobalKeydown`**
|
||||
|
||||
The existing handler has a `switch (e.key)` block. The guard `if (isInputActive() || e.ctrlKey || e.metaKey || e.altKey) return` already runs before the switch, so the Space case only fires when the user isn't typing.
|
||||
|
||||
Inside the `switch (e.key)` block, add this case after the existing `'c'` case:
|
||||
```ts
|
||||
case ' ':
|
||||
e.preventDefault()
|
||||
document.dispatchEvent(new CustomEvent('voice:ptt-toggle'))
|
||||
break
|
||||
```
|
||||
|
||||
#### Template changes
|
||||
|
||||
- [ ] **Step 3: Mount `VoiceOverlay` in the authenticated template**
|
||||
|
||||
Find `<ToastNotification />` near the bottom of the authenticated template block and add `<VoiceOverlay />` directly above it:
|
||||
```html
|
||||
<VoiceOverlay />
|
||||
<ToastNotification />
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Update Space bar description in shortcuts panel**
|
||||
|
||||
Find:
|
||||
```html
|
||||
<span class="shortcut-desc">Hold to speak (voice, when enabled)</span>
|
||||
```
|
||||
Replace with:
|
||||
```html
|
||||
<span class="shortcut-desc">Tap to speak (voice, when enabled)</span>
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Verify TypeScript compiles**
|
||||
|
||||
```bash
|
||||
npx tsc --noEmit
|
||||
```
|
||||
|
||||
Expected: no errors.
|
||||
|
||||
- [ ] **Step 6: Verify full build succeeds**
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
Expected: build completes with no errors.
|
||||
|
||||
- [ ] **Step 7: Manual smoke test**
|
||||
|
||||
1. Start the dev server: `npm run dev`
|
||||
2. Log in — confirm the floating mic button appears in the bottom-right corner
|
||||
3. Ensure voice is enabled in Settings → Voice
|
||||
4. Click the mic button — confirm it turns red with animated amplitude bars
|
||||
5. Speak — bars should animate with your voice
|
||||
6. Stop speaking — after ~1.5 s of silence, the button should switch to purple (transcribing), then green (speaking) as it plays back the response
|
||||
7. Click the mic while recording — confirm it stops immediately
|
||||
8. Press Space (not in an input field) — confirm it starts/stops recording
|
||||
9. Press Space in the chat input — confirm it does NOT trigger voice
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add frontend/src/App.vue
|
||||
git commit -m "feat: mount VoiceOverlay and wire Space bar shortcut in App.vue"
|
||||
```
|
||||
@@ -0,0 +1,162 @@
|
||||
# Research Pipeline — Multi-Note Redesign
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace the single monolithic research note with a set of focused, topic-driven notes plus an index note that links them — making research output browsable, TTS-friendly, and well-organized.
|
||||
|
||||
**Architecture:** Two new LLM calls (outline generation + N parallel section syntheses) replace the single large synthesis call. Public API unchanged — callers receive the index note. Fallback to single-note behavior on any outline failure.
|
||||
|
||||
**Tech Stack:** Python/Quart backend, existing `research.py` service, asyncio.gather for parallelism.
|
||||
|
||||
---
|
||||
|
||||
## Problem
|
||||
|
||||
The current pipeline synthesizes one note with a minimum of 2500 words and 6 sections. This creates:
|
||||
- Notes too large to read or listen to comfortably
|
||||
- No way to navigate directly to a specific sub-topic
|
||||
- TTS failures on long prose (8000-char route limit, unbounded sentence buffers)
|
||||
|
||||
---
|
||||
|
||||
## Pipeline Flow
|
||||
|
||||
Public signature unchanged:
|
||||
```python
|
||||
async def run_research_pipeline(
|
||||
topic: str,
|
||||
user_id: int,
|
||||
model: str,
|
||||
buf=None,
|
||||
project_id: int | None = None,
|
||||
) -> Note: # returns the index note
|
||||
```
|
||||
|
||||
Execution order:
|
||||
|
||||
```
|
||||
1. Generate sub-queries (unchanged)
|
||||
2. Search + fetch sources (unchanged)
|
||||
3. Generate topic outline (NEW — one LLM call → 3–7 section dicts)
|
||||
4. Synthesize each section note (NEW — parallelized via asyncio.gather)
|
||||
5. Create all section notes in DB (sequential, tagged ["research"], same project_id)
|
||||
6. Create index note (NEW — links all sections)
|
||||
7. Return index note
|
||||
```
|
||||
|
||||
Status messages via `buf.append_event("status", ...)`:
|
||||
- `"Generating outline…"`
|
||||
- `"Writing: [Section Title]…"` (one per section, emitted before synthesis starts)
|
||||
- `"Saving [N] notes…"`
|
||||
|
||||
No note content is streamed into chat. After the tool call resolves, the LLM writes a brief conversational summary citing the index note title and section count.
|
||||
|
||||
---
|
||||
|
||||
## Outline Generation
|
||||
|
||||
New function: `_generate_outline(topic, sources, model) -> list[dict]`
|
||||
|
||||
Sends all fetched sources to the model with a prompt requesting a JSON array:
|
||||
|
||||
```json
|
||||
[
|
||||
{"title": "Quantum Entanglement: Mechanisms", "focus": "How entanglement works at the physical level"},
|
||||
{"title": "Quantum Computing Hardware", "focus": "Ion traps, superconducting qubits, photonic approaches"}
|
||||
]
|
||||
```
|
||||
|
||||
**Prompt requirements:**
|
||||
- Produce 3–7 sections covering distinct aspects of the topic
|
||||
- Titles must work as standalone note titles (no "Overview" or "Introduction" generics)
|
||||
- No overlap between sections
|
||||
- `focus` is one sentence describing what this section should specifically cover
|
||||
|
||||
**Guardrails:**
|
||||
- Fewer than 3 sections parsed → fall back to single-note synthesis
|
||||
- JSON parse failure → fall back to single-note synthesis
|
||||
- More than 8 sections → truncate to 8
|
||||
|
||||
**Model params:** `max_tokens=400, num_ctx=16384` (outline is short)
|
||||
|
||||
---
|
||||
|
||||
## Section Synthesis
|
||||
|
||||
New function: `_synthesize_section(section_title, section_focus, sources, model) -> tuple[str, str]`
|
||||
|
||||
Returns `(title, body_markdown)`.
|
||||
|
||||
All sections receive all fetched sources. The `section_focus` field in the prompt directs the model to draw only what's relevant to that section's scope.
|
||||
|
||||
**Prompt requirements:**
|
||||
- 300–600 words of substantive prose
|
||||
- Do NOT include a `# Title` heading (title is set separately)
|
||||
- End with a brief `## Sources` list of relevant URLs from the provided sources
|
||||
- Focus strictly on `section_focus` — ignore source material outside that scope
|
||||
|
||||
**Model params:** `num_predict=2048, num_ctx=16384` (reduced from 8192 — sufficient for 600 words, prevents rambling)
|
||||
|
||||
**Parallelism:** All section synthesis calls run via `asyncio.gather`. Wall-clock time stays close to a single synthesis call despite producing N notes.
|
||||
|
||||
---
|
||||
|
||||
## Note Creation and Index Note
|
||||
|
||||
**Section notes:**
|
||||
- Tags: `["research"]`
|
||||
- `project_id`: same as passed to pipeline (or None)
|
||||
- Title: from outline `title` field
|
||||
- Created sequentially (avoids DB contention)
|
||||
|
||||
**Index note:**
|
||||
- Tags: `["research", "research-index"]`
|
||||
- `project_id`: same as section notes
|
||||
- Title: `"Research: [topic]"`
|
||||
- Created last (after all section notes exist)
|
||||
|
||||
**Index note body format:**
|
||||
```markdown
|
||||
Research overview for **[topic]** — [YYYY-MM-DD]
|
||||
|
||||
Generated from [N] web sources across [M] sections.
|
||||
|
||||
## Sections
|
||||
|
||||
- **[Section 1 Title]** — [focus sentence]
|
||||
- **[Section 2 Title]** — [focus sentence]
|
||||
...
|
||||
|
||||
*Search for any section title to read it.*
|
||||
```
|
||||
|
||||
The index note is what `run_research_pipeline` returns. The existing `research_topic` tool handler uses `note.id` and `note.title` — both remain valid with the index note.
|
||||
|
||||
---
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Scenario | Behaviour |
|
||||
|---|---|
|
||||
| Outline generation raises | Fall back to single-note synthesis (current behaviour) |
|
||||
| Outline JSON unparseable | Fall back to single-note synthesis |
|
||||
| Outline returns < 3 sections | Fall back to single-note synthesis |
|
||||
| Outline returns > 8 sections | Truncate to 8, continue |
|
||||
| A section synthesis raises | Log warning, skip that section; continue with remaining |
|
||||
| All section syntheses fail | Fall back to single-note synthesis |
|
||||
| A section note DB save fails | Log warning, skip from index; index note still created |
|
||||
| No sources fetched | Raise `ValueError` as today — unchanged |
|
||||
|
||||
The fallback in every case is the current single-note pipeline. Research never silently produces nothing.
|
||||
|
||||
---
|
||||
|
||||
## What Is NOT Changing
|
||||
|
||||
- Public function signature of `run_research_pipeline`
|
||||
- Sub-query generation (`_generate_sub_queries`)
|
||||
- SearXNG search and URL fetching
|
||||
- `_search_searxng`, `_search_searxng_images`, `fetch_url_content`
|
||||
- The `research_topic` tool definition and handler in `tools.py`
|
||||
- The `quick_capture` research path
|
||||
- Any frontend component
|
||||
@@ -0,0 +1,278 @@
|
||||
# Web Voice Overlay Polish — Implementation Spec
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Ship the dormant `VoiceOverlay` component by mounting it, wiring the Space bar shortcut, and replacing push-to-talk with click-to-toggle silence detection.
|
||||
|
||||
**Architecture:** A new `useSilenceDetector` composable wraps the Web Audio API `AnalyserNode` and fires a callback when sustained silence is detected. `VoiceOverlay` coordinates `useVoiceRecorder` and `useSilenceDetector`, switching from hold-to-record to click-to-toggle. `App.vue` mounts the overlay and adds the Space bar handler.
|
||||
|
||||
**Tech Stack:** Vue 3 Composition API, Web Audio API (`AnalyserNode`), existing `useVoiceRecorder` / `useVoiceAudio` composables, TypeScript.
|
||||
|
||||
---
|
||||
|
||||
## File Map
|
||||
|
||||
| Action | Path |
|
||||
|--------|------|
|
||||
| Create | `frontend/src/composables/useSilenceDetector.ts` |
|
||||
| Modify | `frontend/src/composables/useVoiceRecorder.ts` |
|
||||
| Modify | `frontend/src/components/VoiceOverlay.vue` |
|
||||
| Modify | `frontend/src/App.vue` |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: `useSilenceDetector` composable
|
||||
|
||||
**Files:**
|
||||
- Create: `frontend/src/composables/useSilenceDetector.ts`
|
||||
|
||||
### Interface
|
||||
|
||||
```ts
|
||||
export interface SilenceDetectorOptions {
|
||||
thresholdDb?: number // default -40
|
||||
silenceDurationMs?: number // default 1500
|
||||
minRecordingMs?: number // default 500
|
||||
}
|
||||
|
||||
export function useSilenceDetector(options?: SilenceDetectorOptions): {
|
||||
amplitude: Readonly<Ref<number>> // 0–1, for visualization
|
||||
start(stream: MediaStream, onSilence: () => void): void
|
||||
stop(): void
|
||||
}
|
||||
```
|
||||
|
||||
### Behaviour
|
||||
|
||||
- `start(stream, onSilence)`:
|
||||
1. Creates `AudioContext`
|
||||
2. `createMediaStreamSource(stream)` → connects to `AnalyserNode` (fftSize 256)
|
||||
3. Records `startedAt = Date.now()`
|
||||
4. Starts a `setInterval` at 100ms that:
|
||||
- Calls `analyser.getByteFrequencyData(dataArray)`
|
||||
- Computes RMS amplitude → maps to 0–1 range for `amplitude.value`
|
||||
- Converts to approximate dB: `db = 20 * log10(rms)` (clamp to -100 when rms === 0)
|
||||
- If `db < thresholdDb`: increments `silenceMs += 100`; else resets `silenceMs = 0`
|
||||
- If `silenceMs >= silenceDurationMs` AND `Date.now() - startedAt >= minRecordingMs`: clears interval, fires `onSilence()`
|
||||
- `stop()`: clears interval, closes `AudioContext`, resets `amplitude.value = 0`
|
||||
- Safe to call `stop()` multiple times (guard with null check)
|
||||
- `amplitude` resets to 0 after `stop()`
|
||||
|
||||
### Full implementation
|
||||
|
||||
```ts
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
export interface SilenceDetectorOptions {
|
||||
thresholdDb?: number
|
||||
silenceDurationMs?: number
|
||||
minRecordingMs?: number
|
||||
}
|
||||
|
||||
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||
const {
|
||||
thresholdDb = -40,
|
||||
silenceDurationMs = 1500,
|
||||
minRecordingMs = 500,
|
||||
} = options
|
||||
|
||||
const amplitude = ref(0)
|
||||
let audioCtx: AudioContext | null = null
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
let silenceMs = 0
|
||||
let startedAt = 0
|
||||
|
||||
function start(stream: MediaStream, onSilence: () => void) {
|
||||
stop()
|
||||
audioCtx = new AudioContext()
|
||||
const source = audioCtx.createMediaStreamSource(stream)
|
||||
const analyser = audioCtx.createAnalyser()
|
||||
analyser.fftSize = 256
|
||||
source.connect(analyser)
|
||||
|
||||
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||
silenceMs = 0
|
||||
startedAt = Date.now()
|
||||
|
||||
intervalId = setInterval(() => {
|
||||
analyser.getByteFrequencyData(data)
|
||||
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||
amplitude.value = rms
|
||||
|
||||
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||
if (db < thresholdDb) {
|
||||
silenceMs += 100
|
||||
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||
stop()
|
||||
onSilence()
|
||||
}
|
||||
} else {
|
||||
silenceMs = 0
|
||||
}
|
||||
}, 100)
|
||||
}
|
||||
|
||||
function stop() {
|
||||
if (intervalId !== null) {
|
||||
clearInterval(intervalId)
|
||||
intervalId = null
|
||||
}
|
||||
if (audioCtx) {
|
||||
audioCtx.close().catch(() => {})
|
||||
audioCtx = null
|
||||
}
|
||||
amplitude.value = 0
|
||||
silenceMs = 0
|
||||
}
|
||||
|
||||
return { amplitude: readonly(amplitude), start, stop }
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Write the file exactly as above
|
||||
- [ ] Verify TypeScript compiles: `cd frontend && npx tsc --noEmit`
|
||||
- [ ] Commit: `git add frontend/src/composables/useSilenceDetector.ts && git commit -m "feat: add useSilenceDetector composable"`
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Expose `stream` from `useVoiceRecorder`
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/composables/useVoiceRecorder.ts`
|
||||
|
||||
Change the `stream` local variable to a `Ref<MediaStream | null>` and export it as readonly.
|
||||
|
||||
- [ ] Change `let stream: MediaStream | null = null` to `const streamRef = ref<MediaStream | null>(null)`
|
||||
- [ ] Replace all `stream` assignments with `streamRef.value`:
|
||||
- `stream = await navigator.mediaDevices.getUserMedia(...)` → `streamRef.value = await ...`
|
||||
- `stream?.getTracks().forEach(...)` → `streamRef.value?.getTracks().forEach(...)`
|
||||
- `stream = null` → `streamRef.value = null`
|
||||
- [ ] Add `stream: readonly(streamRef)` to the return object
|
||||
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||
- [ ] Commit: `git add frontend/src/composables/useVoiceRecorder.ts && git commit -m "feat: expose stream ref from useVoiceRecorder"`
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Wire `VoiceOverlay` — silence detection + click-to-toggle
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/components/VoiceOverlay.vue`
|
||||
|
||||
### Script changes
|
||||
|
||||
- [ ] Import `useSilenceDetector` at the top of `<script setup>`
|
||||
- [ ] Add `const silenceDetector = useSilenceDetector()` after the existing composable instantiations
|
||||
- [ ] In `startPtt()`: after `phase.value = 'recording'`, add:
|
||||
```ts
|
||||
if (recorder.stream.value) {
|
||||
silenceDetector.start(recorder.stream.value, stopPtt)
|
||||
}
|
||||
```
|
||||
- [ ] In `stopPtt()`: add `silenceDetector.stop()` as the first line (before the guard check)
|
||||
- [ ] In `cancelAll()`: add `silenceDetector.stop()` after `recorder.stopRecording().catch(() => {})`
|
||||
|
||||
### Button: click-to-toggle
|
||||
|
||||
Replace the PTT mouse/touch handlers on `.voice-ptt-btn` with click-to-toggle logic:
|
||||
|
||||
- [ ] Remove `@mousedown.prevent="startPtt"` and `@mouseup.prevent="stopPtt"`
|
||||
- [ ] Remove `@touchstart.prevent="startPtt"` and `@touchend.prevent="stopPtt"`
|
||||
- [ ] Replace `@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"` with:
|
||||
```html
|
||||
@click.prevent="onBtnClick"
|
||||
```
|
||||
- [ ] Add `onBtnClick` function in script:
|
||||
```ts
|
||||
function onBtnClick() {
|
||||
if (phase.value === 'error') { phase.value = 'idle'; return }
|
||||
if (phase.value === 'recording') { stopPtt(); return }
|
||||
if (phase.value === 'idle') { startPtt() }
|
||||
}
|
||||
```
|
||||
- [ ] Update `aria-label` and `title` on the button:
|
||||
- `aria-label`: `phase === 'recording' ? 'Click to stop' : 'Click to speak'`
|
||||
- `title`: `phase === 'recording' ? 'Click to stop or wait for silence' : 'Click or press Space to speak'`
|
||||
|
||||
### Amplitude visualization during recording
|
||||
|
||||
Inside the button, when `phase === 'recording'`, replace the static stop icon with animated amplitude bars:
|
||||
|
||||
- [ ] Replace the recording SVG block:
|
||||
```html
|
||||
<svg v-else-if="phase === 'recording'" ...>...</svg>
|
||||
```
|
||||
with:
|
||||
```html
|
||||
<span v-else-if="phase === 'recording'" class="voice-amp-bars">
|
||||
<span
|
||||
v-for="n in 3"
|
||||
:key="n"
|
||||
class="voice-amp-bar"
|
||||
:style="{ transform: `scaleY(${0.3 + silenceDetector.amplitude.value * (0.4 + n * 0.15)})` }"
|
||||
></span>
|
||||
</span>
|
||||
```
|
||||
|
||||
### Hint label
|
||||
|
||||
- [ ] Change the idle hint from `Hold <kbd>Space</kbd> or tap` to `Tap or press <kbd>Space</kbd>`
|
||||
|
||||
### CSS for amplitude bars
|
||||
|
||||
- [ ] Add to `<style scoped>`:
|
||||
```css
|
||||
.voice-amp-bars {
|
||||
display: flex;
|
||||
gap: 3px;
|
||||
align-items: center;
|
||||
height: 22px;
|
||||
}
|
||||
.voice-amp-bar {
|
||||
width: 4px;
|
||||
height: 18px;
|
||||
background: #fff;
|
||||
border-radius: 2px;
|
||||
transform-origin: center;
|
||||
transition: transform 0.08s ease;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||
- [ ] Commit: `git add frontend/src/components/VoiceOverlay.vue && git commit -m "feat: click-to-toggle silence detection in VoiceOverlay"`
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Mount overlay and wire Space bar in `App.vue`
|
||||
|
||||
**Files:**
|
||||
- Modify: `frontend/src/App.vue`
|
||||
|
||||
### Mount VoiceOverlay
|
||||
|
||||
- [ ] Add import at top of `<script setup>`:
|
||||
```ts
|
||||
import VoiceOverlay from '@/components/VoiceOverlay.vue'
|
||||
```
|
||||
- [ ] Add `<VoiceOverlay />` inside the `<template v-if="authStore.isAuthenticated">` block, just before `<ToastNotification />`:
|
||||
```html
|
||||
<VoiceOverlay />
|
||||
<ToastNotification />
|
||||
```
|
||||
|
||||
### Space bar handler
|
||||
|
||||
- [ ] In `onGlobalKeydown`, add a `Space` case inside the `switch (e.key)` block (after the existing cases), only fires when `!isInputActive()`:
|
||||
```ts
|
||||
case ' ':
|
||||
e.preventDefault()
|
||||
document.dispatchEvent(new CustomEvent('voice:ptt-toggle'))
|
||||
break
|
||||
```
|
||||
|
||||
### Shortcuts panel label
|
||||
|
||||
- [ ] Update the Space shortcut description from `Hold to speak (voice, when enabled)` to `Tap to speak (voice, when enabled)`
|
||||
|
||||
- [ ] Verify TypeScript: `npx tsc --noEmit`
|
||||
- [ ] Verify full build: `npm run build`
|
||||
- [ ] Commit: `git add frontend/src/App.vue && git commit -m "feat: mount VoiceOverlay and wire Space bar shortcut"`
|
||||
+10
-4
@@ -25,6 +25,15 @@ function startAppServices() {
|
||||
settingsStore.checkVoiceStatus();
|
||||
// Sync browser timezone to the server on every login/page load.
|
||||
apiPut("/api/settings", { user_timezone: Intl.DateTimeFormat().resolvedOptions().timeZone }).catch(() => {});
|
||||
// Re-check voice status when the tab becomes visible again (model may
|
||||
// have finished loading while the user was away).
|
||||
document.addEventListener("visibilitychange", onVisibilityChange);
|
||||
}
|
||||
|
||||
function onVisibilityChange() {
|
||||
if (document.visibilityState === "visible" && authStore.isAuthenticated) {
|
||||
settingsStore.checkVoiceStatus();
|
||||
}
|
||||
}
|
||||
|
||||
function stopAppServices() {
|
||||
@@ -150,6 +159,7 @@ watch(
|
||||
|
||||
onUnmounted(() => {
|
||||
document.removeEventListener("keydown", onGlobalKeydown);
|
||||
document.removeEventListener("visibilitychange", onVisibilityChange);
|
||||
stopAppServices();
|
||||
});
|
||||
</script>
|
||||
@@ -269,10 +279,6 @@ onUnmounted(() => {
|
||||
<kbd class="shortcut-key">Enter</kbd>
|
||||
<span class="shortcut-desc">New line</span>
|
||||
</div>
|
||||
<div class="shortcut-row">
|
||||
<kbd class="shortcut-key">Space</kbd>
|
||||
<span class="shortcut-desc">Hold to speak (voice, when enabled)</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -649,9 +649,10 @@ export const getVoiceStatus = () => apiGet<VoiceStatusResult>('/api/voice/status
|
||||
export const getVoiceList = () =>
|
||||
apiGet<{ voices: VoiceEntry[] }>('/api/voice/voices').then(r => r.voices)
|
||||
|
||||
export async function transcribeAudio(blob: Blob): Promise<{ transcript: string; duration_ms: number }> {
|
||||
export async function transcribeAudio(blob: Blob, context?: string): Promise<{ transcript: string; duration_ms: number }> {
|
||||
const form = new FormData()
|
||||
form.append('audio', blob, 'audio.webm')
|
||||
if (context) form.append('context', context)
|
||||
const res = await fetch('/api/voice/transcribe', { method: 'POST', body: form })
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: `HTTP ${res.status}` }))
|
||||
|
||||
@@ -77,6 +77,7 @@ router.afterEach(() => {
|
||||
<router-link to="/chat" :class="['nav-link', { 'router-link-active': isChatActive }]">Chat</router-link>
|
||||
<router-link to="/briefing" class="nav-link">Briefing</router-link>
|
||||
<router-link to="/calendar" class="nav-link">Calendar</router-link>
|
||||
<router-link to="/news" class="nav-link">News</router-link>
|
||||
<router-link to="/tasks" class="nav-link">Tasks</router-link>
|
||||
<router-link to="/projects" class="nav-link">Projects</router-link>
|
||||
</div>
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
import { ref, computed, nextTick } from 'vue'
|
||||
import { apiGet, transcribeAudio } from '@/api/client'
|
||||
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
||||
import { useSilenceDetector } from '@/composables/useSilenceDetector'
|
||||
import { useChatStore } from '@/stores/chat'
|
||||
import { useSettingsStore } from '@/stores/settings'
|
||||
import { useToastStore } from '@/stores/toast'
|
||||
import type { Note } from '@/types/note'
|
||||
|
||||
const props = withDefaults(defineProps<{
|
||||
@@ -107,21 +109,46 @@ function removeAttachedNote() {
|
||||
attachedNote.value = null
|
||||
}
|
||||
|
||||
// ── PTT ───────────────────────────────────────────────────────────────────────
|
||||
// ── Voice (click-to-toggle + silence detection) ─────────────────────────────
|
||||
const transcribingVoice = ref(false)
|
||||
const recorder = useVoiceRecorder()
|
||||
const silenceDetector = useSilenceDetector()
|
||||
|
||||
async function startPtt() {
|
||||
if (!voiceEnabled.value || recorder.recording.value) return
|
||||
await recorder.startRecording()
|
||||
async function toggleVoice() {
|
||||
if (transcribingVoice.value) return
|
||||
if (recorder.recording.value) {
|
||||
await stopRecording()
|
||||
} else {
|
||||
await startRecording()
|
||||
}
|
||||
}
|
||||
|
||||
async function stopPtt() {
|
||||
async function startRecording() {
|
||||
if (!voiceEnabled.value || recorder.recording.value) return
|
||||
if (!recorder.isSupported) {
|
||||
useToastStore().show('Microphone requires HTTPS or localhost', 'error')
|
||||
return
|
||||
}
|
||||
await recorder.startRecording()
|
||||
if (recorder.error.value) {
|
||||
useToastStore().show(recorder.error.value, 'error')
|
||||
return
|
||||
}
|
||||
if (recorder.stream.value) {
|
||||
silenceDetector.start(recorder.stream.value, () => stopRecording())
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecording() {
|
||||
silenceDetector.stop()
|
||||
if (!recorder.recording.value) return
|
||||
transcribingVoice.value = true
|
||||
try {
|
||||
const blob = await recorder.stopRecording()
|
||||
const { transcript } = await transcribeAudio(blob)
|
||||
// Pass last assistant message as context to reduce STT mishearings
|
||||
const msgs = store.currentConversation?.messages ?? []
|
||||
const lastAssistant = [...msgs].reverse().find(m => m.role === 'assistant')?.content
|
||||
const { transcript } = await transcribeAudio(blob, lastAssistant)
|
||||
if (transcript.trim()) {
|
||||
messageInput.value = transcript.trim()
|
||||
await nextTick()
|
||||
@@ -205,16 +232,13 @@ defineExpose({ focus, prefill })
|
||||
|
||||
<!-- PTT mic -->
|
||||
<button
|
||||
v-if="voiceEnabled && recorder.isSupported"
|
||||
v-if="voiceEnabled"
|
||||
class="btn-icon btn-mic"
|
||||
:class="{ 'mic-recording': recorder.recording.value, 'mic-transcribing': transcribingVoice }"
|
||||
@mousedown.prevent="startPtt"
|
||||
@mouseup.prevent="stopPtt"
|
||||
@touchstart.prevent="startPtt"
|
||||
@touchend.prevent="stopPtt"
|
||||
@click.prevent="toggleVoice"
|
||||
:disabled="transcribingVoice || !store.chatReady"
|
||||
:title="recorder.recording.value ? 'Release to send' : 'Hold to speak'"
|
||||
aria-label="Push to talk"
|
||||
:title="recorder.recording.value ? 'Click to stop (or wait for silence)' : 'Click to speak'"
|
||||
:aria-label="recorder.recording.value ? 'Stop recording' : 'Start recording'"
|
||||
>
|
||||
<svg v-if="!transcribingVoice" width="17" height="17" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
||||
|
||||
@@ -299,12 +299,22 @@ defineExpose({ focus, prefill, send })
|
||||
<!-- Message list -->
|
||||
<div ref="messagesEl" class="messages-container">
|
||||
<div class="messages-inner">
|
||||
<ChatMessage
|
||||
v-for="msg in store.currentConversation?.messages ?? []"
|
||||
<template
|
||||
v-for="(msg, index) in store.currentConversation?.messages ?? []"
|
||||
:key="msg.id"
|
||||
:message="msg"
|
||||
@save-as-note="handleSaveAsNote"
|
||||
/>
|
||||
>
|
||||
<!-- Briefing slot separator: shown before a 2nd+ briefing slot message -->
|
||||
<div
|
||||
v-if="briefingMode && index > 0 && msg.role === 'assistant' && msg.metadata?.rss_item_ids"
|
||||
class="briefing-slot-separator"
|
||||
>
|
||||
<span>New Briefing Update</span>
|
||||
</div>
|
||||
<ChatMessage
|
||||
:message="msg"
|
||||
@save-as-note="handleSaveAsNote"
|
||||
/>
|
||||
</template>
|
||||
<!-- Streaming bubble -->
|
||||
<ChatStreamingBubble v-if="store.streaming" />
|
||||
<!-- Queued messages -->
|
||||
@@ -523,6 +533,28 @@ defineExpose({ focus, prefill, send })
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
/* Briefing slot separator */
|
||||
.briefing-slot-separator {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
margin: 1.25rem 0 0.5rem;
|
||||
color: var(--color-text-muted, #888);
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
opacity: 0.7;
|
||||
}
|
||||
.briefing-slot-separator::before,
|
||||
.briefing-slot-separator::after {
|
||||
content: '';
|
||||
flex: 1;
|
||||
height: 1px;
|
||||
background: var(--color-border, #333);
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
/* Context sidebar */
|
||||
.context-sidebar {
|
||||
width: 200px;
|
||||
|
||||
@@ -1,538 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
/**
|
||||
* VoiceOverlay — global floating push-to-talk button.
|
||||
*
|
||||
* Full flow: record → transcribe → send to voice conv → stream → TTS → play.
|
||||
* Manages its own "voice" conversation; does NOT touch the chat store so it
|
||||
* never disrupts an open chat session.
|
||||
*
|
||||
* Space bar toggles PTT when no input field is focused (wired from App.vue via
|
||||
* the "voice:ptt-toggle" custom event).
|
||||
*/
|
||||
import { ref, onMounted, onUnmounted, computed } from 'vue'
|
||||
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
||||
import { useVoiceAudio } from '@/composables/useVoiceAudio'
|
||||
import { apiPost, apiSSEStream, getVoiceStatus, transcribeAudio, synthesiseSpeech } from '@/api/client'
|
||||
|
||||
// ─── Voice service availability ──────────────────────────────────────────────
|
||||
const voiceEnabled = ref(false)
|
||||
|
||||
async function checkVoice() {
|
||||
try {
|
||||
const s = await getVoiceStatus()
|
||||
voiceEnabled.value = s.enabled && s.stt && s.tts
|
||||
} catch { /* feature absent */ }
|
||||
}
|
||||
|
||||
// ─── Conversation management ─────────────────────────────────────────────────
|
||||
const STORAGE_KEY = 'voice_overlay_conv_id'
|
||||
const convId = ref<number | null>(Number(localStorage.getItem(STORAGE_KEY)) || null)
|
||||
|
||||
interface VoiceMessage { role: 'user' | 'assistant'; content: string }
|
||||
const messages = ref<VoiceMessage[]>([])
|
||||
|
||||
async function ensureConversation(): Promise<number> {
|
||||
if (convId.value) {
|
||||
// Verify it still exists
|
||||
try {
|
||||
await fetch(`/api/chat/conversations/${convId.value}`)
|
||||
.then((r) => { if (!r.ok) throw new Error('gone') })
|
||||
return convId.value
|
||||
} catch {
|
||||
convId.value = null
|
||||
localStorage.removeItem(STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
const conv = await apiPost<{ id: number }>('/api/chat/conversations', {
|
||||
title: 'Voice',
|
||||
conversation_type: 'voice',
|
||||
})
|
||||
convId.value = conv.id
|
||||
localStorage.setItem(STORAGE_KEY, String(conv.id))
|
||||
return conv.id
|
||||
}
|
||||
|
||||
// ─── State machine ────────────────────────────────────────────────────────────
|
||||
type Phase = 'idle' | 'recording' | 'transcribing' | 'generating' | 'speaking' | 'error'
|
||||
const phase = ref<Phase>('idle')
|
||||
const errorMsg = ref('')
|
||||
const streamContent = ref('')
|
||||
const open = ref(false)
|
||||
|
||||
const isBusy = computed(() => phase.value !== 'idle' && phase.value !== 'error')
|
||||
|
||||
// ─── Composables ─────────────────────────────────────────────────────────────
|
||||
const recorder = useVoiceRecorder()
|
||||
const audio = useVoiceAudio()
|
||||
|
||||
// ─── Core PTT flow ────────────────────────────────────────────────────────────
|
||||
async function startPtt() {
|
||||
if (!voiceEnabled.value || isBusy.value) return
|
||||
// Stop any in-progress TTS playback before opening the mic
|
||||
audio.stop()
|
||||
errorMsg.value = ''
|
||||
open.value = true
|
||||
await recorder.startRecording()
|
||||
if (recorder.error.value) {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = recorder.error.value
|
||||
return
|
||||
}
|
||||
phase.value = 'recording'
|
||||
}
|
||||
|
||||
async function stopPtt() {
|
||||
if (phase.value !== 'recording') return
|
||||
|
||||
phase.value = 'transcribing'
|
||||
let blob: Blob
|
||||
try {
|
||||
blob = await recorder.stopRecording()
|
||||
} catch {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = 'Recording failed'
|
||||
return
|
||||
}
|
||||
|
||||
let transcript: string
|
||||
try {
|
||||
const result = await transcribeAudio(blob)
|
||||
transcript = result.transcript.trim()
|
||||
} catch {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = 'Transcription failed'
|
||||
return
|
||||
}
|
||||
|
||||
if (!transcript) {
|
||||
phase.value = 'idle'
|
||||
return
|
||||
}
|
||||
|
||||
messages.value.push({ role: 'user', content: transcript })
|
||||
scrollToBottom()
|
||||
|
||||
// Send to voice conversation
|
||||
phase.value = 'generating'
|
||||
streamContent.value = ''
|
||||
|
||||
let cid: number
|
||||
try {
|
||||
cid = await ensureConversation()
|
||||
} catch {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = 'Could not create voice conversation'
|
||||
return
|
||||
}
|
||||
|
||||
let assistantMessageId: number
|
||||
try {
|
||||
const resp = await apiPost<{ assistant_message_id: number }>(
|
||||
`/api/chat/conversations/${cid}/messages`,
|
||||
{
|
||||
content: transcript,
|
||||
user_timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
||||
}
|
||||
)
|
||||
assistantMessageId = resp.assistant_message_id
|
||||
} catch {
|
||||
phase.value = 'error'
|
||||
errorMsg.value = 'Failed to send message'
|
||||
return
|
||||
}
|
||||
|
||||
// Stream the response
|
||||
await new Promise<void>((resolve) => {
|
||||
const handle = apiSSEStream(
|
||||
`/api/chat/conversations/${cid}/generation/stream`,
|
||||
(event) => {
|
||||
switch (event.event) {
|
||||
case 'chunk':
|
||||
streamContent.value += event.data.chunk as string
|
||||
break
|
||||
case 'done':
|
||||
handle.close()
|
||||
resolve()
|
||||
break
|
||||
case 'error':
|
||||
handle.close()
|
||||
resolve()
|
||||
break
|
||||
}
|
||||
}
|
||||
)
|
||||
// Safety timeout: 3 minutes
|
||||
setTimeout(() => { handle.close(); resolve() }, 180_000)
|
||||
})
|
||||
|
||||
const responseText = streamContent.value.trim()
|
||||
if (responseText) {
|
||||
messages.value.push({ role: 'assistant', content: responseText })
|
||||
streamContent.value = ''
|
||||
scrollToBottom()
|
||||
}
|
||||
|
||||
// Synthesise and play
|
||||
if (responseText) {
|
||||
phase.value = 'speaking'
|
||||
try {
|
||||
const wavBlob = await synthesiseSpeech(responseText)
|
||||
await audio.play(wavBlob)
|
||||
} catch {
|
||||
// TTS failure is non-critical; show response text
|
||||
}
|
||||
}
|
||||
|
||||
phase.value = 'idle'
|
||||
assistantMessageId // consumed; suppress lint
|
||||
}
|
||||
|
||||
function cancelAll() {
|
||||
recorder.stopRecording().catch(() => {})
|
||||
audio.stop()
|
||||
phase.value = 'idle'
|
||||
streamContent.value = ''
|
||||
errorMsg.value = ''
|
||||
}
|
||||
|
||||
// ─── Space bar PTT (event from App.vue) ──────────────────────────────────────
|
||||
function onPttToggle() {
|
||||
if (!voiceEnabled.value) return
|
||||
if (phase.value === 'recording') {
|
||||
stopPtt()
|
||||
} else if (phase.value === 'idle' || phase.value === 'error') {
|
||||
startPtt()
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Scroll ───────────────────────────────────────────────────────────────────
|
||||
const transcriptEl = ref<HTMLElement | null>(null)
|
||||
function scrollToBottom() {
|
||||
setTimeout(() => {
|
||||
if (transcriptEl.value) {
|
||||
transcriptEl.value.scrollTop = transcriptEl.value.scrollHeight
|
||||
}
|
||||
}, 50)
|
||||
}
|
||||
|
||||
// ─── Lifecycle ────────────────────────────────────────────────────────────────
|
||||
onMounted(() => {
|
||||
checkVoice()
|
||||
document.addEventListener('voice:ptt-toggle', onPttToggle)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
document.removeEventListener('voice:ptt-toggle', onPttToggle)
|
||||
cancelAll()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Teleport to="body">
|
||||
<div v-if="voiceEnabled" class="voice-overlay">
|
||||
|
||||
<!-- Expanded transcript panel -->
|
||||
<Transition name="panel-slide">
|
||||
<div v-if="open && messages.length > 0" class="voice-panel">
|
||||
<div class="voice-panel-header">
|
||||
<span class="voice-panel-title">Voice</span>
|
||||
<button class="voice-panel-close" @click="open = false" aria-label="Close">×</button>
|
||||
</div>
|
||||
<div class="voice-transcript" ref="transcriptEl">
|
||||
<div
|
||||
v-for="(msg, i) in messages.slice(-10)"
|
||||
:key="i"
|
||||
:class="['voice-msg', `voice-msg--${msg.role}`]"
|
||||
>{{ msg.content }}</div>
|
||||
<!-- Live streaming text -->
|
||||
<div v-if="phase === 'generating' && streamContent" class="voice-msg voice-msg--assistant voice-msg--streaming">
|
||||
{{ streamContent }}<span class="voice-cursor">▌</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</Transition>
|
||||
|
||||
<!-- PTT button -->
|
||||
<div class="voice-btn-wrap">
|
||||
<!-- Status label -->
|
||||
<div v-if="phase !== 'idle'" class="voice-status-label">
|
||||
<span v-if="phase === 'recording'">Recording…</span>
|
||||
<span v-else-if="phase === 'transcribing'">Transcribing…</span>
|
||||
<span v-else-if="phase === 'generating'">Thinking…</span>
|
||||
<span v-else-if="phase === 'speaking'">Speaking…</span>
|
||||
<span v-else-if="phase === 'error'" class="voice-error-label">{{ errorMsg || 'Error' }}</span>
|
||||
</div>
|
||||
<div v-else-if="!open || !messages.length" class="voice-status-label voice-hint">
|
||||
Hold <kbd>Space</kbd> or tap
|
||||
</div>
|
||||
|
||||
<!-- Cancel button (shown while busy or speaking) -->
|
||||
<button
|
||||
v-if="isBusy || audio.playing.value"
|
||||
class="voice-cancel"
|
||||
@click="cancelAll"
|
||||
aria-label="Cancel"
|
||||
title="Cancel"
|
||||
>
|
||||
<svg width="13" height="13" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/>
|
||||
</svg>
|
||||
</button>
|
||||
|
||||
<!-- Main PTT button -->
|
||||
<button
|
||||
class="voice-ptt-btn"
|
||||
:class="{
|
||||
'voice-ptt--recording': phase === 'recording',
|
||||
'voice-ptt--busy': phase === 'generating' || phase === 'transcribing',
|
||||
'voice-ptt--speaking': phase === 'speaking' || audio.playing.value,
|
||||
'voice-ptt--error': phase === 'error',
|
||||
}"
|
||||
@mousedown.prevent="startPtt"
|
||||
@mouseup.prevent="stopPtt"
|
||||
@touchstart.prevent="startPtt"
|
||||
@touchend.prevent="stopPtt"
|
||||
@click.prevent="phase === 'error' ? (phase = 'idle') : undefined"
|
||||
:disabled="phase === 'transcribing' || phase === 'generating'"
|
||||
:aria-label="phase === 'recording' ? 'Release to send' : 'Hold to speak'"
|
||||
:title="phase === 'recording' ? 'Release to send' : 'Hold Space or tap to speak'"
|
||||
>
|
||||
<!-- Idle: mic icon -->
|
||||
<svg v-if="phase === 'idle' || phase === 'error'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
||||
</svg>
|
||||
<!-- Recording: waveform / stop icon -->
|
||||
<svg v-else-if="phase === 'recording'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z"/>
|
||||
</svg>
|
||||
<!-- Busy: spinner dots -->
|
||||
<span v-else-if="phase === 'transcribing' || phase === 'generating'" class="voice-spinner">
|
||||
<span></span><span></span><span></span>
|
||||
</span>
|
||||
<!-- Speaking: sound waves -->
|
||||
<svg v-else-if="phase === 'speaking'" width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
|
||||
<path d="M3 9v6h4l5 5V4L7 9H3zm13.5 3c0-1.77-1.02-3.29-2.5-4.03v8.05c1.48-.73 2.5-2.25 2.5-4.02zM14 3.23v2.06c2.89.86 5 3.54 5 6.71s-2.11 5.85-5 6.71v2.06c4.01-.91 7-4.49 7-8.77s-2.99-7.86-7-8.77z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</Teleport>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.voice-overlay {
|
||||
position: fixed;
|
||||
bottom: 2rem;
|
||||
right: 1.5rem;
|
||||
z-index: 8000;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-end;
|
||||
gap: 0.5rem;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* ─── Panel ──────────────────────────────────────────────────────────────── */
|
||||
.voice-panel {
|
||||
pointer-events: all;
|
||||
width: min(320px, 90vw);
|
||||
background: var(--color-bg-card);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 14px;
|
||||
box-shadow: 0 8px 32px var(--color-shadow, rgba(0,0,0,0.22));
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-height: 340px;
|
||||
}
|
||||
|
||||
.voice-panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.6rem 0.85rem 0.5rem;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.voice-panel-title {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.07em;
|
||||
color: var(--color-primary);
|
||||
}
|
||||
.voice-panel-close {
|
||||
background: none;
|
||||
border: none;
|
||||
font-size: 1.3rem;
|
||||
line-height: 1;
|
||||
color: var(--color-text-muted);
|
||||
cursor: pointer;
|
||||
padding: 0 0.1rem;
|
||||
}
|
||||
.voice-panel-close:hover { color: var(--color-text); }
|
||||
|
||||
.voice-transcript {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 0.65rem 0.85rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.45rem;
|
||||
}
|
||||
|
||||
.voice-msg {
|
||||
font-size: 0.875rem;
|
||||
line-height: 1.45;
|
||||
padding: 0.4rem 0.65rem;
|
||||
border-radius: 10px;
|
||||
max-width: 88%;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
.voice-msg--user {
|
||||
align-self: flex-end;
|
||||
background: color-mix(in srgb, var(--color-primary) 12%, transparent);
|
||||
border: 1px solid color-mix(in srgb, var(--color-primary) 25%, transparent);
|
||||
color: var(--color-text);
|
||||
}
|
||||
.voice-msg--assistant {
|
||||
align-self: flex-start;
|
||||
background: var(--color-bg-secondary);
|
||||
border: 1px solid var(--color-border);
|
||||
color: var(--color-text);
|
||||
}
|
||||
.voice-msg--streaming { opacity: 0.85; }
|
||||
.voice-cursor {
|
||||
display: inline-block;
|
||||
animation: blink 0.9s step-end infinite;
|
||||
margin-left: 1px;
|
||||
color: var(--color-primary);
|
||||
}
|
||||
@keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0; } }
|
||||
|
||||
/* ─── Button cluster ─────────────────────────────────────────────────────── */
|
||||
.voice-btn-wrap {
|
||||
pointer-events: all;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: flex-end;
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.voice-status-label {
|
||||
font-size: 0.72rem;
|
||||
color: var(--color-text-muted);
|
||||
background: var(--color-bg-card);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
padding: 0.18rem 0.5rem;
|
||||
white-space: nowrap;
|
||||
box-shadow: 0 2px 6px var(--color-shadow, rgba(0,0,0,0.12));
|
||||
}
|
||||
.voice-hint { opacity: 0.7; }
|
||||
.voice-hint kbd {
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.68rem;
|
||||
padding: 0.05rem 0.25rem;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 3px;
|
||||
background: var(--color-bg-secondary);
|
||||
}
|
||||
.voice-error-label { color: #ef4444; }
|
||||
|
||||
.voice-cancel {
|
||||
background: var(--color-bg-card);
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 50%;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
color: var(--color-text-muted);
|
||||
box-shadow: 0 2px 6px var(--color-shadow, rgba(0,0,0,0.12));
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.voice-cancel:hover { color: #ef4444; border-color: #ef4444; }
|
||||
|
||||
.voice-ptt-btn {
|
||||
width: 58px;
|
||||
height: 58px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
background: linear-gradient(135deg, #6366f1, #4f46e5);
|
||||
color: #fff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 4px 16px rgba(99, 102, 241, 0.45);
|
||||
transition: transform 0.12s, box-shadow 0.12s, background 0.2s;
|
||||
touch-action: none;
|
||||
user-select: none;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.voice-ptt-btn:hover:not(:disabled) {
|
||||
transform: scale(1.06);
|
||||
box-shadow: 0 6px 20px rgba(99, 102, 241, 0.55);
|
||||
}
|
||||
.voice-ptt-btn:disabled { opacity: 0.6; cursor: not-allowed; }
|
||||
|
||||
.voice-ptt--recording {
|
||||
background: linear-gradient(135deg, #ef4444, #dc2626) !important;
|
||||
box-shadow: 0 4px 16px rgba(239, 68, 68, 0.5) !important;
|
||||
animation: ptt-pulse 0.9s ease-in-out infinite;
|
||||
}
|
||||
.voice-ptt--busy {
|
||||
background: linear-gradient(135deg, #8b5cf6, #7c3aed) !important;
|
||||
box-shadow: 0 4px 16px rgba(139, 92, 246, 0.45) !important;
|
||||
}
|
||||
.voice-ptt--speaking {
|
||||
background: linear-gradient(135deg, #10b981, #059669) !important;
|
||||
box-shadow: 0 4px 16px rgba(16, 185, 129, 0.45) !important;
|
||||
animation: ptt-pulse 1.4s ease-in-out infinite;
|
||||
}
|
||||
.voice-ptt--error {
|
||||
background: linear-gradient(135deg, #6b7280, #4b5563) !important;
|
||||
box-shadow: none !important;
|
||||
}
|
||||
|
||||
@keyframes ptt-pulse {
|
||||
0%, 100% { transform: scale(1); }
|
||||
50% { transform: scale(1.08); }
|
||||
}
|
||||
|
||||
/* ─── Spinner dots ───────────────────────────────────────────────────────── */
|
||||
.voice-spinner {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
align-items: center;
|
||||
}
|
||||
.voice-spinner span {
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: #fff;
|
||||
animation: dot-bounce 1.2s ease-in-out infinite;
|
||||
}
|
||||
.voice-spinner span:nth-child(2) { animation-delay: 0.2s; }
|
||||
.voice-spinner span:nth-child(3) { animation-delay: 0.4s; }
|
||||
@keyframes dot-bounce {
|
||||
0%, 80%, 100% { transform: scale(0.7); opacity: 0.5; }
|
||||
40% { transform: scale(1); opacity: 1; }
|
||||
}
|
||||
|
||||
/* ─── Transition ─────────────────────────────────────────────────────────── */
|
||||
.panel-slide-enter-active,
|
||||
.panel-slide-leave-active {
|
||||
transition: opacity 0.2s ease, transform 0.2s ease;
|
||||
}
|
||||
.panel-slide-enter-from,
|
||||
.panel-slide-leave-to {
|
||||
opacity: 0;
|
||||
transform: translateY(8px);
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,69 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
export interface SilenceDetectorOptions {
|
||||
thresholdDb?: number // default -40
|
||||
silenceDurationMs?: number // default 1500
|
||||
minRecordingMs?: number // default 500
|
||||
}
|
||||
|
||||
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||
const {
|
||||
thresholdDb = -40,
|
||||
silenceDurationMs = 1500,
|
||||
minRecordingMs = 500,
|
||||
} = options
|
||||
|
||||
const amplitude = ref(0)
|
||||
let audioCtx: AudioContext | null = null
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
let silenceMs = 0
|
||||
let startedAt = 0
|
||||
|
||||
function start(stream: MediaStream, onSilence: () => void): void {
|
||||
stop()
|
||||
audioCtx = new AudioContext()
|
||||
// Some browsers start AudioContext in "suspended" state — resume so
|
||||
// getByteFrequencyData returns real values instead of all zeros.
|
||||
audioCtx.resume().catch(() => {})
|
||||
const source = audioCtx.createMediaStreamSource(stream)
|
||||
const analyser = audioCtx.createAnalyser()
|
||||
analyser.fftSize = 256
|
||||
source.connect(analyser)
|
||||
|
||||
const data = new Uint8Array(analyser.frequencyBinCount)
|
||||
silenceMs = 0
|
||||
startedAt = Date.now()
|
||||
|
||||
intervalId = setInterval(() => {
|
||||
analyser.getByteFrequencyData(data)
|
||||
const rms = Math.sqrt(data.reduce((s, v) => s + v * v, 0) / data.length) / 255
|
||||
amplitude.value = rms
|
||||
|
||||
const db = rms > 0 ? 20 * Math.log10(rms) : -100
|
||||
if (db < thresholdDb) {
|
||||
silenceMs += 100
|
||||
if (silenceMs >= silenceDurationMs && Date.now() - startedAt >= minRecordingMs) {
|
||||
stop()
|
||||
onSilence()
|
||||
}
|
||||
} else {
|
||||
silenceMs = 0
|
||||
}
|
||||
}, 100)
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (intervalId !== null) {
|
||||
clearInterval(intervalId)
|
||||
intervalId = null
|
||||
}
|
||||
if (audioCtx) {
|
||||
audioCtx.close().catch(() => {})
|
||||
audioCtx = null
|
||||
}
|
||||
amplitude.value = 0
|
||||
silenceMs = 0
|
||||
}
|
||||
|
||||
return { amplitude: readonly(amplitude), start, stop }
|
||||
}
|
||||
@@ -86,11 +86,21 @@ export function useStreamingTts(options: UseStreamingTtsOptions): UseStreamingTt
|
||||
try {
|
||||
blob = await synthesiseSpeech(stripped)
|
||||
} catch (e) {
|
||||
console.warn('[StreamingTTS] Synthesis failed, retrying sentence', { sentence: stripped, error: e })
|
||||
const errMsg = e instanceof Error ? e.message : String(e)
|
||||
console.warn('[StreamingTTS] Synthesis failed, retrying', {
|
||||
chars: stripped.length,
|
||||
preview: stripped.slice(0, 80),
|
||||
error: errMsg,
|
||||
})
|
||||
try {
|
||||
blob = await synthesiseSpeech(stripped)
|
||||
} catch (e2) {
|
||||
console.warn('[StreamingTTS] Retry also failed, skipping sentence', { sentence: stripped, error: e2 })
|
||||
const errMsg2 = e2 instanceof Error ? e2.message : String(e2)
|
||||
console.warn('[StreamingTTS] Retry failed, sentence dropped', {
|
||||
chars: stripped.length,
|
||||
preview: stripped.slice(0, 80),
|
||||
error: errMsg2,
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
pendingCount.value--
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
import { ref, readonly, type Ref } from 'vue'
|
||||
|
||||
/**
|
||||
* Push-to-talk recorder wrapping the browser MediaRecorder API.
|
||||
@@ -15,7 +15,7 @@ export function useVoiceRecorder() {
|
||||
|
||||
let mediaRecorder: MediaRecorder | null = null
|
||||
let chunks: Blob[] = []
|
||||
let stream: MediaStream | null = null
|
||||
const streamRef = ref<MediaStream | null>(null)
|
||||
let resolveStop: ((blob: Blob) => void) | null = null
|
||||
let rejectStop: ((err: Error) => void) | null = null
|
||||
|
||||
@@ -28,7 +28,7 @@ export function useVoiceRecorder() {
|
||||
if (recording.value) return
|
||||
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
streamRef.value = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
} catch (e) {
|
||||
error.value = 'Microphone access denied'
|
||||
return
|
||||
@@ -41,7 +41,7 @@ export function useVoiceRecorder() {
|
||||
? 'audio/webm'
|
||||
: ''
|
||||
|
||||
mediaRecorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream)
|
||||
mediaRecorder = mimeType ? new MediaRecorder(streamRef.value!, { mimeType }) : new MediaRecorder(streamRef.value!)
|
||||
|
||||
mediaRecorder.ondataavailable = (e) => {
|
||||
if (e.data.size > 0) chunks.push(e.data)
|
||||
@@ -50,8 +50,8 @@ export function useVoiceRecorder() {
|
||||
mediaRecorder.onstop = () => {
|
||||
const blob = new Blob(chunks, { type: mediaRecorder?.mimeType ?? 'audio/webm' })
|
||||
chunks = []
|
||||
stream?.getTracks().forEach((t) => t.stop())
|
||||
stream = null
|
||||
streamRef.value?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.value = null
|
||||
recording.value = false
|
||||
resolveStop?.(blob)
|
||||
resolveStop = null
|
||||
@@ -88,5 +88,6 @@ export function useVoiceRecorder() {
|
||||
isSupported,
|
||||
startRecording,
|
||||
stopRecording,
|
||||
stream: readonly(streamRef) as Readonly<Ref<MediaStream | null>>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ export interface Message {
|
||||
context_note_id: number | null;
|
||||
context_note_title?: string | null;
|
||||
tool_calls?: ToolCallRecord[] | null;
|
||||
metadata?: Record<string, unknown> | null;
|
||||
created_at: string;
|
||||
timing?: GenerationTiming;
|
||||
thinking?: string;
|
||||
|
||||
@@ -428,6 +428,28 @@ const notifySecurityAlerts = ref(true);
|
||||
const savingNotifications = ref(false);
|
||||
const notificationsSaved = ref(false);
|
||||
|
||||
// VAPID key reset (admin)
|
||||
const vapidResetting = ref(false);
|
||||
const vapidResetMsg = ref("");
|
||||
const vapidResetError = ref(false);
|
||||
|
||||
async function resetVapidKeys() {
|
||||
if (!confirm("This will regenerate VAPID keys and clear all push subscriptions. You will need to re-enable notifications afterwards. Continue?")) return;
|
||||
vapidResetting.value = true;
|
||||
vapidResetMsg.value = "";
|
||||
vapidResetError.value = false;
|
||||
try {
|
||||
await apiPost("/api/push/reset-vapid", {});
|
||||
vapidResetMsg.value = "Keys regenerated. Please re-enable notifications in this browser.";
|
||||
pushStore.checkSubscription();
|
||||
} catch {
|
||||
vapidResetError.value = true;
|
||||
vapidResetMsg.value = "Reset failed — check server logs.";
|
||||
} finally {
|
||||
vapidResetting.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
// CalDAV settings (per-user)
|
||||
const caldav = ref({
|
||||
caldav_url: "",
|
||||
@@ -478,6 +500,8 @@ async function saveAdminVoice() {
|
||||
adminVoiceSaved.value = true;
|
||||
setTimeout(() => { adminVoiceSaved.value = false; }, 2000);
|
||||
voiceTabLoaded.value = false;
|
||||
// Always update global voice state so mic buttons appear/disappear immediately
|
||||
await store.checkVoiceStatus();
|
||||
if (adminVoiceEnabled.value) {
|
||||
await reloadVoiceModels();
|
||||
}
|
||||
@@ -1882,6 +1906,22 @@ function formatUserDate(iso: string): string {
|
||||
Notifications are blocked. Allow them in your browser site settings to re-enable.
|
||||
</p>
|
||||
</template>
|
||||
<template v-if="authStore.isAdmin">
|
||||
<div class="field-divider" style="margin: 1rem 0;" />
|
||||
<p class="section-desc">
|
||||
If push notifications fail due to a corrupted or misformatted VAPID key, regenerate
|
||||
the key pair here. All existing subscriptions will be cleared — you will need to
|
||||
re-enable notifications in this browser afterwards.
|
||||
</p>
|
||||
<div class="actions" style="margin-top: 0.5rem;">
|
||||
<button class="btn-danger" :disabled="vapidResetting" @click="resetVapidKeys">
|
||||
{{ vapidResetting ? 'Regenerating…' : 'Regenerate VAPID Keys' }}
|
||||
</button>
|
||||
</div>
|
||||
<p v-if="vapidResetMsg" :class="vapidResetError ? 'field-error' : 'field-hint'" style="margin-top: 0.5rem;">
|
||||
{{ vapidResetMsg }}
|
||||
</p>
|
||||
</template>
|
||||
</section>
|
||||
|
||||
<section class="settings-section">
|
||||
|
||||
@@ -27,7 +27,7 @@ class Config:
|
||||
# Lightweight model for background tasks (title generation, tag suggestions,
|
||||
# project summaries, RSS classification). Using a separate model keeps the
|
||||
# main model's KV cache intact between user messages, enabling prefix cache hits.
|
||||
OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:0.5b")
|
||||
OLLAMA_BACKGROUND_MODEL: str = os.environ.get("OLLAMA_BACKGROUND_MODEL", "qwen2.5:3b")
|
||||
# KV cache context window for generation. Keep this as small as practical —
|
||||
# a larger context forces more KV cache into CPU RAM, drastically slowing prefill.
|
||||
# 16384 covers ~30+ message conversations with our system prompt comfortably.
|
||||
|
||||
@@ -453,7 +453,8 @@ async def discuss_article(item_id: int):
|
||||
if get_buffer(conv_id) is not None:
|
||||
return jsonify({"error": "Generation already in progress"}), 409
|
||||
|
||||
article_content = item.content or ""
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
article_content = await _fetch_full_article(item.url) or item.content or ""
|
||||
|
||||
# Store synthetic assistant message with read_article tool result
|
||||
synthetic_tool_calls = [{
|
||||
|
||||
@@ -533,8 +533,9 @@ async def create_conversation_from_article(item_id: int):
|
||||
conv_title = (item.title or "Article discussion")[:80]
|
||||
conv = await create_conversation(uid, title=conv_title, conversation_type="chat")
|
||||
|
||||
from fabledassistant.services.rss import _fetch_full_article
|
||||
source = feed_title or "News"
|
||||
content_body = (item.content or "").strip()
|
||||
content_body = (await _fetch_full_article(item.url) if item.url else None) or (item.content or "").strip()
|
||||
seeded_text = f"**{source}**\n\n**{item.title}**"
|
||||
if content_body:
|
||||
seeded_text += f"\n\n{content_body}"
|
||||
|
||||
@@ -3,9 +3,9 @@ import logging
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from fabledassistant.auth import login_required, get_current_user_id
|
||||
from fabledassistant.auth import login_required, get_current_user_id, admin_required
|
||||
from fabledassistant.config import Config
|
||||
from fabledassistant.services.push import delete_subscription, save_subscription, vapid_enabled
|
||||
from fabledassistant.services.push import delete_subscription, regenerate_vapid_keys, save_subscription, vapid_enabled
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -42,3 +42,13 @@ async def unsubscribe():
|
||||
return jsonify({"error": "endpoint is required"}), 400
|
||||
await delete_subscription(uid, endpoint)
|
||||
return "", 204
|
||||
|
||||
|
||||
@push_bp.route("/reset-vapid", methods=["POST"])
|
||||
@admin_required
|
||||
async def reset_vapid():
|
||||
"""Regenerate VAPID keys and clear all push subscriptions."""
|
||||
ok = await regenerate_vapid_keys()
|
||||
if ok:
|
||||
return jsonify({"publicKey": Config.VAPID_PUBLIC_KEY}), 200
|
||||
return jsonify({"error": "Key regeneration failed"}), 500
|
||||
|
||||
@@ -80,10 +80,12 @@ async def transcribe_audio():
|
||||
return jsonify({"error": "Audio file too large (max 25 MB)"}), 413
|
||||
|
||||
mime_type = audio_file.content_type or "audio/webm"
|
||||
form = await request.form
|
||||
context = (form.get("context") or "").strip() or None
|
||||
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
transcript = await transcribe(audio_bytes, mime_type)
|
||||
transcript = await transcribe(audio_bytes, mime_type, initial_prompt=context)
|
||||
except Exception:
|
||||
logger.exception("STT transcription failed")
|
||||
return jsonify({"error": "Transcription failed"}), 500
|
||||
@@ -117,7 +119,12 @@ async def synthesise_speech():
|
||||
if not text:
|
||||
return jsonify({"error": "text is required"}), 400
|
||||
|
||||
if len(text) > 8000:
|
||||
char_count = len(text)
|
||||
if char_count > 8000:
|
||||
logger.warning(
|
||||
"TTS request rejected: text too long (%d chars, limit 8000). Preview: %r",
|
||||
char_count, text[:120],
|
||||
)
|
||||
return jsonify({"error": "text too long (max 8000 characters)"}), 400
|
||||
|
||||
voice = str(data.get("voice", "af_heart"))
|
||||
@@ -154,11 +161,29 @@ async def synthesise_speech():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
blend_desc = f"blend({len(voice_blend)} voices)" if voice_blend else voice
|
||||
logger.info("TTS synthesis start: %d chars, voice=%s, speed=%.2f", char_count, blend_desc, speed)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
wav_bytes = await synthesise(text, voice=voice, speed=speed, voice_blend=voice_blend)
|
||||
except Exception:
|
||||
logger.exception("TTS synthesis failed")
|
||||
logger.exception(
|
||||
"TTS synthesis failed: %d chars, voice=%s. Preview: %r",
|
||||
char_count, blend_desc, text[:120],
|
||||
)
|
||||
return jsonify({"error": "Synthesis failed"}), 500
|
||||
|
||||
duration_ms = round((time.monotonic() - t0) * 1000)
|
||||
if not wav_bytes:
|
||||
logger.warning(
|
||||
"TTS synthesis returned empty audio: %d chars, voice=%s, %dms. Preview: %r",
|
||||
char_count, blend_desc, duration_ms, text[:120],
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"TTS synthesis complete: %d chars → %d bytes in %dms (voice=%s)",
|
||||
char_count, len(wav_bytes), duration_ms, blend_desc,
|
||||
)
|
||||
|
||||
from quart import Response
|
||||
return Response(wav_bytes, mimetype="audio/wav")
|
||||
|
||||
@@ -118,31 +118,45 @@ _TOOL_LABELS: dict[str, str] = {
|
||||
|
||||
|
||||
async def _generate_title(messages: list[dict], user_id: int) -> str:
|
||||
"""Ask the LLM for a concise conversation title."""
|
||||
# Build conversation text like summarize_conversation_as_note
|
||||
conv_lines = []
|
||||
"""Ask the LLM for a concise conversation title.
|
||||
|
||||
Only uses user messages to avoid feeding tool-call JSON, system prompt
|
||||
fragments, or other noise into the title generator. Caps input length
|
||||
to keep the task fast and focused.
|
||||
"""
|
||||
user_texts = []
|
||||
for m in messages:
|
||||
if m["role"] == "system":
|
||||
continue
|
||||
label = "User" if m["role"] == "user" else "Assistant"
|
||||
conv_lines.append(f"{label}: {m['content']}")
|
||||
# Keep only last 6 pairs worth of text
|
||||
conv_lines = conv_lines[-12:]
|
||||
if m["role"] == "user":
|
||||
content = (m.get("content") or "").strip()
|
||||
if content:
|
||||
user_texts.append(content[:300])
|
||||
if not user_texts:
|
||||
return ""
|
||||
# First + last user messages capture intent best
|
||||
if len(user_texts) > 2:
|
||||
user_texts = [user_texts[0], user_texts[-1]]
|
||||
|
||||
prompt_messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Generate a concise 3-8 word title for this conversation. "
|
||||
"Reply with ONLY the title, no quotes or punctuation."
|
||||
"Generate a concise 3-8 word title for a conversation that started with:\n\n"
|
||||
+ "\n\n".join(user_texts)
|
||||
+ "\n\nReply with ONLY the title. No quotes, no punctuation, no explanation."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": "\n\n".join(conv_lines)},
|
||||
]
|
||||
bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
|
||||
title = await generate_completion(prompt_messages, bg_model, max_tokens=30)
|
||||
title = await generate_completion(prompt_messages, bg_model, max_tokens=30, num_ctx=1024)
|
||||
# Strip common LLM noise: quotes, thinking tags, role labels
|
||||
title = title.strip().strip('"\'').strip()
|
||||
return title[:100] if title else ""
|
||||
for prefix in ("Title:", "title:", "Assistant:", "User:"):
|
||||
if title.startswith(prefix):
|
||||
title = title[len(prefix):].strip()
|
||||
# Drop anything after a newline (model sometimes adds explanation)
|
||||
if "\n" in title:
|
||||
title = title.split("\n")[0].strip()
|
||||
return title[:80] if title else ""
|
||||
|
||||
|
||||
async def _update_message(
|
||||
|
||||
@@ -538,6 +538,11 @@ async def build_context(
|
||||
"Delete tools require an explicit user request. "
|
||||
"Never proactively search notes or comment on absent context."
|
||||
)
|
||||
tool_lines.append(
|
||||
"IMPORTANT: When creating tasks or notes, NEVER infer or guess a project name. "
|
||||
"Only set the project parameter if the user explicitly names a project. "
|
||||
"If the user says 'create a task to buy milk', do NOT assign it to a project."
|
||||
)
|
||||
tool_guidance = "\n".join(tool_lines)
|
||||
|
||||
static_block = (
|
||||
@@ -607,6 +612,25 @@ async def build_context(
|
||||
f"\n\n--- Earlier Conversation ---\n{history_summary}\n--- End Earlier Conversation ---"
|
||||
)
|
||||
|
||||
# Detect briefing conversation — used for both system prompt instruction and article injection
|
||||
_is_briefing_conv = False
|
||||
if conv_id is not None:
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.conversation import Conversation as _Conversation
|
||||
async with _async_session() as _sess:
|
||||
_conv = await _sess.get(_Conversation, conv_id)
|
||||
if _conv and getattr(_conv, "conversation_type", None) == "briefing":
|
||||
_is_briefing_conv = True
|
||||
|
||||
if _is_briefing_conv:
|
||||
system_content += (
|
||||
"\n\nYou are in a briefing conversation. "
|
||||
"The conversation history contains today's briefing — news stories, weather, and tasks. "
|
||||
"When the user asks about a topic, person, or event from the briefing, answer directly "
|
||||
"from the conversation history and the article context that follows. "
|
||||
"Do NOT search the web for information that is already present in the briefing."
|
||||
)
|
||||
|
||||
context_meta: dict = {
|
||||
"context_note_id": None,
|
||||
"context_note_title": None,
|
||||
@@ -769,16 +793,10 @@ async def build_context(
|
||||
)
|
||||
|
||||
# Briefing article context for follow-up Q&A
|
||||
if conv_id is not None:
|
||||
from fabledassistant.models import async_session as _async_session
|
||||
from fabledassistant.models.conversation import Conversation
|
||||
|
||||
async with _async_session() as _sess:
|
||||
_conv = await _sess.get(Conversation, conv_id)
|
||||
if _conv and getattr(_conv, "conversation_type", None) == "briefing":
|
||||
article_context = await _build_briefing_article_context(conv_id)
|
||||
if article_context:
|
||||
user_context_parts.append(article_context.strip())
|
||||
if _is_briefing_conv:
|
||||
article_context = await _build_briefing_article_context(conv_id) # type: ignore[arg-type]
|
||||
if article_context:
|
||||
user_context_parts.append(article_context.strip())
|
||||
|
||||
# Build final user message — context prefix (if any) followed by the actual message
|
||||
if user_context_parts:
|
||||
|
||||
@@ -124,7 +124,7 @@ async def generate_project_summary(user_id: int, project_id: int) -> None:
|
||||
from fabledassistant.services.settings import get_setting
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
bg_model = await get_setting(user_id, "background_model", Config.OLLAMA_BACKGROUND_MODEL)
|
||||
summary = await generate_completion(messages, model=bg_model, max_tokens=400)
|
||||
summary = await generate_completion(messages, model=bg_model, max_tokens=400, num_ctx=2048)
|
||||
if not summary:
|
||||
return
|
||||
|
||||
|
||||
@@ -62,11 +62,14 @@ def ensure_vapid_keys() -> None:
|
||||
v = Vapid01()
|
||||
v.generate_keys()
|
||||
|
||||
private_pem = v.private_key.private_bytes(
|
||||
serialization.Encoding.PEM,
|
||||
# pywebpush expects the private key as a base64url-encoded DER blob
|
||||
# (passed to Vapid.from_string → from_der), NOT a PEM string.
|
||||
private_der = v.private_key.private_bytes(
|
||||
serialization.Encoding.DER,
|
||||
serialization.PrivateFormat.TraditionalOpenSSL,
|
||||
serialization.NoEncryption(),
|
||||
).decode()
|
||||
)
|
||||
private_b64 = base64.urlsafe_b64encode(private_der).rstrip(b"=").decode()
|
||||
|
||||
pub_bytes = v.public_key.public_bytes(
|
||||
serialization.Encoding.X962,
|
||||
@@ -76,15 +79,40 @@ def ensure_vapid_keys() -> None:
|
||||
|
||||
# Persist so they survive container restarts.
|
||||
_VAPID_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
_VAPID_KEYS_FILE.write_text(json.dumps({"private_key": private_pem, "public_key": public_b64}))
|
||||
_VAPID_KEYS_FILE.write_text(json.dumps({"private_key": private_b64, "public_key": public_b64}))
|
||||
|
||||
Config.VAPID_PRIVATE_KEY = private_pem
|
||||
Config.VAPID_PRIVATE_KEY = private_b64
|
||||
Config.VAPID_PUBLIC_KEY = public_b64
|
||||
logger.info("Generated new VAPID keys and saved to %s", _VAPID_KEYS_FILE)
|
||||
except Exception:
|
||||
logger.warning("Failed to generate VAPID keys — push notifications will be unavailable", exc_info=True)
|
||||
|
||||
|
||||
async def regenerate_vapid_keys() -> bool:
|
||||
"""Delete existing VAPID keys, clear all push subscriptions, and generate a fresh pair.
|
||||
|
||||
All existing browser subscriptions are invalidated when keys rotate, so they
|
||||
must be cleared — users will need to re-enable notifications.
|
||||
"""
|
||||
if _VAPID_KEYS_FILE.exists():
|
||||
_VAPID_KEYS_FILE.unlink()
|
||||
Config.VAPID_PRIVATE_KEY = ""
|
||||
Config.VAPID_PUBLIC_KEY = ""
|
||||
|
||||
# Clear all push subscriptions — they are bound to the old public key.
|
||||
async with async_session() as session:
|
||||
await session.execute(delete(PushSubscription))
|
||||
await session.commit()
|
||||
|
||||
ensure_vapid_keys()
|
||||
enabled = vapid_enabled()
|
||||
if enabled:
|
||||
logger.info("VAPID keys regenerated successfully")
|
||||
else:
|
||||
logger.error("VAPID key regeneration failed")
|
||||
return enabled
|
||||
|
||||
|
||||
async def save_subscription(user_id: int, subscription_json: dict, user_agent: str | None = None) -> PushSubscription:
|
||||
"""Upsert a push subscription by endpoint."""
|
||||
endpoint = subscription_json.get("endpoint", "")
|
||||
|
||||
@@ -21,6 +21,106 @@ MAX_SYNTHESIS_SOURCES = 12 # deduplicated sources passed to synthesis LLM
|
||||
CHARS_PER_SOURCE = 2000 # content chars per source sent to synthesis
|
||||
|
||||
|
||||
def _build_sources_block(sources: list[dict]) -> str:
|
||||
"""Format fetched sources into a text block for LLM prompts."""
|
||||
parts = []
|
||||
for i, s in enumerate(sources, 1):
|
||||
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
|
||||
parts.append(
|
||||
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
|
||||
)
|
||||
return "\n\n" + ("─" * 60) + "\n\n".join(parts)
|
||||
|
||||
|
||||
async def _generate_outline(topic: str, sources: list[dict], model: str) -> list[dict]:
|
||||
"""Generate a topic outline from fetched research sources.
|
||||
|
||||
Returns a list of {"title": str, "focus": str} dicts (3–8 entries).
|
||||
Returns [] on failure — callers must fall back to single-note synthesis.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
sources_block = _build_sources_block(sources) if sources else "(no sources)"
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a research organizer. Given research sources on a topic, produce a JSON array "
|
||||
"of section objects that together cover the topic comprehensively from distinct angles.\n\n"
|
||||
"Rules:\n"
|
||||
"- Return exactly 3–7 sections\n"
|
||||
"- Each section must cover a unique angle — no overlap between sections\n"
|
||||
"- Titles must work as standalone note titles (specific, not generic like 'Overview')\n"
|
||||
"- focus: one sentence describing exactly what this section covers\n"
|
||||
"- Respond with ONLY a JSON array, no other text\n\n"
|
||||
'Example: [{"title": "CRISPR: Molecular Mechanisms", "focus": "How Cas9 identifies and cuts DNA at guide-RNA-specified sites"}]'
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Topic: {topic}\n\nSources:\n{sources_block}",
|
||||
},
|
||||
]
|
||||
try:
|
||||
raw = await generate_completion(messages, model, max_tokens=400)
|
||||
raw = raw.strip()
|
||||
raw = re.sub(r"^```(?:json)?\s*", "", raw)
|
||||
raw = re.sub(r"\s*```$", "", raw)
|
||||
idx = raw.find("[")
|
||||
if idx >= 0:
|
||||
parsed, _ = _json.JSONDecoder().raw_decode(raw[idx:])
|
||||
if isinstance(parsed, list):
|
||||
sections = [
|
||||
s for s in parsed
|
||||
if isinstance(s, dict) and s.get("title") and s.get("focus")
|
||||
]
|
||||
if len(sections) >= 3:
|
||||
return sections[:8]
|
||||
except Exception:
|
||||
logger.warning("Outline generation failed for topic '%s'", topic, exc_info=True)
|
||||
return []
|
||||
|
||||
|
||||
async def _synthesize_section(
|
||||
section_title: str,
|
||||
section_focus: str,
|
||||
sources: list[dict],
|
||||
model: str,
|
||||
) -> tuple[str, str]:
|
||||
"""Synthesize one focused note section.
|
||||
|
||||
Returns (section_title, body_markdown). Does not stream.
|
||||
"""
|
||||
sources_block = _build_sources_block(sources) if sources else "(no sources provided)"
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are a focused research writer. Write a single well-structured note section "
|
||||
"on the specific topic provided.\n\n"
|
||||
"Requirements:\n"
|
||||
f"- Focus strictly on: {section_focus}\n"
|
||||
"- 300–600 words of substantive prose\n"
|
||||
"- Use ### for subsections only when they genuinely aid clarity\n"
|
||||
"- Do NOT include a top-level # heading — the title is set separately\n"
|
||||
"- Write in detailed prose paragraphs — not bullet points\n"
|
||||
"- End with a '## Sources' section listing relevant source URLs as markdown hyperlinks\n"
|
||||
"- Ignore source material that falls outside your assigned focus"
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Section title: {section_title}\n"
|
||||
f"Focus: {section_focus}\n\n"
|
||||
f"Sources:\n{sources_block}"
|
||||
),
|
||||
},
|
||||
]
|
||||
raw = await generate_completion(messages, model, max_tokens=2048, num_ctx=16384)
|
||||
return section_title, raw.strip()
|
||||
|
||||
|
||||
async def run_research_pipeline(
|
||||
topic: str,
|
||||
user_id: int,
|
||||
@@ -28,14 +128,17 @@ async def run_research_pipeline(
|
||||
buf=None,
|
||||
project_id: int | None = None,
|
||||
) -> Note:
|
||||
"""Full research pipeline: search → fetch → synthesize → create note.
|
||||
"""Full research pipeline: search → fetch → outline → section notes → index note.
|
||||
|
||||
Emits status events via buf.append_event throughout (when buf is provided).
|
||||
Returns the created Note.
|
||||
Emits status events via buf throughout (when buf is provided).
|
||||
Returns the index note (or a single fallback note on outline failure).
|
||||
"""
|
||||
def _status(msg: str) -> None:
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": msg})
|
||||
|
||||
# Step 1: Generate sub-queries
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": "Generating search queries..."})
|
||||
_status("Generating search queries...")
|
||||
queries = await _generate_sub_queries(topic, model)
|
||||
logger.info("Research: generated %d sub-queries for topic '%s'", len(queries), topic)
|
||||
|
||||
@@ -43,8 +146,7 @@ async def run_research_pipeline(
|
||||
async def _search_with_stagger(i: int, query: str) -> tuple[str, list[dict]]:
|
||||
if i > 0:
|
||||
await asyncio.sleep(0.2 * i)
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": f"Searching: {query}..."})
|
||||
_status(f"Searching: {query}...")
|
||||
results = await _search_searxng(query)
|
||||
logger.info("Research: query '%s' → %d results", query, len(results))
|
||||
return query, results
|
||||
@@ -66,8 +168,7 @@ async def run_research_pipeline(
|
||||
# Fetch all unique URLs in parallel
|
||||
async def _fetch_source(url: str, result: dict, query: str) -> dict:
|
||||
title = result.get("title", url)
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": f"Reading: {title[:60]}..."})
|
||||
_status(f"Reading: {title[:60]}...")
|
||||
content = await fetch_url_content(url)
|
||||
return {
|
||||
"url": url,
|
||||
@@ -84,39 +185,97 @@ async def run_research_pipeline(
|
||||
if not all_sources:
|
||||
raise ValueError(f"No results found for '{topic}'")
|
||||
|
||||
# Step 3: Filter failed fetches
|
||||
good_sources = [
|
||||
s for s in all_sources
|
||||
if not s["content"].startswith("[Failed to fetch")
|
||||
]
|
||||
good_sources = [s for s in all_sources if not s["content"].startswith("[Failed to fetch")]
|
||||
|
||||
if not good_sources:
|
||||
raise ValueError(f"Could not read any sources for '{topic}'")
|
||||
|
||||
# Limit to top N sources for synthesis (already deduplicated by URL)
|
||||
synthesis_sources = good_sources[:MAX_SYNTHESIS_SOURCES]
|
||||
logger.info(
|
||||
"Research: %d/%d sources successfully fetched, using %d for synthesis",
|
||||
len(good_sources), len(all_sources), len(synthesis_sources),
|
||||
)
|
||||
|
||||
# Step 4: Synthesize (streams tokens into chat as the note is being written)
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": f"Synthesizing report from {len(synthesis_sources)} sources..."})
|
||||
title, body = await _synthesize_note(topic, synthesis_sources, model, buf)
|
||||
# Step 3: Generate topic outline
|
||||
_status("Generating outline...")
|
||||
outline = await _generate_outline(topic, synthesis_sources, model)
|
||||
|
||||
# Step 5: Create note
|
||||
if buf is not None:
|
||||
buf.append_event("status", {"status": "Saving note..."})
|
||||
note = await create_note(
|
||||
# Fallback: outline failed or too short → single monolithic note
|
||||
if not outline:
|
||||
logger.warning("Research outline empty, falling back to single note for '%s'", topic)
|
||||
_status("Synthesizing report...")
|
||||
title, body = await _synthesize_note(topic, synthesis_sources, model, buf=None)
|
||||
note = await create_note(
|
||||
user_id=user_id, title=title, body=body, tags=["research"], project_id=project_id,
|
||||
)
|
||||
logger.info("Research (fallback): created note id=%d title='%s'", note.id, note.title)
|
||||
return note
|
||||
|
||||
# Step 4: Synthesize each section in parallel
|
||||
for section in outline:
|
||||
_status(f"Writing: {section['title']}...")
|
||||
|
||||
raw_results = await asyncio.gather(
|
||||
*[_synthesize_section(s["title"], s["focus"], synthesis_sources, model) for s in outline],
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
# Step 5: Create section notes sequentially
|
||||
_status(f"Saving {len(outline)} notes...")
|
||||
section_note_pairs: list[tuple[dict, Note]] = []
|
||||
for section, result in zip(outline, raw_results):
|
||||
if isinstance(result, Exception):
|
||||
logger.warning("Section synthesis failed for '%s': %s", section["title"], result)
|
||||
continue
|
||||
sec_title, sec_body = result
|
||||
try:
|
||||
note = await create_note(
|
||||
user_id=user_id,
|
||||
title=sec_title,
|
||||
body=sec_body,
|
||||
tags=["research"],
|
||||
project_id=project_id,
|
||||
)
|
||||
section_note_pairs.append((section, note))
|
||||
except Exception:
|
||||
logger.warning("Failed to save section note '%s'", sec_title, exc_info=True)
|
||||
|
||||
# All sections failed — fall back to single note
|
||||
if not section_note_pairs:
|
||||
logger.warning("All section syntheses failed, falling back to single note for '%s'", topic)
|
||||
_status("Synthesizing report (fallback)...")
|
||||
title, body = await _synthesize_note(topic, synthesis_sources, model, buf=None)
|
||||
note = await create_note(
|
||||
user_id=user_id, title=title, body=body, tags=["research"], project_id=project_id,
|
||||
)
|
||||
return note
|
||||
|
||||
# Step 6: Create index note
|
||||
from datetime import date as _date
|
||||
index_lines = [
|
||||
f"Research overview for **{topic}** — {_date.today().isoformat()}",
|
||||
"",
|
||||
f"Generated from {len(synthesis_sources)} web sources across {len(section_note_pairs)} sections.",
|
||||
"",
|
||||
"## Sections",
|
||||
"",
|
||||
]
|
||||
for section, note in section_note_pairs:
|
||||
index_lines.append(f"- **{note.title}** — {section['focus']}")
|
||||
index_lines += ["", "*Search for any section title to read it.*"]
|
||||
|
||||
index_note = await create_note(
|
||||
user_id=user_id,
|
||||
title=title,
|
||||
body=body,
|
||||
tags=["research"],
|
||||
title=f"Research: {topic}",
|
||||
body="\n".join(index_lines),
|
||||
tags=["research", "research-index"],
|
||||
project_id=project_id,
|
||||
)
|
||||
logger.info("Research: created note id=%d title='%s'", note.id, note.title)
|
||||
return note
|
||||
logger.info(
|
||||
"Research: created %d section notes + index id=%d for topic '%s'",
|
||||
len(section_note_pairs), index_note.id, topic,
|
||||
)
|
||||
return index_note
|
||||
|
||||
|
||||
async def _generate_sub_queries(topic: str, model: str) -> list[str]:
|
||||
@@ -248,13 +407,7 @@ async def _synthesize_note(
|
||||
When buf is provided, tokens are streamed into the chat buffer in real time
|
||||
so the user can see the note being written. Uses an extended context window.
|
||||
"""
|
||||
sources_text_parts = []
|
||||
for i, s in enumerate(sources, 1):
|
||||
content = (s.get("content") or s.get("snippet") or "")[:CHARS_PER_SOURCE]
|
||||
sources_text_parts.append(
|
||||
f"[Source {i}] {s['title']}\nURL: {s['url']}\nSearch query: {s['query']}\n\n{content}"
|
||||
)
|
||||
sources_block = "\n\n" + ("─" * 60) + "\n\n".join(sources_text_parts)
|
||||
sources_block = _build_sources_block(sources)
|
||||
|
||||
messages = [
|
||||
{
|
||||
|
||||
@@ -55,8 +55,12 @@ def stt_available() -> bool:
|
||||
return _model is not None
|
||||
|
||||
|
||||
async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
|
||||
"""Transcribe audio bytes to text. Runs the model in a thread executor."""
|
||||
async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm", initial_prompt: str | None = None) -> str:
|
||||
"""Transcribe audio bytes to text. Runs the model in a thread executor.
|
||||
|
||||
initial_prompt: optional text to bias the model toward domain-specific vocabulary
|
||||
(e.g. recent conversation context). Reduces mishearings like "gold" for "cold".
|
||||
"""
|
||||
if _model is None:
|
||||
raise RuntimeError("STT model not loaded")
|
||||
|
||||
@@ -73,7 +77,11 @@ async def transcribe(audio_bytes: bytes, mime_type: str = "audio/webm") -> str:
|
||||
f.write(audio_bytes)
|
||||
f.flush()
|
||||
t0 = time.monotonic()
|
||||
segments, _ = _model.transcribe(f.name, beam_size=5) # type: ignore[union-attr]
|
||||
segments, _ = _model.transcribe( # type: ignore[union-attr]
|
||||
f.name,
|
||||
beam_size=5,
|
||||
initial_prompt=initial_prompt or None,
|
||||
)
|
||||
text = " ".join(seg.text.strip() for seg in segments).strip()
|
||||
logger.debug("STT transcription took %.2fs", time.monotonic() - t0)
|
||||
return text
|
||||
|
||||
@@ -101,7 +101,7 @@ _CORE_TOOLS = [
|
||||
},
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Optional project name to assign this task to",
|
||||
"description": "Optional project name to assign this task to. Only set this if the user explicitly named a project. Do NOT infer a project from the task content or context.",
|
||||
},
|
||||
"parent_task": {
|
||||
"type": "string",
|
||||
@@ -147,7 +147,7 @@ _CORE_TOOLS = [
|
||||
},
|
||||
"project": {
|
||||
"type": "string",
|
||||
"description": "Optional project name to assign this note to",
|
||||
"description": "Optional project name to assign this note to. Only set this if the user explicitly named a project. Do NOT infer a project from the note content or context.",
|
||||
},
|
||||
"confirmed": {
|
||||
"type": "boolean",
|
||||
@@ -258,8 +258,8 @@ _CORE_TOOLS = [
|
||||
"function": {
|
||||
"name": "get_note",
|
||||
"description": (
|
||||
"Retrieve the full content of a specific note. Use this when the user asks to read, "
|
||||
"view, or check what a particular note says. Returns the complete note body."
|
||||
"Retrieve the full content of a specific note or task. Use this when the user asks to read, "
|
||||
"view, or check what a particular note or task says. Returns the complete body."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
@@ -316,7 +316,7 @@ _CORE_TOOLS = [
|
||||
"name": "delete_note",
|
||||
"description": (
|
||||
"Delete a note permanently. Use ONLY when the user explicitly asks to delete or remove a note. "
|
||||
"This action requires user confirmation and cannot be undone."
|
||||
"Always confirm with the user first — this cannot be undone."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
@@ -325,6 +325,10 @@ _CORE_TOOLS = [
|
||||
"type": "string",
|
||||
"description": "Title or keyword to find the note to delete",
|
||||
},
|
||||
"confirmed": {
|
||||
"type": "boolean",
|
||||
"description": "Must be true — only set after the user has explicitly confirmed they want this note deleted.",
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
@@ -336,7 +340,7 @@ _CORE_TOOLS = [
|
||||
"name": "delete_task",
|
||||
"description": (
|
||||
"Delete a task permanently. Use ONLY when the user explicitly asks to delete or remove a task. "
|
||||
"This action requires user confirmation and cannot be undone."
|
||||
"Always confirm with the user first — this cannot be undone."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
@@ -345,6 +349,10 @@ _CORE_TOOLS = [
|
||||
"type": "string",
|
||||
"description": "Title or keyword to find the task to delete",
|
||||
},
|
||||
"confirmed": {
|
||||
"type": "boolean",
|
||||
"description": "Must be true — only set after the user has explicitly confirmed they want this task deleted.",
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
@@ -937,6 +945,28 @@ _RAG_TOOLS = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "calculate",
|
||||
"description": (
|
||||
"Evaluate a mathematical expression and return the exact result. Use this for any "
|
||||
"arithmetic, percentages, unit conversions, or multi-step calculations where precision "
|
||||
"matters. Supports standard math operators (+, -, *, /, **, %) and all Python math "
|
||||
"module functions (sqrt, log, sin, cos, floor, ceil, etc.)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"expression": {
|
||||
"type": "string",
|
||||
"description": "A valid Python math expression (e.g. '(450 * 1.13) / 12', 'sqrt(144)', 'log(1000, 10)')",
|
||||
},
|
||||
},
|
||||
"required": ["expression"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
_ENTITY_TOOLS = [
|
||||
@@ -964,6 +994,29 @@ _ENTITY_TOOLS = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "update_person",
|
||||
"description": (
|
||||
"Update details about a saved person. Use this when the user corrects or adds new "
|
||||
"information about someone already in the knowledge base."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Name or keyword to find the person"},
|
||||
"relationship": {"type": "string", "description": "Updated relationship to the user"},
|
||||
"phone": {"type": "string", "description": "Updated phone number"},
|
||||
"email": {"type": "string", "description": "Updated email address"},
|
||||
"birthday": {"type": "string", "description": "Updated birthday in YYYY-MM-DD format"},
|
||||
"address": {"type": "string", "description": "Updated home or mailing address"},
|
||||
"notes": {"type": "string", "description": "Updated free-form notes about this person"},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@@ -987,6 +1040,28 @@ _ENTITY_TOOLS = [
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "update_place",
|
||||
"description": (
|
||||
"Update details about a saved place. Use this when the user corrects or adds new "
|
||||
"information about a location already in the knowledge base."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Name or keyword to find the place"},
|
||||
"address": {"type": "string", "description": "Updated street address"},
|
||||
"phone": {"type": "string", "description": "Updated phone number"},
|
||||
"hours": {"type": "string", "description": "Updated opening hours"},
|
||||
"url": {"type": "string", "description": "Updated website URL"},
|
||||
"notes": {"type": "string", "description": "Updated free-form notes about this place"},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@@ -1051,12 +1126,71 @@ _ENTITY_TOOLS = [
|
||||
]
|
||||
|
||||
|
||||
_PROFILE_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_profile",
|
||||
"description": (
|
||||
"Retrieve the user's stored profile: name, job title, industry, expertise level, "
|
||||
"preferred response style, tone, and interests. Use this when you need to personalise "
|
||||
"a response and the user's profile hasn't already been injected into context."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "update_profile",
|
||||
"description": (
|
||||
"Update the user's stored profile when they share personal information: their name, "
|
||||
"job, industry, expertise, preferred response style, tone, or interests. "
|
||||
"Only set fields the user has explicitly mentioned."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"display_name": {"type": "string", "description": "User's preferred display name"},
|
||||
"job_title": {"type": "string", "description": "Job title (e.g. 'Software Engineer')"},
|
||||
"industry": {"type": "string", "description": "Industry (e.g. 'Healthcare', 'Finance')"},
|
||||
"expertise_level": {
|
||||
"type": "string",
|
||||
"enum": ["novice", "intermediate", "expert"],
|
||||
"description": "User's general expertise level",
|
||||
},
|
||||
"response_style": {
|
||||
"type": "string",
|
||||
"enum": ["concise", "balanced", "detailed"],
|
||||
"description": "Preferred response length/depth",
|
||||
},
|
||||
"tone": {
|
||||
"type": "string",
|
||||
"enum": ["casual", "professional", "technical"],
|
||||
"description": "Preferred communication tone",
|
||||
},
|
||||
"interests": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "List of topics or hobbies the user is interested in",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
async def get_tools_for_user(user_id: int) -> list[dict]:
|
||||
"""Build the tool list for a user based on their configured integrations."""
|
||||
tools = list(_CORE_TOOLS)
|
||||
tools.extend(_URL_TOOLS)
|
||||
tools.extend(_RAG_TOOLS)
|
||||
tools.extend(_ENTITY_TOOLS)
|
||||
tools.extend(_PROFILE_TOOLS)
|
||||
if await is_caldav_configured(user_id):
|
||||
tools.extend(_CALDAV_TOOLS)
|
||||
if Config.searxng_enabled():
|
||||
@@ -1698,6 +1832,12 @@ async def execute_tool(
|
||||
note = notes[0]
|
||||
if note.status is not None:
|
||||
return {"success": False, "error": f"'{note.title}' is a task. Use delete_task instead."}
|
||||
if not arguments.get("confirmed"):
|
||||
return {
|
||||
"success": False,
|
||||
"requires_confirmation": True,
|
||||
"error": f"Deleting '{note.title}' is permanent. Ask the user to confirm, then retry with confirmed=true.",
|
||||
}
|
||||
deleted = await delete_note(user_id, note.id)
|
||||
if not deleted:
|
||||
return {"success": False, "error": "Failed to delete note."}
|
||||
@@ -1717,6 +1857,12 @@ async def execute_tool(
|
||||
note = notes[0]
|
||||
if note.status is None:
|
||||
return {"success": False, "error": f"'{note.title}' is a note. Use delete_note instead."}
|
||||
if not arguments.get("confirmed"):
|
||||
return {
|
||||
"success": False,
|
||||
"requires_confirmation": True,
|
||||
"error": f"Deleting '{note.title}' is permanent. Ask the user to confirm, then retry with confirmed=true.",
|
||||
}
|
||||
deleted = await delete_note(user_id, note.id)
|
||||
if not deleted:
|
||||
return {"success": False, "error": "Failed to delete task."}
|
||||
@@ -2172,6 +2318,72 @@ async def execute_tool(
|
||||
_schedule_embedding(note.id, user_id, name, note.body)
|
||||
return {"success": True, "type": "place", "data": {"id": note.id, "name": name}}
|
||||
|
||||
elif tool_name == "update_person":
|
||||
query = str(arguments.get("query", "")).strip()
|
||||
if not query:
|
||||
return {"success": False, "error": "query is required"}
|
||||
existing, _ = await list_notes(user_id=user_id, q=query, is_task=False, limit=10)
|
||||
target = next((n for n in existing if n.note_type == "person"), None)
|
||||
if target is None:
|
||||
return {"success": False, "error": f"No person found matching '{query}'. Use create_person to add them."}
|
||||
meta = dict(target.entity_meta or {})
|
||||
for field in ("relationship", "phone", "email", "birthday", "address"):
|
||||
val = arguments.get(field)
|
||||
if val is not None:
|
||||
meta[field] = str(val)
|
||||
body_lines = []
|
||||
if meta.get("relationship"):
|
||||
body_lines.append(f"**Relationship:** {meta['relationship']}")
|
||||
if meta.get("phone"):
|
||||
body_lines.append(f"**Phone:** {meta['phone']}")
|
||||
if meta.get("email"):
|
||||
body_lines.append(f"**Email:** {meta['email']}")
|
||||
if meta.get("birthday"):
|
||||
body_lines.append(f"**Birthday:** {meta['birthday']}")
|
||||
if meta.get("address"):
|
||||
body_lines.append(f"**Address:** {meta['address']}")
|
||||
extra_notes = arguments.get("notes")
|
||||
if extra_notes is not None:
|
||||
body_lines.append(f"\n{extra_notes}")
|
||||
new_body = "\n".join(body_lines)
|
||||
updated = await update_note(user_id=user_id, note_id=target.id, body=new_body, entity_meta=meta)
|
||||
if updated is None:
|
||||
return {"success": False, "error": "Failed to update person."}
|
||||
_schedule_embedding(target.id, user_id, target.title, new_body)
|
||||
return {"success": True, "type": "person_updated", "data": {"id": target.id, "name": target.title}}
|
||||
|
||||
elif tool_name == "update_place":
|
||||
query = str(arguments.get("query", "")).strip()
|
||||
if not query:
|
||||
return {"success": False, "error": "query is required"}
|
||||
existing, _ = await list_notes(user_id=user_id, q=query, is_task=False, limit=10)
|
||||
target = next((n for n in existing if n.note_type == "place"), None)
|
||||
if target is None:
|
||||
return {"success": False, "error": f"No place found matching '{query}'. Use create_place to add it."}
|
||||
meta = dict(target.entity_meta or {})
|
||||
for field in ("address", "phone", "hours", "url"):
|
||||
val = arguments.get(field)
|
||||
if val is not None:
|
||||
meta[field] = str(val)
|
||||
body_lines = []
|
||||
if meta.get("address"):
|
||||
body_lines.append(f"**Address:** {meta['address']}")
|
||||
if meta.get("phone"):
|
||||
body_lines.append(f"**Phone:** {meta['phone']}")
|
||||
if meta.get("hours"):
|
||||
body_lines.append(f"**Hours:** {meta['hours']}")
|
||||
if meta.get("url"):
|
||||
body_lines.append(f"**Website:** {meta['url']}")
|
||||
extra_notes = arguments.get("notes")
|
||||
if extra_notes is not None:
|
||||
body_lines.append(f"\n{extra_notes}")
|
||||
new_body = "\n".join(body_lines)
|
||||
updated = await update_note(user_id=user_id, note_id=target.id, body=new_body, entity_meta=meta)
|
||||
if updated is None:
|
||||
return {"success": False, "error": "Failed to update place."}
|
||||
_schedule_embedding(target.id, user_id, target.title, new_body)
|
||||
return {"success": True, "type": "place_updated", "data": {"id": target.id, "name": target.title}}
|
||||
|
||||
elif tool_name == "create_list":
|
||||
name = str(arguments.get("name", "")).strip()
|
||||
if not name:
|
||||
@@ -2235,6 +2447,65 @@ async def execute_tool(
|
||||
_schedule_embedding(target.id, user_id, target.title, cleaned)
|
||||
return {"success": True, "type": "list_cleared", "data": {"id": target.id, "name": target.title}}
|
||||
|
||||
elif tool_name == "get_profile":
|
||||
from fabledassistant.services.user_profile import get_profile as _get_profile
|
||||
profile = await _get_profile(user_id)
|
||||
return {
|
||||
"success": True,
|
||||
"type": "profile",
|
||||
"data": {
|
||||
"display_name": profile.display_name or "",
|
||||
"job_title": profile.job_title or "",
|
||||
"industry": profile.industry or "",
|
||||
"expertise_level": profile.expertise_level or "intermediate",
|
||||
"response_style": profile.response_style or "balanced",
|
||||
"tone": profile.tone or "casual",
|
||||
"interests": profile.interests or [],
|
||||
},
|
||||
}
|
||||
|
||||
elif tool_name == "update_profile":
|
||||
from fabledassistant.services.user_profile import update_profile as _update_profile, VALID_EXPERTISE, VALID_STYLES, VALID_TONES
|
||||
data: dict = {}
|
||||
for field in ("display_name", "job_title", "industry"):
|
||||
val = arguments.get(field)
|
||||
if val is not None:
|
||||
data[field] = str(val)
|
||||
expertise = arguments.get("expertise_level")
|
||||
if expertise in VALID_EXPERTISE:
|
||||
data["expertise_level"] = expertise
|
||||
style = arguments.get("response_style")
|
||||
if style in VALID_STYLES:
|
||||
data["response_style"] = style
|
||||
tone = arguments.get("tone")
|
||||
if tone in VALID_TONES:
|
||||
data["tone"] = tone
|
||||
interests = arguments.get("interests")
|
||||
if isinstance(interests, list):
|
||||
data["interests"] = [str(i) for i in interests if str(i).strip()]
|
||||
if not data:
|
||||
return {"success": False, "error": "No valid fields provided to update."}
|
||||
profile = await _update_profile(user_id, data)
|
||||
return {
|
||||
"success": True,
|
||||
"type": "profile_updated",
|
||||
"data": {"fields_updated": list(data.keys())},
|
||||
}
|
||||
|
||||
elif tool_name == "calculate":
|
||||
import math as _math
|
||||
expr = str(arguments.get("expression", "")).strip()
|
||||
if not expr:
|
||||
return {"success": False, "error": "expression is required"}
|
||||
allowed_names = {k: v for k, v in vars(_math).items() if not k.startswith("_")}
|
||||
allowed_names["abs"] = abs
|
||||
allowed_names["round"] = round
|
||||
try:
|
||||
result = eval(expr, {"__builtins__": {}}, allowed_names) # noqa: S307
|
||||
except Exception as calc_err:
|
||||
return {"success": False, "error": f"Could not evaluate expression: {calc_err}"}
|
||||
return {"success": True, "type": "calculation", "data": {"expression": expr, "result": result}}
|
||||
|
||||
else:
|
||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||
|
||||
|
||||
@@ -239,17 +239,32 @@ async def synthesise(
|
||||
voice_param = _build_voice_param()
|
||||
t0 = time.monotonic()
|
||||
audio_chunks: list = []
|
||||
for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc]
|
||||
if audio is not None:
|
||||
audio_chunks.append(audio)
|
||||
try:
|
||||
for _, _, audio in _pipeline(text, voice=voice_param, speed=speed): # type: ignore[misc]
|
||||
if audio is not None:
|
||||
audio_chunks.append(audio)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Kokoro pipeline error during synthesis: %d chars, preview=%r",
|
||||
len(text), text[:80],
|
||||
)
|
||||
raise
|
||||
|
||||
if not audio_chunks:
|
||||
logger.warning(
|
||||
"Kokoro produced no audio chunks: %d chars, preview=%r",
|
||||
len(text), text[:80],
|
||||
)
|
||||
return b""
|
||||
|
||||
combined = np.concatenate(audio_chunks)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, combined, samplerate=24000, format="WAV", subtype="PCM_16")
|
||||
logger.debug("TTS synthesis took %.2fs for %d chars", time.monotonic() - t0, len(text))
|
||||
elapsed = time.monotonic() - t0
|
||||
logger.info(
|
||||
"Kokoro synthesis: %d chars → %d samples (%.2fs, %.0f chars/s)",
|
||||
len(text), len(combined), elapsed, len(text) / elapsed if elapsed > 0 else 0,
|
||||
)
|
||||
return buf.getvalue()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Tests for the multi-note research pipeline."""
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_outline_parses_valid_json():
|
||||
"""_generate_outline returns parsed sections when model returns valid JSON."""
|
||||
from fabledassistant.services.research import _generate_outline
|
||||
|
||||
outline_json = json.dumps([
|
||||
{"title": "Section One", "focus": "Focus one"},
|
||||
{"title": "Section Two", "focus": "Focus two"},
|
||||
{"title": "Section Three", "focus": "Focus three"},
|
||||
])
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
||||
result = await _generate_outline("test topic", [], "test-model")
|
||||
|
||||
assert len(result) == 3
|
||||
assert result[0]["title"] == "Section One"
|
||||
assert result[0]["focus"] == "Focus one"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_outline_returns_empty_on_parse_failure():
|
||||
"""_generate_outline returns [] when model output is not parseable JSON."""
|
||||
from fabledassistant.services.research import _generate_outline
|
||||
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="not json at all"):
|
||||
result = await _generate_outline("test topic", [], "test-model")
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_outline_returns_empty_when_fewer_than_3_sections():
|
||||
"""_generate_outline returns [] when model returns fewer than 3 valid sections."""
|
||||
from fabledassistant.services.research import _generate_outline
|
||||
|
||||
outline_json = json.dumps([{"title": "Only One", "focus": "Focus"}])
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
||||
result = await _generate_outline("test topic", [], "test-model")
|
||||
|
||||
assert result == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_outline_truncates_to_8():
|
||||
"""_generate_outline truncates results to at most 8 sections."""
|
||||
from fabledassistant.services.research import _generate_outline
|
||||
|
||||
sections = [{"title": f"Section {i}", "focus": f"Focus {i}"} for i in range(10)]
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=json.dumps(sections)):
|
||||
result = await _generate_outline("test topic", [], "test-model")
|
||||
|
||||
assert len(result) == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_outline_skips_malformed_entries():
|
||||
"""_generate_outline filters out entries missing title or focus."""
|
||||
from fabledassistant.services.research import _generate_outline
|
||||
|
||||
outline_json = json.dumps([
|
||||
{"title": "Good One", "focus": "Good focus"},
|
||||
{"title": "Missing focus"},
|
||||
{"focus": "Missing title"},
|
||||
{"title": "Good Two", "focus": "Good focus two"},
|
||||
{"title": "Good Three", "focus": "Good focus three"},
|
||||
])
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value=outline_json):
|
||||
result = await _generate_outline("test topic", [], "test-model")
|
||||
|
||||
assert len(result) == 3
|
||||
assert all(s["title"] and s["focus"] for s in result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_synthesize_section_returns_title_and_body():
|
||||
"""_synthesize_section returns (section_title, body) from model output."""
|
||||
from fabledassistant.services.research import _synthesize_section
|
||||
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="The body of this section."):
|
||||
title, body = await _synthesize_section(
|
||||
section_title="Quantum Entanglement: Mechanisms",
|
||||
section_focus="How entanglement works at the particle level",
|
||||
sources=[],
|
||||
model="test-model",
|
||||
)
|
||||
|
||||
assert title == "Quantum Entanglement: Mechanisms"
|
||||
assert body == "The body of this section."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_synthesize_section_strips_whitespace():
|
||||
"""_synthesize_section strips leading/trailing whitespace from body."""
|
||||
from fabledassistant.services.research import _synthesize_section
|
||||
|
||||
with patch("fabledassistant.services.research.generate_completion", new_callable=AsyncMock, return_value="\n\n body text \n\n"):
|
||||
title, body = await _synthesize_section("Title", "Focus", [], "model")
|
||||
|
||||
assert body == "body text"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_creates_section_notes_and_index():
|
||||
"""run_research_pipeline creates N section notes + 1 index note, returns index."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
outline = [
|
||||
{"title": "Section A", "focus": "Focus A"},
|
||||
{"title": "Section B", "focus": "Focus B"},
|
||||
{"title": "Section C", "focus": "Focus C"},
|
||||
]
|
||||
|
||||
note_id_counter = iter(range(10, 20))
|
||||
|
||||
def _make_note(user_id, title, body, tags, project_id=None):
|
||||
n = MagicMock()
|
||||
n.id = next(note_id_counter)
|
||||
n.title = title
|
||||
return n
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \
|
||||
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=lambda t, f, s, m: (t, f"Body for {t}")), \
|
||||
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, side_effect=_make_note):
|
||||
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
assert result.title == "Research: test topic"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_falls_back_to_single_note_when_outline_empty():
|
||||
"""run_research_pipeline falls back to single-note synthesis when _generate_outline returns []."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
single_note = MagicMock()
|
||||
single_note.id = 99
|
||||
single_note.title = "Research: test topic"
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=[]), \
|
||||
patch("fabledassistant.services.research._synthesize_note", new_callable=AsyncMock, return_value=("Research: test topic", "body")), \
|
||||
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, return_value=single_note):
|
||||
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
assert result.id == 99
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_falls_back_when_all_sections_fail():
|
||||
"""run_research_pipeline falls back to single note when all section syntheses raise."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
outline = [
|
||||
{"title": "Section A", "focus": "Focus A"},
|
||||
{"title": "Section B", "focus": "Focus B"},
|
||||
{"title": "Section C", "focus": "Focus C"},
|
||||
]
|
||||
fallback_note = MagicMock()
|
||||
fallback_note.id = 77
|
||||
fallback_note.title = "Research: test topic"
|
||||
|
||||
with patch("fabledassistant.services.research._generate_sub_queries", new_callable=AsyncMock, return_value=["q1"]), \
|
||||
patch("fabledassistant.services.research._search_searxng", new_callable=AsyncMock, return_value=[{"url": "http://x.com", "title": "X", "snippet": "s"}]), \
|
||||
patch("fabledassistant.services.research.fetch_url_content", new_callable=AsyncMock, return_value="content"), \
|
||||
patch("fabledassistant.services.research._generate_outline", new_callable=AsyncMock, return_value=outline), \
|
||||
patch("fabledassistant.services.research._synthesize_section", new_callable=AsyncMock, side_effect=RuntimeError("synthesis failed")), \
|
||||
patch("fabledassistant.services.research._synthesize_note", new_callable=AsyncMock, return_value=("Research: test topic", "body")), \
|
||||
patch("fabledassistant.services.research.create_note", new_callable=AsyncMock, return_value=fallback_note):
|
||||
|
||||
from fabledassistant.services.research import run_research_pipeline
|
||||
result = await run_research_pipeline("test topic", user_id=1, model="test-model")
|
||||
|
||||
assert result.id == 77
|
||||
Reference in New Issue
Block a user