Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c4ffe418b5 | |||
| 6cf880506d | |||
| aca34e2dfb | |||
| c07a0f0f7e | |||
| 776e5edb68 | |||
| 8b0dd732f7 | |||
| f12d29563a | |||
| a4995606e5 | |||
| 0a8a755909 | |||
| 719de12a6c | |||
| 72018aa389 | |||
| 1e73ea04f1 | |||
| 72708475a3 | |||
| e07d8436b7 |
@@ -68,19 +68,14 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
# Cache node_modules directly (not the npm download cache).
|
||||
# npm ci still has to extract + link every module even with a
|
||||
# warm download cache, which is where the real time goes. Caching
|
||||
# the output directory lets us skip npm ci entirely on hits.
|
||||
- name: Cache node_modules
|
||||
id: npm-cache
|
||||
- name: Cache npm download cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: frontend/node_modules
|
||||
key: node-modules-${{ hashFiles('frontend/package-lock.json') }}
|
||||
path: ~/.npm
|
||||
key: npm-cache-${{ hashFiles('frontend/package-lock.json') }}
|
||||
restore-keys: npm-cache-
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.npm-cache.outputs.cache-hit != 'true'
|
||||
run: npm ci
|
||||
working-directory: frontend
|
||||
|
||||
|
||||
Generated
+393
-1276
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,7 @@
|
||||
"@fullcalendar/interaction": "^6.1.20",
|
||||
"@fullcalendar/timegrid": "^6.1.20",
|
||||
"@fullcalendar/vue3": "^6.1.20",
|
||||
"@ricky0123/vad-web": "^0.0.30",
|
||||
"@tiptap/core": "^3.0.0",
|
||||
"@tiptap/extension-link": "^3.0.0",
|
||||
"@tiptap/extension-list": "^3.0.0",
|
||||
@@ -35,6 +36,7 @@
|
||||
"@vitejs/plugin-vue": "^6.0.0",
|
||||
"typescript": "~5.9.0",
|
||||
"vite": "^7.0.0",
|
||||
"vite-plugin-static-copy": "^4.0.1",
|
||||
"vue-tsc": "^3.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import AppHeader from "@/components/AppHeader.vue";
|
||||
import ToastNotification from "@/components/ToastNotification.vue";
|
||||
import { useTheme } from "@/composables/useTheme";
|
||||
import { useShortcuts } from "@/composables/useShortcuts";
|
||||
import { useOnnxPreloader } from "@/composables/useOnnxPreloader";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import { useChatStore } from "@/stores/chat";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
@@ -12,6 +13,9 @@ import { apiGet, apiPut } from "@/api/client";
|
||||
|
||||
useTheme();
|
||||
|
||||
const { schedulePreload: scheduleVadPreload } = useOnnxPreloader();
|
||||
scheduleVadPreload();
|
||||
|
||||
const router = useRouter();
|
||||
const appVersion = ref("dev");
|
||||
const authStore = useAuthStore();
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, nextTick, onMounted, onUnmounted } from 'vue'
|
||||
import { ref, computed, nextTick, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { apiGet, transcribeAudio } from '@/api/client'
|
||||
import { useVoiceRecorder } from '@/composables/useVoiceRecorder'
|
||||
import { useSilenceDetector } from '@/composables/useSilenceDetector'
|
||||
import { useVad } from '@/composables/useVad'
|
||||
import { useStreamingTts } from '@/composables/useStreamingTts'
|
||||
import { useVoiceAudio, setVoiceVolume } from '@/composables/useVoiceAudio'
|
||||
import { useListenMode } from '@/composables/useListenMode'
|
||||
@@ -138,29 +138,50 @@ function removeAttachedNote() {
|
||||
attachedNote.value = null
|
||||
}
|
||||
|
||||
// ── Voice (click-to-toggle + silence detection) ─────────────────────────────
|
||||
// ── Voice (click-to-toggle + VAD speech detection) ─────────────────────────
|
||||
const transcribingVoice = ref(false)
|
||||
const recorder = useVoiceRecorder()
|
||||
const silenceDetector = useSilenceDetector()
|
||||
// Tracks whether VAD detected speech during the current recording session.
|
||||
// Used to skip the Whisper round-trip when the user manually stops without
|
||||
// ever speaking — the recording is ambient noise and will transcribe to "".
|
||||
const vadSawSpeech = ref(false)
|
||||
|
||||
const vad = useVad({
|
||||
onSpeechEnd: () => {
|
||||
// VAD auto-stop: speech was detected and the user has paused. Proceed
|
||||
// to transcribe the MediaRecorder output.
|
||||
void stopRecording(false)
|
||||
},
|
||||
onNoSpeech: () => {
|
||||
useToastStore().show('No speech detected', 'warning')
|
||||
},
|
||||
onVadError: (msg) => {
|
||||
useToastStore().show(`VAD failed: ${msg}`, 'error')
|
||||
},
|
||||
})
|
||||
|
||||
// Live mic halo. A solid red disc sits behind the mic button and scales
|
||||
// with `silenceDetector.amplitude` (0..1 RMS, already amplified for
|
||||
// visibility). The button itself stays put so the mic icon is always
|
||||
// legible on top. 0.2 baseline breathes on silence; loud speech drives
|
||||
// the disc out to ~2.6× the button size with a softer radial fade.
|
||||
// with `vad.amplitude` (0..1 RMS, already amplified for visibility).
|
||||
const micGlowStyle = computed(() => {
|
||||
if (!recorder.recording.value) return { display: 'none' }
|
||||
const pulse = 0.2 + Math.min(silenceDetector.amplitude.value, 1) * 0.8
|
||||
const pulse = 0.2 + Math.min(vad.amplitude.value, 1) * 0.8
|
||||
return {
|
||||
transform: `translate(-50%, -50%) scale(${1 + pulse * 1.6})`,
|
||||
opacity: String(0.45 + pulse * 0.4),
|
||||
}
|
||||
})
|
||||
|
||||
// vad.speaking flips true on speech-start. Watch it once per session to
|
||||
// capture that speech was detected at some point, without clearing when
|
||||
// speech ends. This drives the no-speech guard in stopRecording.
|
||||
watch(() => vad.speaking.value, (on) => {
|
||||
if (on) vadSawSpeech.value = true
|
||||
})
|
||||
|
||||
async function toggleVoice() {
|
||||
if (transcribingVoice.value) return
|
||||
if (recorder.recording.value) {
|
||||
await stopRecording()
|
||||
await stopRecording(true)
|
||||
} else {
|
||||
await startRecording()
|
||||
}
|
||||
@@ -172,22 +193,32 @@ async function startRecording() {
|
||||
useToastStore().show('Microphone requires HTTPS or localhost', 'error')
|
||||
return
|
||||
}
|
||||
vadSawSpeech.value = false
|
||||
await recorder.startRecording()
|
||||
if (recorder.error.value) {
|
||||
useToastStore().show(recorder.error.value, 'error')
|
||||
return
|
||||
}
|
||||
if (recorder.stream.value) {
|
||||
silenceDetector.start(recorder.stream.value, () => stopRecording())
|
||||
await vad.start(recorder.stream.value)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecording() {
|
||||
silenceDetector.stop()
|
||||
async function stopRecording(manual: boolean) {
|
||||
if (manual) {
|
||||
await vad.stopAndCheck()
|
||||
} else {
|
||||
await vad.stop()
|
||||
}
|
||||
if (!recorder.recording.value) return
|
||||
transcribingVoice.value = true
|
||||
try {
|
||||
const blob = await recorder.stopRecording()
|
||||
// No-speech guard: user manually stopped without VAD seeing speech.
|
||||
// Skip Whisper — the toast was already shown by onNoSpeech.
|
||||
if (manual && !vadSawSpeech.value) {
|
||||
return
|
||||
}
|
||||
// Pass last assistant message as context to reduce STT mishearings
|
||||
const msgs = store.currentConversation?.messages ?? []
|
||||
const lastAssistant = [...msgs].reverse().find(m => m.role === 'assistant')?.content
|
||||
|
||||
@@ -665,7 +665,6 @@ defineExpose({ focus, prefill, send, contextCount, sidebarOpen, toggleContextSid
|
||||
.input-wrapper {
|
||||
grid-column: 2;
|
||||
grid-row: 2;
|
||||
border-top: 1px solid var(--color-border);
|
||||
padding: 0.5rem 1rem 0.75rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -85,22 +85,30 @@ const fetchedAtLabel = computed(() => {
|
||||
Today: {{ weather.today_high }}° / {{ weather.today_low }}°
|
||||
<span v-if="tempDelta" class="weather-delta"> · {{ tempDelta }}</span>
|
||||
</div>
|
||||
<div class="weather-forecast" v-if="weather.forecast.length">
|
||||
<div v-for="day in weather.forecast" :key="day.day" class="weather-forecast-day">
|
||||
<span class="forecast-day-name">{{ day.day }}</span>
|
||||
<span class="forecast-icon">{{ weatherIcon(day.condition) }}</span>
|
||||
<span class="forecast-condition">{{ day.condition }}</span>
|
||||
<span class="forecast-temps">{{ day.high }}° / {{ day.low }}°</span>
|
||||
<span v-if="day.precip_probability != null && day.precip_probability > 0" class="forecast-precip">
|
||||
💧 {{ day.precip_probability }}%
|
||||
</span>
|
||||
<span v-else-if="day.precip_mm != null && day.precip_mm > 0" class="forecast-precip">
|
||||
💧 {{ day.precip_mm.toFixed(1) }}mm
|
||||
</span>
|
||||
<span v-else class="forecast-precip forecast-precip--dry">💧 —</span>
|
||||
<span class="forecast-wind">💨 {{ day.windspeed_max }} {{ weather.wind_unit ?? 'km/h' }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<table class="weather-forecast" v-if="weather.forecast.length">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th></th>
|
||||
<th>Hi / Lo</th>
|
||||
<th>💧</th>
|
||||
<th>💨 {{ weather.wind_unit ?? 'km/h' }}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="day in weather.forecast" :key="day.day">
|
||||
<td class="forecast-day-name">{{ day.day }}</td>
|
||||
<td class="forecast-icon">{{ weatherIcon(day.condition) }}</td>
|
||||
<td class="forecast-temps">{{ day.high }}° / {{ day.low }}°</td>
|
||||
<td class="forecast-precip" :class="{ 'forecast-precip--dry': !(day.precip_probability != null && day.precip_probability > 0) && !(day.precip_mm != null && day.precip_mm > 0) }">
|
||||
<template v-if="day.precip_probability != null && day.precip_probability > 0">{{ day.precip_probability }}%</template>
|
||||
<template v-else-if="day.precip_mm != null && day.precip_mm > 0">{{ day.precip_mm.toFixed(1) }}mm</template>
|
||||
<template v-else>—</template>
|
||||
</td>
|
||||
<td class="forecast-wind">{{ day.windspeed_max }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div v-else class="weather-card weather-unavailable">
|
||||
Weather data unavailable — will retry at next slot.
|
||||
@@ -115,6 +123,7 @@ const fetchedAtLabel = computed(() => {
|
||||
padding: 1rem 1.25rem;
|
||||
margin-bottom: 1rem;
|
||||
font-size: 0.9rem;
|
||||
container-type: inline-size;
|
||||
}
|
||||
|
||||
.weather-header {
|
||||
@@ -142,12 +151,12 @@ const fetchedAtLabel = computed(() => {
|
||||
}
|
||||
|
||||
.weather-icon {
|
||||
font-size: 2rem;
|
||||
font-size: clamp(1.5rem, 5cqi, 2.5rem);
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.weather-temp {
|
||||
font-size: 2rem;
|
||||
font-size: clamp(1.5rem, 5cqi, 2.5rem);
|
||||
font-weight: 700;
|
||||
line-height: 1;
|
||||
}
|
||||
@@ -169,20 +178,31 @@ const fetchedAtLabel = computed(() => {
|
||||
}
|
||||
|
||||
.weather-forecast {
|
||||
display: flex;
|
||||
gap: 0.75rem;
|
||||
overflow-x: auto;
|
||||
padding-top: 0.75rem;
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 0.75rem;
|
||||
border-top: 1px solid var(--color-border);
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.weather-forecast-day {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
min-width: 4.5rem;
|
||||
font-size: 0.8rem;
|
||||
.weather-forecast thead th {
|
||||
font-size: 0.68rem;
|
||||
font-weight: 600;
|
||||
color: var(--color-text-muted);
|
||||
text-align: right;
|
||||
padding: 0.5rem 0.4rem 0.25rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.weather-forecast thead th:first-child,
|
||||
.weather-forecast thead th:nth-child(2) {
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.weather-forecast tbody td {
|
||||
padding: 0.3rem 0.4rem;
|
||||
white-space: nowrap;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.forecast-day-name {
|
||||
@@ -190,26 +210,16 @@ const fetchedAtLabel = computed(() => {
|
||||
}
|
||||
|
||||
.forecast-icon {
|
||||
font-size: 1.2rem;
|
||||
font-size: clamp(0.9rem, 3cqi, 1.3rem);
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.forecast-condition {
|
||||
font-size: 0.7rem;
|
||||
color: var(--color-text-muted);
|
||||
text-align: center;
|
||||
line-height: 1.2;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.forecast-temps {
|
||||
white-space: nowrap;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.forecast-precip,
|
||||
.forecast-wind {
|
||||
font-size: 0.72rem;
|
||||
white-space: nowrap;
|
||||
.forecast-precip {
|
||||
text-align: right;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
@@ -217,6 +227,11 @@ const fetchedAtLabel = computed(() => {
|
||||
opacity: 0.35;
|
||||
}
|
||||
|
||||
.forecast-wind {
|
||||
text-align: right;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.weather-unavailable {
|
||||
color: var(--color-text-muted);
|
||||
font-style: italic;
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
/**
|
||||
* Idle-preload the Silero VAD ONNX model and its companion assets.
|
||||
*
|
||||
* `@ricky0123/vad-web` pulls ~2MB of ONNX model + ~5MB of onnxruntime-web
|
||||
* WASM the first time `MicVAD.new()` runs. Calling `defaultModelFetcher`
|
||||
* during page idle warms the browser cache so the first mic click feels
|
||||
* instant. If the preload fails, VAD simply loads lazily on first click.
|
||||
*/
|
||||
const ready = ref(false)
|
||||
let loading = false
|
||||
|
||||
export function useOnnxPreloader() {
|
||||
async function preload() {
|
||||
if (ready.value || loading) return
|
||||
loading = true
|
||||
try {
|
||||
const { defaultModelFetcher } = await import('@ricky0123/vad-web')
|
||||
// Asset lives at site root because vite-plugin-static-copy drops it
|
||||
// there. Match whatever path MicVAD will use at call time (see useVad).
|
||||
await defaultModelFetcher('/silero_vad_v5.onnx')
|
||||
ready.value = true
|
||||
} catch {
|
||||
// Non-fatal — MicVAD will load the model itself on first use.
|
||||
} finally {
|
||||
loading = false
|
||||
}
|
||||
}
|
||||
|
||||
function schedulePreload() {
|
||||
if (ready.value) return
|
||||
if (typeof window === 'undefined') return
|
||||
if ('requestIdleCallback' in window) {
|
||||
(window as Window & { requestIdleCallback: (cb: () => void, opts: { timeout: number }) => void })
|
||||
.requestIdleCallback(() => void preload(), { timeout: 5000 })
|
||||
} else {
|
||||
setTimeout(() => void preload(), 5000)
|
||||
}
|
||||
}
|
||||
|
||||
return { ready: readonly(ready), schedulePreload }
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
|
||||
export interface SilenceDetectorOptions {
|
||||
/**
|
||||
* Absolute fallback silence threshold in dBFS, used during the first
|
||||
* moments before any real speech has been observed. Once the session peak
|
||||
* clears `dynamicArmDb` we switch to the dynamic threshold instead.
|
||||
*/
|
||||
fallbackThresholdDb?: number // default -45
|
||||
/** How many dB below the session peak counts as "silent" once armed. */
|
||||
dropFromPeakDb?: number // default 15
|
||||
/**
|
||||
* Session peak must reach this level (dBFS) before dynamic thresholding
|
||||
* kicks in. Until then the fallback threshold is used so we don't lock
|
||||
* onto a noise-floor peak.
|
||||
*/
|
||||
dynamicArmDb?: number // default -25
|
||||
/** How long to wait at the start of recording before running silence checks. */
|
||||
graceMs?: number // default 1500
|
||||
silenceDurationMs?: number // default 2000
|
||||
minRecordingMs?: number // default 500
|
||||
}
|
||||
|
||||
/**
|
||||
* Mic silence detector + live amplitude signal.
|
||||
*
|
||||
* Uses `getFloatTimeDomainData()` for honest linear RMS in [-1, 1] space,
|
||||
* then derives dBFS for the silence threshold. The previous implementation
|
||||
* ran RMS over `getByteFrequencyData` bytes — those bytes are already a
|
||||
* dB-scaled quantity, so taking their RMS and re-log'ing it produced
|
||||
* numbers that didn't line up with real dBFS and made any static threshold
|
||||
* unpredictable.
|
||||
*
|
||||
* Silence threshold is dynamic: we track the session peak dBFS and treat
|
||||
* "silent" as "current level is at least [dropFromPeakDb] below peak."
|
||||
* This auto-calibrates to whatever mic and room the user is on. Until the
|
||||
* peak climbs above [dynamicArmDb] we fall back to a conservative static
|
||||
* threshold so a quiet room doesn't spin forever. A grace period at the
|
||||
* start of the recording gives the user time to begin speaking before
|
||||
* silence checks arm.
|
||||
*
|
||||
* `amplitude` is the raw linear RMS scaled ×5 and clamped to 1 so quiet
|
||||
* speech still visibly moves UI that binds to it.
|
||||
*/
|
||||
export function useSilenceDetector(options: SilenceDetectorOptions = {}) {
|
||||
const {
|
||||
fallbackThresholdDb = -45,
|
||||
dropFromPeakDb = 15,
|
||||
dynamicArmDb = -25,
|
||||
graceMs = 1500,
|
||||
silenceDurationMs = 2000,
|
||||
minRecordingMs = 500,
|
||||
} = options
|
||||
|
||||
const amplitude = ref(0)
|
||||
let audioCtx: AudioContext | null = null
|
||||
let intervalId: ReturnType<typeof setInterval> | null = null
|
||||
let silenceMs = 0
|
||||
let startedAt = 0
|
||||
let peakDb = -100
|
||||
|
||||
function start(stream: MediaStream, onSilence: () => void): void {
|
||||
stop()
|
||||
audioCtx = new AudioContext()
|
||||
// Some browsers start AudioContext in "suspended" state — resume so
|
||||
// the analyser returns real values instead of all zeros.
|
||||
audioCtx.resume().catch(() => {})
|
||||
const source = audioCtx.createMediaStreamSource(stream)
|
||||
const analyser = audioCtx.createAnalyser()
|
||||
analyser.fftSize = 1024
|
||||
source.connect(analyser)
|
||||
|
||||
const samples = new Float32Array(analyser.fftSize)
|
||||
silenceMs = 0
|
||||
startedAt = Date.now()
|
||||
peakDb = -100
|
||||
|
||||
intervalId = setInterval(() => {
|
||||
analyser.getFloatTimeDomainData(samples)
|
||||
let sumSq = 0
|
||||
for (let i = 0; i < samples.length; i++) sumSq += samples[i] * samples[i]
|
||||
const rms = Math.sqrt(sumSq / samples.length)
|
||||
amplitude.value = Math.min(rms * 5, 1)
|
||||
|
||||
const db = rms > 1e-7 ? 20 * Math.log10(rms) : -100
|
||||
if (db > peakDb) peakDb = db
|
||||
|
||||
const elapsed = Date.now() - startedAt
|
||||
if (elapsed < graceMs) return
|
||||
|
||||
const threshold =
|
||||
peakDb > dynamicArmDb ? peakDb - dropFromPeakDb : fallbackThresholdDb
|
||||
|
||||
if (db < threshold) {
|
||||
silenceMs += 100
|
||||
if (silenceMs >= silenceDurationMs && elapsed >= minRecordingMs) {
|
||||
stop()
|
||||
onSilence()
|
||||
}
|
||||
} else {
|
||||
silenceMs = 0
|
||||
}
|
||||
}, 100)
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (intervalId !== null) {
|
||||
clearInterval(intervalId)
|
||||
intervalId = null
|
||||
}
|
||||
if (audioCtx) {
|
||||
audioCtx.close().catch(() => {})
|
||||
audioCtx = null
|
||||
}
|
||||
amplitude.value = 0
|
||||
silenceMs = 0
|
||||
peakDb = -100
|
||||
}
|
||||
|
||||
return { amplitude: readonly(amplitude), start, stop }
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { ref, readonly } from 'vue'
|
||||
import type { MicVAD } from '@ricky0123/vad-web'
|
||||
|
||||
export interface UseVadOptions {
|
||||
onSpeechEnd: () => void
|
||||
onNoSpeech: () => void
|
||||
onVadError?: (message: string) => void
|
||||
graceMs?: number
|
||||
minRecordingMs?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Silero VAD-based speech detector.
|
||||
*
|
||||
* Replaces amplitude-based silence detection. Uses the Silero VAD v5 ONNX
|
||||
* model via `@ricky0123/vad-web`. The package ships the model + audio
|
||||
* worklet, and depends on `onnxruntime-web`'s WASM. Vite is configured
|
||||
* (see `vite.config.ts`) to serve these assets from the site root, which
|
||||
* is why we pass `baseAssetPath: "/"` and `onnxWASMBasePath: "/"`.
|
||||
*
|
||||
* `amplitude` is computed separately from a Web Audio API analyser node
|
||||
* on the same stream — it drives the mic pulse animation and is not part
|
||||
* of the stop decision. VAD's `onSpeechEnd` is the stop trigger.
|
||||
*
|
||||
* If VAD never detects speech during a session, calling `stopAndCheck()`
|
||||
* fires `onNoSpeech` instead of treating the recording as transcribable.
|
||||
*/
|
||||
export function useVad(options: UseVadOptions) {
|
||||
const {
|
||||
onSpeechEnd,
|
||||
onNoSpeech,
|
||||
graceMs = 1500,
|
||||
minRecordingMs = 500,
|
||||
} = options
|
||||
|
||||
const speaking = ref(false)
|
||||
const amplitude = ref(0)
|
||||
const loaded = ref(false)
|
||||
|
||||
let micVad: MicVAD | null = null
|
||||
let audioCtx: AudioContext | null = null
|
||||
let analyserInterval: ReturnType<typeof setInterval> | null = null
|
||||
let speechDetected = false
|
||||
let speechStartedAt = 0
|
||||
let recordingStartedAt = 0
|
||||
let stopped = false
|
||||
|
||||
async function start(stream: MediaStream): Promise<void> {
|
||||
await stop()
|
||||
stopped = false
|
||||
speechDetected = false
|
||||
speechStartedAt = 0
|
||||
recordingStartedAt = Date.now()
|
||||
|
||||
// Visual-only amplitude signal. This does NOT drive the stop decision.
|
||||
audioCtx = new AudioContext()
|
||||
await audioCtx.resume().catch(() => {})
|
||||
const source = audioCtx.createMediaStreamSource(stream)
|
||||
const analyser = audioCtx.createAnalyser()
|
||||
analyser.fftSize = 1024
|
||||
source.connect(analyser)
|
||||
const samples = new Float32Array(analyser.fftSize)
|
||||
|
||||
analyserInterval = setInterval(() => {
|
||||
analyser.getFloatTimeDomainData(samples)
|
||||
let sumSq = 0
|
||||
for (let i = 0; i < samples.length; i++) sumSq += samples[i] * samples[i]
|
||||
const rms = Math.sqrt(sumSq / samples.length)
|
||||
amplitude.value = Math.min(rms * 5, 1)
|
||||
}, 100)
|
||||
|
||||
try {
|
||||
const { MicVAD } = await import('@ricky0123/vad-web')
|
||||
micVad = await MicVAD.new({
|
||||
model: 'v5',
|
||||
baseAssetPath: '/',
|
||||
onnxWASMBasePath: '/',
|
||||
// MicVAD manages its own audio graph, so we hand it the same stream
|
||||
// the MediaRecorder is using. It won't consume or alter the stream.
|
||||
getStream: async () => stream,
|
||||
onSpeechStart: () => {
|
||||
speaking.value = true
|
||||
if (!speechDetected) {
|
||||
speechDetected = true
|
||||
speechStartedAt = Date.now()
|
||||
}
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
speaking.value = false
|
||||
if (stopped) return
|
||||
const now = Date.now()
|
||||
const totalElapsed = now - recordingStartedAt
|
||||
const sinceSpeechStart = speechStartedAt > 0 ? now - speechStartedAt : 0
|
||||
if (totalElapsed >= minRecordingMs && sinceSpeechStart >= graceMs) {
|
||||
stopped = true
|
||||
onSpeechEnd()
|
||||
}
|
||||
},
|
||||
onVADMisfire: () => {
|
||||
// Speech-like blip that was too short to count. Reset the local flag
|
||||
// so a true "no speech" session still hits onNoSpeech.
|
||||
speaking.value = false
|
||||
},
|
||||
})
|
||||
await micVad.start()
|
||||
loaded.value = true
|
||||
} catch (e) {
|
||||
console.error('VAD init failed:', e)
|
||||
options.onVadError?.(e instanceof Error ? e.message : String(e))
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<{ hadSpeech: boolean }> {
|
||||
const had = speechDetected
|
||||
stopped = true
|
||||
speaking.value = false
|
||||
amplitude.value = 0
|
||||
|
||||
if (analyserInterval !== null) {
|
||||
clearInterval(analyserInterval)
|
||||
analyserInterval = null
|
||||
}
|
||||
if (audioCtx) {
|
||||
await audioCtx.close().catch(() => {})
|
||||
audioCtx = null
|
||||
}
|
||||
if (micVad) {
|
||||
try {
|
||||
await micVad.pause()
|
||||
await micVad.destroy()
|
||||
} catch {
|
||||
// Already destroyed or failed — nothing to clean up further.
|
||||
}
|
||||
micVad = null
|
||||
}
|
||||
return { hadSpeech: had }
|
||||
}
|
||||
|
||||
async function stopAndCheck(): Promise<void> {
|
||||
const { hadSpeech } = await stop()
|
||||
if (!hadSpeech) {
|
||||
onNoSpeech()
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
speaking: readonly(speaking),
|
||||
amplitude: readonly(amplitude),
|
||||
loaded: readonly(loaded),
|
||||
start,
|
||||
stop,
|
||||
stopAndCheck,
|
||||
}
|
||||
}
|
||||
@@ -56,8 +56,9 @@ const todayConvId = ref<number | null>(null)
|
||||
|
||||
const isToday = computed(() => selectedConvId.value === todayConvId.value)
|
||||
|
||||
// Weather panel (left column)
|
||||
// Weather panel
|
||||
const weatherData = ref<WeatherData[]>([])
|
||||
const selectedWeatherIdx = ref(0)
|
||||
const tempUnit = ref<string>('C')
|
||||
|
||||
interface CurrentConditions {
|
||||
@@ -261,22 +262,7 @@ onMounted(async () => {
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- Left column: Weather -->
|
||||
<div class="briefing-left">
|
||||
<div class="panel-label">Weather</div>
|
||||
<!-- Forecast card -->
|
||||
<template v-if="weatherData.length">
|
||||
<WeatherCard
|
||||
v-for="loc in weatherData"
|
||||
:key="(loc as WeatherData).location"
|
||||
:weather="loc"
|
||||
:temp-unit="tempUnit"
|
||||
/>
|
||||
</template>
|
||||
<div v-else class="panel-empty">No weather configured</div>
|
||||
</div>
|
||||
|
||||
<!-- Center column: Chat -->
|
||||
<!-- Left column: Chat -->
|
||||
<div class="briefing-center">
|
||||
<ChatPanel
|
||||
variant="full"
|
||||
@@ -287,8 +273,27 @@ onMounted(async () => {
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Right column: News -->
|
||||
<!-- Right column: Weather + News -->
|
||||
<div class="briefing-right">
|
||||
<!-- Weather section (sticky) -->
|
||||
<div class="weather-section" v-if="weatherData.length">
|
||||
<div class="weather-tabs" v-if="weatherData.length > 1">
|
||||
<button
|
||||
v-for="(loc, i) in weatherData"
|
||||
:key="(loc as WeatherData).location"
|
||||
class="weather-tab"
|
||||
:class="{ active: selectedWeatherIdx === i }"
|
||||
@click="selectedWeatherIdx = i"
|
||||
>{{ (loc as WeatherData).location }}</button>
|
||||
</div>
|
||||
<WeatherCard
|
||||
:weather="weatherData[selectedWeatherIdx]"
|
||||
:temp-unit="tempUnit"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- News section (scrollable) -->
|
||||
<div class="news-section">
|
||||
<div class="panel-label-row">
|
||||
<div class="panel-label">Today's News</div>
|
||||
<span v-if="newsItems.length" class="news-count">{{ newsItems.length }} items</span>
|
||||
@@ -333,6 +338,7 @@ onMounted(async () => {
|
||||
>💬</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -348,7 +354,7 @@ onMounted(async () => {
|
||||
|
||||
.briefing-shell {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 2fr 1fr;
|
||||
grid-template-columns: 1fr minmax(320px, 35%);
|
||||
grid-template-rows: auto 1fr;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
@@ -421,27 +427,10 @@ onMounted(async () => {
|
||||
}
|
||||
.btn-trigger:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
|
||||
/* ─── Left column (Weather) ──────────────────────────────────────────────── */
|
||||
|
||||
.briefing-left {
|
||||
grid-column: 1;
|
||||
grid-row: 2;
|
||||
border-right: 1px solid var(--color-border);
|
||||
overflow-y: auto;
|
||||
padding: 1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.briefing-left :deep(.weather-card) {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
/* ─── Center column (Chat) ───────────────────────────────────────────────── */
|
||||
/* ─── Left column (Chat) ─────────────────────────────────────────────────── */
|
||||
|
||||
.briefing-center {
|
||||
grid-column: 2;
|
||||
grid-column: 1;
|
||||
grid-row: 2;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -453,14 +442,60 @@ onMounted(async () => {
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
/* ─── Right column (News) ────────────────────────────────────────────────── */
|
||||
/* ─── Right column (Weather + News) ──────────────────────────────────────── */
|
||||
|
||||
.briefing-right {
|
||||
grid-column: 3;
|
||||
grid-column: 2;
|
||||
grid-row: 2;
|
||||
border-left: 1px solid var(--color-border);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.weather-section {
|
||||
flex-shrink: 0;
|
||||
padding: 1rem 1rem 0.5rem;
|
||||
}
|
||||
|
||||
.weather-section :deep(.weather-card) {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.weather-tabs {
|
||||
display: flex;
|
||||
gap: 0.25rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.weather-tab {
|
||||
padding: 0.3rem 0.7rem;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
background: none;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.78rem;
|
||||
font-family: inherit;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
|
||||
.weather-tab:hover {
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
}
|
||||
|
||||
.weather-tab.active {
|
||||
background: color-mix(in srgb, var(--color-primary) 12%, transparent);
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.news-section {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 1rem;
|
||||
padding: 0.75rem 1rem 1rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
@@ -581,29 +616,18 @@ a.news-title:hover { text-decoration: underline; color: var(--color-primary); }
|
||||
@media (max-width: 900px) {
|
||||
.briefing-shell {
|
||||
grid-template-columns: 1fr;
|
||||
grid-template-rows: auto auto 1fr auto;
|
||||
}
|
||||
.briefing-header {
|
||||
grid-column: 1;
|
||||
grid-row: 1;
|
||||
}
|
||||
.briefing-left {
|
||||
grid-column: 1;
|
||||
grid-row: 2;
|
||||
border-right: none;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
max-height: 220px;
|
||||
grid-template-rows: auto 1fr auto;
|
||||
}
|
||||
.briefing-center {
|
||||
grid-column: 1;
|
||||
grid-row: 3;
|
||||
grid-row: 2;
|
||||
}
|
||||
.briefing-right {
|
||||
grid-column: 1;
|
||||
grid-row: 4;
|
||||
grid-row: 3;
|
||||
border-left: none;
|
||||
border-top: 1px solid var(--color-border);
|
||||
max-height: 260px;
|
||||
max-height: 300px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
+31
-1
@@ -1,9 +1,39 @@
|
||||
import { defineConfig } from "vite";
|
||||
import vue from "@vitejs/plugin-vue";
|
||||
import { viteStaticCopy } from "vite-plugin-static-copy";
|
||||
import { resolve } from "path";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [vue()],
|
||||
plugins: [
|
||||
vue(),
|
||||
// @ricky0123/vad-web ships ONNX + audio-worklet assets and depends on
|
||||
// onnxruntime-web's WASM binaries. Copy them into the served root so
|
||||
// MicVAD can load them at runtime via `baseAssetPath: "/"`.
|
||||
viteStaticCopy({
|
||||
targets: [
|
||||
{
|
||||
src: "node_modules/@ricky0123/vad-web/dist/*.onnx",
|
||||
dest: "",
|
||||
rename: { stripBase: true },
|
||||
},
|
||||
{
|
||||
src: "node_modules/@ricky0123/vad-web/dist/vad.worklet.bundle.min.js",
|
||||
dest: "",
|
||||
rename: { stripBase: true },
|
||||
},
|
||||
{
|
||||
src: "node_modules/onnxruntime-web/dist/*.wasm",
|
||||
dest: "",
|
||||
rename: { stripBase: true },
|
||||
},
|
||||
{
|
||||
src: "node_modules/onnxruntime-web/dist/*.mjs",
|
||||
dest: "",
|
||||
rename: { stripBase: true },
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
resolve: {
|
||||
alias: {
|
||||
"@": resolve(__dirname, "src"),
|
||||
|
||||
@@ -143,10 +143,11 @@ def create_app() -> Quart:
|
||||
response.headers.setdefault("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
response.headers.setdefault(
|
||||
"Content-Security-Policy",
|
||||
"default-src 'self'; script-src 'self' 'unsafe-inline'; "
|
||||
"default-src 'self'; "
|
||||
"script-src 'self' 'unsafe-inline' 'wasm-unsafe-eval'; "
|
||||
"style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; "
|
||||
"connect-src 'self'; font-src 'self' data:; object-src 'none'; "
|
||||
"base-uri 'self'; worker-src 'self';"
|
||||
"base-uri 'self'; worker-src 'self' blob:;"
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
@@ -587,18 +587,26 @@ async def build_context(
|
||||
"CRITICAL: Call the tool functions directly. NEVER write out function calls as text or code. NEVER describe what you would do — just do it.",
|
||||
"GROUNDING: When the user asks about their own data — tasks, notes, events, projects, news, anything stored in this system — call the relevant tool to see what actually exists before answering. Never assert facts about the user's data from memory, prior context, or assumption. If you are unsure whether something exists, check with a tool.",
|
||||
"HONESTY WHEN EMPTY: If a tool returns empty results (no matching tasks, no events in the date range, no search hits, no notes found), tell the user plainly that nothing matched. Do not fabricate example items, do not invent plausible-sounding meetings or deadlines to fill the response, and do not hedge with generic suggestions dressed up as real data. A direct 'you don't have anything on your calendar today' is always better than an invented event.",
|
||||
"Available actions: create_task, create_note, update_note, delete_note, delete_task, get_note, list_notes, list_tasks, search_notes.",
|
||||
]
|
||||
actions = [
|
||||
"create_note (also creates tasks — set status='todo')", "update_note", "delete_note",
|
||||
"read_note", "list_notes", "list_tasks", "log_work", "search_notes",
|
||||
"create_project", "list_projects", "get_project", "update_project",
|
||||
"search_projects", "create_milestone", "update_milestone", "list_milestones",
|
||||
"save_person", "save_place", "create_list", "add_to_list", "clear_checked_items",
|
||||
"set_rag_scope", "get_profile", "update_profile", "get_weather", "calculate",
|
||||
"get_rss_items", "add_rss_feed", "read_article",
|
||||
]
|
||||
if has_caldav:
|
||||
tool_lines[-1] = (
|
||||
"Available actions: create_task, create_note, update_note, delete_note, delete_task, get_note, list_notes, list_tasks, search_notes, "
|
||||
"create_event, list_events, search_events, update_event, delete_event, list_calendars."
|
||||
)
|
||||
actions.extend(["create_event", "list_events", "search_events", "update_event", "delete_event", "list_calendars"])
|
||||
tool_lines.append(
|
||||
"For calendar events, use ISO 8601 datetime format with the user's timezone offset (stated in context below). "
|
||||
"Always include the UTC offset in datetime strings (e.g. 2026-09-30T14:00:00+01:00)."
|
||||
)
|
||||
tool_lines.append("When the user says 'remind me' with a time before an event, use the reminder_minutes parameter.")
|
||||
if Config.searxng_enabled():
|
||||
actions.extend(["search_web", "research_topic", "search_images"])
|
||||
tool_lines.append(f"Available actions: {', '.join(actions)}.")
|
||||
tool_lines.append(
|
||||
"For relative dates like 'Friday' or 'next week', resolve them to YYYY-MM-DD format. "
|
||||
"Always include the UTC offset when creating events (user's timezone is stated in context below)."
|
||||
@@ -677,7 +685,7 @@ async def build_context(
|
||||
f"\n\n--- Active Workspace ---\n"
|
||||
f"You are in the \"{wp.title}\" project workspace.\n"
|
||||
f"All notes and tasks you create or update MUST belong to this project.\n"
|
||||
f"Always pass project=\"{wp.title}\" when calling create_note or create_task.\n"
|
||||
f"Always pass project=\"{wp.title}\" when calling create_note.\n"
|
||||
f"--- End Active Workspace ---"
|
||||
)
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user