feat(voice): admin UI to browse + install piper voices from HuggingFace
Building on the kokoro→piper swap (B1), this adds the admin-side voice management story so additional voices can be installed without rebuilding the image. The bundled two voices stay as immediate defaults; everything else is opt-in via a one-click install from the catalog. Backend (services/voice_library.py): - fetch_catalog() pulls voices.json from the piper-voices HF repo with a 24h in-memory TTL. Manual refresh available via ?refresh=1 on the library endpoint. - shape_catalog_for_ui() projects the raw HF dict (~250 voices, lots of nesting) into UI-friendly cards: id, name, language, country, quality, size, install state. Sorted by language_code then name for stable display. Install state distinguishes bundled (read-only) from user (admin-installed, can be removed). - install_voice() downloads .onnx + .onnx.json into /data/voices with atomic .tmp → rename so a failed partial download can't leave a corrupt model around. Idempotent — re-installing an already-present voice is a no-op. - uninstall_voice() removes /data voices; bundled /opt voices raise PermissionError (403 at the route layer). - Strict voice-id regex prevents path traversal in install/uninstall. Routes (admin-only, since these write to shared /data and affect all users on the instance): - GET /api/voice/voices/library - POST /api/voice/voices/install - DELETE /api/voice/voices/<voice_id> Frontend: - New "Voice Library" section in Settings → Voice, visible only to admin users. Collapsed by default; expand to load the catalog on-demand (doesn't hammer HF for non-admins). - Free-text filter across id, language code, language name, country, and dataset name. Refresh button forces a catalog re-fetch. - Per-voice row shows id, language/country/quality/speaker count, size, and either an Install button, a Remove button (user voices), or a "bundled" badge (read-only voices in /opt/piper-voices). - Installs and uninstalls refresh both the library list AND the active voice picker so the new voice is immediately selectable. - VoiceLibraryEntry exported from api/client.ts; new client helpers getVoiceLibrary/installVoice/uninstallVoice. Tests: - Pure-transformation unit tests for shape_catalog_for_ui, _resolve_file_urls, and the voice-id regex (path-traversal coverage). - DB/network paths (fetch_catalog, install_voice) need a real environment — left to CI integration tests or device verification. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -628,11 +628,38 @@ export interface VoiceEntry {
|
||||
sample_rate?: number
|
||||
}
|
||||
|
||||
export interface VoiceLibraryEntry {
|
||||
id: string
|
||||
name: string
|
||||
language_code: string
|
||||
language_name: string
|
||||
country: string
|
||||
quality: string
|
||||
num_speakers: number
|
||||
size_bytes: number
|
||||
installed: boolean
|
||||
installed_source: 'user' | 'bundled' | null
|
||||
}
|
||||
|
||||
export const getVoiceStatus = () => apiGet<VoiceStatusResult>('/api/voice/status')
|
||||
|
||||
export const getVoiceList = () =>
|
||||
apiGet<{ voices: VoiceEntry[] }>('/api/voice/voices').then(r => r.voices)
|
||||
|
||||
export const getVoiceLibrary = (refresh = false) =>
|
||||
apiGet<{ voices: VoiceLibraryEntry[]; count: number }>(
|
||||
refresh ? '/api/voice/voices/library?refresh=1' : '/api/voice/voices/library'
|
||||
)
|
||||
|
||||
export const installVoice = (voiceId: string) =>
|
||||
apiPost<{ id: string; size_bytes: number; skipped: boolean }>(
|
||||
'/api/voice/voices/install',
|
||||
{ voice_id: voiceId }
|
||||
)
|
||||
|
||||
export const uninstallVoice = (voiceId: string) =>
|
||||
apiDelete(`/api/voice/voices/${encodeURIComponent(voiceId)}`)
|
||||
|
||||
export async function transcribeAudio(blob: Blob, context?: string): Promise<{ transcript: string; duration_ms: number }> {
|
||||
const form = new FormData()
|
||||
form.append('audio', blob, 'audio.webm')
|
||||
|
||||
@@ -3,7 +3,7 @@ import { ref, computed, watch, onMounted } from "vue";
|
||||
import { useSettingsStore } from "@/stores/settings";
|
||||
import { useAuthStore } from "@/stores/auth";
|
||||
import { useToastStore } from "@/stores/toast";
|
||||
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, synthesiseSpeech, getProfile, updateProfile, consolidateProfile, clearProfileObservations, listProfileObservations, getJournalConfig, saveJournalConfig, geocodeAddress, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type VoiceStatusResult, type VoiceEntry, type UserProfile, type JournalConfig, type ProfileObservationEntry } from "@/api/client";
|
||||
import { apiGet, apiPost, apiPut, apiDelete, listGroups, createGroup, deleteGroup, listGroupMembers, addGroupMember, removeGroupMember, searchUsers, getFableMcpInfo, listApiKeys, createApiKey as apiCreateApiKey, revokeApiKey as apiRevokeApiKey, getVoiceStatus, getVoiceList, getVoiceLibrary, installVoice, uninstallVoice, synthesiseSpeech, getProfile, updateProfile, consolidateProfile, clearProfileObservations, listProfileObservations, getJournalConfig, saveJournalConfig, geocodeAddress, type ApiKeyEntry, type GroupEntry, type GroupMember, type UserSearchResult, type VoiceStatusResult, type VoiceEntry, type VoiceLibraryEntry, type UserProfile, type JournalConfig, type ProfileObservationEntry } from "@/api/client";
|
||||
import { usePushStore } from "@/stores/push";
|
||||
import type { User } from "@/types/auth";
|
||||
import PaginationBar from "@/components/PaginationBar.vue";
|
||||
@@ -469,6 +469,87 @@ const savingVoice = ref(false);
|
||||
const voiceSaved = ref(false);
|
||||
const voiceTabLoaded = ref(false);
|
||||
|
||||
// Voice library (admin-only): browse + install piper voices from HF
|
||||
const voiceLibrary = ref<VoiceLibraryEntry[]>([]);
|
||||
const voiceLibraryLoading = ref(false);
|
||||
const voiceLibraryError = ref<string | null>(null);
|
||||
const voiceLibraryFilter = ref(""); // free-text language/name filter
|
||||
const voiceLibraryExpanded = ref(false);
|
||||
const installingVoiceIds = ref<Set<string>>(new Set());
|
||||
const uninstallingVoiceIds = ref<Set<string>>(new Set());
|
||||
|
||||
const filteredVoiceLibrary = computed(() => {
|
||||
const q = voiceLibraryFilter.value.trim().toLowerCase();
|
||||
if (!q) return voiceLibrary.value;
|
||||
return voiceLibrary.value.filter(v =>
|
||||
v.id.toLowerCase().includes(q)
|
||||
|| v.language_code.toLowerCase().includes(q)
|
||||
|| v.language_name.toLowerCase().includes(q)
|
||||
|| v.country.toLowerCase().includes(q)
|
||||
|| v.name.toLowerCase().includes(q)
|
||||
);
|
||||
});
|
||||
|
||||
function formatVoiceSize(bytes: number): string {
|
||||
if (!bytes) return "—";
|
||||
const mb = bytes / (1024 * 1024);
|
||||
return mb >= 10 ? `${Math.round(mb)} MB` : `${mb.toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
async function loadVoiceLibrary(refresh = false) {
|
||||
if (!authStore.isAdmin) return;
|
||||
voiceLibraryLoading.value = true;
|
||||
voiceLibraryError.value = null;
|
||||
try {
|
||||
const res = await getVoiceLibrary(refresh);
|
||||
voiceLibrary.value = res.voices;
|
||||
} catch (e) {
|
||||
voiceLibraryError.value = e instanceof Error ? e.message : "Failed to load voice library";
|
||||
} finally {
|
||||
voiceLibraryLoading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshInstalledVoices() {
|
||||
// Reload the active voice list (used after install/uninstall) so the
|
||||
// user's voice picker sees the change without a full page reload.
|
||||
try {
|
||||
availableVoices.value = await getVoiceList();
|
||||
} catch {
|
||||
/* voice feature may be disabled — ignore */
|
||||
}
|
||||
}
|
||||
|
||||
async function installLibraryVoice(voiceId: string) {
|
||||
if (installingVoiceIds.value.has(voiceId)) return;
|
||||
installingVoiceIds.value.add(voiceId);
|
||||
try {
|
||||
await installVoice(voiceId);
|
||||
toastStore.show(`Installed ${voiceId}`, "success");
|
||||
await Promise.all([loadVoiceLibrary(false), refreshInstalledVoices()]);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : "Install failed";
|
||||
toastStore.show(`Failed to install ${voiceId}: ${msg}`, "error");
|
||||
} finally {
|
||||
installingVoiceIds.value.delete(voiceId);
|
||||
}
|
||||
}
|
||||
|
||||
async function uninstallLibraryVoice(voiceId: string) {
|
||||
if (uninstallingVoiceIds.value.has(voiceId)) return;
|
||||
uninstallingVoiceIds.value.add(voiceId);
|
||||
try {
|
||||
await uninstallVoice(voiceId);
|
||||
toastStore.show(`Removed ${voiceId}`, "success");
|
||||
await Promise.all([loadVoiceLibrary(false), refreshInstalledVoices()]);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : "Uninstall failed";
|
||||
toastStore.show(`Failed to remove ${voiceId}: ${msg}`, "error");
|
||||
} finally {
|
||||
uninstallingVoiceIds.value.delete(voiceId);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadVoiceTab() {
|
||||
if (voiceTabLoaded.value) return;
|
||||
voiceTabLoaded.value = true;
|
||||
@@ -2362,6 +2443,96 @@ function formatUserDate(iso: string): string {
|
||||
<!-- Voice Blend section removed with the kokoro → piper migration
|
||||
(2026-05-22). Piper has no voice-blending equivalent. -->
|
||||
|
||||
<!-- Voice Library (admin only) — browse + install piper voices.
|
||||
Voices land in /data/voices and are immediately available to
|
||||
every user's voice picker. Bundled voices in /opt/piper-voices
|
||||
are read-only and show a "bundled" badge instead of a delete. -->
|
||||
<section v-if="authStore.isAdmin && voiceStatus?.enabled" class="settings-section full-width">
|
||||
<div class="voice-library-header">
|
||||
<h2>Voice Library</h2>
|
||||
<button
|
||||
class="btn-secondary"
|
||||
type="button"
|
||||
@click="voiceLibraryExpanded = !voiceLibraryExpanded; if (voiceLibraryExpanded && voiceLibrary.length === 0) loadVoiceLibrary()"
|
||||
>
|
||||
{{ voiceLibraryExpanded ? 'Hide' : 'Browse voices' }}
|
||||
</button>
|
||||
</div>
|
||||
<p class="section-desc">
|
||||
Download additional piper voices from the
|
||||
<a href="https://huggingface.co/rhasspy/piper-voices" target="_blank" rel="noopener">piper-voices catalog</a>.
|
||||
Installed voices appear in every user's voice picker after a page reload.
|
||||
</p>
|
||||
|
||||
<template v-if="voiceLibraryExpanded">
|
||||
<div class="voice-library-toolbar">
|
||||
<input
|
||||
v-model="voiceLibraryFilter"
|
||||
type="search"
|
||||
class="input"
|
||||
placeholder="Filter by language, country, or name (e.g. 'en', 'fr_FR', 'amy')…"
|
||||
/>
|
||||
<button
|
||||
class="btn-secondary"
|
||||
type="button"
|
||||
:disabled="voiceLibraryLoading"
|
||||
@click="loadVoiceLibrary(true)"
|
||||
title="Re-fetch catalog from HuggingFace"
|
||||
>
|
||||
{{ voiceLibraryLoading ? 'Loading…' : 'Refresh' }}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<p v-if="voiceLibraryError" class="field-hint-warn">
|
||||
{{ voiceLibraryError }}
|
||||
</p>
|
||||
|
||||
<div v-if="voiceLibraryLoading && voiceLibrary.length === 0" class="voice-library-empty">
|
||||
Loading catalog…
|
||||
</div>
|
||||
|
||||
<ul v-else-if="filteredVoiceLibrary.length > 0" class="voice-library-list">
|
||||
<li v-for="v in filteredVoiceLibrary" :key="v.id" class="voice-library-row">
|
||||
<div class="voice-library-meta">
|
||||
<div class="voice-library-id">{{ v.id }}</div>
|
||||
<div class="voice-library-sub">
|
||||
{{ v.language_name || v.language_code }}
|
||||
<span v-if="v.country"> · {{ v.country }}</span>
|
||||
· {{ v.quality }}
|
||||
<span v-if="v.num_speakers > 1"> · {{ v.num_speakers }} speakers</span>
|
||||
· {{ formatVoiceSize(v.size_bytes) }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="voice-library-actions">
|
||||
<span v-if="v.installed && v.installed_source === 'bundled'" class="voice-badge voice-badge--bundled">
|
||||
bundled
|
||||
</span>
|
||||
<button
|
||||
v-else-if="v.installed && v.installed_source === 'user'"
|
||||
class="btn-danger btn-sm"
|
||||
:disabled="uninstallingVoiceIds.has(v.id)"
|
||||
@click="uninstallLibraryVoice(v.id)"
|
||||
>
|
||||
{{ uninstallingVoiceIds.has(v.id) ? 'Removing…' : 'Remove' }}
|
||||
</button>
|
||||
<button
|
||||
v-else
|
||||
class="btn-secondary btn-sm"
|
||||
:disabled="installingVoiceIds.has(v.id)"
|
||||
@click="installLibraryVoice(v.id)"
|
||||
>
|
||||
{{ installingVoiceIds.has(v.id) ? 'Installing…' : 'Install' }}
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<div v-else-if="!voiceLibraryLoading" class="voice-library-empty">
|
||||
No voices match the filter.
|
||||
</div>
|
||||
</template>
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<!-- ── API Keys ── -->
|
||||
@@ -4185,6 +4356,78 @@ FABLE_API_KEY={{ effectiveApiKey }}</pre>
|
||||
flex-wrap: wrap;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
|
||||
/* Voice Library (admin) */
|
||||
.voice-library-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
}
|
||||
.voice-library-toolbar {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
align-items: center;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
.voice-library-toolbar .input {
|
||||
flex: 1;
|
||||
}
|
||||
.voice-library-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
border-top: 1px solid var(--color-border);
|
||||
max-height: 480px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.voice-library-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
padding: 0.65rem 0.25rem;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
.voice-library-meta {
|
||||
min-width: 0;
|
||||
flex: 1;
|
||||
}
|
||||
.voice-library-id {
|
||||
font-family: var(--font-mono, monospace);
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
.voice-library-sub {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 0.8rem;
|
||||
margin-top: 0.15rem;
|
||||
}
|
||||
.voice-library-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.voice-library-empty {
|
||||
padding: 1rem 0;
|
||||
color: var(--color-text-muted);
|
||||
text-align: center;
|
||||
font-style: italic;
|
||||
}
|
||||
.voice-badge {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 500;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border-radius: 999px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
.voice-badge--bundled {
|
||||
background: color-mix(in srgb, var(--color-text-muted) 10%, transparent);
|
||||
color: var(--color-text-muted);
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
.status-badge {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
|
||||
@@ -4,7 +4,7 @@ import time
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from fabledassistant.auth import login_required
|
||||
from fabledassistant.auth import admin_required, login_required
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -181,3 +181,78 @@ async def synthesise_speech():
|
||||
|
||||
from quart import Response
|
||||
return Response(wav_bytes, mimetype="audio/wav")
|
||||
|
||||
|
||||
# ── Voice library (admin only) ──────────────────────────────────────────────
|
||||
# Browse the piper-voices catalog, download new voices into /data/voices,
|
||||
# remove user-installed voices. Bundled voices in /opt/piper-voices are
|
||||
# read-only and cannot be touched via these endpoints.
|
||||
# These are admin-only because installs consume shared disk and affect
|
||||
# every user on the instance (voices are picked per-user, but the files
|
||||
# themselves are shared).
|
||||
|
||||
|
||||
@voice_bp.route("/voices/library", methods=["GET"])
|
||||
@admin_required
|
||||
async def voice_library():
|
||||
"""Return the piper-voices catalog with install state annotations.
|
||||
|
||||
Query params:
|
||||
?refresh=1 — force a fresh fetch from HuggingFace (bypass the 24h
|
||||
in-memory cache). Use sparingly; HF doesn't appreciate hammering.
|
||||
"""
|
||||
from fabledassistant.services import voice_library as lib
|
||||
|
||||
force = (request.args.get("refresh") or "").lower() in ("1", "true", "yes")
|
||||
try:
|
||||
catalog = await lib.fetch_catalog(force_refresh=force)
|
||||
except Exception:
|
||||
logger.exception("Voice catalog fetch failed")
|
||||
return jsonify({"error": "Failed to fetch voice catalog"}), 502
|
||||
voices = lib.shape_catalog_for_ui(catalog)
|
||||
return jsonify({"voices": voices, "count": len(voices)})
|
||||
|
||||
|
||||
@voice_bp.route("/voices/install", methods=["POST"])
|
||||
@admin_required
|
||||
async def install_voice_route():
|
||||
"""Download a voice into /data/voices.
|
||||
|
||||
Body: {"voice_id": "en_US-amy-medium"}
|
||||
Idempotent — already-installed voices return {"skipped": true} without
|
||||
re-downloading.
|
||||
"""
|
||||
from fabledassistant.services import voice_library as lib
|
||||
|
||||
data = await request.get_json()
|
||||
voice_id = str((data or {}).get("voice_id") or "").strip()
|
||||
if not voice_id:
|
||||
return jsonify({"error": "voice_id is required"}), 400
|
||||
try:
|
||||
result = await lib.install_voice(voice_id)
|
||||
except ValueError as e:
|
||||
return jsonify({"error": str(e)}), 400
|
||||
except KeyError as e:
|
||||
return jsonify({"error": str(e)}), 404
|
||||
except Exception:
|
||||
logger.exception("Voice install failed: %s", voice_id)
|
||||
return jsonify({"error": "Voice install failed"}), 500
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@voice_bp.route("/voices/<voice_id>", methods=["DELETE"])
|
||||
@admin_required
|
||||
async def uninstall_voice_route(voice_id: str):
|
||||
"""Remove a /data/voices voice. Bundled voices return 403."""
|
||||
from fabledassistant.services import voice_library as lib
|
||||
|
||||
try:
|
||||
result = await lib.uninstall_voice(voice_id)
|
||||
except ValueError as e:
|
||||
return jsonify({"error": str(e)}), 400
|
||||
except PermissionError as e:
|
||||
return jsonify({"error": str(e)}), 403
|
||||
except Exception:
|
||||
logger.exception("Voice uninstall failed: %s", voice_id)
|
||||
return jsonify({"error": "Voice uninstall failed"}), 500
|
||||
return jsonify(result)
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
"""Piper voice library — browse + install voices from the HuggingFace catalog.
|
||||
|
||||
Piper voices live in https://huggingface.co/rhasspy/piper-voices, with a
|
||||
machine-readable catalog at:
|
||||
|
||||
https://huggingface.co/rhasspy/piper-voices/raw/main/voices.json
|
||||
|
||||
That catalog enumerates every available voice (~250+ across many languages)
|
||||
with file sizes and download paths. We cache it in memory with a TTL so
|
||||
the Settings UI's "browse voices" panel doesn't hit HuggingFace every time
|
||||
the page opens, but a manual "refresh" button can force a re-fetch.
|
||||
|
||||
Downloads land in /data/voices so they persist across container restarts.
|
||||
Bundled voices in /opt/piper-voices are read-only and cannot be deleted
|
||||
via the admin UI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CATALOG_URL = (
|
||||
"https://huggingface.co/rhasspy/piper-voices/raw/main/voices.json"
|
||||
)
|
||||
VOICE_BASE_URL = (
|
||||
"https://huggingface.co/rhasspy/piper-voices/resolve/main"
|
||||
)
|
||||
_USER_VOICES_DIR = Path("/data/voices")
|
||||
_BUNDLED_VOICES_DIR = Path("/opt/piper-voices")
|
||||
|
||||
# Voice IDs are like `en_US-amy-medium`, `fr_FR-siwis-low` — strict
|
||||
# allowlist to prevent path traversal in download/delete handlers.
|
||||
_VOICE_ID_RE = re.compile(r"^[a-zA-Z]{2,3}(?:_[a-zA-Z]{2,4})?-[a-zA-Z0-9_+]+-(?:x_low|low|medium|high)$")
|
||||
|
||||
# In-memory cache. Catalog rarely changes (kokoro-style "model update"
|
||||
# checks would be overkill — admin can force-refresh from the UI).
|
||||
_CATALOG_CACHE: dict | None = None
|
||||
_CATALOG_FETCHED_AT: float = 0.0
|
||||
_CATALOG_TTL_SECS = 24 * 3600 # 24h — long, but the manual refresh button overrides
|
||||
_CATALOG_LOCK = asyncio.Lock()
|
||||
|
||||
|
||||
def _is_valid_voice_id(voice_id: str) -> bool:
|
||||
return bool(_VOICE_ID_RE.match(voice_id))
|
||||
|
||||
|
||||
async def fetch_catalog(force_refresh: bool = False) -> dict:
|
||||
"""Return the piper-voices catalog dict, fetching from HF if stale.
|
||||
|
||||
Cached in memory across requests. force_refresh=True bypasses the TTL.
|
||||
Raises on network error so the UI can surface the failure.
|
||||
"""
|
||||
global _CATALOG_CACHE, _CATALOG_FETCHED_AT
|
||||
async with _CATALOG_LOCK:
|
||||
now = time.monotonic()
|
||||
fresh_enough = (
|
||||
_CATALOG_CACHE is not None
|
||||
and (now - _CATALOG_FETCHED_AT) < _CATALOG_TTL_SECS
|
||||
)
|
||||
if fresh_enough and not force_refresh:
|
||||
return _CATALOG_CACHE # type: ignore[return-value]
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _do_fetch() -> dict:
|
||||
with urllib.request.urlopen(CATALOG_URL, timeout=30) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
return json.loads(raw)
|
||||
|
||||
try:
|
||||
catalog = await loop.run_in_executor(None, _do_fetch)
|
||||
except Exception:
|
||||
logger.exception("Failed to fetch piper voices catalog from %s", CATALOG_URL)
|
||||
raise
|
||||
|
||||
_CATALOG_CACHE = catalog
|
||||
_CATALOG_FETCHED_AT = now
|
||||
logger.info("Piper voices catalog refreshed: %d entries", len(catalog))
|
||||
return catalog
|
||||
|
||||
|
||||
def _is_installed(voice_id: str) -> tuple[bool, str | None]:
|
||||
"""Return (installed, source) where source is 'bundled' / 'user' / None."""
|
||||
if (_BUNDLED_VOICES_DIR / f"{voice_id}.onnx").is_file():
|
||||
return True, "bundled"
|
||||
if (_USER_VOICES_DIR / f"{voice_id}.onnx").is_file():
|
||||
return True, "user"
|
||||
return False, None
|
||||
|
||||
|
||||
def shape_catalog_for_ui(catalog: dict) -> list[dict]:
|
||||
"""Project the raw HF catalog into a list of UI-friendly voice cards.
|
||||
|
||||
The HF catalog uses voice IDs as keys with nested file-size metadata.
|
||||
The UI just needs id, language, dataset, quality, size, install state.
|
||||
Sorted by language then dataset for stable display.
|
||||
"""
|
||||
out: list[dict] = []
|
||||
for voice_id, info in catalog.items():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
installed, source = _is_installed(voice_id)
|
||||
lang_info = info.get("language") or {}
|
||||
files = info.get("files") or {}
|
||||
# Sum the file sizes for the .onnx + .onnx.json pair so the UI
|
||||
# can show "≈ 62 MB" before the user clicks Download.
|
||||
total_bytes = 0
|
||||
for f_info in files.values():
|
||||
if isinstance(f_info, dict):
|
||||
total_bytes += int(f_info.get("size_bytes", 0) or 0)
|
||||
out.append({
|
||||
"id": voice_id,
|
||||
"name": info.get("name") or voice_id.split("-")[1],
|
||||
"language_code": lang_info.get("code") or voice_id.split("-")[0],
|
||||
"language_name": lang_info.get("name_native")
|
||||
or lang_info.get("name_english")
|
||||
or lang_info.get("code", ""),
|
||||
"country": lang_info.get("country_english") or lang_info.get("region", ""),
|
||||
"quality": info.get("quality") or voice_id.split("-")[-1],
|
||||
"num_speakers": int(info.get("num_speakers") or 1),
|
||||
"size_bytes": total_bytes,
|
||||
"installed": installed,
|
||||
"installed_source": source,
|
||||
})
|
||||
out.sort(key=lambda v: (v["language_code"], v["name"], v["quality"]))
|
||||
return out
|
||||
|
||||
|
||||
def _resolve_file_urls(catalog: dict, voice_id: str) -> dict[str, str]:
|
||||
"""Return absolute URLs for the .onnx + .onnx.json files of `voice_id`.
|
||||
|
||||
Walks the catalog entry's `files` dict; that's the authoritative path
|
||||
list (the layout in the repo can drift, e.g. some voices live under
|
||||
different sub-paths). Returns {extension: url}.
|
||||
"""
|
||||
info = catalog.get(voice_id)
|
||||
if not isinstance(info, dict):
|
||||
raise KeyError(f"Voice {voice_id!r} not in catalog")
|
||||
files = info.get("files") or {}
|
||||
urls: dict[str, str] = {}
|
||||
for rel_path in files.keys():
|
||||
if rel_path.endswith(".onnx"):
|
||||
urls["onnx"] = f"{VOICE_BASE_URL}/{rel_path}"
|
||||
elif rel_path.endswith(".onnx.json"):
|
||||
urls["onnx.json"] = f"{VOICE_BASE_URL}/{rel_path}"
|
||||
if "onnx" not in urls or "onnx.json" not in urls:
|
||||
raise ValueError(
|
||||
f"Catalog entry for {voice_id!r} is missing .onnx or .onnx.json file"
|
||||
)
|
||||
return urls
|
||||
|
||||
|
||||
async def install_voice(voice_id: str) -> dict:
|
||||
"""Download a voice's .onnx + .onnx.json into /data/voices.
|
||||
|
||||
Idempotent — if already installed under /data, returns success without
|
||||
re-downloading. Bundled voices in /opt are not touched; the same voice
|
||||
can also live under /data if the admin wants to override it.
|
||||
Returns {"id", "size_bytes", "skipped"}.
|
||||
"""
|
||||
if not _is_valid_voice_id(voice_id):
|
||||
raise ValueError(f"Invalid voice id: {voice_id!r}")
|
||||
|
||||
_USER_VOICES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
onnx_path = _USER_VOICES_DIR / f"{voice_id}.onnx"
|
||||
sidecar_path = _USER_VOICES_DIR / f"{voice_id}.onnx.json"
|
||||
|
||||
if onnx_path.is_file() and sidecar_path.is_file():
|
||||
return {
|
||||
"id": voice_id,
|
||||
"size_bytes": onnx_path.stat().st_size + sidecar_path.stat().st_size,
|
||||
"skipped": True,
|
||||
}
|
||||
|
||||
catalog = await fetch_catalog()
|
||||
urls = _resolve_file_urls(catalog, voice_id)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _download() -> int:
|
||||
total = 0
|
||||
# Use atomic write — download to .tmp then rename. Means a failed
|
||||
# partial download doesn't leave a corrupt model in place that
|
||||
# would later cause PiperVoice.load() to throw.
|
||||
for ext, url in urls.items():
|
||||
target = _USER_VOICES_DIR / f"{voice_id}.{ext}"
|
||||
tmp = target.with_suffix(target.suffix + ".tmp")
|
||||
logger.info("Downloading piper voice file %s → %s", url, target)
|
||||
with urllib.request.urlopen(url, timeout=120) as resp:
|
||||
with open(tmp, "wb") as fh:
|
||||
shutil.copyfileobj(resp, fh, length=1024 * 1024)
|
||||
tmp.replace(target)
|
||||
total += target.stat().st_size
|
||||
return total
|
||||
|
||||
try:
|
||||
size = await loop.run_in_executor(None, _download)
|
||||
except Exception:
|
||||
# Clean up partials.
|
||||
for ext in ("onnx", "onnx.json"):
|
||||
for p in (
|
||||
_USER_VOICES_DIR / f"{voice_id}.{ext}",
|
||||
_USER_VOICES_DIR / f"{voice_id}.{ext}.tmp",
|
||||
):
|
||||
try:
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
logger.exception("Voice install failed for %s — cleaned up partials", voice_id)
|
||||
raise
|
||||
|
||||
logger.info("Installed piper voice %s (%d bytes)", voice_id, size)
|
||||
return {"id": voice_id, "size_bytes": size, "skipped": False}
|
||||
|
||||
|
||||
async def uninstall_voice(voice_id: str) -> dict:
|
||||
"""Remove a /data voice. Bundled voices in /opt cannot be removed via API.
|
||||
|
||||
Returns {"id", "removed": bool}.
|
||||
"""
|
||||
if not _is_valid_voice_id(voice_id):
|
||||
raise ValueError(f"Invalid voice id: {voice_id!r}")
|
||||
|
||||
# Refuse to delete a bundled voice. If a /data override exists alongside
|
||||
# a bundled one with the same id, only the /data copy is removed.
|
||||
if not (_USER_VOICES_DIR / f"{voice_id}.onnx").is_file() and not (
|
||||
_USER_VOICES_DIR / f"{voice_id}.onnx.json"
|
||||
).is_file():
|
||||
if (_BUNDLED_VOICES_DIR / f"{voice_id}.onnx").is_file():
|
||||
raise PermissionError(
|
||||
f"Voice {voice_id!r} is bundled with the image — cannot be removed"
|
||||
)
|
||||
return {"id": voice_id, "removed": False}
|
||||
|
||||
removed_any = False
|
||||
for ext in ("onnx", "onnx.json"):
|
||||
p = _USER_VOICES_DIR / f"{voice_id}.{ext}"
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
removed_any = True
|
||||
logger.info("Uninstalled piper voice %s (files removed: %s)", voice_id, removed_any)
|
||||
return {"id": voice_id, "removed": removed_any}
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Unit tests for the voice library catalog shaper + ID validation.
|
||||
|
||||
DB- and network-touching paths (fetch_catalog, install_voice) need a
|
||||
real environment, so we test only the pure transformations here:
|
||||
|
||||
- Voice ID regex accepts valid piper IDs and rejects path-traversal attempts.
|
||||
- shape_catalog_for_ui projects an HF-shaped catalog dict into the UI list.
|
||||
- _resolve_file_urls picks the right .onnx and .onnx.json paths.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from fabledassistant.services import voice_library as lib
|
||||
|
||||
|
||||
def test_voice_id_accepts_real_piper_ids():
|
||||
for vid in [
|
||||
"en_US-amy-medium",
|
||||
"en_US-ryan-medium",
|
||||
"en_GB-alan-low",
|
||||
"fr_FR-siwis-medium",
|
||||
"de_DE-thorsten-high",
|
||||
"es_AR-daniela-high",
|
||||
"zh_CN-huayan-medium",
|
||||
"ar_JO-kareem-low",
|
||||
"ca_ES-upc_ona-x_low",
|
||||
]:
|
||||
assert lib._is_valid_voice_id(vid), f"should accept {vid!r}"
|
||||
|
||||
|
||||
def test_voice_id_rejects_path_traversal_and_garbage():
|
||||
for vid in [
|
||||
"../../etc/passwd",
|
||||
"en_US-amy-medium/../something",
|
||||
"../en_US-amy-medium",
|
||||
"",
|
||||
"en_US-amy", # missing quality
|
||||
"amy-medium", # missing language
|
||||
"en_US--medium", # empty dataset
|
||||
"EN_US-amy-medium", # uppercase language prefix not in catalog convention
|
||||
"en_US-amy-large", # invalid quality
|
||||
".onnx",
|
||||
"en_US-amy-medium.onnx", # has extension; we want the bare id
|
||||
]:
|
||||
assert not lib._is_valid_voice_id(vid), f"should reject {vid!r}"
|
||||
|
||||
|
||||
def test_shape_catalog_projects_expected_fields(monkeypatch, tmp_path):
|
||||
"""Catalog dict → list of UI cards, with install state from disk scan."""
|
||||
# Redirect the install-scan paths to a tmp dir; mark en_US-amy-medium
|
||||
# as installed under /data so we can verify the field flips.
|
||||
user_dir = tmp_path / "data_voices"
|
||||
bundled_dir = tmp_path / "opt_voices"
|
||||
user_dir.mkdir()
|
||||
bundled_dir.mkdir()
|
||||
(user_dir / "en_US-amy-medium.onnx").write_text("model")
|
||||
(user_dir / "en_US-amy-medium.onnx.json").write_text("{}")
|
||||
|
||||
monkeypatch.setattr(lib, "_USER_VOICES_DIR", user_dir)
|
||||
monkeypatch.setattr(lib, "_BUNDLED_VOICES_DIR", bundled_dir)
|
||||
|
||||
catalog = {
|
||||
"en_US-amy-medium": {
|
||||
"name": "amy",
|
||||
"language": {
|
||||
"code": "en_US",
|
||||
"name_native": "English",
|
||||
"country_english": "United States",
|
||||
},
|
||||
"quality": "medium",
|
||||
"num_speakers": 1,
|
||||
"files": {
|
||||
"en/en_US/amy/medium/en_US-amy-medium.onnx": {
|
||||
"size_bytes": 63_500_000,
|
||||
},
|
||||
"en/en_US/amy/medium/en_US-amy-medium.onnx.json": {
|
||||
"size_bytes": 4_300,
|
||||
},
|
||||
},
|
||||
},
|
||||
"fr_FR-siwis-medium": {
|
||||
"name": "siwis",
|
||||
"language": {
|
||||
"code": "fr_FR",
|
||||
"name_native": "Français",
|
||||
"country_english": "France",
|
||||
},
|
||||
"quality": "medium",
|
||||
"num_speakers": 1,
|
||||
"files": {
|
||||
"fr/fr_FR/siwis/medium/fr_FR-siwis-medium.onnx": {
|
||||
"size_bytes": 60_000_000,
|
||||
},
|
||||
"fr/fr_FR/siwis/medium/fr_FR-siwis-medium.onnx.json": {
|
||||
"size_bytes": 3_900,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
out = lib.shape_catalog_for_ui(catalog)
|
||||
assert len(out) == 2
|
||||
by_id = {v["id"]: v for v in out}
|
||||
|
||||
amy = by_id["en_US-amy-medium"]
|
||||
assert amy["name"] == "amy"
|
||||
assert amy["language_code"] == "en_US"
|
||||
assert amy["quality"] == "medium"
|
||||
assert amy["size_bytes"] == 63_500_000 + 4_300
|
||||
assert amy["installed"] is True
|
||||
assert amy["installed_source"] == "user"
|
||||
|
||||
siwis = by_id["fr_FR-siwis-medium"]
|
||||
assert siwis["language_code"] == "fr_FR"
|
||||
assert siwis["installed"] is False
|
||||
assert siwis["installed_source"] is None
|
||||
|
||||
# Sorted by language_code, then name — en_US before fr_FR.
|
||||
assert out[0]["id"] == "en_US-amy-medium"
|
||||
assert out[1]["id"] == "fr_FR-siwis-medium"
|
||||
|
||||
|
||||
def test_resolve_file_urls_picks_onnx_and_sidecar():
|
||||
catalog = {
|
||||
"en_US-amy-medium": {
|
||||
"files": {
|
||||
"en/en_US/amy/medium/en_US-amy-medium.onnx": {"size_bytes": 1},
|
||||
"en/en_US/amy/medium/en_US-amy-medium.onnx.json": {"size_bytes": 2},
|
||||
},
|
||||
}
|
||||
}
|
||||
urls = lib._resolve_file_urls(catalog, "en_US-amy-medium")
|
||||
assert urls["onnx"].endswith("en_US-amy-medium.onnx")
|
||||
assert urls["onnx.json"].endswith("en_US-amy-medium.onnx.json")
|
||||
assert urls["onnx"].startswith("https://huggingface.co/rhasspy/piper-voices/resolve/main/")
|
||||
|
||||
|
||||
def test_resolve_file_urls_raises_when_voice_missing():
|
||||
with pytest.raises(KeyError):
|
||||
lib._resolve_file_urls({}, "en_US-amy-medium")
|
||||
|
||||
|
||||
def test_resolve_file_urls_raises_when_files_incomplete():
|
||||
catalog = {
|
||||
"en_US-amy-medium": {
|
||||
"files": {
|
||||
# Only the .onnx, missing the sidecar — that's malformed.
|
||||
"en/en_US/amy/medium/en_US-amy-medium.onnx": {"size_bytes": 1},
|
||||
}
|
||||
}
|
||||
}
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
lib._resolve_file_urls(catalog, "en_US-amy-medium")
|
||||
Reference in New Issue
Block a user