From 2a0e37a73d2bdb8239c58a1b96e146ba4771c60d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 26 Jan 2026 11:38:51 -0500 Subject: [PATCH] improve ux, moved functionality for better ui experience on app and extension. --- backend/app/api/settings.py | 73 +++++++++-------- backend/app/models/setting.py | 3 + backend/app/tasks/celery_app.py | 6 +- backend/app/tasks/maintenance.py | 127 +++++++++++++++++++++++++++++ extension/background/background.js | 38 ++++++++- extension/popup/popup.css | 6 ++ extension/popup/popup.js | 103 +++++++++++++++++++---- frontend/src/services/api.js | 1 + frontend/src/views/Credentials.vue | 5 +- frontend/src/views/Dashboard.vue | 71 +++++++++++++--- frontend/src/views/Downloads.vue | 9 +- frontend/src/views/Settings.vue | 5 +- 12 files changed, 369 insertions(+), 78 deletions(-) create mode 100644 backend/app/tasks/maintenance.py diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 0ac0575..ce0014b 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -1,54 +1,37 @@ """Settings API endpoints.""" import json -import os from pathlib import Path from quart import Blueprint, request, jsonify, current_app from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.models.setting import Setting, DEFAULT_SETTINGS, EXTENSION_API_KEY_SETTING, generate_api_key +from app.models.setting import Setting, DEFAULT_SETTINGS, EXTENSION_API_KEY_SETTING, STORAGE_STATS_SETTING, generate_api_key from app.config import get_settings bp = Blueprint("settings", __name__) -def get_storage_stats(): - """Calculate storage statistics for the downloads directory.""" - config = get_settings() - downloads_path = Path(config.download_path) +async def get_cached_storage_stats(session: AsyncSession) -> dict: + """Get cached storage statistics from the database. - if not downloads_path.exists(): - return {"total_size": 0, "total_size_formatted": "0 B", "file_count": 0} + Returns cached stats if available, otherwise returns placeholder. + Stats are calculated by a background Celery task. + """ + result = await session.execute( + select(Setting).where(Setting.key == STORAGE_STATS_SETTING) + ) + setting = result.scalar_one_or_none() - total_size = 0 - file_count = 0 - - try: - for root, dirs, files in os.walk(downloads_path): - for f in files: - fp = os.path.join(root, f) - try: - total_size += os.path.getsize(fp) - file_count += 1 - except OSError: - pass - except Exception as e: - current_app.logger.error(f"Error calculating storage stats: {e}") - return {"total_size": 0, "total_size_formatted": "Error", "file_count": 0} - - # Format size - def format_size(size): - for unit in ['B', 'KB', 'MB', 'GB', 'TB']: - if size < 1024: - return f"{size:.1f} {unit}" - size /= 1024 - return f"{size:.1f} PB" + if setting and setting.value: + return setting.value + # No cached stats yet - return placeholder return { - "total_size": total_size, - "total_size_formatted": format_size(total_size), - "file_count": file_count, + "total_size": 0, + "total_size_formatted": "Calculating...", + "file_count": 0, + "calculated_at": None, } @@ -112,8 +95,8 @@ async def get_all_settings(): api_key = await get_or_create_extension_api_key(session) settings_dict[EXTENSION_API_KEY_SETTING] = api_key - # Include storage statistics - storage_stats = get_storage_stats() + # Get cached storage statistics (calculated by background task) + storage_stats = await get_cached_storage_stats(session) return jsonify({"settings": settings_dict, "storage_stats": storage_stats}) @@ -231,6 +214,24 @@ async def update_gallery_dl_config(): return jsonify({"error": f"Failed to write config: {e}"}), 500 +@bp.route("/storage-stats/refresh", methods=["POST"]) +async def refresh_storage_stats(): + """Trigger a refresh of storage statistics. + + This queues a background task to recalculate storage stats. + """ + from app.tasks.maintenance import update_storage_stats + + try: + # Queue the task + update_storage_stats.delay() + current_app.logger.info("Queued storage stats refresh") + return jsonify({"message": "Storage stats refresh queued"}) + except Exception as e: + current_app.logger.error(f"Failed to queue storage stats refresh: {e}") + return jsonify({"error": str(e)}), 500 + + @bp.route("/logs", methods=["GET"]) async def get_logs(): """Get recent download logs from download metadata. diff --git a/backend/app/models/setting.py b/backend/app/models/setting.py index 5430094..70ab73e 100644 --- a/backend/app/models/setting.py +++ b/backend/app/models/setting.py @@ -47,3 +47,6 @@ DEFAULT_SETTINGS = { # Key for the extension API key setting EXTENSION_API_KEY_SETTING = "security.extension_api_key" + +# Key for cached storage stats +STORAGE_STATS_SETTING = "cache.storage_stats" diff --git a/backend/app/tasks/celery_app.py b/backend/app/tasks/celery_app.py index 69abda9..8aa27d1 100644 --- a/backend/app/tasks/celery_app.py +++ b/backend/app/tasks/celery_app.py @@ -11,7 +11,7 @@ celery_app = Celery( "gallery_subscriber", broker=settings.redis_url, backend=settings.redis_url, - include=["app.tasks.downloads"], + include=["app.tasks.downloads", "app.tasks.maintenance"], ) # Celery configuration @@ -40,4 +40,8 @@ celery_app.conf.beat_schedule = { "task": "app.tasks.downloads.cleanup_old_downloads", "schedule": crontab(hour=3, minute=0), # Daily at 3 AM }, + "update-storage-stats": { + "task": "tasks.update_storage_stats", + "schedule": crontab(minute="*/30"), # Every 30 minutes + }, } diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py new file mode 100644 index 0000000..5934f1a --- /dev/null +++ b/backend/app/tasks/maintenance.py @@ -0,0 +1,127 @@ +"""System maintenance Celery tasks.""" + +import asyncio +import logging +import os +from datetime import datetime +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker + +from app.config import get_settings +from app.tasks.celery_app import celery_app +from app.models.setting import Setting, STORAGE_STATS_SETTING + +logger = logging.getLogger(__name__) + +settings = get_settings() + + +def get_async_session(): + """Get async session factory for Celery tasks.""" + engine = create_async_engine(settings.async_database_url, echo=False) + session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + return session_factory, engine + + +async def _cleanup_engine(engine): + """Properly dispose of the async engine.""" + await engine.dispose() + + +def format_size(size_bytes: int) -> str: + """Format bytes as human-readable size.""" + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size_bytes < 1024: + return f"{size_bytes:.1f} {unit}" + size_bytes /= 1024 + return f"{size_bytes:.1f} PB" + + +def calculate_storage_stats() -> dict: + """Calculate storage statistics for the downloads directory. + + This runs synchronously since it's a filesystem operation. + """ + downloads_path = Path(settings.download_path) + + if not downloads_path.exists(): + return { + "total_size": 0, + "total_size_formatted": "0 B", + "file_count": 0, + "calculated_at": datetime.utcnow().isoformat(), + } + + total_size = 0 + file_count = 0 + + try: + for root, dirs, files in os.walk(downloads_path): + for f in files: + fp = os.path.join(root, f) + try: + total_size += os.path.getsize(fp) + file_count += 1 + except OSError: + pass + except Exception as e: + logger.error(f"Error calculating storage stats: {e}") + return { + "total_size": 0, + "total_size_formatted": "Error", + "file_count": 0, + "error": str(e), + "calculated_at": datetime.utcnow().isoformat(), + } + + return { + "total_size": total_size, + "total_size_formatted": format_size(total_size), + "file_count": file_count, + "calculated_at": datetime.utcnow().isoformat(), + } + + +async def _update_storage_stats_async() -> dict: + """Async implementation to calculate and store storage stats.""" + session_factory, engine = get_async_session() + + try: + # Calculate stats (synchronous filesystem operation) + stats = calculate_storage_stats() + logger.info(f"Calculated storage stats: {stats['total_size_formatted']}, {stats['file_count']} files") + + # Store in database + async with session_factory() as session: + # Check if setting exists + result = await session.execute( + select(Setting).where(Setting.key == STORAGE_STATS_SETTING) + ) + setting = result.scalar_one_or_none() + + if setting: + setting.value = stats + else: + setting = Setting(key=STORAGE_STATS_SETTING, value=stats) + session.add(setting) + + await session.commit() + + return stats + + finally: + await _cleanup_engine(engine) + + +@celery_app.task(name="tasks.update_storage_stats") +def update_storage_stats() -> dict: + """Celery task to calculate and cache storage stats. + + This task should be scheduled to run periodically (e.g., every hour). + """ + logger.info("Starting storage stats calculation...") + result = asyncio.run(_update_storage_stats_async()) + logger.info(f"Storage stats updated: {result}") + return result diff --git a/extension/background/background.js b/extension/background/background.js index 2292487..1759af7 100644 --- a/extension/background/background.js +++ b/extension/background/background.js @@ -6,18 +6,41 @@ let discordToken = null; let discordTokenCapturedAt = null; +// Track initialization state +let initialized = false; + +/** + * Ensure extension is initialized + */ +async function ensureInitialized() { + if (!initialized) { + console.log('Initializing extension...'); + try { + await api.init(); + await loadDiscordToken(); + initialized = true; + console.log('Extension initialized successfully'); + } catch (error) { + console.error('Failed to initialize extension:', error); + throw error; + } + } +} + // Initialize API on startup browser.runtime.onInstalled.addListener(async () => { console.log('GallerySubscriber extension installed'); - await api.init(); - await loadDiscordToken(); + await ensureInitialized(); }); browser.runtime.onStartup.addListener(async () => { - await api.init(); - await loadDiscordToken(); + console.log('GallerySubscriber extension startup'); + await ensureInitialized(); }); +// Also initialize immediately in case background wakes up from idle +ensureInitialized().catch(err => console.error('Initial init failed:', err)); + /** * Load Discord token from storage */ @@ -88,6 +111,13 @@ browser.runtime.onMessage.addListener((message, sender, sendResponse) => { * @returns {Promise} - Response data */ async function handleMessage(message) { + console.log('Handling message:', message.type); + + // Ensure initialized for operations that need it + if (['EXPORT_COOKIES', 'EXPORT_ALL_COOKIES', 'GET_PLATFORM_STATUS', 'TEST_CONNECTION'].includes(message.type)) { + await ensureInitialized(); + } + switch (message.type) { case 'EXPORT_COOKIES': return await exportCookies(message.platform); diff --git a/extension/popup/popup.css b/extension/popup/popup.css index 16b4e68..5c27bbd 100644 --- a/extension/popup/popup.css +++ b/extension/popup/popup.css @@ -302,6 +302,12 @@ body { color: var(--alert-warning-text); } +.alert-error { + background: var(--status-error-bg); + border: 1px solid var(--status-error-border); + color: var(--status-error-text); +} + .status-message { padding: 10px 12px; border-radius: 6px; diff --git a/extension/popup/popup.js b/extension/popup/popup.js index 926caa8..e7b0114 100644 --- a/extension/popup/popup.js +++ b/extension/popup/popup.js @@ -3,38 +3,107 @@ */ document.addEventListener('DOMContentLoaded', async () => { - await init(); + try { + await init(); + } catch (error) { + console.error('Popup init error:', error); + // Show error in UI + document.getElementById('setup-required').classList.remove('hidden'); + const alertDiv = document.querySelector('#setup-required .alert'); + if (alertDiv) { + alertDiv.innerHTML = `Error

${error.message}

`; + alertDiv.className = 'alert alert-error'; + } + } }); +// Rate limit connection tests (max once per 2 minutes) +const CONNECTION_TEST_INTERVAL = 2 * 60 * 1000; // 2 minutes + /** * Initialize the popup */ async function init() { - // Check configuration - const config = await browser.runtime.sendMessage({ type: 'GET_CONFIG' }); + console.log('Popup initializing...'); - if (!config.apiUrl || !config.apiKey) { + // Check configuration (this is fast - just reads from storage) + let config; + try { + config = await browser.runtime.sendMessage({ type: 'GET_CONFIG' }); + console.log('Got config:', config); + } catch (error) { + console.error('Failed to get config:', error); + throw new Error(`Failed to communicate with background script: ${error.message}`); + } + + if (!config || !config.apiUrl || !config.apiKey) { + console.log('Config missing, showing setup'); showSetupRequired(); return; } - // Test connection - const connectionStatus = await browser.runtime.sendMessage({ type: 'TEST_CONNECTION' }); - updateConnectionIndicator(connectionStatus.connected); - - // Show main content + // Show main content IMMEDIATELY - don't wait for connection test or platform status document.getElementById('setup-required').classList.add('hidden'); document.getElementById('main-content').classList.remove('hidden'); - if (!connectionStatus.connected) { - showError(`Cannot connect to backend: ${connectionStatus.error}`); - } - - // Load platform status - await loadPlatformStatus(); - - // Setup event listeners + // Setup event listeners right away setupEventListeners(); + + // Show loading state for platforms + const platformsContainer = document.getElementById('platforms-list'); + platformsContainer.innerHTML = '
Loading platforms...
'; + + // Now do background tasks in parallel (non-blocking) + // Check if we should test connection (rate limited) + testConnectionIfNeeded(); + + // Load platform status (this can take time) + loadPlatformStatus().catch(error => { + console.error('Failed to load platform status:', error); + showError(`Failed to load platforms: ${error.message}`); + }); +} + +/** + * Test connection only if enough time has passed since last test + */ +async function testConnectionIfNeeded() { + try { + // Check last test time from storage + const stored = await browser.storage.local.get(['lastConnectionTest', 'lastConnectionStatus']); + const now = Date.now(); + const lastTest = stored.lastConnectionTest || 0; + + // Use cached status if recent + if (now - lastTest < CONNECTION_TEST_INTERVAL && stored.lastConnectionStatus !== undefined) { + console.log('Using cached connection status'); + updateConnectionIndicator(stored.lastConnectionStatus); + if (!stored.lastConnectionStatus) { + showError('Cannot connect to backend (cached status)'); + } + return; + } + + // Test connection in background + console.log('Testing connection...'); + const connectionStatus = await browser.runtime.sendMessage({ type: 'TEST_CONNECTION' }); + console.log('Connection status:', connectionStatus); + + // Cache the result + await browser.storage.local.set({ + lastConnectionTest: now, + lastConnectionStatus: connectionStatus.connected + }); + + updateConnectionIndicator(connectionStatus.connected); + + if (!connectionStatus.connected) { + showError(`Cannot connect to backend: ${connectionStatus.error}`); + } + } catch (error) { + console.error('Failed to test connection:', error); + updateConnectionIndicator(false); + } } /** diff --git a/frontend/src/services/api.js b/frontend/src/services/api.js index 8b078da..d7750fe 100644 --- a/frontend/src/services/api.js +++ b/frontend/src/services/api.js @@ -56,6 +56,7 @@ export const settingsApi = { getApiKey: () => api.get('/settings/api-key'), regenerateApiKey: () => api.post('/settings/api-key/regenerate'), getLogs: (params = {}) => api.get('/settings/logs', { params }), + refreshStorageStats: () => api.post('/settings/storage-stats/refresh'), } // Platforms API diff --git a/frontend/src/views/Credentials.vue b/frontend/src/views/Credentials.vue index c967231..dee71f9 100644 --- a/frontend/src/views/Credentials.vue +++ b/frontend/src/views/Credentials.vue @@ -208,8 +208,9 @@ const deleteDialog = ref(false) const deletePlatform = ref('') const verifyingPlatform = ref('') -onMounted(async () => { - await loadData() +onMounted(() => { + // Don't block UI - load data in background + loadData() }) async function loadData() { diff --git a/frontend/src/views/Dashboard.vue b/frontend/src/views/Dashboard.vue index 914ca78..108a671 100644 --- a/frontend/src/views/Dashboard.vue +++ b/frontend/src/views/Dashboard.vue @@ -82,12 +82,29 @@ Retry All Failed ({{ failedCount }}) -
+
mdi-harddisk Storage: {{ storageStats?.total_size_formatted || 'Calculating...' }} ({{ storageStats.file_count.toLocaleString() }} files) + + + Updated: {{ formatDate(storageStats.calculated_at) }} + + + mdi-refresh +
@@ -270,6 +287,10 @@ const checkingAll = ref(false) const retryingAll = ref(false) const storageStats = ref(null) const runningDownloads = ref([]) +const loadingStats = ref(true) +const loadingActivity = ref(true) +const loadingStorage = ref(true) +const refreshingStorage = ref(false) let refreshInterval = null // Computed @@ -280,8 +301,9 @@ const sourcesWithErrors = computed(() => ) const failedCount = computed(() => stats.value?.failed || 0) -onMounted(async () => { - await loadDashboardData() +onMounted(() => { + // Load data in background - DON'T await, let UI render immediately + loadDashboardData() // Start auto-refresh for running downloads startAutoRefresh() }) @@ -290,14 +312,23 @@ onUnmounted(() => { stopAutoRefresh() }) -async function loadDashboardData() { - await Promise.all([ - sourcesStore.fetchSources(), - downloadsStore.fetchStats(), - downloadsStore.fetchRecentActivity({ limit: 10 }), - fetchRunningDownloads(), - fetchStorageStats(), - ]) +function loadDashboardData() { + // Fire off all requests in parallel, don't block UI + // Each section handles its own loading state + sourcesStore.fetchSources().catch(e => console.error('Failed to fetch sources:', e)) + + downloadsStore.fetchStats() + .catch(e => console.error('Failed to fetch stats:', e)) + .finally(() => loadingStats.value = false) + + downloadsStore.fetchRecentActivity({ limit: 10 }) + .catch(e => console.error('Failed to fetch recent activity:', e)) + .finally(() => loadingActivity.value = false) + + fetchRunningDownloads() + + // Storage stats can be slow - load independently + fetchStorageStats() } async function fetchRunningDownloads() { @@ -310,11 +341,29 @@ async function fetchRunningDownloads() { } async function fetchStorageStats() { + loadingStorage.value = true try { const response = await settingsApi.get() storageStats.value = response.data.storage_stats } catch (e) { console.error('Failed to fetch storage stats:', e) + } finally { + loadingStorage.value = false + } +} + +async function refreshStorageStats() { + refreshingStorage.value = true + try { + await settingsApi.refreshStorageStats() + notifications.success('Storage stats refresh queued') + // Wait a moment then fetch updated stats + setTimeout(() => fetchStorageStats(), 3000) + } catch (e) { + console.error('Failed to refresh storage stats:', e) + notifications.error('Failed to refresh storage stats') + } finally { + refreshingStorage.value = false } } diff --git a/frontend/src/views/Downloads.vue b/frontend/src/views/Downloads.vue index 51e9b60..4992471 100644 --- a/frontend/src/views/Downloads.vue +++ b/frontend/src/views/Downloads.vue @@ -241,11 +241,10 @@ const headers = [ { title: 'Actions', key: 'actions', width: 100, sortable: false }, ] -onMounted(async () => { - await Promise.all([ - loadDownloads(), - sourcesStore.fetchSources(), - ]) +onMounted(() => { + // Load data in background - don't block UI rendering + loadDownloads() + sourcesStore.fetchSources() }) watch([filterStatus, filterSourceId, filterFromDate, filterToDate], () => { diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue index 571850d..32f4e51 100644 --- a/frontend/src/views/Settings.vue +++ b/frontend/src/views/Settings.vue @@ -330,8 +330,9 @@ const apiHealthy = ref(true) const showApiKey = ref(false) const regenerateDialog = ref(false) -onMounted(async () => { - await loadAll() +onMounted(() => { + // Don't block UI - load data in background + loadAll() }) async function loadAll() {