tuning job log views and deep scan archive processing.

This commit is contained in:
Bryan Van Deusen
2026-02-02 18:48:01 -05:00
parent 042a69f9c3
commit 09883960d4
9 changed files with 1078 additions and 103 deletions
+457 -40
View File
@@ -4,7 +4,44 @@
{% block content %}
<h1 style="text-align:center; margin-top: 1rem;">Settings</h1>
<!-- Import Queue Status (Full Width at Top) -->
<!-- System Stats (Full Width at Top) -->
<div class="settings-grid-full">
<div class="settings-container">
<div class="settings-section">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem;">
<h2 style="margin: 0;">System Overview</h2>
<div style="display: flex; align-items: center; gap: 0.75rem;">
<span id="statsUpdatedAt" style="font-size: 0.75rem; color: var(--text-muted);"></span>
<button id="refreshStatsBtn" class="btn secondary-btn" style="padding: 0.3rem 0.6rem; font-size: 0.8rem;" title="Refresh stats">Refresh</button>
</div>
</div>
<div class="stats-grid" style="grid-template-columns: repeat(5, 1fr);">
<div class="stat-item">
<span class="stat-value" id="statTotalImages">-</span>
<span class="stat-label">Total Images</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statTotalTags">-</span>
<span class="stat-label">Total Tags</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statStorage">-</span>
<span class="stat-label">Storage Used</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statPendingFiles">-</span>
<span class="stat-label">Files to Import</span>
</div>
<div class="stat-item">
<span class="stat-value" id="statPendingArchives">-</span>
<span class="stat-label">Archives to Import</span>
</div>
</div>
</div>
</div>
</div>
<!-- Import Queue Status (Full Width) -->
<div class="settings-grid-full">
<div class="settings-container">
<div class="settings-section">
@@ -25,20 +62,28 @@
<span class="stat-value" id="queueProcessing">-</span>
<span class="stat-label">Processing</span>
</div>
<div class="stat-item">
<div class="stat-item clickable-stat" id="completeStatItem" title="Click for breakdown">
<span class="stat-value" id="queueComplete">-</span>
<span class="stat-label">Complete</span>
</div>
<div class="stat-item">
<div class="stat-item clickable-stat" id="skippedStatItem" title="Click for breakdown">
<span class="stat-value" id="queueSkipped">-</span>
<span class="stat-label">Skipped</span>
</div>
<div class="stat-item">
<div class="stat-item clickable-stat" id="failedStatItem" title="Click for breakdown">
<span class="stat-value" id="queueFailed">-</span>
<span class="stat-label">Failed</span>
</div>
</div>
<!-- Age breakdown tooltip -->
<div id="ageBreakdown" style="display: none; margin-top: 0.5rem; padding: 0.5rem; background: rgba(255,255,255,0.05); border-radius: 4px; font-size: 0.8rem;">
<span id="ageBreakdownTitle" style="color: var(--text-muted);"></span>:
<span id="ageBreakdownToday">-</span> today,
<span id="ageBreakdownWeek">-</span> this week,
<span id="ageBreakdownOlder">-</span> older
</div>
<div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
<h3>Active Batch</h3>
<p>Processing <strong id="batchDir">-</strong></p>
@@ -51,9 +96,36 @@
</div>
<div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
<button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
<button id="triggerImportBtn" class="btn primary-btn">Quick Scan</button>
<button id="triggerDeepScanBtn" class="btn warning-btn" title="Full reprocessing: re-extracts metadata, runs pHash comparison, merges archive tags">Deep Scan</button>
<button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
<button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
</div>
<!-- Task Cleanup Controls -->
<div class="cleanup-controls" style="margin-top: 1rem; padding: 0.75rem; background: rgba(255,255,255,0.03); border-radius: 6px;">
<div style="display: flex; align-items: center; gap: 0.75rem; flex-wrap: wrap;">
<span style="color: var(--text-muted); font-size: 0.85rem;">Clear tasks:</span>
<select id="clearStatusSelect" class="form-input" style="width: auto; min-width: 140px;">
<option value="complete,skipped">Complete + Skipped</option>
<option value="complete">Complete only</option>
<option value="skipped">Skipped only</option>
<option value="failed">Failed only</option>
<option value="complete,skipped,failed">All finished</option>
</select>
<select id="clearAgeSelect" class="form-input" style="width: auto; min-width: 120px;">
<option value="0">All</option>
<option value="1">Older than 1 day</option>
<option value="7" selected>Older than 7 days</option>
<option value="30">Older than 30 days</option>
</select>
<button id="clearTasksBtn" class="btn secondary-btn">Clear</button>
</div>
<p style="color: var(--text-muted); font-size: 0.75rem; margin-top: 0.5rem;">
Note: Failed/skipped tasks older than 7 days are auto-cleaned daily.
</p>
</div>
<div class="queue-actions" style="margin-top: 0.75rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
<button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
<button id="reapplyArtistTagsBtn" class="btn secondary-btn">Reapply Artist Tags from Paths</button>
<button id="cleanupOrphanedTagsBtn" class="btn secondary-btn">Cleanup Orphaned Tags</button>
@@ -67,25 +139,76 @@
</div>
</div>
<!-- Recent Tasks Table -->
<div id="recentTasksSection" style="margin-top: 1.5rem;">
<h3>Recent Tasks</h3>
<div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
<!-- Task List with Filters -->
<div id="taskListSection" style="margin-top: 1.5rem;">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.75rem; flex-wrap: wrap; gap: 0.5rem;">
<h3 style="margin: 0;">Import Tasks</h3>
<div style="display: flex; gap: 0.5rem; align-items: center; flex-wrap: wrap;">
<select id="taskStatusFilter" class="form-input" style="width: auto; min-width: 120px; padding: 0.4rem;">
<option value="">All statuses</option>
<option value="failed">Failed</option>
<option value="skipped">Skipped</option>
<option value="complete">Complete</option>
<option value="processing">Processing</option>
<option value="queued">Queued</option>
<option value="pending">Pending</option>
</select>
<input type="text" id="taskSearchInput" class="form-input" placeholder="Search..." style="width: 150px; padding: 0.4rem;">
<button id="taskSearchBtn" class="btn secondary-btn" style="padding: 0.4rem 0.75rem;">Search</button>
</div>
</div>
<div id="taskListInfo" style="font-size: 0.8rem; color: var(--text-muted); margin-bottom: 0.5rem;">
Showing <span id="taskListShowing">0</span> of <span id="taskListTotal">0</span> tasks
</div>
<div class="tasks-table-wrapper" style="max-height: 400px; overflow-y: auto;">
<table class="tasks-table">
<thead>
<tr>
<th>File</th>
<th>Type</th>
<th>Status</th>
<th>Error</th>
<th>Time</th>
<th style="width: 40%;">File</th>
<th style="width: 10%;">Type</th>
<th style="width: 10%;">Status</th>
<th style="width: 30%;">Error/Reason</th>
<th style="width: 10%;">Time</th>
</tr>
</thead>
<tbody id="recentTasksBody">
<tbody id="taskListBody">
<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
</tbody>
</table>
</div>
<div style="margin-top: 0.75rem; display: flex; gap: 0.5rem;">
<button id="loadMoreTasksBtn" class="btn secondary-btn" style="display: none;">Load More</button>
</div>
</div>
<!-- Error Detail Modal -->
<div id="errorDetailModal" class="modal" style="display: none;">
<div class="modal-content" style="max-width: 600px;">
<div class="modal-header">
<h3>Task Details</h3>
<button class="modal-close" onclick="document.getElementById('errorDetailModal').style.display='none'">&times;</button>
</div>
<div class="modal-body">
<div style="margin-bottom: 1rem;">
<strong>File:</strong>
<div id="errorDetailPath" style="word-break: break-all; font-family: monospace; font-size: 0.85rem; background: rgba(0,0,0,0.2); padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem;"></div>
</div>
<div style="margin-bottom: 1rem;">
<strong>Status:</strong> <span id="errorDetailStatus"></span>
</div>
<div style="margin-bottom: 1rem;">
<strong>Error/Reason:</strong>
<div id="errorDetailMessage" style="word-break: break-word; font-family: monospace; font-size: 0.85rem; background: rgba(255,100,100,0.1); padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem; white-space: pre-wrap;"></div>
</div>
<div style="display: flex; gap: 1rem; font-size: 0.85rem; color: var(--text-muted);">
<span><strong>Retries:</strong> <span id="errorDetailRetries"></span></span>
<span><strong>Created:</strong> <span id="errorDetailCreated"></span></span>
</div>
</div>
</div>
</div>
</div>
</div>
@@ -282,6 +405,18 @@
</div>
<style>
.clickable-stat {
cursor: pointer;
transition: background 0.2s;
border-radius: 4px;
}
.clickable-stat:hover {
background: rgba(255, 255, 255, 0.08);
}
.cleanup-controls select {
padding: 0.4rem 0.6rem;
font-size: 0.85rem;
}
.tasks-table {
width: 100%;
border-collapse: collapse;
@@ -560,6 +695,55 @@
background: var(--btn-danger);
color: #fff;
}
/* Error detail modal */
#errorDetailModal {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.7);
z-index: 1000;
display: flex;
align-items: center;
justify-content: center;
}
#errorDetailModal .modal-content {
background: var(--surface-elevated);
border-radius: 8px;
max-height: 80vh;
overflow-y: auto;
}
#errorDetailModal .modal-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem;
border-bottom: 1px solid rgba(255,255,255,0.1);
}
#errorDetailModal .modal-header h3 {
margin: 0;
}
#errorDetailModal .modal-close {
background: none;
border: none;
font-size: 1.5rem;
cursor: pointer;
color: var(--text-muted);
}
#errorDetailModal .modal-body {
padding: 1rem;
}
.tasks-table .error-cell {
max-width: 250px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
cursor: pointer;
}
.tasks-table .error-cell:hover {
text-decoration: underline;
}
</style>
<script>
@@ -595,27 +779,121 @@ document.addEventListener('DOMContentLoaded', () => {
batchEl.style.display = 'none';
}
// Update recent tasks table
const tbody = document.getElementById('recentTasksBody');
if (data.recent_tasks && data.recent_tasks.length > 0) {
tbody.innerHTML = data.recent_tasks.map(t => `
<tr>
<td title="${t.source_path || ''}">${t.source_path || '-'}</td>
<td>${t.task_type || '-'}</td>
<td class="status-${t.status}">${t.status || '-'}</td>
<td title="${t.error_message || ''}">${t.error_message ? t.error_message.substring(0, 30) + '...' : '-'}</td>
<td>${t.created_at ? new Date(t.created_at).toLocaleString() : '-'}</td>
</tr>
`).join('');
} else {
tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No recent tasks</td></tr>';
}
}
} catch (e) {
console.error('Failed to load queue status:', e);
}
}
// ========================================
// Task List with Filtering
// ========================================
let taskListOffset = 0;
const taskListLimit = 50;
let taskListTasks = [];
async function loadTaskList(append = false) {
const status = document.getElementById('taskStatusFilter').value;
const search = document.getElementById('taskSearchInput').value;
if (!append) {
taskListOffset = 0;
taskListTasks = [];
}
try {
const params = new URLSearchParams({
limit: taskListLimit,
offset: taskListOffset,
order: 'desc'
});
if (status) params.append('status', status);
if (search) params.append('search', search);
const r = await fetch('/api/import/tasks?' + params.toString());
const data = await r.json();
if (data.ok) {
taskListTasks = append ? [...taskListTasks, ...data.tasks] : data.tasks;
const tbody = document.getElementById('taskListBody');
if (taskListTasks.length > 0) {
tbody.innerHTML = taskListTasks.map(t => `
<tr>
<td title="${escapeHtml(t.source_path || '')}">${escapeHtml(t.filename || '-')}</td>
<td>${t.task_type === 'import_image' ? 'Image' : t.task_type === 'import_archive' ? 'Archive' : t.task_type || '-'}</td>
<td class="status-${t.status}">${t.status || '-'}</td>
<td class="error-cell" onclick="window.showErrorDetail(${t.id})" title="${t.error_message ? escapeHtml(t.error_message) : 'Click for details'}">${t.error_message ? escapeHtml(t.error_message.substring(0, 50)) + (t.error_message.length > 50 ? '...' : '') : '-'}</td>
<td>${t.completed_at ? new Date(t.completed_at).toLocaleString() : (t.created_at ? new Date(t.created_at).toLocaleString() : '-')}</td>
</tr>
`).join('');
} else {
tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No tasks found</td></tr>';
}
// Update info
document.getElementById('taskListShowing').textContent = taskListTasks.length;
document.getElementById('taskListTotal').textContent = data.total;
// Show/hide load more button
const loadMoreBtn = document.getElementById('loadMoreTasksBtn');
if (taskListTasks.length < data.total) {
loadMoreBtn.style.display = 'inline-block';
} else {
loadMoreBtn.style.display = 'none';
}
}
} catch (e) {
console.error('Failed to load task list:', e);
}
}
function escapeHtml(text) {
if (!text) return '';
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// Store tasks for detail view
let taskDetailCache = {};
// Make showErrorDetail globally accessible for onclick handlers
window.showErrorDetail = function(taskId) {
const task = taskListTasks.find(t => t.id === taskId);
if (!task) return;
document.getElementById('errorDetailPath').textContent = task.source_path || '-';
document.getElementById('errorDetailStatus').textContent = task.status || '-';
document.getElementById('errorDetailStatus').className = 'status-' + task.status;
document.getElementById('errorDetailMessage').textContent = task.error_message || 'No error message';
document.getElementById('errorDetailRetries').textContent = task.retry_count || 0;
document.getElementById('errorDetailCreated').textContent = task.created_at ? new Date(task.created_at).toLocaleString() : '-';
document.getElementById('errorDetailModal').style.display = 'flex';
};
// Close modal on backdrop click
document.getElementById('errorDetailModal').addEventListener('click', (e) => {
if (e.target.id === 'errorDetailModal') {
e.target.style.display = 'none';
}
});
// Task list event listeners
document.getElementById('taskStatusFilter').addEventListener('change', () => loadTaskList());
document.getElementById('taskSearchBtn').addEventListener('click', () => loadTaskList());
document.getElementById('taskSearchInput').addEventListener('keypress', (e) => {
if (e.key === 'Enter') loadTaskList();
});
document.getElementById('loadMoreTasksBtn').addEventListener('click', () => {
taskListOffset += taskListLimit;
loadTaskList(true);
});
// Initial load
loadTaskList();
async function loadCeleryStatus() {
try {
const r = await fetch('/api/celery/status');
@@ -634,13 +912,106 @@ document.addEventListener('DOMContentLoaded', () => {
}
}
// System stats
let taskAgeStats = {};
async function loadSystemStats(triggerRefresh = false) {
try {
const url = triggerRefresh ? '/api/system/stats?refresh=true' : '/api/system/stats';
const r = await fetch(url);
const data = await r.json();
if (data.ok) {
document.getElementById('statTotalImages').textContent = data.images.total.toLocaleString();
document.getElementById('statTotalTags').textContent = data.tags.total.toLocaleString();
document.getElementById('statStorage').textContent = data.images.storage_human;
document.getElementById('statPendingFiles').textContent = data.import_folder.pending_files.toLocaleString();
document.getElementById('statPendingArchives').textContent = data.import_folder.pending_archives.toLocaleString();
// Show last updated time
const updatedEl = document.getElementById('statsUpdatedAt');
if (data.updated_at) {
const updated = new Date(data.updated_at);
const now = new Date();
const diffMs = now - updated;
const diffMins = Math.floor(diffMs / 60000);
if (diffMins < 1) {
updatedEl.textContent = 'Updated just now';
} else if (diffMins < 60) {
updatedEl.textContent = `Updated ${diffMins}m ago`;
} else {
const diffHours = Math.floor(diffMins / 60);
updatedEl.textContent = `Updated ${diffHours}h ago`;
}
} else if (data.message) {
updatedEl.textContent = data.message;
} else {
updatedEl.textContent = '';
}
}
} catch (e) {
console.error('Failed to load system stats:', e);
}
}
// Refresh stats button handler
document.getElementById('refreshStatsBtn').addEventListener('click', async () => {
const btn = document.getElementById('refreshStatsBtn');
btn.disabled = true;
btn.textContent = 'Refreshing...';
// Trigger refresh and reload after a short delay
await loadSystemStats(true);
await new Promise(r => setTimeout(r, 2000)); // Wait for task to run
await loadSystemStats();
btn.disabled = false;
btn.textContent = 'Refresh';
});
async function loadTaskAgeStats() {
try {
const r = await fetch('/api/import/task-stats');
const data = await r.json();
if (data.ok) {
taskAgeStats = data.stats;
}
} catch (e) {
console.error('Failed to load task age stats:', e);
}
}
function showAgeBreakdown(status) {
const stats = taskAgeStats[status];
if (!stats) return;
const breakdown = document.getElementById('ageBreakdown');
const title = status.charAt(0).toUpperCase() + status.slice(1);
document.getElementById('ageBreakdownTitle').textContent = title;
document.getElementById('ageBreakdownToday').textContent = stats.today;
document.getElementById('ageBreakdownWeek').textContent = stats.this_week;
document.getElementById('ageBreakdownOlder').textContent = stats.older;
breakdown.style.display = 'block';
}
// Add click handlers for stat items
document.getElementById('completeStatItem').addEventListener('click', () => showAgeBreakdown('complete'));
document.getElementById('skippedStatItem').addEventListener('click', () => showAgeBreakdown('skipped'));
document.getElementById('failedStatItem').addEventListener('click', () => showAgeBreakdown('failed'));
// Load queue status on page load and refresh periodically
loadQueueStatus();
loadCeleryStatus();
loadSystemStats();
loadTaskAgeStats();
setInterval(loadQueueStatus, 5000); // Refresh every 5 seconds
setInterval(loadCeleryStatus, 10000); // Refresh every 10 seconds
setInterval(loadSystemStats, 30000); // Refresh every 30 seconds
setInterval(loadTaskAgeStats, 30000); // Refresh every 30 seconds
// Trigger Import Scan
// Quick Scan (default)
document.getElementById('triggerImportBtn').addEventListener('click', async () => {
const btn = document.getElementById('triggerImportBtn');
btn.disabled = true;
@@ -651,7 +1022,7 @@ document.addEventListener('DOMContentLoaded', () => {
const data = await r.json();
if (data.ok) {
alert('Import scan started! Task ID: ' + data.task_id);
alert('Quick scan started! Task ID: ' + data.task_id);
loadQueueStatus();
} else {
alert('Failed to start import: ' + (data.error || 'Unknown error'));
@@ -660,7 +1031,38 @@ document.addEventListener('DOMContentLoaded', () => {
alert('Failed to start import: ' + e.message);
} finally {
btn.disabled = false;
btn.textContent = 'Trigger Import Scan';
btn.textContent = 'Quick Scan';
}
});
// Deep Scan (full reprocessing)
document.getElementById('triggerDeepScanBtn').addEventListener('click', async () => {
if (!confirm('Deep scan will reprocess ALL files in the import folder.\n\nThis includes:\n- Full pHash similarity detection\n- Archive tag merging for duplicates\n- Thumbnail regeneration\n- Sidecar metadata re-application\n\nThis may take a long time. Continue?')) {
return;
}
const btn = document.getElementById('triggerDeepScanBtn');
btn.disabled = true;
btn.textContent = 'Starting...';
try {
const formData = new FormData();
formData.append('deep', 'true');
const r = await fetch('/api/import/trigger', { method: 'POST', body: formData });
const data = await r.json();
if (data.ok) {
alert('Deep scan started! Task ID: ' + data.task_id + '\n\nThis will reprocess all files and may take a while.');
loadQueueStatus();
} else {
alert('Failed to start deep scan: ' + (data.error || 'Unknown error'));
}
} catch (e) {
alert('Failed to start deep scan: ' + e.message);
} finally {
btn.disabled = false;
btn.textContent = 'Deep Scan';
}
});
@@ -688,19 +1090,34 @@ document.addEventListener('DOMContentLoaded', () => {
}
});
// Clear Completed Tasks
document.getElementById('clearCompletedBtn').addEventListener('click', async () => {
const btn = document.getElementById('clearCompletedBtn');
// Clear Tasks with age/status options
document.getElementById('clearTasksBtn').addEventListener('click', async () => {
const btn = document.getElementById('clearTasksBtn');
const statuses = document.getElementById('clearStatusSelect').value;
const olderThanDays = document.getElementById('clearAgeSelect').value;
const statusLabel = document.getElementById('clearStatusSelect').selectedOptions[0].text;
const ageLabel = document.getElementById('clearAgeSelect').selectedOptions[0].text;
if (!confirm(`Clear ${statusLabel} tasks (${ageLabel.toLowerCase()})?`)) {
return;
}
btn.disabled = true;
btn.textContent = 'Clearing...';
try {
const r = await fetch('/api/import/clear-completed', { method: 'POST' });
const formData = new FormData();
formData.append('statuses', statuses);
formData.append('older_than_days', olderThanDays);
const r = await fetch('/api/import/clear-completed', { method: 'POST', body: formData });
const data = await r.json();
if (data.ok) {
alert(`Cleared ${data.deleted} completed/skipped task(s).`);
alert(`Cleared ${data.deleted} task(s).`);
loadQueueStatus();
loadTaskAgeStats();
} else {
alert('Failed to clear: ' + (data.error || 'Unknown error'));
}
@@ -708,7 +1125,7 @@ document.addEventListener('DOMContentLoaded', () => {
alert('Failed to clear: ' + e.message);
} finally {
btn.disabled = false;
btn.textContent = 'Clear Completed';
btn.textContent = 'Clear';
}
});