tuning job log views and deep scan archive processing.
This commit is contained in:
Bryan Van Deusen
+198
-9
@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload, aliased
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch, AppSettings
|
||||
from app import db
|
||||
import os
|
||||
import shutil
|
||||
@@ -1225,6 +1225,91 @@ def import_queue_status():
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/tasks')
|
||||
def list_import_tasks():
|
||||
"""
|
||||
List import tasks with filtering and pagination.
|
||||
|
||||
Query params:
|
||||
status: Filter by status (comma-separated, e.g., 'failed,skipped')
|
||||
task_type: Filter by task type ('import_image', 'import_archive')
|
||||
search: Search in source_path or error_message
|
||||
limit: Max results (default 50, max 200)
|
||||
offset: Skip first N results
|
||||
sort: Sort field ('created_at', 'completed_at', default: 'created_at')
|
||||
order: Sort order ('asc', 'desc', default: 'desc')
|
||||
"""
|
||||
try:
|
||||
# Parse parameters
|
||||
status_filter = request.args.get('status', '')
|
||||
task_type = request.args.get('task_type', '')
|
||||
search = request.args.get('search', '')
|
||||
limit = min(int(request.args.get('limit', 50)), 200)
|
||||
offset = int(request.args.get('offset', 0))
|
||||
sort_field = request.args.get('sort', 'created_at')
|
||||
sort_order = request.args.get('order', 'desc')
|
||||
|
||||
# Build query
|
||||
query = ImportTask.query
|
||||
|
||||
# Filter by status
|
||||
if status_filter:
|
||||
statuses = [s.strip() for s in status_filter.split(',') if s.strip()]
|
||||
if statuses:
|
||||
query = query.filter(ImportTask.status.in_(statuses))
|
||||
|
||||
# Filter by task type
|
||||
if task_type:
|
||||
query = query.filter_by(task_type=task_type)
|
||||
|
||||
# Search in source_path or error_message
|
||||
if search:
|
||||
search_pattern = f'%{search}%'
|
||||
query = query.filter(
|
||||
db.or_(
|
||||
ImportTask.source_path.ilike(search_pattern),
|
||||
ImportTask.error_message.ilike(search_pattern)
|
||||
)
|
||||
)
|
||||
|
||||
# Get total count before pagination
|
||||
total = query.count()
|
||||
|
||||
# Sort
|
||||
sort_col = getattr(ImportTask, sort_field, ImportTask.created_at)
|
||||
if sort_order == 'asc':
|
||||
query = query.order_by(sort_col.asc())
|
||||
else:
|
||||
query = query.order_by(sort_col.desc())
|
||||
|
||||
# Paginate
|
||||
tasks = query.offset(offset).limit(limit).all()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'total': total,
|
||||
'offset': offset,
|
||||
'limit': limit,
|
||||
'tasks': [
|
||||
{
|
||||
'id': t.id,
|
||||
'source_path': t.source_path,
|
||||
'filename': os.path.basename(t.source_path) if t.source_path else None,
|
||||
'task_type': t.task_type,
|
||||
'status': t.status,
|
||||
'error_message': t.error_message, # Full error message
|
||||
'file_size': t.file_size,
|
||||
'retry_count': t.retry_count,
|
||||
'created_at': t.created_at.isoformat() if t.created_at else None,
|
||||
'completed_at': t.completed_at.isoformat() if t.completed_at else None
|
||||
}
|
||||
for t in tasks
|
||||
]
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/task/<int:task_id>')
|
||||
def get_import_task(task_id):
|
||||
"""
|
||||
@@ -1308,25 +1393,129 @@ def retry_failed_import_tasks():
|
||||
@main.route('/api/import/clear-completed', methods=['POST'])
|
||||
def clear_completed_tasks():
|
||||
"""
|
||||
Clear completed and skipped import tasks from the database.
|
||||
Keeps failed tasks for review.
|
||||
"""
|
||||
try:
|
||||
deleted = ImportTask.query.filter(
|
||||
ImportTask.status.in_(['complete', 'skipped'])
|
||||
).delete(synchronize_session=False)
|
||||
Clear import tasks from the database with optional age filtering.
|
||||
|
||||
POST params:
|
||||
statuses: Comma-separated statuses to clear (default: complete,skipped)
|
||||
older_than_days: Only clear tasks older than N days (default: 0 = all)
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
try:
|
||||
# Parse parameters
|
||||
statuses_param = request.form.get('statuses', 'complete,skipped')
|
||||
older_than_days = int(request.form.get('older_than_days', 0))
|
||||
|
||||
# Validate statuses
|
||||
allowed_statuses = {'complete', 'skipped', 'failed'}
|
||||
statuses = [s.strip() for s in statuses_param.split(',') if s.strip() in allowed_statuses]
|
||||
if not statuses:
|
||||
statuses = ['complete', 'skipped']
|
||||
|
||||
# Build query
|
||||
query = ImportTask.query.filter(ImportTask.status.in_(statuses))
|
||||
|
||||
# Apply age filter if specified
|
||||
if older_than_days > 0:
|
||||
cutoff = datetime.utcnow() - timedelta(days=older_than_days)
|
||||
query = query.filter(ImportTask.completed_at < cutoff)
|
||||
|
||||
deleted = query.delete(synchronize_session=False)
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'deleted': deleted
|
||||
'deleted': deleted,
|
||||
'statuses': statuses,
|
||||
'older_than_days': older_than_days
|
||||
})
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/task-stats')
|
||||
def import_task_stats():
|
||||
"""
|
||||
Get detailed task statistics with age breakdown.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
try:
|
||||
now = datetime.utcnow()
|
||||
today = now - timedelta(days=1)
|
||||
week_ago = now - timedelta(days=7)
|
||||
|
||||
stats = {}
|
||||
for status in ['complete', 'skipped', 'failed']:
|
||||
total = ImportTask.query.filter_by(status=status).count()
|
||||
today_count = ImportTask.query.filter(
|
||||
ImportTask.status == status,
|
||||
ImportTask.completed_at >= today
|
||||
).count()
|
||||
week_count = ImportTask.query.filter(
|
||||
ImportTask.status == status,
|
||||
ImportTask.completed_at >= week_ago,
|
||||
ImportTask.completed_at < today
|
||||
).count()
|
||||
older = total - today_count - week_count
|
||||
|
||||
stats[status] = {
|
||||
'total': total,
|
||||
'today': today_count,
|
||||
'this_week': week_count,
|
||||
'older': older
|
||||
}
|
||||
|
||||
return jsonify({'ok': True, 'stats': stats})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/system/stats')
|
||||
def system_stats():
|
||||
"""
|
||||
Get system-wide statistics for the dashboard.
|
||||
|
||||
Returns cached stats computed by the update_system_stats Celery task.
|
||||
Stats are updated every 5 minutes by Celery Beat.
|
||||
|
||||
Query params:
|
||||
refresh: If 'true', triggers an immediate stats update (async)
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
# Check for refresh request
|
||||
if request.args.get('refresh') == 'true':
|
||||
from app.tasks.scan import update_system_stats
|
||||
update_system_stats.delay()
|
||||
|
||||
# Get cached stats from AppSettings
|
||||
setting = AppSettings.query.filter_by(key='system_stats_cache').first()
|
||||
|
||||
if setting and setting.value:
|
||||
stats = json.loads(setting.value)
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'cached': True,
|
||||
**stats
|
||||
})
|
||||
|
||||
# No cached stats - return empty with flag to indicate first run
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'cached': False,
|
||||
'images': {'total': 0, 'storage_bytes': 0, 'storage_human': '0 B'},
|
||||
'tags': {'total': 0, 'by_kind': {}},
|
||||
'import_folder': {'pending_files': 0, 'pending_archives': 0},
|
||||
'message': 'Stats not yet computed. Will be available shortly.'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/thumbnails/regenerate', methods=['POST'])
|
||||
def regenerate_thumbnails_celery():
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user