changes to mobile styling in modal view, complete reword of backend worker system
This commit is contained in:
Bryan Van Deusen
+297
-1
@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch
|
||||
from app import db
|
||||
import os
|
||||
import shutil
|
||||
@@ -987,3 +987,299 @@ def delete_filtered_images():
|
||||
deleted_count=deleted_count,
|
||||
errors=errors if errors else None
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Import Task Queue API (Celery)
|
||||
# ----------------------------
|
||||
|
||||
@main.route('/api/import/trigger', methods=['POST'])
|
||||
def trigger_import_celery():
|
||||
"""
|
||||
Trigger a directory scan and import via Celery.
|
||||
Replaces the old flag-file mechanism.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.scan import scan_directory
|
||||
|
||||
source_dir = request.form.get('source_dir', '/import')
|
||||
dest_dir = request.form.get('dest_dir', '/images')
|
||||
|
||||
result = scan_directory.delay(source_dir, dest_dir)
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Import scan started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/status')
|
||||
def import_queue_status():
|
||||
"""
|
||||
Get current import queue status.
|
||||
Returns counts by task status and recent task info.
|
||||
"""
|
||||
try:
|
||||
# Count tasks by status
|
||||
status_counts = dict(
|
||||
db.session.query(
|
||||
ImportTask.status,
|
||||
func.count(ImportTask.id)
|
||||
).group_by(ImportTask.status).all()
|
||||
)
|
||||
|
||||
# Get active batch info
|
||||
active_batch = ImportBatch.query.filter_by(status='running').order_by(
|
||||
ImportBatch.started_at.desc()
|
||||
).first()
|
||||
|
||||
# Recent tasks (last 20)
|
||||
recent_tasks = ImportTask.query.order_by(
|
||||
ImportTask.created_at.desc()
|
||||
).limit(20).all()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'status_counts': {
|
||||
'pending': status_counts.get('pending', 0),
|
||||
'queued': status_counts.get('queued', 0),
|
||||
'processing': status_counts.get('processing', 0),
|
||||
'complete': status_counts.get('complete', 0),
|
||||
'failed': status_counts.get('failed', 0),
|
||||
'skipped': status_counts.get('skipped', 0),
|
||||
},
|
||||
'active_batch': {
|
||||
'id': active_batch.id,
|
||||
'source_directory': active_batch.source_directory,
|
||||
'total_files': active_batch.total_files,
|
||||
'processed_files': active_batch.processed_files,
|
||||
'imported_count': active_batch.imported_count,
|
||||
'skipped_count': active_batch.skipped_count,
|
||||
'failed_count': active_batch.failed_count,
|
||||
'started_at': active_batch.started_at.isoformat() if active_batch.started_at else None
|
||||
} if active_batch else None,
|
||||
'recent_tasks': [
|
||||
{
|
||||
'id': t.id,
|
||||
'source_path': os.path.basename(t.source_path) if t.source_path else None,
|
||||
'task_type': t.task_type,
|
||||
'status': t.status,
|
||||
'error_message': t.error_message[:100] if t.error_message else None,
|
||||
'created_at': t.created_at.isoformat() if t.created_at else None
|
||||
}
|
||||
for t in recent_tasks
|
||||
]
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/task/<int:task_id>')
|
||||
def get_import_task(task_id):
|
||||
"""
|
||||
Get detailed status for a specific import task.
|
||||
"""
|
||||
task = ImportTask.query.get_or_404(task_id)
|
||||
|
||||
# Check Celery task status if still processing
|
||||
celery_status = None
|
||||
if task.celery_task_id:
|
||||
try:
|
||||
from app.celery_app import celery
|
||||
result = celery.AsyncResult(task.celery_task_id)
|
||||
celery_status = result.status
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task': {
|
||||
'id': task.id,
|
||||
'source_path': task.source_path,
|
||||
'file_hash': task.file_hash,
|
||||
'file_size': task.file_size,
|
||||
'task_type': task.task_type,
|
||||
'status': task.status,
|
||||
'celery_status': celery_status,
|
||||
'celery_task_id': task.celery_task_id,
|
||||
'error_message': task.error_message,
|
||||
'retry_count': task.retry_count,
|
||||
'context': task.context,
|
||||
'batch_id': task.batch_id,
|
||||
'result_image_id': task.result_image_id,
|
||||
'created_at': task.created_at.isoformat() if task.created_at else None,
|
||||
'started_at': task.started_at.isoformat() if task.started_at else None,
|
||||
'completed_at': task.completed_at.isoformat() if task.completed_at else None
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@main.route('/api/import/retry-failed', methods=['POST'])
|
||||
def retry_failed_import_tasks():
|
||||
"""
|
||||
Re-queue all failed import tasks.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
|
||||
failed_tasks = ImportTask.query.filter_by(status='failed').all()
|
||||
retried = 0
|
||||
|
||||
for task in failed_tasks:
|
||||
task.status = 'pending'
|
||||
task.error_message = None
|
||||
task.retry_count = 0
|
||||
task.started_at = None
|
||||
task.completed_at = None
|
||||
|
||||
if task.task_type == 'import_image':
|
||||
result = import_media_file.delay(task.id)
|
||||
elif task.task_type == 'import_archive':
|
||||
result = import_archive.delay(task.id)
|
||||
else:
|
||||
continue
|
||||
|
||||
task.celery_task_id = result.id
|
||||
task.status = 'queued'
|
||||
retried += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'retried': retried
|
||||
})
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/clear-completed', methods=['POST'])
|
||||
def clear_completed_tasks():
|
||||
"""
|
||||
Clear completed and skipped import tasks from the database.
|
||||
Keeps failed tasks for review.
|
||||
"""
|
||||
try:
|
||||
deleted = ImportTask.query.filter(
|
||||
ImportTask.status.in_(['complete', 'skipped'])
|
||||
).delete(synchronize_session=False)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'deleted': deleted
|
||||
})
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/thumbnails/regenerate', methods=['POST'])
|
||||
def regenerate_thumbnails_celery():
|
||||
"""
|
||||
Trigger thumbnail regeneration for all images via Celery.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.thumbnail import regenerate_all_thumbnails
|
||||
|
||||
overwrite = request.form.get('overwrite', 'true').lower() == 'true'
|
||||
result = regenerate_all_thumbnails.delay(overwrite=overwrite)
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Thumbnail regeneration started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/thumbnails/regenerate-missing', methods=['POST'])
|
||||
def regenerate_missing_thumbnails_celery():
|
||||
"""
|
||||
Generate thumbnails only for images that don't have them.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.thumbnail import regenerate_missing_thumbnails
|
||||
|
||||
result = regenerate_missing_thumbnails.delay()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Missing thumbnail generation started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/celery/status')
|
||||
def celery_cluster_status():
|
||||
"""
|
||||
Get Celery cluster status (workers, queues, etc.).
|
||||
"""
|
||||
try:
|
||||
from app.celery_app import celery
|
||||
|
||||
inspect = celery.control.inspect()
|
||||
|
||||
# Get worker stats
|
||||
active = inspect.active() or {}
|
||||
scheduled = inspect.scheduled() or {}
|
||||
reserved = inspect.reserved() or {}
|
||||
stats = inspect.stats() or {}
|
||||
|
||||
# Count active tasks
|
||||
active_count = sum(len(tasks) for tasks in active.values())
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'workers': list(stats.keys()),
|
||||
'worker_count': len(stats),
|
||||
'active_tasks': active_count,
|
||||
'active': active,
|
||||
'scheduled': scheduled,
|
||||
'reserved': reserved
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
'ok': False,
|
||||
'error': str(e),
|
||||
'message': 'Celery may not be running'
|
||||
}), 503
|
||||
|
||||
|
||||
@main.route('/api/import/batches')
|
||||
def list_import_batches():
|
||||
"""
|
||||
List recent import batches with statistics.
|
||||
"""
|
||||
limit = request.args.get('limit', 10, type=int)
|
||||
|
||||
batches = ImportBatch.query.order_by(
|
||||
ImportBatch.started_at.desc()
|
||||
).limit(limit).all()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'batches': [
|
||||
{
|
||||
'id': b.id,
|
||||
'source_directory': b.source_directory,
|
||||
'status': b.status,
|
||||
'total_files': b.total_files,
|
||||
'processed_files': b.processed_files,
|
||||
'imported_count': b.imported_count,
|
||||
'skipped_count': b.skipped_count,
|
||||
'failed_count': b.failed_count,
|
||||
'started_at': b.started_at.isoformat() if b.started_at else None,
|
||||
'completed_at': b.completed_at.isoformat() if b.completed_at else None
|
||||
}
|
||||
for b in batches
|
||||
]
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user