changes to mobile styling in modal view, complete reword of backend worker system
This commit is contained in:
Bryan Van Deusen
@@ -13,6 +13,9 @@ import sqlite3
|
||||
db = SQLAlchemy()
|
||||
migrate = Migrate()
|
||||
|
||||
# Celery instance - will be configured with app context in create_app
|
||||
celery = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Commands
|
||||
@@ -25,6 +28,34 @@ def version_check_command():
|
||||
from app.utils.version_migration import check_and_run_migrations
|
||||
check_and_run_migrations()
|
||||
|
||||
|
||||
@click.command("celery-worker")
|
||||
@click.option('--queues', '-Q', default='scan,import,thumbnail,sidecar,default',
|
||||
help='Comma-separated list of queues to consume')
|
||||
@click.option('--concurrency', '-c', default=2, type=int,
|
||||
help='Number of worker processes')
|
||||
@with_appcontext
|
||||
def celery_worker_command(queues, concurrency):
|
||||
"""Start a Celery worker for processing import tasks."""
|
||||
from app.celery_app import celery as celery_app
|
||||
celery_app.worker_main([
|
||||
'worker',
|
||||
f'--queues={queues}',
|
||||
f'--concurrency={concurrency}',
|
||||
'--loglevel=info'
|
||||
])
|
||||
|
||||
|
||||
@click.command("celery-beat")
|
||||
@with_appcontext
|
||||
def celery_beat_command():
|
||||
"""Start Celery Beat scheduler for periodic tasks."""
|
||||
from app.celery_app import celery as celery_app
|
||||
celery_app.worker_main([
|
||||
'beat',
|
||||
'--loglevel=info'
|
||||
])
|
||||
|
||||
@event.listens_for(Engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_connection, connection_record):
|
||||
# Only apply to SQLite connections
|
||||
@@ -54,5 +85,13 @@ def create_app(config_class='config.Config'):
|
||||
|
||||
# Register CLI commands
|
||||
app.cli.add_command(version_check_command)
|
||||
app.cli.add_command(celery_worker_command)
|
||||
app.cli.add_command(celery_beat_command)
|
||||
|
||||
# Initialize Celery with Flask app context
|
||||
from app.celery_app import make_celery
|
||||
global celery
|
||||
celery = make_celery(app)
|
||||
app.celery = celery
|
||||
|
||||
return app
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# app/celery_app.py
|
||||
"""
|
||||
Celery application factory and configuration.
|
||||
Provides task queue functionality for the ImageRepo import pipeline.
|
||||
"""
|
||||
import os
|
||||
from celery import Celery
|
||||
|
||||
# Default broker/backend URLs
|
||||
DEFAULT_BROKER = 'redis://redis:6379/0'
|
||||
DEFAULT_BACKEND = 'redis://redis:6379/0'
|
||||
|
||||
|
||||
def make_celery(app=None):
|
||||
"""
|
||||
Create Celery instance with Flask app context support.
|
||||
|
||||
Args:
|
||||
app: Flask application instance (optional)
|
||||
|
||||
Returns:
|
||||
Configured Celery instance
|
||||
"""
|
||||
# Get broker/backend from app config or environment
|
||||
if app:
|
||||
broker = app.config.get('CELERY_BROKER_URL', DEFAULT_BROKER)
|
||||
backend = app.config.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
|
||||
else:
|
||||
broker = os.environ.get('CELERY_BROKER_URL', DEFAULT_BROKER)
|
||||
backend = os.environ.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
|
||||
|
||||
celery = Celery(
|
||||
'imagerepo',
|
||||
broker=broker,
|
||||
backend=backend,
|
||||
include=[
|
||||
'app.tasks.scan',
|
||||
'app.tasks.import_file',
|
||||
'app.tasks.thumbnail',
|
||||
'app.tasks.sidecar'
|
||||
]
|
||||
)
|
||||
|
||||
# Worker concurrency from environment
|
||||
concurrency = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2'))
|
||||
|
||||
celery.conf.update(
|
||||
# Serialization
|
||||
task_serializer='json',
|
||||
accept_content=['json'],
|
||||
result_serializer='json',
|
||||
timezone='UTC',
|
||||
enable_utc=True,
|
||||
|
||||
# Concurrency - intentionally low for steady background processing
|
||||
worker_concurrency=concurrency,
|
||||
worker_prefetch_multiplier=1, # One task at a time per worker process
|
||||
|
||||
# Task routing - separate queues for different task types
|
||||
task_routes={
|
||||
'app.tasks.scan.*': {'queue': 'scan'},
|
||||
'app.tasks.import_file.*': {'queue': 'import'},
|
||||
'app.tasks.thumbnail.*': {'queue': 'thumbnail'},
|
||||
'app.tasks.sidecar.*': {'queue': 'sidecar'},
|
||||
},
|
||||
|
||||
# Task default queue for unrouted tasks
|
||||
task_default_queue='default',
|
||||
|
||||
# Result backend settings
|
||||
result_expires=86400, # 24 hours
|
||||
|
||||
# Task execution settings for resume capability
|
||||
task_acks_late=True, # Acknowledge after task completes
|
||||
task_reject_on_worker_lost=True, # Requeue if worker dies
|
||||
|
||||
# Time limits
|
||||
task_soft_time_limit=300, # 5 minutes soft limit
|
||||
task_time_limit=600, # 10 minutes hard limit (archives may need more)
|
||||
|
||||
# Periodic task schedule (Celery Beat)
|
||||
beat_schedule={
|
||||
'periodic-import-scan': {
|
||||
'task': 'app.tasks.scan.scan_directory',
|
||||
'schedule': int(os.environ.get('IMPORT_EVERY_SECONDS', '28800')), # 8 hours default
|
||||
'args': ('/import', '/images'),
|
||||
},
|
||||
'recover-interrupted-tasks': {
|
||||
'task': 'app.tasks.scan.recover_interrupted_tasks',
|
||||
'schedule': 300, # Every 5 minutes
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
if app:
|
||||
# Don't pass Flask config directly to Celery - it contains old-style keys
|
||||
# that conflict with Celery's new lowercase format.
|
||||
# The broker/backend are already set above from app.config.
|
||||
|
||||
# Create a task base class that runs within Flask app context
|
||||
class ContextTask(celery.Task):
|
||||
def __call__(self, *args, **kwargs):
|
||||
with app.app_context():
|
||||
return self.run(*args, **kwargs)
|
||||
|
||||
celery.Task = ContextTask
|
||||
|
||||
return celery
|
||||
|
||||
|
||||
def create_celery_with_app():
|
||||
"""
|
||||
Create Celery instance with Flask app context for standalone workers.
|
||||
This is used when running `celery -A app.celery_app:celery worker`.
|
||||
"""
|
||||
from app import create_app
|
||||
flask_app = create_app()
|
||||
return make_celery(flask_app)
|
||||
|
||||
|
||||
# Create celery instance with Flask app context
|
||||
# This ensures workers have access to the database and Flask extensions
|
||||
celery = create_celery_with_app()
|
||||
+297
-1
@@ -7,7 +7,7 @@ from sqlalchemy.orm import joinedload
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags, ImportTask, ImportBatch
|
||||
from app import db
|
||||
import os
|
||||
import shutil
|
||||
@@ -987,3 +987,299 @@ def delete_filtered_images():
|
||||
deleted_count=deleted_count,
|
||||
errors=errors if errors else None
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Import Task Queue API (Celery)
|
||||
# ----------------------------
|
||||
|
||||
@main.route('/api/import/trigger', methods=['POST'])
|
||||
def trigger_import_celery():
|
||||
"""
|
||||
Trigger a directory scan and import via Celery.
|
||||
Replaces the old flag-file mechanism.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.scan import scan_directory
|
||||
|
||||
source_dir = request.form.get('source_dir', '/import')
|
||||
dest_dir = request.form.get('dest_dir', '/images')
|
||||
|
||||
result = scan_directory.delay(source_dir, dest_dir)
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Import scan started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/status')
|
||||
def import_queue_status():
|
||||
"""
|
||||
Get current import queue status.
|
||||
Returns counts by task status and recent task info.
|
||||
"""
|
||||
try:
|
||||
# Count tasks by status
|
||||
status_counts = dict(
|
||||
db.session.query(
|
||||
ImportTask.status,
|
||||
func.count(ImportTask.id)
|
||||
).group_by(ImportTask.status).all()
|
||||
)
|
||||
|
||||
# Get active batch info
|
||||
active_batch = ImportBatch.query.filter_by(status='running').order_by(
|
||||
ImportBatch.started_at.desc()
|
||||
).first()
|
||||
|
||||
# Recent tasks (last 20)
|
||||
recent_tasks = ImportTask.query.order_by(
|
||||
ImportTask.created_at.desc()
|
||||
).limit(20).all()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'status_counts': {
|
||||
'pending': status_counts.get('pending', 0),
|
||||
'queued': status_counts.get('queued', 0),
|
||||
'processing': status_counts.get('processing', 0),
|
||||
'complete': status_counts.get('complete', 0),
|
||||
'failed': status_counts.get('failed', 0),
|
||||
'skipped': status_counts.get('skipped', 0),
|
||||
},
|
||||
'active_batch': {
|
||||
'id': active_batch.id,
|
||||
'source_directory': active_batch.source_directory,
|
||||
'total_files': active_batch.total_files,
|
||||
'processed_files': active_batch.processed_files,
|
||||
'imported_count': active_batch.imported_count,
|
||||
'skipped_count': active_batch.skipped_count,
|
||||
'failed_count': active_batch.failed_count,
|
||||
'started_at': active_batch.started_at.isoformat() if active_batch.started_at else None
|
||||
} if active_batch else None,
|
||||
'recent_tasks': [
|
||||
{
|
||||
'id': t.id,
|
||||
'source_path': os.path.basename(t.source_path) if t.source_path else None,
|
||||
'task_type': t.task_type,
|
||||
'status': t.status,
|
||||
'error_message': t.error_message[:100] if t.error_message else None,
|
||||
'created_at': t.created_at.isoformat() if t.created_at else None
|
||||
}
|
||||
for t in recent_tasks
|
||||
]
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/task/<int:task_id>')
|
||||
def get_import_task(task_id):
|
||||
"""
|
||||
Get detailed status for a specific import task.
|
||||
"""
|
||||
task = ImportTask.query.get_or_404(task_id)
|
||||
|
||||
# Check Celery task status if still processing
|
||||
celery_status = None
|
||||
if task.celery_task_id:
|
||||
try:
|
||||
from app.celery_app import celery
|
||||
result = celery.AsyncResult(task.celery_task_id)
|
||||
celery_status = result.status
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task': {
|
||||
'id': task.id,
|
||||
'source_path': task.source_path,
|
||||
'file_hash': task.file_hash,
|
||||
'file_size': task.file_size,
|
||||
'task_type': task.task_type,
|
||||
'status': task.status,
|
||||
'celery_status': celery_status,
|
||||
'celery_task_id': task.celery_task_id,
|
||||
'error_message': task.error_message,
|
||||
'retry_count': task.retry_count,
|
||||
'context': task.context,
|
||||
'batch_id': task.batch_id,
|
||||
'result_image_id': task.result_image_id,
|
||||
'created_at': task.created_at.isoformat() if task.created_at else None,
|
||||
'started_at': task.started_at.isoformat() if task.started_at else None,
|
||||
'completed_at': task.completed_at.isoformat() if task.completed_at else None
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@main.route('/api/import/retry-failed', methods=['POST'])
|
||||
def retry_failed_import_tasks():
|
||||
"""
|
||||
Re-queue all failed import tasks.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
|
||||
failed_tasks = ImportTask.query.filter_by(status='failed').all()
|
||||
retried = 0
|
||||
|
||||
for task in failed_tasks:
|
||||
task.status = 'pending'
|
||||
task.error_message = None
|
||||
task.retry_count = 0
|
||||
task.started_at = None
|
||||
task.completed_at = None
|
||||
|
||||
if task.task_type == 'import_image':
|
||||
result = import_media_file.delay(task.id)
|
||||
elif task.task_type == 'import_archive':
|
||||
result = import_archive.delay(task.id)
|
||||
else:
|
||||
continue
|
||||
|
||||
task.celery_task_id = result.id
|
||||
task.status = 'queued'
|
||||
retried += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'retried': retried
|
||||
})
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/import/clear-completed', methods=['POST'])
|
||||
def clear_completed_tasks():
|
||||
"""
|
||||
Clear completed and skipped import tasks from the database.
|
||||
Keeps failed tasks for review.
|
||||
"""
|
||||
try:
|
||||
deleted = ImportTask.query.filter(
|
||||
ImportTask.status.in_(['complete', 'skipped'])
|
||||
).delete(synchronize_session=False)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'deleted': deleted
|
||||
})
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/thumbnails/regenerate', methods=['POST'])
|
||||
def regenerate_thumbnails_celery():
|
||||
"""
|
||||
Trigger thumbnail regeneration for all images via Celery.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.thumbnail import regenerate_all_thumbnails
|
||||
|
||||
overwrite = request.form.get('overwrite', 'true').lower() == 'true'
|
||||
result = regenerate_all_thumbnails.delay(overwrite=overwrite)
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Thumbnail regeneration started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/thumbnails/regenerate-missing', methods=['POST'])
|
||||
def regenerate_missing_thumbnails_celery():
|
||||
"""
|
||||
Generate thumbnails only for images that don't have them.
|
||||
"""
|
||||
try:
|
||||
from app.tasks.thumbnail import regenerate_missing_thumbnails
|
||||
|
||||
result = regenerate_missing_thumbnails.delay()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'task_id': result.id,
|
||||
'message': 'Missing thumbnail generation started'
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'ok': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@main.route('/api/celery/status')
|
||||
def celery_cluster_status():
|
||||
"""
|
||||
Get Celery cluster status (workers, queues, etc.).
|
||||
"""
|
||||
try:
|
||||
from app.celery_app import celery
|
||||
|
||||
inspect = celery.control.inspect()
|
||||
|
||||
# Get worker stats
|
||||
active = inspect.active() or {}
|
||||
scheduled = inspect.scheduled() or {}
|
||||
reserved = inspect.reserved() or {}
|
||||
stats = inspect.stats() or {}
|
||||
|
||||
# Count active tasks
|
||||
active_count = sum(len(tasks) for tasks in active.values())
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'workers': list(stats.keys()),
|
||||
'worker_count': len(stats),
|
||||
'active_tasks': active_count,
|
||||
'active': active,
|
||||
'scheduled': scheduled,
|
||||
'reserved': reserved
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
'ok': False,
|
||||
'error': str(e),
|
||||
'message': 'Celery may not be running'
|
||||
}), 503
|
||||
|
||||
|
||||
@main.route('/api/import/batches')
|
||||
def list_import_batches():
|
||||
"""
|
||||
List recent import batches with statistics.
|
||||
"""
|
||||
limit = request.args.get('limit', 10, type=int)
|
||||
|
||||
batches = ImportBatch.query.order_by(
|
||||
ImportBatch.started_at.desc()
|
||||
).limit(limit).all()
|
||||
|
||||
return jsonify({
|
||||
'ok': True,
|
||||
'batches': [
|
||||
{
|
||||
'id': b.id,
|
||||
'source_directory': b.source_directory,
|
||||
'status': b.status,
|
||||
'total_files': b.total_files,
|
||||
'processed_files': b.processed_files,
|
||||
'imported_count': b.imported_count,
|
||||
'skipped_count': b.skipped_count,
|
||||
'failed_count': b.failed_count,
|
||||
'started_at': b.started_at.isoformat() if b.started_at else None,
|
||||
'completed_at': b.completed_at.isoformat() if b.completed_at else None
|
||||
}
|
||||
for b in batches
|
||||
]
|
||||
})
|
||||
|
||||
@@ -76,3 +76,80 @@ class AppSettings(db.Model):
|
||||
key = db.Column(db.String(64), unique=True, nullable=False)
|
||||
value = db.Column(db.Text, nullable=True)
|
||||
updated_at = db.Column(db.DateTime, default=db.func.now(), onupdate=db.func.now())
|
||||
|
||||
|
||||
class ImportBatch(db.Model):
|
||||
"""
|
||||
Groups related import tasks for batch tracking and reporting.
|
||||
One batch per scan operation.
|
||||
"""
|
||||
__tablename__ = "import_batch"
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
source_directory = db.Column(db.String(1024), nullable=False)
|
||||
status = db.Column(db.String(32), default='running', index=True) # running, complete, failed, cancelled
|
||||
|
||||
# Statistics
|
||||
total_files = db.Column(db.Integer, default=0)
|
||||
processed_files = db.Column(db.Integer, default=0)
|
||||
imported_count = db.Column(db.Integer, default=0)
|
||||
skipped_count = db.Column(db.Integer, default=0)
|
||||
failed_count = db.Column(db.Integer, default=0)
|
||||
|
||||
# Timestamps
|
||||
started_at = db.Column(db.DateTime, default=db.func.now())
|
||||
completed_at = db.Column(db.DateTime, nullable=True)
|
||||
|
||||
|
||||
class ImportTask(db.Model):
|
||||
"""
|
||||
Tracks the state of each file being processed through the import pipeline.
|
||||
Enables resume capability and parallel processing.
|
||||
"""
|
||||
__tablename__ = "import_task"
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# File identification
|
||||
source_path = db.Column(db.String(1024), nullable=False, index=True)
|
||||
file_hash = db.Column(db.String(64), nullable=True, index=True) # SHA256, computed during processing
|
||||
file_size = db.Column(db.BigInteger, nullable=True)
|
||||
|
||||
# Classification
|
||||
# Values: 'scan', 'import_image', 'import_archive', 'thumbnail', 'sidecar'
|
||||
task_type = db.Column(db.String(32), nullable=False, index=True)
|
||||
|
||||
# State tracking
|
||||
# Values: 'pending', 'queued', 'processing', 'complete', 'failed', 'skipped'
|
||||
status = db.Column(db.String(32), nullable=False, default='pending', index=True)
|
||||
|
||||
# Celery task tracking
|
||||
celery_task_id = db.Column(db.String(64), nullable=True, index=True)
|
||||
|
||||
# Context/metadata (JSON)
|
||||
# For images: {"artist": "...", "dest_dir": "..."}
|
||||
# For archives: {"artist": "...", "dest_dir": "...", "archive_tag": "..."}
|
||||
# For sidecars: {"image_id": 123, "sidecar_path": "..."}
|
||||
context = db.Column(db.JSON, nullable=True)
|
||||
|
||||
# Batch relationship
|
||||
batch_id = db.Column(db.Integer, db.ForeignKey('import_batch.id', ondelete='SET NULL'), nullable=True)
|
||||
batch = db.relationship('ImportBatch', backref=db.backref('tasks', lazy='dynamic'))
|
||||
|
||||
# Result tracking
|
||||
result_image_id = db.Column(db.Integer, db.ForeignKey('image_record.id', ondelete='SET NULL'), nullable=True)
|
||||
result_image = db.relationship('ImageRecord', foreign_keys=[result_image_id])
|
||||
error_message = db.Column(db.Text, nullable=True)
|
||||
retry_count = db.Column(db.Integer, default=0)
|
||||
|
||||
# Timestamps
|
||||
created_at = db.Column(db.DateTime, default=db.func.now(), index=True)
|
||||
started_at = db.Column(db.DateTime, nullable=True)
|
||||
completed_at = db.Column(db.DateTime, nullable=True)
|
||||
|
||||
# Unique constraint: prevent duplicate tasks for same file+type combination
|
||||
# Note: file_hash can be NULL initially (computed during processing)
|
||||
__table_args__ = (
|
||||
db.Index('ix_import_task_status_type', 'status', 'task_type'),
|
||||
db.Index('ix_import_task_batch_status', 'batch_id', 'status'),
|
||||
)
|
||||
|
||||
@@ -53,7 +53,9 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
archive: '🗜️',
|
||||
character: '👤',
|
||||
series: '📺',
|
||||
rating: '⚠️'
|
||||
rating: '⚠️',
|
||||
source: '🌐',
|
||||
post: '📌'
|
||||
};
|
||||
return icons[kind] || '#';
|
||||
}
|
||||
@@ -303,12 +305,17 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
modal.classList.add('active');
|
||||
updateImage(index);
|
||||
history.replaceState({ modalIndex: index }, '', window.location.href);
|
||||
// Focus the tag input after modal opens
|
||||
if (tagInput) {
|
||||
// Focus the tag input after modal opens (desktop only - avoids keyboard popup on mobile)
|
||||
if (tagInput && !isTouchDevice()) {
|
||||
setTimeout(() => tagInput.focus(), 100);
|
||||
}
|
||||
}
|
||||
|
||||
// Detect touch devices to avoid auto-focus keyboard popup
|
||||
function isTouchDevice() {
|
||||
return ('ontouchstart' in window) || (navigator.maxTouchPoints > 0);
|
||||
}
|
||||
|
||||
function closeModal() {
|
||||
modal.classList.remove('active');
|
||||
modalImageWrapper.innerHTML = '';
|
||||
|
||||
@@ -286,7 +286,9 @@
|
||||
archive: '🗜️',
|
||||
character: '👤',
|
||||
series: '📺',
|
||||
rating: '⚠️'
|
||||
rating: '⚠️',
|
||||
source: '🌐',
|
||||
post: '📌'
|
||||
};
|
||||
|
||||
const icon = icons[tag.kind];
|
||||
|
||||
+50
-9
@@ -369,6 +369,28 @@ header {
|
||||
.modal-body {
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
height: 100%;
|
||||
overflow-y: auto;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
}
|
||||
}
|
||||
|
||||
/* Mobile landscape: prioritize image, minimize tag editor */
|
||||
@media (max-width: 900px) and (orientation: landscape) {
|
||||
.modal-body {
|
||||
flex-direction: row;
|
||||
align-items: stretch;
|
||||
}
|
||||
.modal-image-wrapper {
|
||||
max-width: calc(100% - 200px);
|
||||
max-height: calc(100vh - 2rem);
|
||||
height: calc(100vh - 2rem);
|
||||
}
|
||||
.tag-editor {
|
||||
width: 180px;
|
||||
min-width: 180px;
|
||||
max-height: calc(100vh - 2rem);
|
||||
overflow-y: auto;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,12 +437,17 @@ header {
|
||||
width: 90vw; height: auto; max-height: none;
|
||||
}
|
||||
|
||||
/* On smaller screens, full width for image wrapper */
|
||||
@media (max-width: 900px) {
|
||||
/* On smaller screens (portrait), full width for image wrapper */
|
||||
@media (max-width: 900px) and (orientation: portrait) {
|
||||
.modal-image-wrapper {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
max-height: 60vh;
|
||||
max-height: 55vh;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.modal-image-wrapper img,
|
||||
.modal-image-wrapper video {
|
||||
max-height: 55vh;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,6 +480,17 @@ header {
|
||||
.modal-prev { left: 1rem; }
|
||||
.modal-next { right: 1rem; }
|
||||
|
||||
/* Mobile: always show nav buttons, larger touch targets */
|
||||
@media (max-width: 900px) {
|
||||
.modal-button {
|
||||
opacity: 0.8;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
}
|
||||
.modal-prev { left: 0.5rem; }
|
||||
.modal-next { right: 0.5rem; }
|
||||
}
|
||||
|
||||
/* Close button */
|
||||
.modal-close-btn {
|
||||
position: absolute;
|
||||
@@ -535,13 +573,15 @@ header {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
/* On smaller screens, make tag editor full width */
|
||||
@media (max-width: 900px) {
|
||||
/* On smaller screens (portrait), make tag editor full width and scrollable */
|
||||
@media (max-width: 900px) and (orientation: portrait) {
|
||||
.tag-editor {
|
||||
width: 100%;
|
||||
max-width: 500px;
|
||||
min-width: unset;
|
||||
max-height: unset;
|
||||
max-height: 35vh;
|
||||
overflow-y: auto;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -594,7 +634,7 @@ header {
|
||||
border-radius: 6px;
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
color: var(--text);
|
||||
font-size: 0.9rem;
|
||||
font-size: 16px; /* Must be 16px+ to prevent iOS auto-zoom on focus */
|
||||
}
|
||||
.tag-form input::placeholder {
|
||||
color: var(--text-muted);
|
||||
@@ -714,8 +754,9 @@ header {
|
||||
|
||||
.tag-thumb {
|
||||
flex: 1;
|
||||
background-size: cover;
|
||||
background-position: center;
|
||||
min-width: 0;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
background-color: var(--bg-elevated);
|
||||
transition: filter 0.2s ease;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# app/tasks/__init__.py
|
||||
"""
|
||||
Celery task modules for the ImageRepo import pipeline.
|
||||
|
||||
Task Types:
|
||||
- scan: Directory scanning and task queuing
|
||||
- import_file: Individual file import (images, videos, archives)
|
||||
- thumbnail: Thumbnail generation
|
||||
- sidecar: Gallery-DL metadata extraction and application
|
||||
"""
|
||||
@@ -0,0 +1,507 @@
|
||||
# app/tasks/import_file.py
|
||||
"""
|
||||
File import tasks for images, videos, and archives.
|
||||
|
||||
These tasks handle the actual import of individual files,
|
||||
including filtering, duplicate detection, and storage.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
import imagehash
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImportTask, ImageRecord, Tag, ArchiveRecord
|
||||
|
||||
log = logging.getLogger('celery.tasks.import_file')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.import_file.import_media_file',
|
||||
max_retries=3, default_retry_delay=60)
|
||||
def import_media_file(self, task_id: int):
|
||||
"""
|
||||
Import a single media file (image or video).
|
||||
|
||||
Steps:
|
||||
1. Load ImportTask record
|
||||
2. Apply filters (dimensions, transparency, single-color)
|
||||
3. Check for duplicates (hash and pHash)
|
||||
4. Copy to content-addressed storage
|
||||
5. Create ImageRecord
|
||||
6. Queue thumbnail task
|
||||
7. Queue sidecar metadata task
|
||||
8. Update ImportTask status
|
||||
|
||||
Args:
|
||||
task_id: ID of the ImportTask record
|
||||
|
||||
Returns:
|
||||
dict with status and result info
|
||||
"""
|
||||
from app.tasks.thumbnail import generate_thumbnail_task
|
||||
from app.tasks.sidecar import apply_sidecar_metadata
|
||||
from app.utils.image_importer import (
|
||||
extract_metadata, calculate_hash, calculate_perceptual_hash,
|
||||
is_mostly_transparent, is_mostly_single_color, find_similar_image,
|
||||
build_hashed_dest_path, load_import_settings, supersede_image,
|
||||
generate_thumbnail, generate_video_thumbnail_mirrored
|
||||
)
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
task = ImportTask.query.get(task_id)
|
||||
if not task:
|
||||
log.error(f"Task {task_id} not found")
|
||||
return {'error': 'Task not found'}
|
||||
|
||||
task.status = 'processing'
|
||||
task.started_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
try:
|
||||
src_path = task.source_path
|
||||
context = task.context or {}
|
||||
artist = context.get('artist', 'unknown')
|
||||
dest_dir = context.get('dest_dir', '/images')
|
||||
|
||||
log.info(f"Processing: {src_path}")
|
||||
|
||||
# Load settings
|
||||
settings = load_import_settings()
|
||||
|
||||
# Validate file exists
|
||||
if not os.path.exists(src_path):
|
||||
return _fail_task(task, 'Source file not found')
|
||||
|
||||
filename = os.path.basename(src_path)
|
||||
lower = filename.lower()
|
||||
|
||||
# Extract metadata first for filtering
|
||||
metadata = extract_metadata(src_path)
|
||||
if not metadata.get('width') or not metadata.get('height'):
|
||||
return _skip_task(task, 'Missing dimension data')
|
||||
|
||||
# Apply dimension filter
|
||||
min_width = settings.get('min_width', 0)
|
||||
min_height = settings.get('min_height', 0)
|
||||
if min_width > 0 and metadata['width'] < min_width:
|
||||
return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
|
||||
if min_height > 0 and metadata['height'] < min_height:
|
||||
return _skip_task(task, f"Too small: {metadata['width']}x{metadata['height']}")
|
||||
|
||||
# Apply transparency filter
|
||||
if settings.get('skip_transparent') and lower.endswith(('.png', '.gif', '.webp')):
|
||||
threshold = settings.get('transparency_threshold', 0.9)
|
||||
if is_mostly_transparent(src_path, threshold):
|
||||
return _skip_task(task, 'Mostly transparent')
|
||||
|
||||
# Apply single-color filter
|
||||
if settings.get('skip_single_color', True) and lower.endswith(('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp')):
|
||||
if is_mostly_single_color(src_path,
|
||||
settings.get('single_color_threshold', 0.95),
|
||||
settings.get('single_color_tolerance', 30)):
|
||||
return _skip_task(task, 'Mostly single color')
|
||||
|
||||
# Compute content hash
|
||||
file_hash = calculate_hash(src_path)
|
||||
task.file_hash = file_hash
|
||||
|
||||
# Check for exact duplicate by hash
|
||||
existing = ImageRecord.query.filter_by(hash=file_hash).first()
|
||||
if existing:
|
||||
# Still try to enrich with sidecar metadata
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(existing.id, sidecar_path)
|
||||
return _skip_task(task, 'Duplicate by hash', result_image_id=existing.id)
|
||||
|
||||
# Compute pHash for similarity check
|
||||
phash = calculate_perceptual_hash(src_path)
|
||||
phash_threshold = settings.get('phash_threshold', 10)
|
||||
|
||||
if phash and phash_threshold > 0:
|
||||
# Load existing pHashes for comparison
|
||||
existing_phashes = [
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
|
||||
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash.isnot(None)).all()
|
||||
]
|
||||
|
||||
relationship, similar_id = find_similar_image(
|
||||
phash, metadata['width'], metadata['height'],
|
||||
existing_phashes, threshold=phash_threshold
|
||||
)
|
||||
|
||||
if relationship == 'larger_exists':
|
||||
return _skip_task(task, 'Similar larger image exists')
|
||||
|
||||
# Handle supersede case - new image is larger than existing
|
||||
if relationship == 'smaller_exists' and settings.get('supersede_smaller', True) and similar_id:
|
||||
old_record = ImageRecord.query.get(similar_id)
|
||||
if old_record:
|
||||
return _supersede_existing(
|
||||
task, old_record, src_path, file_hash, phash,
|
||||
metadata, artist, dest_dir, filename
|
||||
)
|
||||
|
||||
# Normal import - copy to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
log.warning(f"Thumbnail generation failed for {filename}: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Create ImageRecord
|
||||
record = ImageRecord(
|
||||
filename=filename,
|
||||
filepath=dest_path,
|
||||
thumb_path=thumb_path,
|
||||
hash=file_hash,
|
||||
perceptual_hash=str(phash) if phash else None,
|
||||
file_size=metadata['file_size'],
|
||||
width=metadata['width'],
|
||||
height=metadata['height'],
|
||||
format=metadata['format'],
|
||||
camera_model=metadata.get('camera_model'),
|
||||
taken_at=metadata.get('taken_at'),
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
# Add artist tag
|
||||
_add_artist_tag(record, artist)
|
||||
|
||||
db.session.add(record)
|
||||
db.session.commit()
|
||||
|
||||
# Complete task
|
||||
task.status = 'complete'
|
||||
task.result_image_id = record.id
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
# Queue sidecar metadata enrichment
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.info(f"Imported: {filename} -> {dest_path}")
|
||||
|
||||
return {
|
||||
'status': 'imported',
|
||||
'image_id': record.id,
|
||||
'dest_path': dest_path
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Import failed for task {task_id}: {e}", exc_info=True)
|
||||
|
||||
task.retry_count += 1
|
||||
if task.retry_count < 3:
|
||||
task.status = 'queued'
|
||||
db.session.commit()
|
||||
raise self.retry(exc=e)
|
||||
else:
|
||||
return _fail_task(task, str(e))
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.import_file.import_archive',
|
||||
max_retries=2, default_retry_delay=120,
|
||||
soft_time_limit=1800, time_limit=3600) # 30min soft, 60min hard for large archives
|
||||
def import_archive(self, task_id: int):
|
||||
"""
|
||||
Import an archive file.
|
||||
|
||||
Steps:
|
||||
1. Check if archive already processed (by hash)
|
||||
2. Extract to temp directory
|
||||
3. Create ImportTask for each media file inside
|
||||
4. Queue import tasks for extracted files
|
||||
5. Create archive tag if files were found
|
||||
6. Clean up is deferred until child tasks complete
|
||||
|
||||
Args:
|
||||
task_id: ID of the ImportTask record
|
||||
|
||||
Returns:
|
||||
dict with status and counts
|
||||
"""
|
||||
from app.utils.image_importer import calculate_hash, compute_next_archive_tag_name
|
||||
from pyunpack import Archive
|
||||
|
||||
task = ImportTask.query.get(task_id)
|
||||
if not task:
|
||||
return {'error': 'Task not found'}
|
||||
|
||||
task.status = 'processing'
|
||||
task.started_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
tmpdir = None
|
||||
try:
|
||||
archive_path = task.source_path
|
||||
context = task.context or {}
|
||||
artist = context.get('artist', 'unknown')
|
||||
dest_dir = context.get('dest_dir', '/images')
|
||||
|
||||
log.info(f"Processing archive: {archive_path}")
|
||||
|
||||
# Compute archive hash
|
||||
archive_hash = calculate_hash(archive_path)
|
||||
task.file_hash = archive_hash
|
||||
|
||||
# Check if already processed
|
||||
existing = ArchiveRecord.query.filter_by(hash=archive_hash).first()
|
||||
if existing:
|
||||
return _skip_task(task, 'Archive already imported')
|
||||
|
||||
# Get archive size
|
||||
archive_size = os.path.getsize(archive_path)
|
||||
|
||||
# Extract archive
|
||||
tmpdir = tempfile.mkdtemp(prefix='extract_')
|
||||
log.info(f"Extracting to: {tmpdir}")
|
||||
|
||||
try:
|
||||
Archive(archive_path).extractall(tmpdir)
|
||||
except Exception as e:
|
||||
log.error(f"Extraction failed: {e}")
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _fail_task(task, f'Extraction failed: {e}')
|
||||
|
||||
# Find media files in extracted content
|
||||
media_files = []
|
||||
for root, _, files in os.walk(tmpdir):
|
||||
for filename in files:
|
||||
lower = filename.lower()
|
||||
if lower.endswith(('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp',
|
||||
'.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
media_files.append(os.path.join(root, filename))
|
||||
|
||||
if not media_files:
|
||||
log.info(f"No media files found in archive: {archive_path}")
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _skip_task(task, 'No media files in archive')
|
||||
|
||||
log.info(f"Found {len(media_files)} media files in archive")
|
||||
|
||||
# Create archive tag
|
||||
tag_name = compute_next_archive_tag_name(artist)
|
||||
archive_tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not archive_tag:
|
||||
archive_tag = Tag(name=tag_name, kind='archive')
|
||||
db.session.add(archive_tag)
|
||||
db.session.flush()
|
||||
|
||||
# Create ArchiveRecord
|
||||
archive_record = ArchiveRecord(
|
||||
filename=archive_path,
|
||||
file_size=archive_size,
|
||||
hash=archive_hash,
|
||||
artist=artist,
|
||||
tag_id=archive_tag.id
|
||||
)
|
||||
db.session.add(archive_record)
|
||||
db.session.commit()
|
||||
|
||||
# Queue tasks for each media file
|
||||
queued_count = 0
|
||||
for media_path in media_files:
|
||||
child_task = ImportTask(
|
||||
source_path=media_path,
|
||||
task_type='import_image',
|
||||
batch_id=task.batch_id,
|
||||
context={
|
||||
'artist': artist,
|
||||
'dest_dir': dest_dir,
|
||||
'archive_task_id': task.id,
|
||||
'archive_tag_id': archive_tag.id,
|
||||
'temp_dir': tmpdir
|
||||
},
|
||||
status='pending'
|
||||
)
|
||||
db.session.add(child_task)
|
||||
db.session.flush()
|
||||
|
||||
result = import_media_file.delay(child_task.id)
|
||||
child_task.celery_task_id = result.id
|
||||
child_task.status = 'queued'
|
||||
queued_count += 1
|
||||
|
||||
# Store child task info for cleanup tracking
|
||||
task.context = task.context or {}
|
||||
task.context['temp_dir'] = tmpdir
|
||||
task.context['archive_tag_id'] = archive_tag.id
|
||||
task.context['child_count'] = queued_count
|
||||
|
||||
task.status = 'complete'
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Archive processed: queued {queued_count} files from {archive_path}")
|
||||
|
||||
# Note: temp directory cleanup happens via a separate periodic task
|
||||
# or when all child tasks complete
|
||||
|
||||
return {
|
||||
'status': 'extracted',
|
||||
'queued_files': queued_count,
|
||||
'archive_tag': tag_name
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Archive import failed: {e}", exc_info=True)
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
return _fail_task(task, str(e))
|
||||
|
||||
|
||||
def _fail_task(task: ImportTask, error: str, **kwargs) -> dict:
|
||||
"""Mark task as failed with error message."""
|
||||
task.status = 'failed'
|
||||
task.error_message = error
|
||||
task.completed_at = datetime.utcnow()
|
||||
for key, value in kwargs.items():
|
||||
setattr(task, key, value)
|
||||
db.session.commit()
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.error(f"Task {task.id} failed: {error}")
|
||||
return {'status': 'failed', 'error': error}
|
||||
|
||||
|
||||
def _skip_task(task: ImportTask, reason: str, **kwargs) -> dict:
|
||||
"""Mark task as skipped with reason."""
|
||||
task.status = 'skipped'
|
||||
task.error_message = reason
|
||||
task.completed_at = datetime.utcnow()
|
||||
for key, value in kwargs.items():
|
||||
setattr(task, key, value)
|
||||
db.session.commit()
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
log.info(f"Task {task.id} skipped: {reason}")
|
||||
return {'status': 'skipped', 'reason': reason}
|
||||
|
||||
|
||||
def _add_artist_tag(record: ImageRecord, artist: str):
|
||||
"""Add artist tag to image record."""
|
||||
if not artist:
|
||||
return
|
||||
|
||||
tag_name = f"artist:{artist}"
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind='artist')
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
|
||||
|
||||
def _add_archive_tag(record: ImageRecord, archive_tag_id: int):
|
||||
"""Add archive tag to image record."""
|
||||
if not archive_tag_id:
|
||||
return
|
||||
|
||||
tag = Tag.query.get(archive_tag_id)
|
||||
if tag and tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
|
||||
|
||||
def _supersede_existing(task: ImportTask, old_record: ImageRecord, src_path: str,
|
||||
file_hash: str, phash, metadata: dict, artist: str,
|
||||
dest_dir: str, filename: str) -> dict:
|
||||
"""
|
||||
Replace an existing smaller image with a larger version.
|
||||
|
||||
Preserves tags and metadata from the old record.
|
||||
"""
|
||||
from app.utils.image_importer import (
|
||||
build_hashed_dest_path, generate_thumbnail, generate_video_thumbnail_mirrored,
|
||||
supersede_image
|
||||
)
|
||||
from app.utils.metadata_enrichment import find_sidecar_json, enrich_from_sidecar
|
||||
from app.tasks.sidecar import apply_sidecar_metadata
|
||||
|
||||
log.info(f"Superseding {old_record.filename} ({old_record.width}x{old_record.height}) "
|
||||
f"with {filename} ({metadata['width']}x{metadata['height']})")
|
||||
|
||||
# Enrich metadata from sidecar if available
|
||||
sidecar_path = find_sidecar_json(src_path)
|
||||
if sidecar_path:
|
||||
enriched = enrich_from_sidecar(sidecar_path, {'taken_at': metadata.get('taken_at')})
|
||||
if enriched.get('taken_at'):
|
||||
metadata['taken_at'] = enriched['taken_at']
|
||||
|
||||
# Copy new file to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate new thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv')):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
log.warning(f"Thumbnail generation failed: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Supersede the old record
|
||||
record = supersede_image(
|
||||
old_record, dest_path, thumb_path, file_hash, phash, metadata, filename
|
||||
)
|
||||
|
||||
db.session.commit()
|
||||
|
||||
# Complete task
|
||||
task.status = 'complete'
|
||||
task.result_image_id = record.id
|
||||
task.completed_at = datetime.utcnow()
|
||||
db.session.commit()
|
||||
|
||||
# Apply sidecar metadata if available
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
|
||||
# Update batch stats
|
||||
if task.batch_id:
|
||||
from app.tasks.scan import update_batch_stats
|
||||
update_batch_stats.delay(task.batch_id)
|
||||
|
||||
return {
|
||||
'status': 'superseded',
|
||||
'image_id': record.id,
|
||||
'dest_path': dest_path
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
# app/tasks/scan.py
|
||||
"""
|
||||
Directory scanning task for the import pipeline.
|
||||
|
||||
This task walks the import directory, detects new/changed files,
|
||||
creates ImportTask records, and queues appropriate processing tasks.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImportTask, ImportBatch
|
||||
|
||||
log = logging.getLogger('celery.tasks.scan')
|
||||
|
||||
# Supported file extensions (imported from image_importer for consistency)
|
||||
ALLOWED_MEDIA_EXTS = (
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.tiff', '.tif',
|
||||
'.mp4', '.mov', '.webm', '.avi', '.mkv'
|
||||
)
|
||||
ALLOWED_ARCHIVE_EXTS = (
|
||||
'.zip', '.rar', '.7z', '.tar', '.tar.gz', '.tgz', '.tar.bz2', '.tbz2',
|
||||
'.cbr', '.cbz'
|
||||
)
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.scan.scan_directory')
|
||||
def scan_directory(self, source_dir: str = '/import', dest_dir: str = '/images'):
|
||||
"""
|
||||
Scan source directory for new/changed files.
|
||||
|
||||
Creates ImportTask records for each file and queues them for processing.
|
||||
|
||||
Strategy:
|
||||
1. Walk directory tree under each artist folder
|
||||
2. For each file, check if already in ImportTask with same path+size
|
||||
3. If new/changed, create ImportTask and queue appropriate task
|
||||
4. Handle archives specially (first volume only, queue as atomic unit)
|
||||
|
||||
Args:
|
||||
source_dir: Root directory to scan (default: /import)
|
||||
dest_dir: Destination for imported files (default: /images)
|
||||
|
||||
Returns:
|
||||
dict with batch_id and statistics
|
||||
"""
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.utils.image_importer import is_first_volume
|
||||
|
||||
log.info(f"Starting directory scan: {source_dir}")
|
||||
|
||||
# Create batch record
|
||||
batch = ImportBatch(source_directory=source_dir)
|
||||
db.session.add(batch)
|
||||
db.session.commit()
|
||||
|
||||
stats = {
|
||||
'total_files': 0,
|
||||
'queued_media': 0,
|
||||
'queued_archives': 0,
|
||||
'skipped_existing': 0,
|
||||
'skipped_non_first_volume': 0,
|
||||
'skipped_non_media': 0,
|
||||
}
|
||||
|
||||
try:
|
||||
# Iterate through artist directories
|
||||
for artist_dir in os.listdir(source_dir):
|
||||
artist_path = os.path.join(source_dir, artist_dir)
|
||||
if not os.path.isdir(artist_path):
|
||||
continue
|
||||
|
||||
log.debug(f"Scanning artist directory: {artist_dir}")
|
||||
|
||||
# Walk all files in artist directory (including subdirectories)
|
||||
for root, _, files in os.walk(artist_path):
|
||||
for filename in files:
|
||||
full_path = os.path.join(root, filename)
|
||||
lower = filename.lower()
|
||||
stats['total_files'] += 1
|
||||
|
||||
# Skip sidecar JSON files (handled during import)
|
||||
if lower.endswith('.json'):
|
||||
continue
|
||||
|
||||
# Check if archive
|
||||
if lower.endswith(ALLOWED_ARCHIVE_EXTS):
|
||||
# Only process first volume of multi-part archives
|
||||
if not is_first_volume(full_path):
|
||||
stats['skipped_non_first_volume'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue archive task
|
||||
task_record = _create_task_if_new(
|
||||
source_path=full_path,
|
||||
task_type='import_archive',
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
)
|
||||
|
||||
if task_record:
|
||||
result = import_archive.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_archives'] += 1
|
||||
log.debug(f"Queued archive: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
continue
|
||||
|
||||
# Check if media file
|
||||
if not lower.endswith(ALLOWED_MEDIA_EXTS):
|
||||
stats['skipped_non_media'] += 1
|
||||
continue
|
||||
|
||||
# Create and queue media file task
|
||||
task_record = _create_task_if_new(
|
||||
source_path=full_path,
|
||||
task_type='import_image',
|
||||
batch_id=batch.id,
|
||||
context={'artist': artist_dir, 'dest_dir': dest_dir}
|
||||
)
|
||||
|
||||
if task_record:
|
||||
result = import_media_file.delay(task_record.id)
|
||||
task_record.celery_task_id = result.id
|
||||
task_record.status = 'queued'
|
||||
db.session.commit()
|
||||
stats['queued_media'] += 1
|
||||
log.debug(f"Queued media: {filename}")
|
||||
else:
|
||||
stats['skipped_existing'] += 1
|
||||
|
||||
# Update batch with totals
|
||||
batch.total_files = stats['total_files']
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Scan complete. Queued {stats['queued_media']} media, "
|
||||
f"{stats['queued_archives']} archives. "
|
||||
f"Skipped {stats['skipped_existing']} existing.")
|
||||
|
||||
return {
|
||||
'batch_id': batch.id,
|
||||
'stats': stats
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Scan failed: {e}", exc_info=True)
|
||||
batch.status = 'failed'
|
||||
db.session.commit()
|
||||
raise
|
||||
|
||||
|
||||
def _create_task_if_new(source_path: str, task_type: str, batch_id: int, context: dict) -> ImportTask:
|
||||
"""
|
||||
Create ImportTask if no existing task for this file.
|
||||
|
||||
Uses file path and size for change detection.
|
||||
Hash is computed later during actual processing.
|
||||
|
||||
Args:
|
||||
source_path: Full path to source file
|
||||
task_type: Type of task ('import_image', 'import_archive', etc.)
|
||||
batch_id: ID of the current ImportBatch
|
||||
context: Context dict with artist, dest_dir, etc.
|
||||
|
||||
Returns:
|
||||
New ImportTask record if created, None if already exists
|
||||
"""
|
||||
try:
|
||||
file_size = os.path.getsize(source_path)
|
||||
except OSError as e:
|
||||
log.warning(f"Cannot access file {source_path}: {e}")
|
||||
return None
|
||||
|
||||
# Check for existing pending/queued/processing task for same file
|
||||
existing_active = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type
|
||||
).filter(
|
||||
ImportTask.status.in_(['pending', 'queued', 'processing'])
|
||||
).first()
|
||||
|
||||
if existing_active:
|
||||
log.debug(f"Task already active for: {source_path}")
|
||||
return None
|
||||
|
||||
# Check for completed task with same file path and size
|
||||
# If size changed, we should re-import
|
||||
completed_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='complete'
|
||||
).first()
|
||||
|
||||
if completed_same:
|
||||
log.debug(f"Already imported (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Also check 'skipped' status - don't re-queue skipped files unless size changed
|
||||
skipped_same = ImportTask.query.filter_by(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
status='skipped'
|
||||
).first()
|
||||
|
||||
if skipped_same:
|
||||
log.debug(f"Previously skipped (same size): {source_path}")
|
||||
return None
|
||||
|
||||
# Create new task
|
||||
task = ImportTask(
|
||||
source_path=source_path,
|
||||
task_type=task_type,
|
||||
file_size=file_size,
|
||||
batch_id=batch_id,
|
||||
context=context,
|
||||
status='pending'
|
||||
)
|
||||
db.session.add(task)
|
||||
db.session.commit()
|
||||
|
||||
return task
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.scan.recover_interrupted_tasks')
|
||||
def recover_interrupted_tasks():
|
||||
"""
|
||||
Find and re-queue tasks that were interrupted.
|
||||
|
||||
Called periodically by Celery Beat to recover from worker crashes.
|
||||
Tasks with status 'processing' or 'queued' that have no active
|
||||
Celery task are re-queued.
|
||||
|
||||
Returns:
|
||||
dict with count of recovered tasks
|
||||
"""
|
||||
from app.tasks.import_file import import_media_file, import_archive
|
||||
from app.tasks.thumbnail import generate_thumbnail_task
|
||||
|
||||
log.info("Checking for interrupted tasks...")
|
||||
|
||||
# Find tasks that are stuck in processing/queued state
|
||||
# We look for tasks that have been in this state for more than 10 minutes
|
||||
cutoff = datetime.utcnow()
|
||||
|
||||
interrupted = ImportTask.query.filter(
|
||||
ImportTask.status.in_(['processing', 'queued'])
|
||||
).all()
|
||||
|
||||
recovered = 0
|
||||
for task in interrupted:
|
||||
# Check if Celery task is still active
|
||||
if task.celery_task_id:
|
||||
from app.celery_app import celery as celery_app
|
||||
result = celery_app.AsyncResult(task.celery_task_id)
|
||||
# If task is still pending/started, don't re-queue
|
||||
if result.state in ('PENDING', 'STARTED', 'RETRY'):
|
||||
continue
|
||||
|
||||
log.info(f"Recovering task {task.id}: {task.source_path}")
|
||||
|
||||
# Reset task state
|
||||
task.status = 'pending'
|
||||
task.celery_task_id = None
|
||||
task.started_at = None
|
||||
task.retry_count += 1
|
||||
|
||||
# Re-queue based on type
|
||||
if task.task_type == 'import_image':
|
||||
result = import_media_file.delay(task.id)
|
||||
elif task.task_type == 'import_archive':
|
||||
result = import_archive.delay(task.id)
|
||||
elif task.task_type == 'thumbnail':
|
||||
# Thumbnail tasks store image_id in context
|
||||
image_id = task.context.get('image_id') if task.context else None
|
||||
if image_id:
|
||||
result = generate_thumbnail_task.delay(image_id)
|
||||
else:
|
||||
task.status = 'failed'
|
||||
task.error_message = 'Missing image_id in context'
|
||||
db.session.commit()
|
||||
continue
|
||||
else:
|
||||
log.warning(f"Unknown task type: {task.task_type}")
|
||||
continue
|
||||
|
||||
task.celery_task_id = result.id
|
||||
task.status = 'queued'
|
||||
recovered += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
if recovered > 0:
|
||||
log.info(f"Recovered {recovered} interrupted tasks")
|
||||
|
||||
return {'recovered': recovered}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.scan.update_batch_stats')
|
||||
def update_batch_stats(batch_id: int):
|
||||
"""
|
||||
Update statistics for an ImportBatch based on its tasks.
|
||||
|
||||
Args:
|
||||
batch_id: ID of the batch to update
|
||||
"""
|
||||
batch = ImportBatch.query.get(batch_id)
|
||||
if not batch:
|
||||
return
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
# Count tasks by status
|
||||
status_counts = dict(
|
||||
db.session.query(
|
||||
ImportTask.status,
|
||||
func.count(ImportTask.id)
|
||||
).filter(
|
||||
ImportTask.batch_id == batch_id
|
||||
).group_by(ImportTask.status).all()
|
||||
)
|
||||
|
||||
batch.processed_files = sum(
|
||||
status_counts.get(s, 0)
|
||||
for s in ['complete', 'skipped', 'failed']
|
||||
)
|
||||
batch.imported_count = status_counts.get('complete', 0)
|
||||
batch.skipped_count = status_counts.get('skipped', 0)
|
||||
batch.failed_count = status_counts.get('failed', 0)
|
||||
|
||||
# Check if batch is complete
|
||||
pending_or_active = sum(
|
||||
status_counts.get(s, 0)
|
||||
for s in ['pending', 'queued', 'processing']
|
||||
)
|
||||
|
||||
if pending_or_active == 0 and batch.status == 'running':
|
||||
batch.status = 'complete'
|
||||
batch.completed_at = datetime.utcnow()
|
||||
log.info(f"Batch {batch_id} complete: {batch.imported_count} imported, "
|
||||
f"{batch.skipped_count} skipped, {batch.failed_count} failed")
|
||||
|
||||
db.session.commit()
|
||||
@@ -0,0 +1,181 @@
|
||||
# app/tasks/sidecar.py
|
||||
"""
|
||||
Sidecar metadata extraction and application tasks.
|
||||
|
||||
Handles finding and applying metadata from Gallery-DL sidecar JSON files
|
||||
to imported images.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImageRecord, Tag
|
||||
|
||||
log = logging.getLogger('celery.tasks.sidecar')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.sidecar.apply_sidecar_metadata',
|
||||
max_retries=2, default_retry_delay=30)
|
||||
def apply_sidecar_metadata(self, image_id: int, sidecar_path: str):
|
||||
"""
|
||||
Apply metadata from a sidecar JSON file to an image record.
|
||||
|
||||
Extracts platform info, artist, post ID, dates, and generates
|
||||
appropriate tags (source:platform, post:platform:artist:id).
|
||||
|
||||
Args:
|
||||
image_id: ID of the ImageRecord to update
|
||||
sidecar_path: Path to the sidecar JSON file
|
||||
|
||||
Returns:
|
||||
dict with status and applied metadata info
|
||||
"""
|
||||
from app.utils.metadata_enrichment import (
|
||||
enrich_from_sidecar, resolve_date_conflict, load_sidecar_metadata
|
||||
)
|
||||
|
||||
record = ImageRecord.query.get(image_id)
|
||||
if not record:
|
||||
log.warning(f"Image {image_id} not found")
|
||||
return {'error': 'Image not found'}
|
||||
|
||||
try:
|
||||
# Load sidecar data
|
||||
sidecar_data = load_sidecar_metadata(sidecar_path)
|
||||
if not sidecar_data:
|
||||
return {'status': 'no_metadata', 'reason': 'Failed to load sidecar'}
|
||||
|
||||
# Get enriched metadata
|
||||
enriched = enrich_from_sidecar(sidecar_path, {'taken_at': record.taken_at})
|
||||
|
||||
if not enriched.get('raw_metadata'):
|
||||
return {'status': 'no_metadata', 'reason': 'No metadata extracted'}
|
||||
|
||||
tags_added = 0
|
||||
|
||||
# Update date if earlier
|
||||
new_date = resolve_date_conflict(record.taken_at, enriched.get('taken_at'))
|
||||
if new_date != record.taken_at:
|
||||
record.taken_at = new_date
|
||||
log.debug(f"Updated date for {image_id}: {new_date}")
|
||||
|
||||
# Apply tags from metadata
|
||||
for tag_info in enriched.get('tags', []):
|
||||
tag_name = tag_info.get('name')
|
||||
tag_kind = tag_info.get('kind')
|
||||
if not tag_name:
|
||||
continue
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind=tag_kind)
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
log.debug(f"Created tag: {tag_name} ({tag_kind})")
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
tags_added += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
log.info(f"Applied sidecar metadata to {image_id}: "
|
||||
f"platform={enriched.get('platform')}, tags_added={tags_added}")
|
||||
|
||||
return {
|
||||
'status': 'applied',
|
||||
'platform': enriched.get('platform'),
|
||||
'artist': enriched.get('artist'),
|
||||
'post_id': enriched.get('post_id'),
|
||||
'tags_added': tags_added
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Sidecar application failed for {image_id}: {e}", exc_info=True)
|
||||
raise self.retry(exc=e)
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.sidecar.scan_and_match_sidecars')
|
||||
def scan_and_match_sidecars(batch_id: int = None, limit: int = 1000):
|
||||
"""
|
||||
Scan for sidecar files and match them to imported images.
|
||||
|
||||
Looks at recently imported images and tries to find corresponding
|
||||
sidecar JSON files for metadata enrichment.
|
||||
|
||||
Args:
|
||||
batch_id: Optional batch ID to limit scope
|
||||
limit: Maximum number of images to process
|
||||
|
||||
Returns:
|
||||
dict with count of matched sidecars
|
||||
"""
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
log.info(f"Scanning for unmatched sidecars (batch={batch_id}, limit={limit})")
|
||||
|
||||
# Build query for images that might need sidecar matching
|
||||
query = ImageRecord.query.filter(
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.imported_at.desc())
|
||||
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
|
||||
matched = 0
|
||||
already_has_source = 0
|
||||
|
||||
for record in query.all():
|
||||
# Check if already has source tag (indicates sidecar was processed)
|
||||
has_source_tag = any(
|
||||
tag.kind == 'source' or tag.kind == 'post'
|
||||
for tag in record.tags
|
||||
)
|
||||
if has_source_tag:
|
||||
already_has_source += 1
|
||||
continue
|
||||
|
||||
# Try to find sidecar
|
||||
sidecar_path = find_sidecar_json(record.filepath)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
matched += 1
|
||||
|
||||
log.info(f"Sidecar scan complete: matched={matched}, already_tagged={already_has_source}")
|
||||
return {'matched': matched, 'already_tagged': already_has_source}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.sidecar.reprocess_sidecars_for_tag')
|
||||
def reprocess_sidecars_for_tag(tag_name: str):
|
||||
"""
|
||||
Reprocess sidecar metadata for all images with a specific tag.
|
||||
|
||||
Useful for re-running sidecar extraction after updates to
|
||||
the metadata extraction logic.
|
||||
|
||||
Args:
|
||||
tag_name: Name of the tag to filter by
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
from app.utils.metadata_enrichment import find_sidecar_json
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
return {'error': f'Tag not found: {tag_name}'}
|
||||
|
||||
queued = 0
|
||||
for record in tag.images:
|
||||
if not record.filepath:
|
||||
continue
|
||||
|
||||
sidecar_path = find_sidecar_json(record.filepath)
|
||||
if sidecar_path:
|
||||
apply_sidecar_metadata.delay(record.id, sidecar_path)
|
||||
queued += 1
|
||||
|
||||
log.info(f"Queued {queued} sidecar reprocessing tasks for tag: {tag_name}")
|
||||
return {'queued': queued, 'tag': tag_name}
|
||||
@@ -0,0 +1,153 @@
|
||||
# app/tasks/thumbnail.py
|
||||
"""
|
||||
Thumbnail generation tasks.
|
||||
|
||||
Handles generating thumbnails for images and videos,
|
||||
both individually and in bulk.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from app.celery_app import celery
|
||||
from app import db
|
||||
from app.models import ImageRecord, ImportTask
|
||||
|
||||
log = logging.getLogger('celery.tasks.thumbnail')
|
||||
|
||||
|
||||
@celery.task(bind=True, name='app.tasks.thumbnail.generate_thumbnail_task',
|
||||
max_retries=2, default_retry_delay=30)
|
||||
def generate_thumbnail_task(self, image_id: int, overwrite: bool = False):
|
||||
"""
|
||||
Generate thumbnail for a single image/video.
|
||||
|
||||
Args:
|
||||
image_id: ID of the ImageRecord
|
||||
overwrite: Whether to overwrite existing thumbnail
|
||||
|
||||
Returns:
|
||||
dict with status and thumbnail path
|
||||
"""
|
||||
from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail_mirrored
|
||||
|
||||
record = ImageRecord.query.get(image_id)
|
||||
if not record:
|
||||
log.warning(f"Image {image_id} not found")
|
||||
return {'error': 'Image not found'}
|
||||
|
||||
try:
|
||||
filepath = record.filepath
|
||||
if not filepath:
|
||||
return {'error': 'No filepath'}
|
||||
|
||||
# Check if thumbnail already exists and we're not overwriting
|
||||
if record.thumb_path and not overwrite:
|
||||
import os
|
||||
if os.path.exists(record.thumb_path):
|
||||
return {'status': 'exists', 'thumb_path': record.thumb_path}
|
||||
|
||||
log.debug(f"Generating thumbnail for: {filepath}")
|
||||
|
||||
is_video = filepath.lower().endswith(('.mp4', '.mov', '.webm', '.avi', '.mkv'))
|
||||
|
||||
if is_video:
|
||||
thumb_path = generate_video_thumbnail_mirrored(filepath)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(filepath, overwrite=overwrite)
|
||||
|
||||
if thumb_path:
|
||||
record.thumb_path = thumb_path
|
||||
db.session.commit()
|
||||
log.info(f"Generated thumbnail: {thumb_path}")
|
||||
return {'status': 'success', 'thumb_path': thumb_path}
|
||||
else:
|
||||
return {'status': 'failed', 'error': 'Thumbnail generation returned None'}
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log.error(f"Thumbnail generation failed for {image_id}: {e}", exc_info=True)
|
||||
raise self.retry(exc=e)
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.thumbnail.regenerate_all_thumbnails')
|
||||
def regenerate_all_thumbnails(batch_size: int = 100, overwrite: bool = True):
|
||||
"""
|
||||
Queue thumbnail regeneration for all images.
|
||||
|
||||
Processes images in batches to avoid memory issues.
|
||||
|
||||
Args:
|
||||
batch_size: Number of images to process per batch
|
||||
overwrite: Whether to overwrite existing thumbnails
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
log.info("Starting bulk thumbnail regeneration")
|
||||
|
||||
count = 0
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
records = ImageRecord.query.filter(
|
||||
ImageRecord.id > last_id,
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.id.asc()).limit(batch_size).all()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for record in records:
|
||||
generate_thumbnail_task.delay(record.id, overwrite=overwrite)
|
||||
count += 1
|
||||
|
||||
last_id = records[-1].id
|
||||
log.info(f"Queued {count} thumbnail tasks so far...")
|
||||
|
||||
log.info(f"Thumbnail regeneration complete: queued {count} tasks")
|
||||
return {'queued': count}
|
||||
|
||||
|
||||
@celery.task(name='app.tasks.thumbnail.regenerate_missing_thumbnails')
|
||||
def regenerate_missing_thumbnails(batch_size: int = 100):
|
||||
"""
|
||||
Generate thumbnails only for images that don't have them.
|
||||
|
||||
Args:
|
||||
batch_size: Number of images to process per batch
|
||||
|
||||
Returns:
|
||||
dict with count of queued tasks
|
||||
"""
|
||||
import os
|
||||
|
||||
log.info("Checking for missing thumbnails")
|
||||
|
||||
count = 0
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
records = ImageRecord.query.filter(
|
||||
ImageRecord.id > last_id,
|
||||
ImageRecord.filepath.isnot(None)
|
||||
).order_by(ImageRecord.id.asc()).limit(batch_size).all()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for record in records:
|
||||
# Check if thumbnail is missing
|
||||
needs_thumb = False
|
||||
if not record.thumb_path:
|
||||
needs_thumb = True
|
||||
elif not os.path.exists(record.thumb_path):
|
||||
needs_thumb = True
|
||||
|
||||
if needs_thumb:
|
||||
generate_thumbnail_task.delay(record.id, overwrite=False)
|
||||
count += 1
|
||||
|
||||
last_id = records[-1].id
|
||||
|
||||
log.info(f"Missing thumbnail check complete: queued {count} tasks")
|
||||
return {'queued': count}
|
||||
@@ -206,7 +206,140 @@
|
||||
</div>
|
||||
</div><!-- end settings-grid -->
|
||||
|
||||
<!-- Import Queue Status (Full Width) -->
|
||||
<div class="settings-container" style="max-width: 1200px; margin: 2rem auto;">
|
||||
<div class="settings-section">
|
||||
<h2>Import Queue Status</h2>
|
||||
<p class="settings-description">Monitor and control the Celery-based import queue system.</p>
|
||||
|
||||
<div id="queueStatusSection">
|
||||
<div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queuePending">-</span>
|
||||
<span class="stat-label">Pending</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queueQueued">-</span>
|
||||
<span class="stat-label">Queued</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queueProcessing">-</span>
|
||||
<span class="stat-label">Processing</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queueComplete">-</span>
|
||||
<span class="stat-label">Complete</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queueSkipped">-</span>
|
||||
<span class="stat-label">Skipped</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-value" id="queueFailed">-</span>
|
||||
<span class="stat-label">Failed</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
|
||||
<h3>Active Batch</h3>
|
||||
<p>Processing <strong id="batchDir">-</strong></p>
|
||||
<div class="batch-progress">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
|
||||
</div>
|
||||
<span id="batchProgressText">0 / 0</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
|
||||
<button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
|
||||
<button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
|
||||
<button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
|
||||
<button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
|
||||
<a href="/flower" target="_blank" class="btn secondary-btn" style="text-decoration: none;">Open Flower Dashboard</a>
|
||||
</div>
|
||||
|
||||
<div id="celeryWorkerInfo" style="margin-top: 1rem;">
|
||||
<p style="color: var(--text-muted); font-size: 0.85rem;">
|
||||
Workers: <strong id="workerCount">-</strong> |
|
||||
Active Tasks: <strong id="activeTasks">-</strong>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Recent Tasks Table -->
|
||||
<div id="recentTasksSection" style="margin-top: 1.5rem;">
|
||||
<h3>Recent Tasks</h3>
|
||||
<div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
|
||||
<table class="tasks-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>File</th>
|
||||
<th>Type</th>
|
||||
<th>Status</th>
|
||||
<th>Error</th>
|
||||
<th>Time</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="recentTasksBody">
|
||||
<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.tasks-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
.tasks-table th,
|
||||
.tasks-table td {
|
||||
padding: 0.5rem 0.75rem;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
.tasks-table th {
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
color: var(--text-dim);
|
||||
font-weight: 500;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
}
|
||||
.tasks-table td {
|
||||
color: var(--text);
|
||||
}
|
||||
.tasks-table .status-pending { color: #9ca3af; }
|
||||
.tasks-table .status-queued { color: #60a5fa; }
|
||||
.tasks-table .status-processing { color: #fbbf24; }
|
||||
.tasks-table .status-complete { color: #34d399; }
|
||||
.tasks-table .status-skipped { color: #a78bfa; }
|
||||
.tasks-table .status-failed { color: #f87171; }
|
||||
.batch-progress {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
margin-top: 0.5rem;
|
||||
}
|
||||
.progress-bar {
|
||||
flex: 1;
|
||||
height: 8px;
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
background: var(--btn-primary);
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
.queue-actions .btn {
|
||||
min-width: auto;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.settings-description {
|
||||
color: var(--text-muted);
|
||||
margin-bottom: 1rem;
|
||||
@@ -363,6 +496,177 @@
|
||||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// ========================================
|
||||
// Celery Queue Status
|
||||
// ========================================
|
||||
async function loadQueueStatus() {
|
||||
try {
|
||||
const r = await fetch('/api/import/status');
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
// Update status counts
|
||||
document.getElementById('queuePending').textContent = data.status_counts.pending || 0;
|
||||
document.getElementById('queueQueued').textContent = data.status_counts.queued || 0;
|
||||
document.getElementById('queueProcessing').textContent = data.status_counts.processing || 0;
|
||||
document.getElementById('queueComplete').textContent = data.status_counts.complete || 0;
|
||||
document.getElementById('queueSkipped').textContent = data.status_counts.skipped || 0;
|
||||
document.getElementById('queueFailed').textContent = data.status_counts.failed || 0;
|
||||
|
||||
// Update active batch info
|
||||
const batchEl = document.getElementById('activeBatchInfo');
|
||||
if (data.active_batch) {
|
||||
batchEl.style.display = 'block';
|
||||
document.getElementById('batchDir').textContent = data.active_batch.source_directory;
|
||||
const processed = data.active_batch.processed_files || 0;
|
||||
const total = data.active_batch.total_files || 1;
|
||||
const pct = Math.round((processed / total) * 100);
|
||||
document.getElementById('batchProgressFill').style.width = pct + '%';
|
||||
document.getElementById('batchProgressText').textContent = `${processed} / ${total}`;
|
||||
} else {
|
||||
batchEl.style.display = 'none';
|
||||
}
|
||||
|
||||
// Update recent tasks table
|
||||
const tbody = document.getElementById('recentTasksBody');
|
||||
if (data.recent_tasks && data.recent_tasks.length > 0) {
|
||||
tbody.innerHTML = data.recent_tasks.map(t => `
|
||||
<tr>
|
||||
<td title="${t.source_path || ''}">${t.source_path || '-'}</td>
|
||||
<td>${t.task_type || '-'}</td>
|
||||
<td class="status-${t.status}">${t.status || '-'}</td>
|
||||
<td title="${t.error_message || ''}">${t.error_message ? t.error_message.substring(0, 30) + '...' : '-'}</td>
|
||||
<td>${t.created_at ? new Date(t.created_at).toLocaleString() : '-'}</td>
|
||||
</tr>
|
||||
`).join('');
|
||||
} else {
|
||||
tbody.innerHTML = '<tr><td colspan="5" style="text-align: center; color: var(--text-muted);">No recent tasks</td></tr>';
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Failed to load queue status:', e);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadCeleryStatus() {
|
||||
try {
|
||||
const r = await fetch('/api/celery/status');
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
document.getElementById('workerCount').textContent = data.worker_count || 0;
|
||||
document.getElementById('activeTasks').textContent = data.active_tasks || 0;
|
||||
} else {
|
||||
document.getElementById('workerCount').textContent = '?';
|
||||
document.getElementById('activeTasks').textContent = '?';
|
||||
}
|
||||
} catch (e) {
|
||||
document.getElementById('workerCount').textContent = 'offline';
|
||||
document.getElementById('activeTasks').textContent = '-';
|
||||
}
|
||||
}
|
||||
|
||||
// Load queue status on page load and refresh periodically
|
||||
loadQueueStatus();
|
||||
loadCeleryStatus();
|
||||
setInterval(loadQueueStatus, 5000); // Refresh every 5 seconds
|
||||
setInterval(loadCeleryStatus, 10000); // Refresh every 10 seconds
|
||||
|
||||
// Trigger Import Scan
|
||||
document.getElementById('triggerImportBtn').addEventListener('click', async () => {
|
||||
const btn = document.getElementById('triggerImportBtn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Starting...';
|
||||
|
||||
try {
|
||||
const r = await fetch('/api/import/trigger', { method: 'POST' });
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
alert('Import scan started! Task ID: ' + data.task_id);
|
||||
loadQueueStatus();
|
||||
} else {
|
||||
alert('Failed to start import: ' + (data.error || 'Unknown error'));
|
||||
}
|
||||
} catch (e) {
|
||||
alert('Failed to start import: ' + e.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Trigger Import Scan';
|
||||
}
|
||||
});
|
||||
|
||||
// Retry Failed Tasks
|
||||
document.getElementById('retryFailedBtn').addEventListener('click', async () => {
|
||||
const btn = document.getElementById('retryFailedBtn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Retrying...';
|
||||
|
||||
try {
|
||||
const r = await fetch('/api/import/retry-failed', { method: 'POST' });
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
alert(`Retried ${data.retried} failed task(s).`);
|
||||
loadQueueStatus();
|
||||
} else {
|
||||
alert('Failed to retry: ' + (data.error || 'Unknown error'));
|
||||
}
|
||||
} catch (e) {
|
||||
alert('Failed to retry: ' + e.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Retry Failed Tasks';
|
||||
}
|
||||
});
|
||||
|
||||
// Clear Completed Tasks
|
||||
document.getElementById('clearCompletedBtn').addEventListener('click', async () => {
|
||||
const btn = document.getElementById('clearCompletedBtn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Clearing...';
|
||||
|
||||
try {
|
||||
const r = await fetch('/api/import/clear-completed', { method: 'POST' });
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
alert(`Cleared ${data.deleted} completed/skipped task(s).`);
|
||||
loadQueueStatus();
|
||||
} else {
|
||||
alert('Failed to clear: ' + (data.error || 'Unknown error'));
|
||||
}
|
||||
} catch (e) {
|
||||
alert('Failed to clear: ' + e.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Clear Completed';
|
||||
}
|
||||
});
|
||||
|
||||
// Regenerate Missing Thumbnails
|
||||
document.getElementById('regenMissingThumbsBtn').addEventListener('click', async () => {
|
||||
const btn = document.getElementById('regenMissingThumbsBtn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Starting...';
|
||||
|
||||
try {
|
||||
const r = await fetch('/api/thumbnails/regenerate-missing', { method: 'POST' });
|
||||
const data = await r.json();
|
||||
|
||||
if (data.ok) {
|
||||
alert('Missing thumbnail generation started! Task ID: ' + data.task_id);
|
||||
} else {
|
||||
alert('Failed to start: ' + (data.error || 'Unknown error'));
|
||||
}
|
||||
} catch (e) {
|
||||
alert('Failed to start: ' + e.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = 'Regenerate Missing Thumbnails';
|
||||
}
|
||||
});
|
||||
|
||||
// Load import settings
|
||||
fetch('/api/import-settings')
|
||||
.then(r => r.json())
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
{%- elif active_kind == 'series' -%}Series
|
||||
{%- elif active_kind == 'rating' -%}Ratings
|
||||
{%- elif active_kind == 'archive' -%}Archives
|
||||
{%- elif active_kind == 'source' -%}Sources
|
||||
{%- elif active_kind == 'post' -%}Posts
|
||||
{%- elif active_kind == 'user' -%}User Tags
|
||||
{%- else -%}Tags{%- endif -%}
|
||||
{% endblock %}
|
||||
@@ -16,6 +18,8 @@
|
||||
{%- elif active_kind == 'series' -%}Series
|
||||
{%- elif active_kind == 'rating' -%}Ratings
|
||||
{%- elif active_kind == 'archive' -%}Archives
|
||||
{%- elif active_kind == 'source' -%}Sources
|
||||
{%- elif active_kind == 'post' -%}Posts
|
||||
{%- elif active_kind == 'user' -%}User Tags
|
||||
{%- else -%}All Tags{%- endif -%}
|
||||
</h1>
|
||||
@@ -46,6 +50,14 @@
|
||||
href="{{ url_for('main.tag_list', kind='archive') }}"
|
||||
aria-current="{{ 'page' if active_kind=='archive' else 'false' }}">🗜️ Archives</a>
|
||||
|
||||
<a class="filter-btn {{ 'is-active' if active_kind=='source' }}"
|
||||
href="{{ url_for('main.tag_list', kind='source') }}"
|
||||
aria-current="{{ 'page' if active_kind=='source' else 'false' }}">🌐 Sources</a>
|
||||
|
||||
<a class="filter-btn {{ 'is-active' if active_kind=='post' }}"
|
||||
href="{{ url_for('main.tag_list', kind='post') }}"
|
||||
aria-current="{{ 'page' if active_kind=='post' else 'false' }}">📌 Posts</a>
|
||||
|
||||
<a class="filter-btn {{ 'is-active' if active_kind=='user' }}"
|
||||
href="{{ url_for('main.tag_list', kind='user') }}"
|
||||
aria-current="{{ 'page' if active_kind=='user' else 'false' }}"># User</a>
|
||||
@@ -58,9 +70,10 @@
|
||||
<a href="{{ url_for('main.gallery', tag=tag.tag) }}" class="tag-card-link">
|
||||
<div class="tag-preview">
|
||||
{% for image in tag.images %}
|
||||
<div class="tag-thumb"
|
||||
style="background-image: url('{{ url_for('main.serve_image', filename=(image.thumb_path or image.filepath) | replace('/images/', '') ) }}');">
|
||||
</div>
|
||||
<img class="tag-thumb"
|
||||
src="{{ url_for('main.serve_image', filename=(image.thumb_path or image.filepath) | replace('/images/', '') ) }}"
|
||||
loading="lazy"
|
||||
alt="">
|
||||
{% endfor %}
|
||||
{% if tag.images|length == 0 %}
|
||||
<div class="tag-thumb tag-thumb-empty"></div>
|
||||
@@ -73,6 +86,8 @@
|
||||
{%- elif tag.kind == 'series' -%}📺
|
||||
{%- elif tag.kind == 'rating' -%}⚠️
|
||||
{%- elif tag.kind == 'archive' -%}🗜️
|
||||
{%- elif tag.kind == 'source' -%}🌐
|
||||
{%- elif tag.kind == 'post' -%}📌
|
||||
{%- else -%}#{%- endif -%}
|
||||
</span>
|
||||
<span class="tag-name" title="{{ tag.name }}">{{ tag.name }}</span>
|
||||
@@ -109,6 +124,8 @@
|
||||
<option value="series">📺 Series</option>
|
||||
<option value="rating">⚠️ Rating</option>
|
||||
<option value="archive">🗜️ Archive</option>
|
||||
<option value="source">🌐 Source</option>
|
||||
<option value="post">📌 Post</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -28,12 +28,10 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
"""
|
||||
Find the Gallery-DL sidecar JSON file for an image.
|
||||
|
||||
Gallery-DL creates JSON files with the UUID portion of the filename.
|
||||
For example:
|
||||
Image: 01_BBEFD45F-8775-4DF9-8BCA-99E6063E3616.jpg
|
||||
JSON: BBEFD45F-8775-4DF9-8BCA-99E6063E3616.json
|
||||
|
||||
Also checks for direct {filename}.json pattern.
|
||||
Gallery-DL creates JSON files with various naming patterns:
|
||||
- Patreon: {filename}.json in same directory
|
||||
- HentaiFoundry: {artist}-{index}-{title}.json (may differ from image filename)
|
||||
- UUID-based: {uuid}.json where UUID is extracted from filename
|
||||
|
||||
Returns the path to the sidecar JSON if found, None otherwise.
|
||||
"""
|
||||
@@ -41,12 +39,18 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
parent_dir = image_path.parent
|
||||
stem = image_path.stem # filename without extension
|
||||
|
||||
# Pattern 1: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
|
||||
# Pattern 1: Same stem with .json extension (most common)
|
||||
# e.g., "Artist-12345-title.jpg" -> "Artist-12345-title.json"
|
||||
stem_json = parent_dir / f"{stem}.json"
|
||||
if stem_json.exists():
|
||||
return str(stem_json)
|
||||
|
||||
# Pattern 2: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
|
||||
direct_json = image_path.with_suffix(image_path.suffix + ".json")
|
||||
if direct_json.exists():
|
||||
return str(direct_json)
|
||||
|
||||
# Pattern 2: Extract UUID from filename and look for {uuid}.json
|
||||
# Pattern 3: Extract UUID from filename and look for {uuid}.json
|
||||
# Common patterns: "01_UUID", "UUID", "{num}_{UUID}"
|
||||
uuid_pattern = re.compile(r'([A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12})', re.IGNORECASE)
|
||||
match = uuid_pattern.search(stem)
|
||||
@@ -56,7 +60,7 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
if uuid_json.exists():
|
||||
return str(uuid_json)
|
||||
|
||||
# Pattern 3: Just the stem without prefix numbers
|
||||
# Pattern 4: Just the stem without prefix numbers
|
||||
# e.g., "01_filename" -> look for "filename.json"
|
||||
stem_no_prefix = re.sub(r'^\d+_', '', stem)
|
||||
if stem_no_prefix != stem:
|
||||
@@ -64,6 +68,18 @@ def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
if alt_json.exists():
|
||||
return str(alt_json)
|
||||
|
||||
# Pattern 5: Extract numeric ID from filename and find JSON with same ID
|
||||
# Handles HentaiFoundry where image is "hentaifoundry_1000220_title.jpg"
|
||||
# but JSON is "Artist-1000220-title.json"
|
||||
# Look for a 5-7 digit number that could be a post ID
|
||||
id_match = re.search(r'[_-](\d{5,8})[_-]', stem)
|
||||
if id_match:
|
||||
post_id = id_match.group(1)
|
||||
# Search for any JSON file in the directory containing this ID
|
||||
for json_file in parent_dir.glob("*.json"):
|
||||
if f"-{post_id}-" in json_file.name or f"_{post_id}_" in json_file.name:
|
||||
return str(json_file)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -122,9 +138,14 @@ def extract_artist(metadata: dict) -> Optional[str]:
|
||||
|
||||
def extract_post_id(metadata: dict) -> Optional[str]:
|
||||
"""Extract the post ID from metadata."""
|
||||
# Patreon, general
|
||||
if "id" in metadata:
|
||||
return str(metadata["id"])
|
||||
|
||||
# HentaiFoundry uses "index" as the post ID
|
||||
if "index" in metadata:
|
||||
return str(metadata["index"])
|
||||
|
||||
if "post_id" in metadata:
|
||||
return str(metadata["post_id"])
|
||||
|
||||
|
||||
@@ -28,3 +28,7 @@ class Config:
|
||||
|
||||
SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
|
||||
# Celery configuration
|
||||
CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/0')
|
||||
CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0')
|
||||
CELERY_WORKER_CONCURRENCY = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2'))
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# docker-compose.sqlite.yml
|
||||
# Development/testing configuration using SQLite instead of PostgreSQL
|
||||
# Usage: docker-compose -f docker-compose.sqlite.yml up
|
||||
#
|
||||
# NOTE: SQLite does not support concurrent writes well.
|
||||
# This configuration is for development/testing only.
|
||||
# For production, use the main docker-compose.yml with PostgreSQL.
|
||||
|
||||
services:
|
||||
# Redis message broker for Celery
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# Flask web application (SQLite mode)
|
||||
web:
|
||||
build: .
|
||||
ports:
|
||||
- "5000:5000"
|
||||
environment:
|
||||
- DB_TYPE=sqlite
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
- RUN_IMAGE_IMPORTER=0 # Disable old polling worker
|
||||
volumes:
|
||||
- ./imagerepo/db/:/db
|
||||
- ./imagerepo/images:/images
|
||||
- ./import:/import
|
||||
- ./migrations:/app/migrations
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# Single Celery worker (SQLite needs single worker to avoid write conflicts)
|
||||
celery-worker:
|
||||
build: .
|
||||
command: celery -A app.celery_app:celery worker --loglevel=info -Q scan,import,thumbnail,sidecar,default --concurrency=1
|
||||
environment:
|
||||
- DB_TYPE=sqlite
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
volumes:
|
||||
- ./imagerepo/db/:/db
|
||||
- ./imagerepo/images:/images
|
||||
- ./import:/import
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# Celery Beat scheduler (optional for development)
|
||||
celery-beat:
|
||||
build: .
|
||||
command: celery -A app.celery_app:celery beat --loglevel=info
|
||||
environment:
|
||||
- DB_TYPE=sqlite
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
- IMPORT_EVERY_SECONDS=28800
|
||||
volumes:
|
||||
- ./imagerepo/db/:/db
|
||||
depends_on:
|
||||
- redis
|
||||
- celery-worker
|
||||
restart: unless-stopped
|
||||
|
||||
# Flower monitoring (optional)
|
||||
flower:
|
||||
build: .
|
||||
command: celery -A app.celery_app:celery flower --port=5555
|
||||
ports:
|
||||
- "5555:5555"
|
||||
environment:
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
depends_on:
|
||||
- redis
|
||||
restart: unless-stopped
|
||||
+97
-4
@@ -1,10 +1,103 @@
|
||||
services:
|
||||
# Redis message broker for Celery
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# PostgreSQL database (production-ready)
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: ${DB_USER:-imagerepo}
|
||||
POSTGRES_PASSWORD: ${DB_PASS:-postgres}
|
||||
POSTGRES_DB: ${DB_NAME:-imagerepo}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-imagerepo}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# Flask web application
|
||||
web:
|
||||
build: .
|
||||
ports:
|
||||
- "5000:5000"
|
||||
environment:
|
||||
- DB_TYPE=postgresql
|
||||
- DB_USER=${DB_USER:-imagerepo}
|
||||
- DB_PASS=${DB_PASS:-postgres}
|
||||
- DB_HOST=postgres
|
||||
- DB_PORT=5432
|
||||
- DB_NAME=${DB_NAME:-imagerepo}
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
volumes:
|
||||
- ./imagerepo/db/:/db # SQLite db storage
|
||||
- ./imagerepo/images:/images # where the app will store it's imported images
|
||||
- ./import:/import # where it will import images from
|
||||
- ./migrations:/app/migrations # for dev
|
||||
- ./imagerepo/db/:/db # Legacy SQLite path (not used with PostgreSQL)
|
||||
- ./imagerepo/images:/images
|
||||
- ./import:/import
|
||||
- ./migrations:/app/migrations # For dev
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# Celery worker for processing import tasks
|
||||
celery-worker:
|
||||
build: .
|
||||
command: celery -A app.celery_app:celery worker --loglevel=info -Q scan,import,thumbnail,sidecar,default --concurrency=${CELERY_WORKER_CONCURRENCY:-2}
|
||||
environment:
|
||||
- DB_TYPE=postgresql
|
||||
- DB_USER=${DB_USER:-imagerepo}
|
||||
- DB_PASS=${DB_PASS:-postgres}
|
||||
- DB_HOST=postgres
|
||||
- DB_PORT=5432
|
||||
- DB_NAME=${DB_NAME:-imagerepo}
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
- CELERY_WORKER_CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-2}
|
||||
volumes:
|
||||
- ./imagerepo/images:/images
|
||||
- ./import:/import
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
# Celery Beat scheduler for periodic tasks
|
||||
celery-beat:
|
||||
build: .
|
||||
command: celery -A app.celery_app:celery beat --loglevel=info
|
||||
environment:
|
||||
- DB_TYPE=postgresql
|
||||
- DB_USER=${DB_USER:-imagerepo}
|
||||
- DB_PASS=${DB_PASS:-postgres}
|
||||
- DB_HOST=postgres
|
||||
- DB_PORT=5432
|
||||
- DB_NAME=${DB_NAME:-imagerepo}
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
- IMPORT_EVERY_SECONDS=${IMPORT_EVERY_SECONDS:-28800}
|
||||
depends_on:
|
||||
- redis
|
||||
- celery-worker
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
redis_data:
|
||||
postgres_data:
|
||||
|
||||
+8
-6
@@ -5,6 +5,14 @@ set -e
|
||||
export FLASK_APP=run.py
|
||||
export FLASK_ENV=production
|
||||
|
||||
# If arguments are passed and first arg is NOT "web", execute them directly
|
||||
# This allows: docker run <image> celery -A app.celery_app:celery worker ...
|
||||
if [ $# -gt 0 ] && [ "$1" != "web" ]; then
|
||||
exec "$@"
|
||||
fi
|
||||
|
||||
# --- Web service mode (default or explicit "web" argument) ---
|
||||
|
||||
# --- Migrations with simple retry (no pg_isready needed) ---
|
||||
if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then
|
||||
ATTEMPTS="${MIGRATION_MAX_ATTEMPTS:-30}" # ~90s default with 3s sleep
|
||||
@@ -26,12 +34,6 @@ if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then
|
||||
flask version-check
|
||||
fi
|
||||
|
||||
# --- Start the image import worker exactly once per container ---
|
||||
if [ "${RUN_IMAGE_IMPORTER:-1}" = "1" ]; then
|
||||
echo "Starting image import worker (background)…"
|
||||
python image_import_worker.py &
|
||||
fi
|
||||
|
||||
# --- Start Gunicorn ---
|
||||
GUNICORN_BIND="${GUNICORN_BIND:-0.0.0.0:5000}"
|
||||
GUNICORN_WORKERS="${GUNICORN_WORKERS:-8}"
|
||||
|
||||
@@ -1,451 +0,0 @@
|
||||
# image_import_worker.py
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app import create_app
|
||||
from app.utils.image_importer import import_images_task
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Paths & basic config
|
||||
# -------------------------------------------------------------------
|
||||
THUMBNAIL_FLAG = "/import/thumbnail.flag"
|
||||
TRIGGER_FLAG = "/import/trigger.flag"
|
||||
SOURCE_DIR = "/import"
|
||||
DEST_DIR = "/images"
|
||||
|
||||
CHECK_INTERVAL = int(os.getenv("CHECK_INTERVAL", "10")) # seconds
|
||||
|
||||
# Periodic import interval (default 8 hours)
|
||||
IMPORT_EVERY_SECONDS = int(os.getenv("IMPORT_EVERY_SECONDS", str(8 * 60 * 60)))
|
||||
|
||||
# Thumbnail verbosity controls
|
||||
THUMBS_VERBOSE = os.getenv("THUMBS_VERBOSE", "0").lower() in ("1", "true", "yes")
|
||||
THUMBS_LOG_EVERY = int(os.getenv("THUMBS_LOG_EVERY", "50"))
|
||||
THUMBS_BATCH_SIZE = int(os.getenv("THUMBS_BATCH_SIZE", "500"))
|
||||
|
||||
MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Logging
|
||||
# -------------------------------------------------------------------
|
||||
logging.basicConfig(
|
||||
level=os.getenv("LOG_LEVEL", "INFO").upper(),
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
log = logging.getLogger("image-import-worker")
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Job / flag helpers
|
||||
# -------------------------------------------------------------------
|
||||
@dataclass(frozen=True)
|
||||
class FlagJob:
|
||||
name: str
|
||||
flag_path: str
|
||||
|
||||
@property
|
||||
def run_path(self) -> str:
|
||||
return self.flag_path + ".run"
|
||||
|
||||
@property
|
||||
def err_path(self) -> str:
|
||||
return self.flag_path + ".err"
|
||||
|
||||
|
||||
def claim_job(job: FlagJob) -> bool:
|
||||
"""
|
||||
Atomically claim a job by renaming <flag> -> <flag>.run.
|
||||
Returns True if claimed, False otherwise.
|
||||
"""
|
||||
try:
|
||||
os.replace(job.flag_path, job.run_path)
|
||||
log.info("[%s] Claimed job -> %s", job.name, job.run_path)
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
# Flag disappeared between exists() check and replace()
|
||||
return False
|
||||
except Exception as e:
|
||||
log.warning("[%s] Could not claim job: %s", job.name, e)
|
||||
return False
|
||||
|
||||
|
||||
def complete_job(job: FlagJob) -> None:
|
||||
"""Remove the .run file on success."""
|
||||
try:
|
||||
os.remove(job.run_path)
|
||||
log.info("[%s] Cleared run flag.", job.name)
|
||||
except FileNotFoundError:
|
||||
log.info("[%s] Run flag already cleared.", job.name)
|
||||
except Exception as e:
|
||||
log.warning("[%s] Could not remove run flag: %s", job.name, e)
|
||||
|
||||
|
||||
def fail_job(job: FlagJob, error_text: Optional[str] = None) -> None:
|
||||
"""
|
||||
Move .run -> .err and write error details into .err.
|
||||
"""
|
||||
import datetime
|
||||
|
||||
try:
|
||||
os.replace(job.run_path, job.err_path)
|
||||
log.error("[%s] Marked failure -> %s", job.name, job.err_path)
|
||||
|
||||
if error_text:
|
||||
try:
|
||||
with open(job.err_path, "a", encoding="utf-8") as f:
|
||||
f.write("\n")
|
||||
f.write(datetime.datetime.utcnow().isoformat() + "Z\n")
|
||||
f.write(error_text)
|
||||
f.write("\n")
|
||||
except Exception as e:
|
||||
log.warning("[%s] Could not write error details: %s", job.name, e)
|
||||
|
||||
except FileNotFoundError:
|
||||
log.error("[%s] Could not mark failure: run flag missing.", job.name)
|
||||
except Exception as e:
|
||||
log.error("[%s] Could not mark failure: %s", job.name, e)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# DB helpers
|
||||
# -------------------------------------------------------------------
|
||||
def db_ping() -> None:
|
||||
"""Validate DB session connection quickly."""
|
||||
from sqlalchemy import text
|
||||
from app import db
|
||||
|
||||
db.session.execute(text("SELECT 1"))
|
||||
|
||||
|
||||
def recover_db_session(dispose_engine: bool = False) -> None:
|
||||
"""
|
||||
Rollback session and optionally dispose engine pool to force fresh connections.
|
||||
"""
|
||||
try:
|
||||
from app import db
|
||||
db.session.rollback()
|
||||
if dispose_engine:
|
||||
db.engine.dispose()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_with_retries(
|
||||
job: FlagJob,
|
||||
fn: Callable[[], None],
|
||||
*,
|
||||
dispose_engine_on_operational_error: bool = False,
|
||||
max_retries: int = MAX_RETRIES,
|
||||
) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Run fn() with retry/backoff for OperationalError.
|
||||
Returns (success, error_text). error_text contains the last traceback on failure.
|
||||
"""
|
||||
import traceback
|
||||
|
||||
attempt = 0
|
||||
last_error_text: Optional[str] = None
|
||||
|
||||
while attempt < max_retries:
|
||||
try:
|
||||
# Catch stale pool conns early
|
||||
db_ping()
|
||||
|
||||
fn()
|
||||
return True, None
|
||||
|
||||
except OperationalError as e:
|
||||
attempt += 1
|
||||
last_error_text = "".join(traceback.format_exception(type(e), e, e.__traceback__))
|
||||
log.warning(
|
||||
"[%s] DB error (attempt %d/%d): %s",
|
||||
job.name,
|
||||
attempt,
|
||||
max_retries,
|
||||
e,
|
||||
)
|
||||
|
||||
recover_db_session(dispose_engine=dispose_engine_on_operational_error)
|
||||
|
||||
if attempt >= max_retries:
|
||||
return False, last_error_text
|
||||
|
||||
sleep_time = 2 ** attempt
|
||||
log.info("[%s] Retrying in %ss...", job.name, sleep_time)
|
||||
time.sleep(sleep_time)
|
||||
|
||||
except Exception as e:
|
||||
last_error_text = "".join(traceback.format_exception(type(e), e, e.__traceback__))
|
||||
log.exception("[%s] Unexpected error: %s", job.name, e)
|
||||
recover_db_session(dispose_engine=True)
|
||||
return False, last_error_text
|
||||
|
||||
return False, last_error_text
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Actual job bodies
|
||||
# -------------------------------------------------------------------
|
||||
def do_import() -> None:
|
||||
result = import_images_task(SOURCE_DIR, DEST_DIR)
|
||||
log.info("[Import] %s", result)
|
||||
|
||||
|
||||
def do_thumbnail_regen() -> None:
|
||||
"""
|
||||
Your thumbnail regeneration logic, moved into a function.
|
||||
Behavior is unchanged: chunked, progress logging, commit per batch.
|
||||
"""
|
||||
from pathlib import Path
|
||||
from app import db
|
||||
from app.models import ImageRecord
|
||||
from app.utils.image_importer import generate_thumbnail, generate_video_thumbnail
|
||||
|
||||
total = db.session.query(ImageRecord).count()
|
||||
log.info("[Thumbs] Starting regen; ImageRecord count = %d", total)
|
||||
|
||||
images_root = Path(DEST_DIR).resolve()
|
||||
batch_size = THUMBS_BATCH_SIZE
|
||||
|
||||
processed = 0
|
||||
writes = 0
|
||||
path_updates = 0
|
||||
failures = 0
|
||||
last_id = 0
|
||||
t_start = time.time()
|
||||
|
||||
while True:
|
||||
rows = (
|
||||
ImageRecord.query
|
||||
.filter(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(batch_size)
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
batch_start_id = rows[0].id
|
||||
batch_end_id = rows[-1].id
|
||||
batch_processed = 0
|
||||
batch_writes = 0
|
||||
batch_updates = 0
|
||||
batch_failures = 0
|
||||
bt0 = time.time()
|
||||
|
||||
for img in rows:
|
||||
try:
|
||||
processed += 1
|
||||
batch_processed += 1
|
||||
|
||||
ext = (img.format or "").lower()
|
||||
is_video = ext in ("mp4", "mov") or img.filepath.lower().endswith((".mp4", ".mov"))
|
||||
|
||||
if is_video:
|
||||
out_path = img.thumb_path
|
||||
if not out_path:
|
||||
try:
|
||||
rel = Path(img.filepath).resolve().relative_to(images_root)
|
||||
except Exception:
|
||||
batch_failures += 1
|
||||
failures += 1
|
||||
if THUMBS_VERBOSE:
|
||||
log.info(
|
||||
"[Thumbs] SKIP (not under %s): id=%s %s",
|
||||
images_root,
|
||||
img.id,
|
||||
img.filepath,
|
||||
)
|
||||
continue
|
||||
out_path = str((images_root / "thumbs" / rel).with_suffix(".jpg"))
|
||||
|
||||
new_path = generate_video_thumbnail(img.filepath, out_path)
|
||||
else:
|
||||
new_path = generate_thumbnail(img.filepath, overwrite=True)
|
||||
|
||||
if new_path:
|
||||
writes += 1
|
||||
batch_writes += 1
|
||||
|
||||
if new_path and new_path != img.thumb_path:
|
||||
img.thumb_path = new_path
|
||||
path_updates += 1
|
||||
batch_updates += 1
|
||||
|
||||
if THUMBS_VERBOSE and (processed <= 10 or processed % THUMBS_LOG_EVERY == 0):
|
||||
kind = "video" if is_video else "image"
|
||||
log.info(
|
||||
"[Thumbs] #%d (%s) id=%s name='%s' -> %s",
|
||||
processed,
|
||||
kind,
|
||||
img.id,
|
||||
img.filename,
|
||||
new_path or "None",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
batch_failures += 1
|
||||
log.warning(
|
||||
"[Thumbs] Failed for id=%s name='%s': %s",
|
||||
img.id,
|
||||
img.filename,
|
||||
e,
|
||||
)
|
||||
|
||||
db.session.commit()
|
||||
last_id = rows[-1].id
|
||||
db.session.expunge_all()
|
||||
|
||||
bt = time.time() - bt0
|
||||
rate = batch_processed / bt if bt > 0 else 0
|
||||
log.info(
|
||||
"[Thumbs] Batch %s-%s: processed=%d writes=%d db_updates=%d failures=%d (%.1f items/s) "
|
||||
"— totals: processed=%d writes=%d db_updates=%d failures=%d (last_id=%d)",
|
||||
batch_start_id,
|
||||
batch_end_id,
|
||||
batch_processed,
|
||||
batch_writes,
|
||||
batch_updates,
|
||||
batch_failures,
|
||||
rate,
|
||||
processed,
|
||||
writes,
|
||||
path_updates,
|
||||
failures,
|
||||
last_id,
|
||||
)
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
overall_rate = processed / elapsed if elapsed > 0 else 0
|
||||
log.info(
|
||||
"[Thumbs] COMPLETE in %.1fs — processed=%d writes=%d db_updates=%d failures=%d (%.1f items/s).",
|
||||
elapsed,
|
||||
processed,
|
||||
writes,
|
||||
path_updates,
|
||||
failures,
|
||||
overall_rate,
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Orchestration helpers
|
||||
# -------------------------------------------------------------------
|
||||
def handle_flag_job(
|
||||
job: FlagJob,
|
||||
fn: Callable[[], None],
|
||||
*,
|
||||
dispose_engine_on_operational_error: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
If <flag> exists, claim it and run fn with retries.
|
||||
Returns True if it did anything (including another worker claiming it).
|
||||
"""
|
||||
if not os.path.exists(job.flag_path):
|
||||
return False
|
||||
|
||||
log.info("[%s] Flag detected: %s", job.name, job.flag_path)
|
||||
|
||||
if not claim_job(job):
|
||||
# It existed, but someone else may have claimed it.
|
||||
return True
|
||||
|
||||
ok, err_text = run_with_retries(
|
||||
job,
|
||||
fn,
|
||||
dispose_engine_on_operational_error=dispose_engine_on_operational_error,
|
||||
)
|
||||
if ok:
|
||||
complete_job(job)
|
||||
else:
|
||||
fail_job(job, error_text=err_text)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Main loop
|
||||
# -------------------------------------------------------------------
|
||||
def monitor_and_import() -> None:
|
||||
thumbnail_job = FlagJob(name="Thumbs", flag_path=THUMBNAIL_FLAG)
|
||||
import_job = FlagJob(name="Import", flag_path=TRIGGER_FLAG)
|
||||
|
||||
# next time we should run periodic import (monotonic avoids wall-clock jumps)
|
||||
next_periodic_import = time.monotonic() + IMPORT_EVERY_SECONDS
|
||||
|
||||
with app.app_context():
|
||||
log.info(
|
||||
"[Worker] Started. Poll=%ss, periodic import every=%ss (%.2fh).",
|
||||
CHECK_INTERVAL,
|
||||
IMPORT_EVERY_SECONDS,
|
||||
IMPORT_EVERY_SECONDS / 3600.0,
|
||||
)
|
||||
|
||||
while True:
|
||||
did_work = False
|
||||
|
||||
# 1) Thumbnail regen (priority)
|
||||
did_work |= handle_flag_job(thumbnail_job, do_thumbnail_regen)
|
||||
|
||||
# 2) Flag-triggered import
|
||||
did_work |= handle_flag_job(
|
||||
import_job,
|
||||
do_import,
|
||||
dispose_engine_on_operational_error=True,
|
||||
)
|
||||
|
||||
# 3) Periodic import (every IMPORT_EVERY_SECONDS)
|
||||
now = time.monotonic()
|
||||
if IMPORT_EVERY_SECONDS > 0 and now >= next_periodic_import:
|
||||
next_periodic_import = now + IMPORT_EVERY_SECONDS
|
||||
|
||||
# Skip if an import is already in progress or in error
|
||||
if os.path.exists(import_job.run_path):
|
||||
log.info("[Import] Periodic run skipped (import already running).")
|
||||
elif os.path.exists(import_job.err_path):
|
||||
log.info(
|
||||
"[Import] Periodic run skipped (import in error state: %s).",
|
||||
import_job.err_path,
|
||||
)
|
||||
else:
|
||||
# Create a .run lock for periodic run (no trigger.flag required)
|
||||
tmp = import_job.run_path + ".tmp"
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.write("periodic\n")
|
||||
os.replace(tmp, import_job.run_path)
|
||||
log.info("[Import] Periodic run claimed -> %s", import_job.run_path)
|
||||
|
||||
ok, err_text = run_with_retries(
|
||||
import_job,
|
||||
do_import,
|
||||
dispose_engine_on_operational_error=True,
|
||||
)
|
||||
if ok:
|
||||
complete_job(import_job)
|
||||
else:
|
||||
fail_job(import_job, error_text=err_text)
|
||||
|
||||
did_work = True
|
||||
except Exception as e:
|
||||
log.warning("[Import] Could not start periodic run: %s", e)
|
||||
try:
|
||||
if os.path.exists(tmp):
|
||||
os.remove(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not did_work:
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
monitor_and_import()
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Add ImportTask and ImportBatch tables for Celery task queue
|
||||
|
||||
Revision ID: c26012001
|
||||
Revises: b26011901
|
||||
Create Date: 2026-01-20
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'c26012001'
|
||||
down_revision = 'b26011901'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Create import_batch table
|
||||
op.create_table(
|
||||
'import_batch',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_directory', sa.String(length=1024), nullable=False),
|
||||
sa.Column('status', sa.String(length=32), nullable=True),
|
||||
sa.Column('total_files', sa.Integer(), nullable=True),
|
||||
sa.Column('processed_files', sa.Integer(), nullable=True),
|
||||
sa.Column('imported_count', sa.Integer(), nullable=True),
|
||||
sa.Column('skipped_count', sa.Integer(), nullable=True),
|
||||
sa.Column('failed_count', sa.Integer(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('completed_at', sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False)
|
||||
|
||||
# Create import_task table
|
||||
op.create_table(
|
||||
'import_task',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_path', sa.String(length=1024), nullable=False),
|
||||
sa.Column('file_hash', sa.String(length=64), nullable=True),
|
||||
sa.Column('file_size', sa.BigInteger(), nullable=True),
|
||||
sa.Column('task_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('status', sa.String(length=32), nullable=False),
|
||||
sa.Column('celery_task_id', sa.String(length=64), nullable=True),
|
||||
sa.Column('context', sa.JSON(), nullable=True),
|
||||
sa.Column('batch_id', sa.Integer(), nullable=True),
|
||||
sa.Column('result_image_id', sa.Integer(), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('retry_count', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('completed_at', sa.DateTime(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['batch_id'], ['import_batch.id'], ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['result_image_id'], ['image_record.id'], ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index(op.f('ix_import_task_celery_task_id'), 'import_task', ['celery_task_id'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_created_at'), 'import_task', ['created_at'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_file_hash'), 'import_task', ['file_hash'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_source_path'), 'import_task', ['source_path'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_task_type'), 'import_task', ['task_type'], unique=False)
|
||||
op.create_index('ix_import_task_batch_status', 'import_task', ['batch_id', 'status'], unique=False)
|
||||
op.create_index('ix_import_task_status_type', 'import_task', ['status', 'task_type'], unique=False)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index('ix_import_task_status_type', table_name='import_task')
|
||||
op.drop_index('ix_import_task_batch_status', table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_task_type'), table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_status'), table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_source_path'), table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_file_hash'), table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_created_at'), table_name='import_task')
|
||||
op.drop_index(op.f('ix_import_task_celery_task_id'), table_name='import_task')
|
||||
op.drop_table('import_task')
|
||||
op.drop_index(op.f('ix_import_batch_status'), table_name='import_batch')
|
||||
op.drop_table('import_batch')
|
||||
@@ -8,3 +8,7 @@ imagehash
|
||||
pyunpack
|
||||
patool
|
||||
gunicorn
|
||||
|
||||
# Celery task queue
|
||||
celery[redis]>=5.3.0
|
||||
redis>=5.0.0
|
||||
|
||||
Reference in New Issue
Block a user