bvandeusen/imagerepo
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added transcoding to mp4 in import to ensure web playback.

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-01-20 16:10:19 -05:00
parent cb3897c0b0
commit 87bcb633f6
4 changed files with 175 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/celery_app.py
+3 -3
View File
@@ -74,9 +74,9 @@ def make_celery(app=None):
task_acks_late=True, # Acknowledge after task completes
task_reject_on_worker_lost=True, # Requeue if worker dies
# Time limits
task_soft_time_limit=300, # 5 minutes soft limit
task_time_limit=600, # 10 minutes hard limit (archives may need more)
# Time limits (video transcoding may need longer)
task_soft_time_limit=600, # 10 minutes soft limit
task_time_limit=900, # 15 minutes hard limit
# Periodic task schedule (Celery Beat)
beat_schedule={
app/main.py
+50 -17
View File
@@ -2,8 +2,8 @@
from flask import Blueprint, render_template, flash, redirect, url_for, abort, send_from_directory, request, jsonify
from sqlalchemy import desc, func, text, extract
from sqlalchemy.orm import joinedload
from sqlalchemy import desc, func, text, extract, and_
from sqlalchemy.orm import joinedload, aliased
from collections import OrderedDict
from datetime import datetime
@@ -400,11 +400,15 @@ def tag_list():
/tags?kind=series -> only series tags
/tags?kind=rating -> only rating tags
Shows a few preview images per tag.
Optimized to use 2 queries instead of N+1:
1. Get all tags with counts
2. Get preview images for all tags at once using window function
"""
images_per_tag = 3
kind = request.args.get('kind') # 'artist' | 'archive' | 'user' | 'character' | 'series' | 'rating' | None
kind = request.args.get('kind')
# Get tags with image counts
# Query 1: Get tags with image counts
q = db.session.query(
Tag,
func.count(image_tags.c.image_id).label('img_count')
@@ -415,26 +419,55 @@ def tag_list():
tags_with_counts = q.order_by(Tag.name.asc()).all()
tag_data = []
# Build tag_id list and initialize tag_data dict
tag_ids = [t.id for t, _ in tags_with_counts]
tag_map = {}
for t, count in tags_with_counts:
imgs = (
ImageRecord.query
.join(ImageRecord.tags)
.filter(Tag.id == t.id)
.order_by(desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at)))
.limit(images_per_tag)
.all()
)
# Display label: for prefixed kinds use the part after ':', else full name
label = t.name.split(":", 1)[1] if (":" in t.name) else t.name
tag_data.append({
tag_map[t.id] = {
"id": t.id,
"name": label,
"tag": t.name,
"images": imgs,
"images": [],
"kind": t.kind,
"count": count
})
}
# Query 2: Get preview images for all tags at once using a subquery with row_number
# This fetches the top N images per tag in a single query
if tag_ids:
# Subquery to rank images within each tag
row_num = func.row_number().over(
partition_by=image_tags.c.tag_id,
order_by=desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at))
).label('rn')
subq = (
db.session.query(
image_tags.c.tag_id,
ImageRecord.id.label('image_id'),
row_num
)
.join(ImageRecord, ImageRecord.id == image_tags.c.image_id)
.filter(image_tags.c.tag_id.in_(tag_ids))
.subquery()
)
# Get only the top N images per tag
preview_rows = (
db.session.query(subq.c.tag_id, ImageRecord)
.join(ImageRecord, ImageRecord.id == subq.c.image_id)
.filter(subq.c.rn <= images_per_tag)
.all()
)
# Group images by tag_id
for tag_id, img in preview_rows:
if tag_id in tag_map:
tag_map[tag_id]["images"].append(img)
# Convert to list maintaining original order
tag_data = [tag_map[t.id] for t, _ in tags_with_counts]
return render_template('tags_list.html',
tag_data=tag_data,
app/tasks/import_file.py
+35 -8
View File
@@ -48,7 +48,8 @@ def import_media_file(self, task_id: int):
extract_metadata, calculate_hash, calculate_perceptual_hash,
is_mostly_transparent, is_mostly_single_color, find_similar_image,
build_hashed_dest_path, load_import_settings, supersede_image,
generate_thumbnail, generate_video_thumbnail_mirrored
generate_thumbnail, generate_video_thumbnail_mirrored,
transcode_video_to_mp4, needs_transcode
)
from app.utils.metadata_enrichment import find_sidecar_json
@@ -149,8 +150,33 @@ def import_media_file(self, task_id: int):
# Normal import - copy to destination
target_dir = os.path.join(dest_dir, artist)
os.makedirs(target_dir, exist_ok=True)
dest_path = build_hashed_dest_path(target_dir, filename, file_hash)
shutil.copy2(src_path, dest_path)
# Transcode video if needed (non-MP4 formats to H.264 MP4)
transcoded_temp = None
actual_src = src_path
actual_filename = filename
if needs_transcode(src_path):
log.info(f"Transcoding video: {filename}")
# Transcode to temp location first
transcoded_temp = transcode_video_to_mp4(src_path)
if transcoded_temp:
actual_src = transcoded_temp
# Update filename to .mp4 extension
actual_filename = os.path.splitext(filename)[0] + '.mp4'
# Recalculate hash for transcoded file
file_hash = calculate_hash(transcoded_temp)
task.file_hash = file_hash
log.info(f"Transcoded successfully: {actual_filename}")
else:
log.warning(f"Transcode failed for {filename}, importing original")
dest_path = build_hashed_dest_path(target_dir, actual_filename, file_hash)
shutil.copy2(actual_src, dest_path)
# Clean up transcoded temp file
if transcoded_temp and os.path.exists(transcoded_temp):
os.remove(transcoded_temp)
# Generate thumbnail
try:
@@ -159,20 +185,21 @@ def import_media_file(self, task_id: int):
else:
thumb_path = generate_thumbnail(dest_path)
except Exception as e:
log.warning(f"Thumbnail generation failed for {filename}: {e}")
log.warning(f"Thumbnail generation failed for {actual_filename}: {e}")
thumb_path = None
# Create ImageRecord
# Use actual_filename in case video was transcoded to .mp4
record = ImageRecord(
filename=filename,
filename=actual_filename,
filepath=dest_path,
thumb_path=thumb_path,
hash=file_hash,
perceptual_hash=str(phash) if phash else None,
file_size=metadata['file_size'],
file_size=os.path.getsize(dest_path), # Use actual file size after transcode
width=metadata['width'],
height=metadata['height'],
format=metadata['format'],
format=metadata['format'] if not needs_transcode(src_path) else 'mp4',
camera_model=metadata.get('camera_model'),
taken_at=metadata.get('taken_at'),
imported_at=datetime.utcnow()
@@ -200,7 +227,7 @@ def import_media_file(self, task_id: int):
from app.tasks.scan import update_batch_stats
update_batch_stats.delay(task.batch_id)
log.info(f"Imported: {filename} -> {dest_path}")
log.info(f"Imported: {actual_filename} -> {dest_path}")
return {
'status': 'imported',
app/utils/image_importer.py
+87 -3
View File
@@ -25,6 +25,7 @@ from app import db
from app.models import ImageRecord, Tag, ArchiveRecord
FFMPEG_THUMB_TIMEOUT = int(os.environ.get("THUMBS_FFMPEG_TIMEOUT", "60")) # seconds
FFMPEG_TRANSCODE_TIMEOUT = int(os.environ.get("FFMPEG_TRANSCODE_TIMEOUT", "600")) # 10 min default
Image.MAX_IMAGE_PIXELS = int(os.environ.get("PIL_MAX_IMAGE_PIXELS", "178956970"))
ARCHIVE_NUM_WIDTH = int(os.environ.get("ARCHIVE_NUM_WIDTH", "4"))
@@ -36,9 +37,10 @@ ARCHIVE_NUM_WIDTH = int(os.environ.get("ARCHIVE_NUM_WIDTH", "4"))
THUMB_SIZE = (400, 400)
# Media and archive extensions we handle
ALLOWED_MEDIA_EXTS = (
".jpg", ".jpeg", ".jfif", ".png", ".gif", ".bmp", ".tiff", ".webp", ".mp4", ".mov"
)
IMAGE_EXTS = (".jpg", ".jpeg", ".jfif", ".png", ".gif", ".bmp", ".tiff", ".webp")
VIDEO_EXTS = (".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv")
VIDEO_EXTS_NEED_TRANSCODE = (".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv") # Non-MP4 formats
ALLOWED_MEDIA_EXTS = IMAGE_EXTS + VIDEO_EXTS
ALLOWED_ARCHIVE_EXTS = (
".zip", ".rar", ".cbr", ".7z",
".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz"
@@ -1025,6 +1027,88 @@ def generate_video_thumbnail_mirrored(video_path: str, time_position: str = "00:
return generate_video_thumbnail(str(video_path_p), str(thumb_path), time_position=time_position)
def transcode_video_to_mp4(input_path: str, output_path: str = None) -> str | None:
"""
Transcode a video to H.264 MP4 for universal browser playback.
Args:
input_path: Path to source video file
output_path: Optional destination path. If None, replaces extension with .mp4
in a temp location.
Returns:
Path to transcoded MP4 file on success, None on failure.
"""
input_p = Path(input_path)
# Determine output path
if output_path is None:
# Create temp file with .mp4 extension
output_path = str(input_p.with_suffix(".mp4"))
if output_path == input_path:
# Already .mp4, use temp suffix
output_path = str(input_p.with_suffix(".transcoded.mp4"))
try:
os.makedirs(os.path.dirname(output_path), exist_ok=True)
# FFmpeg command for H.264/AAC transcoding with good browser compatibility
# -movflags +faststart: Enables progressive playback (moov atom at start)
# -preset medium: Balance between speed and compression
# -crf 23: Good quality (lower = better, 18-28 typical range)
# -c:v libx264: H.264 video codec
# -c:a aac: AAC audio codec
# -pix_fmt yuv420p: Pixel format for maximum compatibility
cmd = [
"ffmpeg", "-y",
"-i", str(input_path),
"-c:v", "libx264",
"-preset", "medium",
"-crf", "23",
"-pix_fmt", "yuv420p",
"-c:a", "aac",
"-b:a", "128k",
"-movflags", "+faststart",
str(output_path),
]
print(f"[INFO] Transcoding video: {input_path} -> {output_path}")
result = subprocess.run(
cmd,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=FFMPEG_TRANSCODE_TIMEOUT
)
print(f"[INFO] Transcoding complete: {output_path}")
return output_path
except subprocess.TimeoutExpired:
print(f"[WARN] Video transcode timed out after {FFMPEG_TRANSCODE_TIMEOUT}s: {input_path}")
# Clean up partial output
if os.path.exists(output_path):
os.remove(output_path)
return None
except subprocess.CalledProcessError as e:
print(f"[WARN] Failed to transcode video {input_path}: {e}")
if e.stderr:
print(f"[WARN] FFmpeg stderr: {e.stderr.decode('utf-8', errors='ignore')[:500]}")
# Clean up partial output
if os.path.exists(output_path):
os.remove(output_path)
return None
except Exception as e:
print(f"[WARN] Unexpected error transcoding video {input_path}: {e}")
if os.path.exists(output_path):
os.remove(output_path)
return None
def needs_transcode(file_path: str) -> bool:
"""Check if a video file needs transcoding to MP4."""
ext = Path(file_path).suffix.lower()
return ext in VIDEO_EXTS_NEED_TRANSCODE
def calculate_hash(file_path: str) -> str:
hash_func = hashlib.sha256()