polish on archive import processing

app/models.py

@@ -3,6 +3,14 @@ from flask_login import UserMixin
 from . import login_manager
 from datetime import datetime
 
+# tag to object relationship table
+image_tags = db.Table(
+    'image_tags',
+    db.Column('image_id', db.Integer, db.ForeignKey('image_record.id'), primary_key=True),
+    db.Column('tag_id', db.Integer, db.ForeignKey('tag.id'), primary_key=True),
+    db.UniqueConstraint('image_id', 'tag_id', name='uq_image_tag')  # prevent duplicate tags per image
+)
+
 class User(UserMixin, db.Model):
     id = db.Column(db.Integer, primary_key=True)
     username = db.Column(db.String(64), unique=True, nullable=False)
@@ -26,8 +34,34 @@ class ImageRecord(db.Model):
     taken_at = db.Column(db.DateTime, nullable=True)
     imported_at = db.Column(db.DateTime, default=db.func.now())
     is_thumbnail = db.Column(db.Boolean, default=False)
+    tags = db.relationship('Tag', secondary=image_tags, back_populates='images')
 
+class Tag(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    # Tag names are unique so we can upsert quickly (e.g. "zip:Artist/ArchiveName.zip")
+    name = db.Column(db.String(255), unique=True, nullable=False)
+    # Optional “kind” if you want to filter by tag type later ("zip", "import-source", etc.)
+    kind = db.Column(db.String(64), nullable=True)
 
+    images = db.relationship('ImageRecord', secondary=image_tags, back_populates='tags')
+
+class ArchiveRecord(db.Model):
+    """
+    Tracks archives we've processed so we can skip re-importing the same large file.
+    Also holds a Tag that marks images that came from this archive.
+    """
+    id = db.Column(db.Integer, primary_key=True)
+    filename = db.Column(db.String(512), nullable=False)  # full path or logical name
+    file_size = db.Column(db.BigInteger, nullable=False)
+    hash = db.Column(db.String(64), unique=True, nullable=False)  # SHA256 of the archive file
+    imported_at = db.Column(db.DateTime, default=db.func.now())
+
+    # optional: where this archive lived in the source tree (e.g., artist dir)
+    artist = db.Column(db.String(255), nullable=True)
+
+    # Link to the tag we created for this archive so we can reuse it if the same archive returns
+    tag_id = db.Column(db.Integer, db.ForeignKey('tag.id'), nullable=False)
+    tag = db.relationship('Tag', backref=db.backref('archives', lazy='dynamic'))
 
 @login_manager.user_loader
 def load_user(user_id):

app/utils/image_importer.py

@@ -1,27 +1,184 @@
 # app/utils/image_importer.py
+
+from __future__ import annotations
+
 from datetime import datetime
-import os, shutil, hashlib, time
+from pathlib import Path
+import hashlib
+import json
+import mimetypes
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+import uuid
+
 from PIL import Image
 import exifread
-import mimetypes
 import imagehash
-import uuid
-import subprocess
 
 from app import db
-from app.models import ImageRecord
+from app.models import ImageRecord, Tag, ArchiveRecord
+
+
+# =============================================================================
+# Configuration & Constants
+# =============================================================================
 
 THUMB_SIZE = (400, 400)
 
-def import_images_task(source_dir, dest_dir):
+# Media and archive extensions we handle
-    imported = []
+ALLOWED_MEDIA_EXTS = (
+    ".jpg", ".jpeg", ".jfif", ".png", ".gif", ".bmp", ".tiff", ".webp", ".mp4", ".mov"
+)
+ALLOWED_ARCHIVE_EXTS = (
+    ".zip", ".rar", ".cbr", ".7z",
+    ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz"
+)
+
+# Regex for multi-part detection
+RAR_PART_RE    = re.compile(r"\.part(\d+)\.rar$", re.IGNORECASE)
+SEVENZ_PART_RE = re.compile(r"\.7z\.(\d{3})$", re.IGNORECASE)
+OLD_RAR_VOL_RE = re.compile(r"\.r\d{2}$", re.IGNORECASE)  # .r00, .r01, ...
+
+# Environment configuration (optional)
+ENV_TMPDIR = "ARCHIVE_TMPDIR"          # where to extract archives (default: system tmp)
+ENV_MINFREE = "ARCHIVE_MIN_FREE_GB"    # min free GB required to start extraction (float; 0 = disabled)
+
+
+# =============================================================================
+# Utilities: Temp space management & free-space guard
+# =============================================================================
+
+def get_tmp_base() -> Path:
+    """Return the base temp directory for archive extraction."""
+    return Path(os.environ.get(ENV_TMPDIR, tempfile.gettempdir())).resolve()
+
+
+def cleanup_stale_tempdirs(prefix: str = "extract_") -> None:
+    """Delete any leftover extraction directories from previous crashes."""
+    base = get_tmp_base()
+    base.mkdir(parents=True, exist_ok=True)
+    for entry in base.glob(f"{prefix}*"):
+        if entry.is_dir():
+            try:
+                shutil.rmtree(entry, ignore_errors=True)
+            except Exception:
+                pass
+
+
+def ensure_free_space_or_raise(path: Path, min_free_gb: float) -> None:
+    """Raise if there is not enough free space at path."""
+    if min_free_gb <= 0:
+        return
+    total, used, free = shutil.disk_usage(str(path))
+    free_gb = free / (1024 ** 3)
+    if free_gb < min_free_gb:
+        raise RuntimeError(
+            f"Insufficient free space at {path}: {free_gb:.1f} GB free "
+            f"(required >= {min_free_gb:.1f} GB)."
+        )
+
+
+# =============================================================================
+# Utilities: External extractors with resilient fallback
+# =============================================================================
+
+class ExtractError(Exception):
+    """Raised when archive extraction fails with all strategies."""
+
+
+def _run_cmd(cmd: list[str]) -> tuple[int, str, str]:
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    return p.returncode, p.stdout, p.stderr
+
+
+def extract_with_unar(archive_path: str, out_dir: str) -> None:
+    os.makedirs(out_dir, exist_ok=True)
+    rc, out, err = _run_cmd(["unar", "-o", out_dir, archive_path])
+    if rc != 0:
+        raise ExtractError(f"unar failed (rc={rc})\nSTDOUT:\n{out}\nSTDERR:\n{err}")
+
+
+def extract_with_7z(archive_path: str, out_dir: str) -> None:
+    os.makedirs(out_dir, exist_ok=True)
+    rc, out, err = _run_cmd(["7z", "x", "-y", f"-o{out_dir}", archive_path])
+    if rc != 0:
+        raise ExtractError(f"7z failed (rc={rc})\nSTDOUT:\n{out}\nSTDERR:\n{err}")
+
+
+def looks_password_protected_with_lsar(archive_path: str) -> bool:
+    """Best-effort pre-check for 'Encrypted/Password' in lsar output."""
+    try:
+        rc, out, err = _run_cmd(["lsar", archive_path])
+        txt = (out + "\n" + err).lower()
+        return ("encrypted" in txt) or ("password" in txt)
+    except Exception:
+        return False
+
+
+def extract_archive_resilient(archive_path: str, out_dir: str) -> None:
+    """
+    Prefer unar for RAR/CBR (RAR5-friendly); fallback to 7z.
+    For other formats, try 7z first then unar.
+    Surfaces stderr so logs show CRC/password/permissions issues.
+    """
+    lower = archive_path.lower()
+    errors = []
+
+    # Optional password hint to skip early
+    if lower.endswith((".rar", ".cbr")) and looks_password_protected_with_lsar(archive_path):
+        raise ExtractError("Archive appears password-protected; skipping (no password provided).")
+
+    if lower.endswith((".rar", ".cbr")):
+        order = (extract_with_unar, extract_with_7z)
+    else:
+        order = (extract_with_7z, extract_with_unar)
+
+    for fn in order:
+        try:
+            fn(archive_path, out_dir)
+            return
+        except ExtractError as e:
+            errors.append(str(e))
+
+    raise ExtractError("Both extractors failed:\n" + "\n---\n".join(errors))
+
+
+# =============================================================================
+# Core: Import task & archive processing
+# =============================================================================
+
+def import_images_task(source_dir: str, dest_dir: str) -> str:
+    """
+    Walk the import tree, process archives first (only first volumes),
+    then media files. Batches DB commits for efficiency.
+    """
+    # Cleanup any stale extraction dirs from previous runs/crashes
+    cleanup_stale_tempdirs()
+
+    imported: list[str] = []
     batch_size = 10
     batch_counter = 0
+
+    # Preload existing pHashes to avoid N+1 lookups
     existing_phashes = [
         (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height)
         for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
     ]
 
+    # Ensure thumbs base exists
+    (Path(dest_dir) / "thumbs").mkdir(parents=True, exist_ok=True)
+
+    # Optional free-space guard
+    min_free = float(os.environ.get(ENV_MINFREE, "0") or "0")
+    try:
+        ensure_free_space_or_raise(get_tmp_base(), min_free)
+    except Exception as e:
+        print(f"[WARN] Free-space check failed: {e}")
+
     for artist_dir in os.listdir(source_dir):
         artist_path = os.path.join(source_dir, artist_dir)
         if not os.path.isdir(artist_path):
@@ -29,97 +186,215 @@ def import_images_task(source_dir, dest_dir):
 
         for root, _, files in os.walk(artist_path):
             for file in files:
-                if file.lower().endswith(('.jpg', '.jpeg', '.jfif', '.png', '.gif', '.bmp', '.tiff', '.webp', '.mp4', '.mov')):
+                full_path = os.path.join(root, file)
-                    full_path = os.path.join(root, file)
+                lower = file.lower()
-                    filename = os.path.basename(full_path)
 
-                    print(f"[INFO] Processing: {full_path}")
+                # Archives first
-                    file_size = os.path.getsize(full_path)
+                if lower.endswith(ALLOWED_ARCHIVE_EXTS):
-
+                    if not is_first_volume(full_path):
-                    if ImageRecord.query.filter_by(filename=filename, file_size=file_size).first():
+                        print(f"[SKIP] Not first archive volume: {file}")
-                        print(f"[SKIP] Duplicate by name+size: {filename}")
                         continue
-                    
-                    file_hash = calculate_hash(full_path)
-                    if ImageRecord.query.filter_by(hash=file_hash).first():
-                        print(f"[SKIP] Duplicate by hash: {filename}")
-                        continue
-
-                    metadata = extract_metadata(full_path)
-                    if not metadata['width'] or not metadata['height']:
-                        print(f"[SKIP] Missing dimension data for {filename}")
-                        continue
-
-                    phash = calculate_perceptual_hash(full_path)
-                    if phash and is_similar_image(phash, metadata['width'], metadata['height'], existing_phashes):
-                        print(f"[SKIP] {filename} is visually similar to an existing larger image.")
-                        continue
-
-                    target_dir = os.path.join(dest_dir, artist_dir)
-                    os.makedirs(target_dir, exist_ok=True)
-                    dest_path = os.path.join(target_dir, filename)
-                    shutil.copy2(full_path, dest_path)
-
                     try:
-                        if dest_path.lower().endswith(('.mp4', '.mov')):
+                        count = process_archive(
-                            thumb_filename = f"{uuid.uuid4().hex}.jpg"
+                            archive_path=full_path,
-                            thumb_path = os.path.join("/images/thumbs", thumb_filename)
+                            dest_dir=dest_dir,
-                            thumb_path = generate_video_thumbnail(dest_path, thumb_path)
+                            artist=artist_dir,
-                        else:
+                            existing_phashes=existing_phashes
-                            thumb_path = generate_thumbnail(dest_path)
+                        )
+                        imported.append(f"[archive]{file}:{count}")
                     except Exception as e:
-                        print(f"[WARN] Failed to generate thumbnail for {filename}: {e}")
+                        print(f"[WARN] Failed processing archive {file}: {e}")
-                        thumb_path = None
+                    continue
 
-                    record = ImageRecord(
+                # Non-archive media
-                        filename=filename,
+                if not lower.endswith(ALLOWED_MEDIA_EXTS):
-                        filepath=dest_path,
+                    continue
-                        thumb_path=thumb_path,
-                        hash=file_hash,
-                        perceptual_hash=str(phash) if phash else None,
-                        artist=artist_dir,
-                        file_size=metadata['file_size'],
-                        width=metadata['width'],
-                        height=metadata['height'],
-                        format=metadata['format'],
-                        camera_model=metadata['camera_model'],
-                        taken_at=metadata['taken_at'],
-                        imported_at=datetime.utcnow()
-                    )
-                    db.session.add(record)
-                    imported.append(filename)
 
-                    if phash:
+                added, phash, _rec = import_single_file(
-                        existing_phashes.append((phash, metadata['width'], metadata['height']))
+                    src_path=full_path,
+                    dest_dir=dest_dir,
+                    artist=artist_dir,
+                    existing_phashes=existing_phashes
+                )
+                if added:
+                    imported.append(file)
 
-                    batch_counter += 1
+                if phash:
+                    md = extract_metadata(full_path)
+                    if md.get("width") and md.get("height"):
+                        existing_phashes.append((phash, md["width"], md["height"]))
 
-                    if batch_counter >= batch_size:
+                batch_counter += 1
-                        db.session.commit()
+                if batch_counter >= batch_size:
-                        print(f"[INFO] Committed batch of {batch_size} images.")
+                    db.session.commit()
-                        batch_counter = 0
+                    print(f"[INFO] Committed batch of {batch_size} items.")
-
+                    batch_counter = 0
-                        if 'sqlite' in db.engine.url.drivername:
+                    if "sqlite" in db.engine.url.drivername:
-                            time.sleep(1)
+                        time.sleep(1)
 
     if batch_counter > 0:
         db.session.commit()
-        print(f"[INFO] Committed final batch of {batch_counter} images.")
+        print(f"[INFO] Committed final batch of {batch_counter} items.")
 
-    print(f"[INFO] Import complete. Total images imported: {len(imported)}")
+    summary = f"Imported {len(imported)} items."
-    return f"Imported {len(imported)} images."
+    print(f"[INFO] {summary}")
+    return summary
 
 
-def generate_thumbnail(image_path, size=(400, 400), overwrite=False):
+def process_archive(archive_path: str, dest_dir: str, artist: str, existing_phashes) -> int:
-    from pathlib import Path
+    """
+    Extract archive into a temp dir, import/tag media, and record ArchiveRecord for de-dup.
+    Returns number of items imported or tagged.
+    """
+    archive_size = os.path.getsize(archive_path)
+    archive_hash = calculate_hash(archive_path)
 
-    images_root = Path("/images").resolve()
+    # De-dup the archive itself
-    image_path = Path(image_path).resolve()
+    existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
+    if existing_archive:
+        print(f"[SKIP] Archive already imported: {archive_path}")
+        return 0
+
+    # Create/reuse tag for this archive
+    tag_name = f"archive:{artist}/{os.path.basename(archive_path)}"
+    tag = Tag.query.filter_by(name=tag_name).first()
+    if not tag:
+        tag = Tag(name=tag_name, kind="archive")
+        db.session.add(tag)
+        db.session.flush()  # ensure tag.id
+
+    # Create an extraction dir under the configured temp base
+    tmpdir = tempfile.mkdtemp(prefix="extract_", dir=str(get_tmp_base()))
+    imported_or_tagged = 0
 
     try:
-        rel_path = image_path.relative_to(images_root)
+        # Resilient extraction (unar → fallback 7z for RAR; 7z → fallback unar for others)
-    except ValueError:
+        extract_archive_resilient(archive_path, tmpdir)
-        raise ValueError(f"{image_path} is not under {images_root}")
+
+        # Walk extracted tree and process media files
+        for root, _, files in os.walk(tmpdir):
+            for file in files:
+                if not file.lower().endswith(ALLOWED_MEDIA_EXTS):
+                    continue
+                src_path = os.path.join(root, file)
+
+                added, _phash, rec = import_single_file(
+                    src_path=src_path,
+                    dest_dir=dest_dir,
+                    artist=artist,
+                    existing_phashes=existing_phashes,
+                    commit=False  # batch after loop
+                )
+                if rec is not None and tag not in rec.tags:
+                    rec.tags.append(tag)
+                    imported_or_tagged += 1
+
+        # Record this archive as processed
+        arch = ArchiveRecord(
+            filename=archive_path,
+            file_size=archive_size,
+            hash=archive_hash,
+            artist=artist,
+            tag=tag
+        )
+        db.session.add(arch)
+        db.session.commit()
+        print(f"[INFO] Archive processed: {archive_path} items={imported_or_tagged}")
+        return imported_or_tagged
+
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+# =============================================================================
+# Core: Single-file import path
+# =============================================================================
+
+def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, commit: bool = True):
+    """
+    Import a single media file. If an equivalent file already exists, return that
+    record so callers can tag it (no duplicate import).
+    Returns (added: bool, phash_or_None, record_or_None).
+    """
+    filename = os.path.basename(src_path)
+    print(f"[INFO] Processing: {src_path}")
+    file_size = os.path.getsize(src_path)
+
+    # Check by name + size
+    existing = ImageRecord.query.filter_by(filename=filename, file_size=file_size).first()
+    if existing:
+        print(f"[SKIP] Duplicate by name+size: {filename}")
+        return (False, None, existing)
+
+    # Check by content hash
+    file_hash = calculate_hash(src_path)
+    existing = ImageRecord.query.filter_by(hash=file_hash).first()
+    if existing:
+        print(f"[SKIP] Duplicate by hash: {filename}")
+        return (False, None, existing)
+
+    # Metadata & pHash similarity
+    metadata = extract_metadata(src_path)
+    if not metadata["width"] or not metadata["height"]:
+        print(f"[SKIP] Missing dimension data for {filename}")
+        return (False, None, None)
+
+    phash = calculate_perceptual_hash(src_path)
+    if phash and is_similar_image(phash, metadata["width"], metadata["height"], existing_phashes):
+        print(f"[SKIP] {filename} is visually similar to an existing larger image.")
+        return (False, phash, None)
+
+    # Copy into /images/<artist>/<filename>
+    target_dir = os.path.join(dest_dir, artist)
+    os.makedirs(target_dir, exist_ok=True)
+    dest_path = os.path.join(target_dir, filename)
+    shutil.copy2(src_path, dest_path)
+
+    # Thumbnail
+    try:
+        if dest_path.lower().endswith((".mp4", ".mov")):
+            thumb_filename = f"{uuid.uuid4().hex}.jpg"
+            thumb_path = os.path.join(dest_dir, "thumbs", thumb_filename)
+            thumb_path = generate_video_thumbnail(dest_path, thumb_path)
+        else:
+            thumb_path = generate_thumbnail(dest_path)
+    except Exception as e:
+        print(f"[WARN] Failed to generate thumbnail for {filename}: {e}")
+        thumb_path = None
+
+    # Create DB record
+    record = ImageRecord(
+        filename=filename,
+        filepath=dest_path,
+        thumb_path=thumb_path,
+        hash=file_hash,
+        perceptual_hash=str(phash) if phash else None,
+        artist=artist,
+        file_size=metadata["file_size"],
+        width=metadata["width"],
+        height=metadata["height"],
+        format=metadata["format"],
+        camera_model=metadata["camera_model"],
+        taken_at=metadata["taken_at"],
+        imported_at=datetime.utcnow()
+    )
+    db.session.add(record)
+    if commit:
+        db.session.commit()
+    return (True, phash, record)
+
+
+# =============================================================================
+# Helpers: Thumbnails, metadata, hashing, similarity
+# =============================================================================
+
+def generate_thumbnail(image_path: str, size: tuple[int, int] = (400, 400), overwrite: bool = False) -> str:
+    """
+    Save thumbnails mirrored under /images/thumbs/<artist>/<filename>.
+    """
+    images_root = Path("/images").resolve()
+    image_path = Path(image_path).resolve()
+    rel_path = image_path.relative_to(images_root)  # raises if not under /images
 
     thumb_path = images_root / "thumbs" / rel_path
     thumb_path.parent.mkdir(parents=True, exist_ok=True)
@@ -133,18 +408,13 @@ def generate_thumbnail(image_path, size=(400, 400), overwrite=False):
 
     return str(thumb_path)
 
-def generate_video_thumbnail(video_path, output_path, time_position="00:00:01"):
+
+def generate_video_thumbnail(video_path: str, output_path: str, time_position: str = "00:00:01") -> str | None:
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
-
     command = [
-        "ffmpeg",
+        "ffmpeg", "-i", video_path, "-ss", time_position,
-        "-i", video_path,
+        "-vframes", "1", "-vf", f"scale={THUMB_SIZE[0]}:-1", output_path
-        "-ss", time_position,
-        "-vframes", "1",
-        "-vf", f"scale={THUMB_SIZE[0]}:-1",
-        output_path
     ]
-
     try:
         subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         return output_path
@@ -152,22 +422,25 @@ def generate_video_thumbnail(video_path, output_path, time_position="00:00:01"):
         print(f"[WARN] Failed to extract video thumbnail for {video_path}: {e}")
         return None
 
-def calculate_hash(file_path):
+
+def calculate_hash(file_path: str) -> str:
     hash_func = hashlib.sha256()
-    with open(file_path, 'rb') as f:
+    with open(file_path, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b''):
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):  # 1MB chunks work well for big files
             hash_func.update(chunk)
     return hash_func.hexdigest()
 
-def calculate_perceptual_hash(file_path):
+
+def calculate_perceptual_hash(file_path: str):
     try:
         with Image.open(file_path) as img:
             return imagehash.phash(img)
-    except Exception as e:
+    except Exception:
-        print(f"[WARN] Failed to compute perceptual hash for {file_path}: {e}")
+        # videos or unreadable images land here (phash not applicable)
         return None
 
-def is_similar_image(phash, width, height, existing_phashes, threshold=2):
+
+def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 2) -> bool:
     for existing, ew, eh in existing_phashes:
         try:
             distance = phash - existing
@@ -178,59 +451,97 @@ def is_similar_image(phash, width, height, existing_phashes, threshold=2):
     return False
 
 
-def extract_metadata(file_path):
+def extract_metadata(file_path: str) -> dict:
     metadata = {
-        'file_size': None,
+        "file_size": None,
-        'width': None,
+        "width": None,
-        'height': None,
+        "height": None,
-        'format': None,
+        "format": None,
-        'camera_model': None,
+        "camera_model": None,
-        'taken_at': None
+        "taken_at": None
     }
 
-    metadata['file_size'] = os.path.getsize(file_path)
+    metadata["file_size"] = os.path.getsize(file_path)
     ext = os.path.splitext(file_path)[1].lower()
 
-    if ext in ('.mp4', '.mov'):
+    # Video: use ffprobe
-        # Handle video dimensions using ffprobe
+    if ext in (".mp4", ".mov"):
         try:
-            import subprocess, json
             cmd = [
                 "ffprobe", "-v", "error",
                 "-select_streams", "v:0",
                 "-show_entries", "stream=width,height",
                 "-of", "json", file_path
             ]
-            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
+            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=False)
-            data = json.loads(result.stdout)
+            data = json.loads(result.stdout or "{}")
             stream = data.get("streams", [{}])[0]
-            metadata['width'] = stream.get('width')
+            metadata["width"] = stream.get("width")
-            metadata['height'] = stream.get('height')
+            metadata["height"] = stream.get("height")
-            metadata['format'] = 'mp4'
+            metadata["format"] = ext.lstrip(".")
         except Exception as e:
             print(f"[WARN] Failed to extract video metadata: {file_path} – {e}")
         return metadata
 
+    # Image: PIL + EXIF (best-effort)
     try:
-        metadata['file_size'] = os.path.getsize(file_path)
         with Image.open(file_path) as img:
-            metadata['width'], metadata['height'] = img.size
+            metadata["width"], metadata["height"] = img.size
-            metadata['format'] = img.format
+            metadata["format"] = (img.format or "").lower()
     except Exception as e:
-        ext = os.path.splitext(file_path)[1]
         mime_type, _ = mimetypes.guess_type(file_path)
-        print(f"[WARN] Failed to read image metadata: {file_path}")
+        print(f"[WARN] Failed to read image metadata: {file_path} ({e}) ext={ext} mime={mime_type}")
-        print(f"[WARN] Reason: {e}")
-        print(f"[INFO] File extension: {ext}, MIME type: {mime_type}")
 
     try:
-        with open(file_path, 'rb') as f:
+        with open(file_path, "rb") as f:
             tags = exifread.process_file(f, stop_tag="UNDEF", details=False)
-        if 'EXIF DateTimeOriginal' in tags:
+        if "EXIF DateTimeOriginal" in tags:
-            metadata['taken_at'] = datetime.strptime(str(tags['EXIF DateTimeOriginal']), "%Y:%m:%d %H:%M:%S")
+            metadata["taken_at"] = datetime.strptime(str(tags["EXIF DateTimeOriginal"]), "%Y:%m:%d %H:%M:%S")
-        if 'Image Model' in tags:
+        if "Image Model" in tags:
-            metadata['camera_model'] = str(tags['Image Model'])
+            metadata["camera_model"] = str(tags["Image Model"])
-    except Exception as e:
+    except Exception:
-        print(f"[WARN] Failed to read EXIF data: {file_path} – {e}")
+        pass
 
     return metadata
+
+
+# =============================================================================
+# Helpers: Multi-part detection
+# =============================================================================
+
+def is_first_volume(path: str) -> bool:
+    """
+    Determine if 'path' is the first volume of a multi-part archive.
+    We only attempt extraction from the first volume.
+    """
+    name = os.path.basename(path).lower()
+
+    # RAR: .partN.rar (only N==1 is first)
+    m = RAR_PART_RE.search(name)
+    if m:
+        n = m.group(1)
+        return n in ("1", "01", "001")
+
+    # RAR old-style: first is .rar, subsequent are .r00, .r01...
+    if name.endswith(".rar"):
+        return True
+    if OLD_RAR_VOL_RE.search(name):
+        return False
+
+    # CBR is typically a single-volume RAR
+    if name.endswith(".cbr"):
+        return True
+
+    # 7z multi-part: .7z.001 is first
+    m = SEVENZ_PART_RE.search(name)
+    if m:
+        n = m.group(1)
+        return n in ("1", "01", "001")
+    if name.endswith(".7z"):
+        return True  # single-volume 7z
+
+    # ZIP and tarballs: treat as single-volume (we don't include .z01 in our allowlist)
+    if name.endswith((".zip", ".tar", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
+        return True
+
+    return True

dockerfile

@@ -3,10 +3,14 @@ FROM python:slim
 
 WORKDIR /app
 
-# Only what you actually need (ffmpeg kept for video/thumbs)
+# System deps for archives and video thumbs (all FOSS)
-RUN apt-get update && \
+RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-get install -y --no-install-recommends ffmpeg && \
+    p7zip-full \
-    rm -rf /var/lib/apt/lists/*
+    unar \
+    file \
+    ffmpeg \
+ && rm -rf /var/lib/apt/lists/*
+
 
 # Install deps first for better layer caching
 COPY requirements.txt .

entrypoint.sh

@@ -30,9 +30,9 @@ fi
 
 # --- Start Gunicorn ---
 GUNICORN_BIND="${GUNICORN_BIND:-0.0.0.0:5000}"
-GUNICORN_WORKERS="${GUNICORN_WORKERS:-2}"
+GUNICORN_WORKERS="${GUNICORN_WORKERS:-4}"
-GUNICORN_THREADS="${GUNICORN_THREADS:-4}"
+GUNICORN_THREADS="${GUNICORN_THREADS:-8}"
-GUNICORN_TIMEOUT="${GUNICORN_TIMEOUT:-120}"
+GUNICORN_TIMEOUT="${GUNICORN_TIMEOUT:-180}"
 GUNICORN_OPTS="${GUNICORN_OPTS:-}"
 
 echo "Starting Gunicorn on ${GUNICORN_BIND} …"

migrations/versions/fe4dfdd6616c_add_tag_image_tags_archiverecord.py

@@ -0,0 +1,56 @@
+"""Add Tag, image_tags, ArchiveRecord
+
+Revision ID: fe4dfdd6616c
+Revises: 861f9cae4fe5
+Create Date: 2025-08-14 17:57:24.191794
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'fe4dfdd6616c'
+down_revision = '861f9cae4fe5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tag',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('kind', sa.String(length=64), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('archive_record',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('filename', sa.String(length=512), nullable=False),
+    sa.Column('file_size', sa.BigInteger(), nullable=False),
+    sa.Column('hash', sa.String(length=64), nullable=False),
+    sa.Column('imported_at', sa.DateTime(), nullable=True),
+    sa.Column('artist', sa.String(length=255), nullable=True),
+    sa.Column('tag_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('hash')
+    )
+    op.create_table('image_tags',
+    sa.Column('image_id', sa.Integer(), nullable=False),
+    sa.Column('tag_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], ),
+    sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ),
+    sa.PrimaryKeyConstraint('image_id', 'tag_id'),
+    sa.UniqueConstraint('image_id', 'tag_id', name='uq_image_tag')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('image_tags')
+    op.drop_table('archive_record')
+    op.drop_table('tag')
+    # ### end Alembic commands ###

requirements.txt

@@ -8,4 +8,6 @@ email-validator
 pillow
 exifread
 imagehash
+pyunpack
+patool
 gunicorn