fixed metadata time import error and moved quest status widget to top of settings.

app/main.py

@@ -490,55 +490,6 @@ def settings():
     return render_template('settings.html')
 
 
-@main.route('/trigger-thumbnail-generation', methods=['POST'])
-def trigger_thumbnail_generation():
-    try:
-        with open('/import/thumbnail.flag', 'w') as f:
-            f.write("trigger")
-        flash("Thumbnail regeneration has been triggered.", "success")
-    except Exception as e:
-        flash(f"Failed to trigger thumbnail regeneration: {e}", "danger")
-
-    return redirect(url_for('main.settings'))
-
-
-@main.route('/import-images')
-def trigger_image_import():
-    with open('/import/trigger.flag', 'w') as f:
-        f.write('start')
-    flash("Image import triggered.", "success")
-    return redirect(url_for('main.settings'))
-
-
-@main.route('/reset-db', methods=['POST'])
-def reset_db():
-    """
-    Safe reset that works with Postgres (TRUNCATE ... CASCADE).
-    Falls back to ordered deletes if TRUNCATE isn't supported (e.g., SQLite).
-    """
-    try:
-        # Postgres fast path
-        db.session.execute(text("""
-            TRUNCATE TABLE image_tags, archive_record, image_record, tag
-            RESTART IDENTITY CASCADE
-        """))
-        db.session.commit()
-    except Exception:
-        # Fallback: manual delete order
-        db.session.execute(image_tags.delete())
-        ArchiveRecord.query.delete()
-        ImageRecord.query.delete()
-        Tag.query.delete()
-        db.session.commit()
-
-    # Optional: clear thumbs on disk to regenerate fresh
-    shutil.rmtree("/images/thumbs", ignore_errors=True)
-    os.makedirs("/images/thumbs", exist_ok=True)
-
-    flash("Database has been reset. All records removed.", "info")
-    return redirect(url_for('main.settings'))
-
-
 # ----------------------------
 # Tag add/remove endpoints
 # ----------------------------
@@ -870,28 +821,6 @@ def save_import_settings():
         return jsonify(ok=False, error=str(e)), 500
 
 
-@main.get("/api/import-stats")
-def get_import_stats():
-    """Get import statistics including filtered image counts."""
-    import json
-    stats_path = "/import/stats.json"
-    try:
-        if os.path.exists(stats_path):
-            with open(stats_path, "r") as f:
-                stats = json.load(f)
-        else:
-            stats = {
-                "total_processed": 0,
-                "total_imported": 0,
-                "filtered_by_dimension": 0,
-                "filtered_by_transparency": 0,
-                "filtered_by_duplicate": 0,
-            }
-        return jsonify(ok=True, stats=stats)
-    except Exception as e:
-        return jsonify(ok=False, error=str(e)), 500
-
-
 # ----------------------------
 # Image filter/cleanup endpoints
 # ----------------------------

app/static/style.css

@@ -953,37 +953,30 @@ header {
 ------------------------------------------------------------------------------*/
 .settings-grid {
   display: grid;
-  grid-template-columns: repeat(3, 1fr);
+  grid-template-columns: repeat(2, 1fr);
   gap: 1.5rem;
   max-width: 1600px;
   margin: 1.5rem auto;
   padding: 0 1.5rem;
   align-items: start;
 }
+.settings-grid-full {
+  max-width: 1600px;
+  margin: 1.5rem auto;
+  padding: 0 1.5rem;
+}
 .settings-column {
   display: flex;
   flex-direction: column;
   gap: 1.5rem;
 }
-@media (max-width: 1200px) {
-  .settings-grid {
-    grid-template-columns: repeat(2, 1fr);
-  }
-  .settings-column:first-child {
-    grid-column: 1 / -1;
-    flex-direction: row;
-  }
-  .settings-column:first-child > .settings-container {
-    flex: 1;
-  }
-}
 @media (max-width: 768px) {
   .settings-grid {
     grid-template-columns: 1fr;
     padding: 0 1rem;
   }
-  .settings-column:first-child {
+  .settings-grid-full {
-    flex-direction: column;
+    padding: 0 1rem;
   }
 }
 .settings-container {

app/tasks/import_file.py

@@ -273,8 +273,9 @@ def import_archive(self, task_id: int):
     Returns:
         dict with status and counts
     """
-    from app.utils.image_importer import calculate_hash, compute_next_archive_tag_name
+    from app.utils.image_importer import (
-    from pyunpack import Archive
+        calculate_hash, compute_next_archive_tag_name, extract_archive_resilient, ExtractError
+    )
 
     task = ImportTask.query.get(task_id)
     if not task:
@@ -305,13 +306,13 @@ def import_archive(self, task_id: int):
         # Get archive size
         archive_size = os.path.getsize(archive_path)
 
-        # Extract archive
+        # Extract archive using resilient extractor (tries unar then 7z for RAR)
         tmpdir = tempfile.mkdtemp(prefix='extract_')
         log.info(f"Extracting to: {tmpdir}")
 
         try:
-            Archive(archive_path).extractall(tmpdir)
+            extract_archive_resilient(archive_path, tmpdir)
-        except Exception as e:
+        except ExtractError as e:
             log.error(f"Extraction failed: {e}")
             if tmpdir and os.path.exists(tmpdir):
                 shutil.rmtree(tmpdir, ignore_errors=True)

app/templates/settings.html

@@ -4,38 +4,93 @@
 {% block content %}
 <h1 style="text-align:center; margin-top: 1rem;">Settings</h1>
 
-<div class="settings-grid">
+<!-- Import Queue Status (Full Width at Top) -->
-  <!-- Column 1: Info & Admin Actions -->
+<div class="settings-grid-full">
-  <div class="settings-column">
+  <div class="settings-container">
-    <div class="settings-container">
+    <div class="settings-section">
-      <div class="settings-info">
+      <h2>Import Queue Status</h2>
-        <h2>About These Tools</h2>
+      <p class="settings-description">Monitor and control the Celery-based import queue system.</p>
-        <p>Use these tools to manage your image library. You can trigger imports, configure filters, and clean up unwanted images.</p>
+
-        <p style="margin-top: 0.75rem;"><strong>Note:</strong> Resetting the database does not delete the actual image files from disk.</p>
+      <div id="queueStatusSection">
+        <div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
+          <div class="stat-item">
+            <span class="stat-value" id="queuePending">-</span>
+            <span class="stat-label">Pending</span>
+          </div>
+          <div class="stat-item">
+            <span class="stat-value" id="queueQueued">-</span>
+            <span class="stat-label">Queued</span>
+          </div>
+          <div class="stat-item">
+            <span class="stat-value" id="queueProcessing">-</span>
+            <span class="stat-label">Processing</span>
+          </div>
+          <div class="stat-item">
+            <span class="stat-value" id="queueComplete">-</span>
+            <span class="stat-label">Complete</span>
+          </div>
+          <div class="stat-item">
+            <span class="stat-value" id="queueSkipped">-</span>
+            <span class="stat-label">Skipped</span>
+          </div>
+          <div class="stat-item">
+            <span class="stat-value" id="queueFailed">-</span>
+            <span class="stat-label">Failed</span>
+          </div>
+        </div>
+
+        <div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
+          <h3>Active Batch</h3>
+          <p>Processing <strong id="batchDir">-</strong></p>
+          <div class="batch-progress">
+            <div class="progress-bar">
+              <div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
+            </div>
+            <span id="batchProgressText">0 / 0</span>
+          </div>
+        </div>
+
+        <div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
+          <button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
+          <button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
+          <button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
+          <button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
+        </div>
+
+        <div id="celeryWorkerInfo" style="margin-top: 1rem;">
+          <p style="color: var(--text-muted); font-size: 0.85rem;">
+            Workers: <strong id="workerCount">-</strong> |
+            Active Tasks: <strong id="activeTasks">-</strong>
+          </p>
+        </div>
       </div>
-    </div>
 
-    <div class="settings-container">
+      <!-- Recent Tasks Table -->
-      <div class="settings-section">
+      <div id="recentTasksSection" style="margin-top: 1.5rem;">
-        <h2>Admin Actions</h2>
+        <h3>Recent Tasks</h3>
-        <div class="settings-actions">
+        <div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
-          <form action="{{ url_for('main.trigger_image_import') }}" method="get">
+          <table class="tasks-table">
-            <button type="submit" class="btn primary-btn">Trigger Image Import</button>
+            <thead>
-          </form>
+              <tr>
-
+                <th>File</th>
-          <form action="{{ url_for('main.trigger_thumbnail_generation') }}" method="post" onsubmit="return confirm('Are you sure you want to regenerate all thumbnails?');">
+                <th>Type</th>
-            <button type="submit" class="btn warning-btn">Regenerate Thumbnails</button>
+                <th>Status</th>
-          </form>
+                <th>Error</th>
-
+                <th>Time</th>
-          <form action="{{ url_for('main.reset_db') }}" method="post" onsubmit="return confirm('Are you sure you want to delete all image records? This action cannot be undone.');">
+              </tr>
-            <button type="submit" class="btn danger-btn">Reset Image Database</button>
+            </thead>
-          </form>
+            <tbody id="recentTasksBody">
+              <tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
+            </tbody>
+          </table>
         </div>
       </div>
     </div>
   </div>
+</div>
 
-  <!-- Column 2: Import Filters -->
+<div class="settings-grid">
+  <!-- Column 1: Import Filters -->
   <div class="settings-column">
     <div class="settings-container">
       <div class="settings-section">
@@ -81,33 +136,11 @@
 
           <button type="submit" class="btn primary-btn" style="width: 100%;">Save Filter Settings</button>
         </form>
-
-        <div id="importStats" class="import-stats">
-          <h3>Last Import Statistics</h3>
-          <div class="stats-grid">
-            <div class="stat-item">
-              <span class="stat-value" id="statProcessed">-</span>
-              <span class="stat-label">Processed</span>
-            </div>
-            <div class="stat-item">
-              <span class="stat-value" id="statImported">-</span>
-              <span class="stat-label">Imported</span>
-            </div>
-            <div class="stat-item">
-              <span class="stat-value" id="statDimension">-</span>
-              <span class="stat-label">Filtered (size)</span>
-            </div>
-            <div class="stat-item">
-              <span class="stat-value" id="statTransparency">-</span>
-              <span class="stat-label">Filtered (transparent)</span>
-            </div>
-          </div>
-        </div>
       </div>
     </div>
   </div>
 
-  <!-- Column 3: Deletion Tools -->
+  <!-- Column 2: Deletion Tools -->
   <div class="settings-column">
     <div class="settings-container">
       <div class="settings-section">
@@ -246,89 +279,6 @@
   </div>
 </div>
 
-<!-- Import Queue Status (Full Width) -->
-<div class="settings-container" style="max-width: 1200px; margin: 2rem auto;">
-  <div class="settings-section">
-    <h2>Import Queue Status</h2>
-    <p class="settings-description">Monitor and control the Celery-based import queue system.</p>
-
-    <div id="queueStatusSection">
-      <div class="stats-grid" style="grid-template-columns: repeat(6, 1fr);">
-        <div class="stat-item">
-          <span class="stat-value" id="queuePending">-</span>
-          <span class="stat-label">Pending</span>
-        </div>
-        <div class="stat-item">
-          <span class="stat-value" id="queueQueued">-</span>
-          <span class="stat-label">Queued</span>
-        </div>
-        <div class="stat-item">
-          <span class="stat-value" id="queueProcessing">-</span>
-          <span class="stat-label">Processing</span>
-        </div>
-        <div class="stat-item">
-          <span class="stat-value" id="queueComplete">-</span>
-          <span class="stat-label">Complete</span>
-        </div>
-        <div class="stat-item">
-          <span class="stat-value" id="queueSkipped">-</span>
-          <span class="stat-label">Skipped</span>
-        </div>
-        <div class="stat-item">
-          <span class="stat-value" id="queueFailed">-</span>
-          <span class="stat-label">Failed</span>
-        </div>
-      </div>
-
-      <div id="activeBatchInfo" class="import-stats" style="margin-top: 1rem; display: none;">
-        <h3>Active Batch</h3>
-        <p>Processing <strong id="batchDir">-</strong></p>
-        <div class="batch-progress">
-          <div class="progress-bar">
-            <div class="progress-fill" id="batchProgressFill" style="width: 0%"></div>
-          </div>
-          <span id="batchProgressText">0 / 0</span>
-        </div>
-      </div>
-
-      <div class="queue-actions" style="margin-top: 1rem; display: flex; gap: 0.75rem; flex-wrap: wrap;">
-        <button id="triggerImportBtn" class="btn primary-btn">Trigger Import Scan</button>
-        <button id="retryFailedBtn" class="btn warning-btn">Retry Failed Tasks</button>
-        <button id="clearCompletedBtn" class="btn secondary-btn">Clear Completed</button>
-        <button id="regenMissingThumbsBtn" class="btn secondary-btn">Regenerate Missing Thumbnails</button>
-      </div>
-
-      <div id="celeryWorkerInfo" style="margin-top: 1rem;">
-        <p style="color: var(--text-muted); font-size: 0.85rem;">
-          Workers: <strong id="workerCount">-</strong> |
-          Active Tasks: <strong id="activeTasks">-</strong>
-        </p>
-      </div>
-    </div>
-
-    <!-- Recent Tasks Table -->
-    <div id="recentTasksSection" style="margin-top: 1.5rem;">
-      <h3>Recent Tasks</h3>
-      <div class="tasks-table-wrapper" style="max-height: 300px; overflow-y: auto;">
-        <table class="tasks-table">
-          <thead>
-            <tr>
-              <th>File</th>
-              <th>Type</th>
-              <th>Status</th>
-              <th>Error</th>
-              <th>Time</th>
-            </tr>
-          </thead>
-          <tbody id="recentTasksBody">
-            <tr><td colspan="5" style="text-align: center; color: var(--text-muted);">Loading...</td></tr>
-          </tbody>
-        </table>
-      </div>
-    </div>
-  </div>
-</div>
-
 <style>
   .tasks-table {
     width: 100%;
@@ -797,18 +747,6 @@ document.addEventListener('DOMContentLoaded', () => {
       }
     });
 
-  // Load import stats
-  fetch('/api/import-stats')
-    .then(r => r.json())
-    .then(data => {
-      if (data.ok) {
-        document.getElementById('statProcessed').textContent = data.stats.total_processed || 0;
-        document.getElementById('statImported').textContent = data.stats.total_imported || 0;
-        document.getElementById('statDimension').textContent = data.stats.filtered_by_dimension || 0;
-        document.getElementById('statTransparency').textContent = data.stats.filtered_by_transparency || 0;
-      }
-    });
-
   // Load tags for deletion dropdown
   loadTagsForDeletion();
 

app/utils/image_importer.py

@@ -12,15 +12,12 @@ import re
 import shutil
 import subprocess
 import tempfile
-import time
 import uuid
 
 from PIL import Image, ImageOps
 import exifread
 import imagehash
 
-from pyunpack import Archive
-
 from app import db
 from app.models import ImageRecord, Tag, ArchiveRecord
 
@@ -48,7 +45,6 @@ ALLOWED_ARCHIVE_EXTS = (
 
 # Import filter settings (loaded from file or env)
 IMPORT_SETTINGS_PATH = "/import/settings.json"
-IMPORT_STATS_PATH = "/import/stats.json"
 
 
 def load_import_settings() -> dict:
@@ -92,15 +88,6 @@ def load_import_settings() -> dict:
     return defaults
 
 
-def save_import_stats(stats: dict) -> None:
-    """Save import statistics to file."""
-    try:
-        with open(IMPORT_STATS_PATH, "w") as f:
-            json.dump(stats, f, indent=2)
-    except Exception as e:
-        print(f"[WARN] Failed to save import stats: {e}")
-
-
 def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool:
     """
     Check if an image is mostly transparent (> threshold % of pixels are transparent).
@@ -398,298 +385,7 @@ def extract_archive_resilient(archive_path: str, out_dir: str) -> None:
 
 
 # =============================================================================
-# Core: Import task & archive processing
+# Content-addressed storage helpers
-# =============================================================================
-
-def import_images_task(source_dir: str, dest_dir: str) -> str:
-    """
-    Walk the import tree, process archives first (only first volumes),
-    then media files. Batches DB commits for efficiency.
-    """
-    # Cleanup any stale extraction dirs from previous runs/crashes
-    cleanup_stale_tempdirs()
-
-    # Load import filter settings
-    settings = load_import_settings()
-    min_width = settings.get("min_width", 0)
-    min_height = settings.get("min_height", 0)
-    skip_transparent = settings.get("skip_transparent", False)
-    transparency_threshold = settings.get("transparency_threshold", 0.9)
-    skip_single_color = settings.get("skip_single_color", True)
-    single_color_threshold = settings.get("single_color_threshold", 0.95)
-    single_color_tolerance = settings.get("single_color_tolerance", 30)
-    phash_threshold = settings.get("phash_threshold", 2)
-    supersede_smaller = settings.get("supersede_smaller", True)
-
-    print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, "
-          f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, "
-          f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, "
-          f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}")
-
-    # Statistics tracking
-    stats = {
-        "total_processed": 0,
-        "total_imported": 0,
-        "total_superseded": 0,
-        "filtered_by_dimension": 0,
-        "filtered_by_transparency": 0,
-        "filtered_by_single_color": 0,
-        "filtered_by_duplicate": 0,
-    }
-
-    imported: list[str] = []
-    batch_size = 10
-    batch_counter = 0
-
-    # Preload existing pHashes to avoid N+1 lookups
-    # Format: (phash, width, height, image_id) for supersede functionality
-    existing_phashes = [
-        (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
-        for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
-    ]
-
-    # Ensure thumbs base exists
-    (Path(dest_dir) / "thumbs").mkdir(parents=True, exist_ok=True)
-
-    # Optional free-space guard
-    min_free = float(os.environ.get(ENV_MINFREE, "0") or "0")
-    try:
-        ensure_free_space_or_raise(get_tmp_base(), min_free)
-    except Exception as e:
-        print(f"[WARN] Free-space check failed: {e}")
-
-    for artist_dir in os.listdir(source_dir):
-        artist_path = os.path.join(source_dir, artist_dir)
-        if not os.path.isdir(artist_path):
-            continue
-
-        for root, _, files in os.walk(artist_path):
-            for file in files:
-                full_path = os.path.join(root, file)
-                lower = file.lower()
-
-                # Archives first
-                if lower.endswith(ALLOWED_ARCHIVE_EXTS):
-                    if not is_first_volume(full_path):
-                        print(f"[SKIP] Not first archive volume: {file}")
-                        continue
-                    try:
-                        count = process_archive(
-                            archive_path=full_path,
-                            dest_dir=dest_dir,
-                            artist=artist_dir,
-                            existing_phashes=existing_phashes,
-                            settings=settings,
-                            stats=stats
-                        )
-                        imported.append(f"[archive]{file}:{count}")
-                    except Exception as e:
-                        print(f"[WARN] Failed processing archive {file}: {e}")
-                    continue
-
-                # Non-archive media
-                if not lower.endswith(ALLOWED_MEDIA_EXTS):
-                    continue
-
-                stats["total_processed"] += 1
-
-                # Apply dimension filter
-                if min_width > 0 or min_height > 0:
-                    md = extract_metadata(full_path)
-                    if md.get("width") and md.get("height"):
-                        if md["width"] < min_width or md["height"] < min_height:
-                            print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
-                            stats["filtered_by_dimension"] += 1
-                            continue
-
-                # Apply transparency filter
-                if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
-                    if is_mostly_transparent(full_path, transparency_threshold):
-                        print(f"[SKIP] Mostly transparent: {file}")
-                        stats["filtered_by_transparency"] += 1
-                        continue
-
-                # Apply single-color filter (skip images that are mostly one solid color)
-                if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
-                    if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance):
-                        print(f"[SKIP] Mostly single color: {file}")
-                        stats["filtered_by_single_color"] += 1
-                        continue
-
-                added, phash, _rec, superseded = import_single_file(
-                    src_path=full_path,
-                    dest_dir=dest_dir,
-                    artist=artist_dir,
-                    existing_phashes=existing_phashes,
-                    phash_threshold=phash_threshold,
-                    supersede_smaller=supersede_smaller
-                )
-                if added:
-                    imported.append(file)
-                    if superseded:
-                        stats["total_superseded"] += 1
-                    else:
-                        stats["total_imported"] += 1
-                elif _rec is None and phash is None:
-                    # Was a duplicate
-                    stats["filtered_by_duplicate"] += 1
-
-                # Add new image's phash to the tracking list (for fresh imports only, not supersedes)
-                if phash and _rec and not superseded:
-                    existing_phashes.append((phash, _rec.width, _rec.height, _rec.id))
-
-                batch_counter += 1
-                if batch_counter >= batch_size:
-                    db.session.commit()
-                    print(f"[INFO] Committed batch of {batch_size} items.")
-                    batch_counter = 0
-                    if "sqlite" in db.engine.url.drivername:
-                        time.sleep(1)
-
-    if batch_counter > 0:
-        db.session.commit()
-        print(f"[INFO] Committed final batch of {batch_counter} items.")
-
-    # Save stats
-    save_import_stats(stats)
-
-    summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. "
-               f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, "
-               f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.")
-    print(f"[INFO] {summary}")
-    return summary
-
-
-def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=None, stats=None):
-    """
-    Extracts archive to temp dir, imports/tag existing images, and records ArchiveRecord.
-    Creates an archive:* tag ONLY if at least one image was imported or tagged.
-    Returns number of items imported or tagged.
-    """
-    if settings is None:
-        settings = load_import_settings()
-    if stats is None:
-        stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0,
-                 "filtered_by_dimension": 0, "filtered_by_transparency": 0,
-                 "filtered_by_single_color": 0, "filtered_by_duplicate": 0}
-
-    min_width = settings.get("min_width", 0)
-    min_height = settings.get("min_height", 0)
-    skip_transparent = settings.get("skip_transparent", False)
-    transparency_threshold = settings.get("transparency_threshold", 0.9)
-    skip_single_color = settings.get("skip_single_color", True)
-    single_color_threshold = settings.get("single_color_threshold", 0.95)
-    single_color_tolerance = settings.get("single_color_tolerance", 30)
-    phash_threshold = settings.get("phash_threshold", 2)
-    supersede_smaller = settings.get("supersede_smaller", True)
-
-    archive_size = os.path.getsize(archive_path)
-    archive_hash = calculate_hash(archive_path)
-
-    # Skip if this exact archive file was already processed (hash-based)
-    existing_archive = ArchiveRecord.query.filter_by(hash=archive_hash).first()
-    if existing_archive:
-        print(f"[SKIP] Archive already imported: {archive_path}")
-        return 0
-
-    tmpdir = tempfile.mkdtemp(prefix="extract_")
-    touched_records = []   # ImageRecord objects we imported or matched
-    imported_or_tagged = 0
-    created_tag = None
-
-    try:
-        # Extract using pyunpack
-        Archive(archive_path).extractall(tmpdir)
-
-        # Import / collect all media
-        for root, _, files in os.walk(tmpdir):
-            for file in files:
-                lower = file.lower()
-                if not lower.endswith(ALLOWED_MEDIA_EXTS):
-                    continue
-                src_path = os.path.join(root, file)
-                stats["total_processed"] += 1
-
-                # Apply dimension filter
-                if min_width > 0 or min_height > 0:
-                    md = extract_metadata(src_path)
-                    if md.get("width") and md.get("height"):
-                        if md["width"] < min_width or md["height"] < min_height:
-                            print(f"[SKIP] Too small ({md['width']}x{md['height']}): {file}")
-                            stats["filtered_by_dimension"] += 1
-                            continue
-
-                # Apply transparency filter
-                if skip_transparent and lower.endswith((".png", ".gif", ".webp")):
-                    if is_mostly_transparent(src_path, transparency_threshold):
-                        print(f"[SKIP] Mostly transparent: {file}")
-                        stats["filtered_by_transparency"] += 1
-                        continue
-
-                # Apply single-color filter
-                if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
-                    if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance):
-                        print(f"[SKIP] Mostly single color: {file}")
-                        stats["filtered_by_single_color"] += 1
-                        continue
-
-                added, _phash, rec, superseded = import_single_file(
-                    src_path=src_path,
-                    dest_dir=dest_dir,
-                    artist=artist,  # still used for auto artist:<name> tag
-                    existing_phashes=existing_phashes,
-                    commit=False,
-                    phash_threshold=phash_threshold,
-                    supersede_smaller=supersede_smaller
-                )
-                if rec is not None:
-                    touched_records.append(rec)
-                    if added:
-                        imported_or_tagged += 1
-                        if superseded:
-                            stats["total_superseded"] += 1
-                        else:
-                            stats["total_imported"] += 1
-                elif rec is None and _phash is None:
-                    stats["filtered_by_duplicate"] += 1
-
-        # Only now: create/attach the archive tag if we actually touched images
-        if touched_records:
-            # Incremental name like 'archive:{artist}/0001'
-            tag_name = compute_next_archive_tag_name(artist)
-            created_tag = Tag.query.filter_by(name=tag_name).first()
-            if not created_tag:
-                created_tag = Tag(name=tag_name, kind='archive')
-                db.session.add(created_tag)
-                db.session.flush()
-
-            newly_tagged = 0
-            for rec in touched_records:
-                if created_tag not in rec.tags:
-                    rec.tags.append(created_tag)
-                    newly_tagged += 1
-            imported_or_tagged += newly_tagged
-
-        # Record this archive as processed (tag may be None if no media)
-        arch = ArchiveRecord(
-            filename=archive_path,
-            file_size=archive_size,
-            hash=archive_hash,
-            artist=artist,
-            tag=created_tag
-        )
-        db.session.add(arch)
-        db.session.commit()
-
-        print(f"[INFO] Archive processed: {archive_path} items={imported_or_tagged}")
-        return imported_or_tagged
-
-    finally:
-        shutil.rmtree(tmpdir, ignore_errors=True)
-
-
-# =============================================================================
-# Core: Single-file import path (content-addressed storage)
 # =============================================================================
 
 def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: str) -> str:
@@ -712,225 +408,6 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st
         i += 1
 
 
-def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes,
-                       commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True):
-    """
-    Import a single media file. If an equivalent file already exists, return that
-    record so callers can tag it (no duplicate import).
-
-    If a smaller visually-similar image exists and supersede_smaller is True,
-    the smaller image will be replaced with the larger one (preserving tags).
-
-    Also enriches metadata from Gallery-DL sidecar JSON files if present.
-
-    Returns (added: bool, phash_or_None, record_or_None, superseded: bool).
-    The superseded flag indicates if an existing smaller image was replaced.
-    """
-    original_name = os.path.basename(src_path)
-    print(f"[INFO] Processing: {src_path}")
-    file_size = os.path.getsize(src_path)
-
-    # Compute content hash first (authoritative de-dup + used in filepath)
-    file_hash = calculate_hash(src_path)
-    existing = ImageRecord.query.filter_by(hash=file_hash).first()
-    if existing:
-        print(f"[SKIP] Duplicate by hash: {original_name}")
-        # Even for duplicates, try to enrich with sidecar metadata
-        _enrich_existing_record(existing, src_path)
-        return (False, None, existing, False)
-
-    # Optional info: same name+size (but different content) → we still import with hashed name
-    existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
-    if existing_ns:
-        print(f"[INFO] Same name+size exists but different hash; storing with hashed name: {original_name}")
-
-    # Metadata & pHash similarity
-    metadata = extract_metadata(src_path)
-    if not metadata["width"] or not metadata["height"]:
-        print(f"[SKIP] Missing dimension data for {original_name}")
-        return (False, None, None, False)
-
-    phash = calculate_perceptual_hash(src_path)
-
-    # Check for similar images
-    superseded = False
-    if phash and phash_threshold > 0:
-        relationship, similar_id = find_similar_image(
-            phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold
-        )
-
-        if relationship == "larger_exists":
-            print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
-            return (False, phash, None, False)
-
-        elif relationship == "smaller_exists" and supersede_smaller and similar_id:
-            # Found a smaller version - supersede it
-            old_record = ImageRecord.query.get(similar_id)
-            if old_record:
-                # Enrich metadata from Gallery-DL sidecar JSON if present
-                enriched = _get_enriched_metadata(src_path, metadata)
-                metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"]
-
-                # Copy new file to destination
-                target_dir = os.path.join(dest_dir, artist)
-                os.makedirs(target_dir, exist_ok=True)
-                dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
-                shutil.copy2(src_path, dest_path)
-
-                # Generate new thumbnail
-                try:
-                    if dest_path.lower().endswith((".mp4", ".mov")):
-                        thumb_path = generate_video_thumbnail_mirrored(dest_path)
-                    else:
-                        thumb_path = generate_thumbnail(dest_path)
-                except Exception as e:
-                    print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
-                    thumb_path = None
-
-                # Supersede the old record (preserves tags)
-                record = supersede_image(
-                    old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name
-                )
-
-                # Apply any new tags from sidecar
-                _apply_enriched_tags(record, enriched)
-
-                if commit:
-                    db.session.commit()
-
-                # Update existing_phashes list with new dimensions
-                for i, (ph, w, h, img_id) in enumerate(existing_phashes):
-                    if img_id == similar_id:
-                        existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id)
-                        break
-
-                return (True, phash, record, True)  # superseded=True
-
-    # No similar image found (or supersede disabled) - normal import
-    # Enrich metadata from Gallery-DL sidecar JSON if present
-    enriched = _get_enriched_metadata(src_path, metadata)
-
-    # Copy into /images/<artist>/<base>__<hash[:10]><ext>
-    target_dir = os.path.join(dest_dir, artist)
-    os.makedirs(target_dir, exist_ok=True)
-    dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
-    shutil.copy2(src_path, dest_path)
-
-    # Thumbnails (mirrored for both images and videos)
-    try:
-        if dest_path.lower().endswith((".mp4", ".mov")):
-            thumb_path = generate_video_thumbnail_mirrored(dest_path)
-        else:
-            thumb_path = generate_thumbnail(dest_path)
-    except Exception as e:
-        print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
-        thumb_path = None
-
-    # Create DB record (keep display name as original; filepath is content-addressed)
-    # Use enriched taken_at (earliest date wins)
-    record = ImageRecord(
-        filename=original_name,
-        filepath=dest_path,
-        thumb_path=thumb_path,
-        hash=file_hash,
-        perceptual_hash=str(phash) if phash else None,
-        file_size=metadata["file_size"],
-        width=metadata["width"],
-        height=metadata["height"],
-        format=metadata["format"],
-        camera_model=metadata["camera_model"],
-        taken_at=enriched.get("taken_at") or metadata["taken_at"],
-        imported_at=datetime.utcnow()
-    )
-
-    # Auto-tag with artist:<name> (treat artist as a tag only)
-    if artist:
-        artist_tag_name = f"artist:{artist}"
-        artist_tag = Tag.query.filter_by(name=artist_tag_name).first()
-        if not artist_tag:
-            artist_tag = Tag(name=artist_tag_name, kind="artist")
-            db.session.add(artist_tag)
-            db.session.flush()
-        if artist_tag not in record.tags:
-            record.tags.append(artist_tag)
-
-    # Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id)
-    _apply_enriched_tags(record, enriched)
-
-    db.session.add(record)
-    if commit:
-        db.session.commit()
-    return (True, phash, record, False)  # superseded=False (new import)
-
-
-def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict:
-    """
-    Get enriched metadata from Gallery-DL sidecar JSON.
-    Returns empty dict if no sidecar found.
-    """
-    try:
-        from app.utils.metadata_enrichment import enrich_from_sidecar
-        return enrich_from_sidecar(src_path, existing_metadata)
-    except ImportError:
-        return {}
-    except Exception as e:
-        print(f"[WARN] Failed to enrich metadata for {src_path}: {e}")
-        return {}
-
-
-def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None:
-    """
-    Apply tags from enriched metadata to an ImageRecord.
-    Creates tags if they don't exist.
-    """
-    if not enriched or "tags" not in enriched:
-        return
-
-    for tag_info in enriched.get("tags", []):
-        tag_name = tag_info.get("name")
-        tag_kind = tag_info.get("kind")
-        if not tag_name:
-            continue
-
-        tag = Tag.query.filter_by(name=tag_name).first()
-        if not tag:
-            tag = Tag(name=tag_name, kind=tag_kind)
-            db.session.add(tag)
-            db.session.flush()
-
-        if tag not in record.tags:
-            record.tags.append(tag)
-            print(f"[INFO] Added tag: {tag_name}")
-
-
-def _enrich_existing_record(record: ImageRecord, src_path: str) -> None:
-    """
-    Enrich an existing record with metadata from sidecar JSON.
-    Used when a duplicate is found - we still want to merge metadata.
-
-    - Updates taken_at if sidecar date is earlier
-    - Adds source/post tags if not already present
-    """
-    try:
-        from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict
-
-        enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at})
-        if not enriched.get("raw_metadata"):
-            return  # No sidecar found
-
-        # Update date if sidecar has an earlier date
-        new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at"))
-        if new_date != record.taken_at:
-            print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}")
-            record.taken_at = new_date
-
-        # Add any new tags
-        _apply_enriched_tags(record, enriched)
-
-    except Exception as e:
-        print(f"[WARN] Failed to enrich existing record: {e}")
-
-
 # =============================================================================
 # Helpers: Thumbnails, metadata, hashing, similarity
 # =============================================================================

app/utils/metadata_enrichment.py

@@ -301,6 +301,18 @@ def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[
     if new_date is None:
         return existing_date
 
+    # Normalize timezone awareness before comparison
+    # If one is aware and one is naive, strip timezone from the aware one
+    existing_aware = existing_date.tzinfo is not None
+    new_aware = new_date.tzinfo is not None
+
+    if existing_aware and not new_aware:
+        # Strip timezone from existing to compare
+        existing_date = existing_date.replace(tzinfo=None)
+    elif new_aware and not existing_aware:
+        # Strip timezone from new to compare
+        new_date = new_date.replace(tzinfo=None)
+
     # Return the earlier date
     return min(existing_date, new_date)
 

requirements.txt

@@ -5,8 +5,6 @@ psycopg2-binary
 pillow
 exifread
 imagehash
-pyunpack
-patool
 gunicorn
 
 # Celery task queue