tuning job log views and deep scan archive processing.

2026-02-02 18:48:01 -05:00
parent 042a69f9c3
commit 09883960d4
9 changed files with 1078 additions and 103 deletions
@@ -633,3 +633,105 @@ def update_batch_stats(batch_id: int):
                 f"{batch.skipped_count} skipped, {batch.failed_count} failed")

    db.session.commit()
+
+
+@celery.task(name='app.tasks.scan.update_system_stats')
+def update_system_stats():
+    """
+    Compute and cache system-wide statistics for the dashboard.
+
+    Called periodically by Celery Beat to avoid expensive queries on page load.
+    Stores results in AppSettings as JSON for quick retrieval.
+
+    Stats computed:
+    - Total images and storage usage
+    - Tag counts by kind
+    - Pending files/archives in import folder
+
+    Returns:
+        dict with computed stats
+    """
+    import json
+    from sqlalchemy import func
+    from app.models import ImageRecord, Tag, AppSettings
+
+    log.info("Updating system stats cache...")
+
+    try:
+        # Database stats
+        total_images = ImageRecord.query.count()
+        total_tags = Tag.query.count()
+
+        # Tag kind breakdown
+        tag_kinds = dict(
+            db.session.query(Tag.kind, func.count(Tag.id))
+            .group_by(Tag.kind).all()
+        )
+
+        # Storage stats (sum of file_size column)
+        storage_result = db.session.query(func.sum(ImageRecord.file_size)).scalar()
+        total_storage_bytes = storage_result or 0
+
+        # Import folder stats
+        import_dir = '/import'
+        pending_files = 0
+        pending_archives = 0
+        if os.path.exists(import_dir):
+            for artist_dir in os.listdir(import_dir):
+                artist_path = os.path.join(import_dir, artist_dir)
+                if not os.path.isdir(artist_path):
+                    continue
+                for root, _, files in os.walk(artist_path):
+                    for f in files:
+                        lower = f.lower()
+                        if lower.endswith(ALLOWED_MEDIA_EXTS):
+                            pending_files += 1
+                        elif lower.endswith(ALLOWED_ARCHIVE_EXTS):
+                            pending_archives += 1
+
+        # Format storage for display
+        storage_human = _format_bytes(total_storage_bytes)
+
+        # Build stats dict
+        stats = {
+            'images': {
+                'total': total_images,
+                'storage_bytes': total_storage_bytes,
+                'storage_human': storage_human
+            },
+            'tags': {
+                'total': total_tags,
+                'by_kind': tag_kinds
+            },
+            'import_folder': {
+                'pending_files': pending_files,
+                'pending_archives': pending_archives
+            },
+            'updated_at': datetime.utcnow().isoformat()
+        }
+
+        # Store in AppSettings
+        setting = AppSettings.query.filter_by(key='system_stats_cache').first()
+        if not setting:
+            setting = AppSettings(key='system_stats_cache')
+            db.session.add(setting)
+        setting.value = json.dumps(stats)
+        db.session.commit()
+
+        log.info(f"System stats updated: {total_images} images, {total_tags} tags, "
+                 f"{storage_human} storage, {pending_files + pending_archives} pending")
+
+        return stats
+
+    except Exception as e:
+        log.error(f"Failed to update system stats: {e}", exc_info=True)
+        raise
+
+
+def _format_bytes(size_bytes: int) -> str:
+    """Format bytes to human readable string."""
+    for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
+        if size_bytes < 1024:
+            return f"{size_bytes:.1f} {unit}"
+        size_bytes /= 1024
+    return f"{size_bytes:.1f} PB"