diff --git a/app/__init__.py b/app/__init__.py index e69b946..2e3152f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,6 +1,8 @@ # app/__init__.py +import click from flask import Flask +from flask.cli import with_appcontext from flask_sqlalchemy import SQLAlchemy from flask_migrate import Migrate from werkzeug.middleware.proxy_fix import ProxyFix @@ -11,6 +13,18 @@ import sqlite3 db = SQLAlchemy() migrate = Migrate() + +# ============================================================================= +# CLI Commands +# ============================================================================= + +@click.command("version-check") +@with_appcontext +def version_check_command(): + """Check app version and run data migrations if needed (e.g., phash recomputation).""" + from app.utils.version_migration import check_and_run_migrations + check_and_run_migrations() + @event.listens_for(Engine, "connect") def set_sqlite_pragma(dbapi_connection, connection_record): # Only apply to SQLite connections @@ -38,4 +52,7 @@ def create_app(config_class='config.Config'): def datetimeformat(value, format="%Y-%m-%d"): return value.strftime(format) if value else "" + # Register CLI commands + app.cli.add_command(version_check_command) + return app diff --git a/app/main.py b/app/main.py index ce83867..3a48c0b 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,10 @@ from flask import Blueprint, render_template, flash, redirect, url_for, abort, send_from_directory, request, jsonify -from sqlalchemy import desc, func, text +from sqlalchemy import desc, func, text, extract from sqlalchemy.orm import joinedload +from collections import OrderedDict +from datetime import datetime from app.models import ImageRecord, Tag, ArchiveRecord, image_tags from app import db @@ -78,35 +80,47 @@ def random_images_api(): @main.route('/gallery') def gallery(): """ - Gallery with pagination and optional tag filter (?tag=artist:NAME or any tag name). - Keeps your pagination shape so modal-pagination.js continues to work. + Gallery with infinite scroll and year-month date dividers. + Server renders initial batch, then JS takes over for infinite scroll. + Optional tag filter via ?tag=artist:NAME or any tag name. """ - page = request.args.get('page', 1, type=int) - per_page = request.args.get('per_page', 35, type=int) + initial_limit = 30 tag_name = request.args.get('tag') - # Base query + eager-load tags (cheap and future-proofs tag chips) + # Base query + eager-load tags q = ImageRecord.query.options(joinedload(ImageRecord.tags)) # Optional tag filter if tag_name: q = q.join(ImageRecord.tags).filter(Tag.name == tag_name) - images = q.order_by(desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at))) \ - .paginate(page=page, per_page=per_page) + # Fetch initial batch ordered by date descending + images = q.order_by( + desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at)) + ).limit(initial_limit + 1).all() - # Preserve next/prev URLs (carry tag= so modal pagination fetches stay filtered) - next_url = url_for('main.gallery', page=images.next_num, per_page=per_page, tag=tag_name) if images.has_next else '' - prev_url = url_for('main.gallery', page=images.prev_num, per_page=per_page, tag=tag_name) if images.has_prev else '' + # Check if there are more images + has_more = len(images) > initial_limit + if has_more: + images = images[:initial_limit] + + # Group by year-month for initial render + grouped_images = _group_images_by_month(images) + + # Compute initial cursor for JS to continue from + initial_cursor = None + if images and has_more: + last_img = images[-1] + last_dt = _get_image_date(last_img) + if last_dt: + initial_cursor = last_dt.isoformat() return render_template( 'gallery.html', - images=images, + grouped_images=grouped_images, active_tag=tag_name, - has_next=images.has_next, - next_page_url=next_url, - has_prev=images.has_prev, - prev_page_url=prev_url + initial_cursor=initial_cursor, + has_more=has_more ) @@ -115,6 +129,265 @@ def serve_image(filename): return send_from_directory('/images', filename) +# ---------------------------- +# Gallery Infinite Scroll API +# ---------------------------- + +def _get_image_date(img): + """Get the effective date for an image (taken_at or imported_at).""" + return img.taken_at or img.imported_at + + +def _format_year_month(dt): + """Format a datetime as year-month key and label.""" + if not dt: + return 'unknown', 'Unknown Date' + return dt.strftime('%Y-%m'), dt.strftime('%B %Y') + + +def _group_images_by_month(images): + """ + Group images by year-month, preserving order. + Returns list of (group_info, images) tuples. + """ + groups = OrderedDict() + + for img in images: + dt = _get_image_date(img) + key, label = _format_year_month(dt) + + if key not in groups: + groups[key] = {'key': key, 'label': label, 'images': []} + + groups[key]['images'].append(img) + + return [(info['key'], info['label'], info['images']) for info in groups.values()] + + +def _serialize_image(img): + """Serialize an ImageRecord for JSON response.""" + is_video = img.filename.lower().endswith(('.mp4', '.mov')) + thumb_path = (img.thumb_path or img.filepath).replace('/images/', '') + full_path = img.filepath.replace('/images/', '') + dt = _get_image_date(img) + key, label = _format_year_month(dt) + + return { + 'id': img.id, + 'filename': img.filename, + 'thumb_url': url_for('main.serve_image', filename=thumb_path), + 'full_url': url_for('main.serve_image', filename=full_path), + 'is_video': is_video, + 'date': dt.strftime('%Y-%m-%d') if dt else '', + 'year_month': key, + 'year_month_label': label, + 'tags': [{'name': t.name, 'kind': t.kind} for t in img.tags if t.kind != 'archive'] + } + + +@main.route('/api/gallery/scroll') +def gallery_scroll(): + """ + Cursor-based infinite scroll endpoint with date grouping. + + Query params: + - cursor: ISO datetime string for pagination (fetch images older than this) + - limit: number of images per batch (default 30) + - tag: optional tag filter + + Returns: + - images: list of image objects with year_month grouping info + - next_cursor: datetime string for next batch (null if no more) + - date_groups: summary of year-month groups in this batch + - has_more: boolean indicating if more images exist + """ + cursor_str = request.args.get('cursor') + limit = request.args.get('limit', 30, type=int) + tag_name = request.args.get('tag') + + # Build base query + q = ImageRecord.query.options(joinedload(ImageRecord.tags)) + + # Optional tag filter + if tag_name: + q = q.join(ImageRecord.tags).filter(Tag.name == tag_name) + + # Apply cursor filter (fetch images older than cursor) + if cursor_str: + try: + cursor_dt = datetime.fromisoformat(cursor_str.replace('Z', '+00:00')) + q = q.filter( + func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at) < cursor_dt + ) + except ValueError: + pass # Invalid cursor, ignore + + # Order by date descending and fetch with limit + 1 to check has_more + images = q.order_by( + desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at)) + ).limit(limit + 1).all() + + # Check if there are more images + has_more = len(images) > limit + if has_more: + images = images[:limit] + + # Serialize images + serialized = [_serialize_image(img) for img in images] + + # Build date_groups summary + groups_seen = OrderedDict() + for s in serialized: + key = s['year_month'] + if key not in groups_seen: + groups_seen[key] = {'key': key, 'label': s['year_month_label'], 'count': 0} + groups_seen[key]['count'] += 1 + + # Compute next cursor + next_cursor = None + if images and has_more: + last_img = images[-1] + last_dt = _get_image_date(last_img) + if last_dt: + next_cursor = last_dt.isoformat() + + return jsonify( + images=serialized, + next_cursor=next_cursor, + has_more=has_more, + date_groups=list(groups_seen.values()) + ) + + +@main.route('/api/gallery/timeline') +def gallery_timeline(): + """ + Returns all year-month groups with counts for timeline navigation. + + Query params: + - tag: optional tag filter (same as gallery) + + Returns list of {key, label, count, year} ordered newest to oldest. + """ + tag_name = request.args.get('tag') + + # Use SQL to aggregate by year-month + date_expr = func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at) + + q = db.session.query( + extract('year', date_expr).label('year'), + extract('month', date_expr).label('month'), + func.count(ImageRecord.id).label('count') + ) + + if tag_name: + q = q.join(ImageRecord.tags).filter(Tag.name == tag_name) + + q = q.group_by( + extract('year', date_expr), + extract('month', date_expr) + ).order_by( + extract('year', date_expr).desc(), + extract('month', date_expr).desc() + ) + + results = q.all() + + # Format results + groups = [] + for row in results: + if row.year and row.month: + year = int(row.year) + month = int(row.month) + key = f"{year:04d}-{month:02d}" + # Create a date object to format the month name + dt = datetime(year, month, 1) + label = dt.strftime('%B %Y') + short_label = dt.strftime('%b') # Short month name for sidebar + groups.append({ + 'key': key, + 'label': label, + 'short_label': short_label, + 'year': year, + 'month': month, + 'count': row.count + }) + + return jsonify(groups=groups) + + +@main.route('/api/gallery/jump') +def gallery_jump(): + """ + Fetch images starting from a specific year-month for timeline jump navigation. + + Query params: + - year_month: target year-month (e.g., '2024-03') + - limit: number of images (default 30) + - tag: optional tag filter + + Returns same format as /api/gallery/scroll but starting from the target month. + """ + year_month = request.args.get('year_month', '') + limit = request.args.get('limit', 30, type=int) + tag_name = request.args.get('tag') + + if not year_month: + return jsonify(error='year_month is required'), 400 + + try: + # Parse year-month to get start of that month + target_dt = datetime.strptime(year_month + '-01', '%Y-%m-%d') + # Get end of month (start of next month) + if target_dt.month == 12: + next_month = datetime(target_dt.year + 1, 1, 1) + else: + next_month = datetime(target_dt.year, target_dt.month + 1, 1) + except ValueError: + return jsonify(error='Invalid year_month format'), 400 + + # Build query for images in that month and earlier + q = ImageRecord.query.options(joinedload(ImageRecord.tags)) + + if tag_name: + q = q.join(ImageRecord.tags).filter(Tag.name == tag_name) + + # Get images from start of target month onwards (but ordered descending) + date_expr = func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at) + q = q.filter(date_expr < next_month) + + images = q.order_by(desc(date_expr)).limit(limit + 1).all() + + has_more = len(images) > limit + if has_more: + images = images[:limit] + + serialized = [_serialize_image(img) for img in images] + + # Build date_groups summary + groups_seen = OrderedDict() + for s in serialized: + key = s['year_month'] + if key not in groups_seen: + groups_seen[key] = {'key': key, 'label': s['year_month_label'], 'count': 0} + groups_seen[key]['count'] += 1 + + next_cursor = None + if images and has_more: + last_img = images[-1] + last_dt = _get_image_date(last_img) + if last_dt: + next_cursor = last_dt.isoformat() + + return jsonify( + images=serialized, + next_cursor=next_cursor, + has_more=has_more, + date_groups=list(groups_seen.values()), + target_year_month=year_month + ) + + @main.route('/tags') def tag_list(): """ diff --git a/app/models.py b/app/models.py index 2e28ce6..03ea892 100644 --- a/app/models.py +++ b/app/models.py @@ -23,7 +23,7 @@ class ImageRecord(db.Model): filepath = db.Column(db.String(512), nullable=False) thumb_path = db.Column(db.String(512)) hash = db.Column(db.String(64), unique=True, nullable=False) # SHA256 - perceptual_hash = db.Column(db.String(16), nullable=True) # hex of 64-bit hash + perceptual_hash = db.Column(db.String(64), nullable=True) # hex of 256-bit hash file_size = db.Column(db.Integer, nullable=True) width = db.Column(db.Integer, nullable=True) height = db.Column(db.Integer, nullable=True) @@ -64,3 +64,15 @@ class ArchiveRecord(db.Model): artist = db.Column(db.String(255), nullable=True) tag_id = db.Column(db.Integer, db.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True) tag = db.relationship("Tag") + + +class AppSettings(db.Model): + """ + Key-value store for application settings and version tracking. + Used to persist configuration and trigger data migrations on version changes. + """ + __tablename__ = "app_settings" + id = db.Column(db.Integer, primary_key=True) + key = db.Column(db.String(64), unique=True, nullable=False) + value = db.Column(db.Text, nullable=True) + updated_at = db.Column(db.DateTime, default=db.func.now(), onupdate=db.func.now()) diff --git a/app/static/js/gallery-infinite.js b/app/static/js/gallery-infinite.js new file mode 100644 index 0000000..d7cd46d --- /dev/null +++ b/app/static/js/gallery-infinite.js @@ -0,0 +1,475 @@ +// /app/static/js/gallery-infinite.js +// Infinite scroll gallery with year-month date dividers and timeline navigation + +(function() { + 'use strict'; + + // Configuration + const CONFIG = { + SCROLL_THRESHOLD: 1200, // px from bottom to trigger load + BATCH_SIZE: 30, // images per API request + DEBOUNCE_SCROLL: 100, // ms debounce for scroll handler + DEBOUNCE_RESIZE: 200 // ms debounce for resize handler + }; + + // State + const state = { + isLoading: false, + hasMore: true, + cursor: null, + activeTag: null, + loadedSections: new Set(), // year_month keys of loaded sections + timelineData: [], + currentVisibleSection: null, + isJumping: false // true during jump navigation + }; + + // DOM references + const dom = { + container: null, + loading: null, + end: null, + timeline: null, + timelineTrack: null + }; + + // Initialize + function init() { + dom.container = document.getElementById('galleryInfinite'); + dom.loading = document.getElementById('galleryLoading'); + dom.end = document.getElementById('galleryEnd'); + dom.timeline = document.getElementById('timelineSidebar'); + dom.timelineTrack = document.getElementById('timelineTrack'); + + if (!dom.container) return; + + // Parse initial state from window.galleryState (set by template) + if (window.galleryState) { + state.cursor = window.galleryState.cursor; + state.hasMore = window.galleryState.hasMore; + state.activeTag = window.galleryState.activeTag; + } + + // Track initially loaded sections + parseInitialSections(); + + // Load timeline data + loadTimeline(); + + // Setup event listeners + setupScrollListener(); + setupIntersectionObserver(); + + // Check if we need to load more on init (short content) + setTimeout(checkScrollPosition, 100); + } + + // Parse initially rendered sections + function parseInitialSections() { + const sections = dom.container.querySelectorAll('.date-section'); + sections.forEach(section => { + const yearMonth = section.dataset.yearMonth; + if (yearMonth) { + state.loadedSections.add(yearMonth); + } + }); + } + + // Fetch more images + async function loadMore() { + if (state.isLoading || !state.hasMore) return; + + state.isLoading = true; + dom.loading.style.display = 'flex'; + + try { + const params = new URLSearchParams({ + limit: CONFIG.BATCH_SIZE + }); + + if (state.cursor) { + params.set('cursor', state.cursor); + } + if (state.activeTag) { + params.set('tag', state.activeTag); + } + + const response = await fetch(`/api/gallery/scroll?${params}`); + const data = await response.json(); + + if (data.images && data.images.length > 0) { + appendImages(data.images); + state.cursor = data.next_cursor; + state.hasMore = data.has_more; + + // Notify modal system + dispatchGalleryUpdate(); + } + + if (!data.has_more) { + state.hasMore = false; + dom.end.style.display = 'block'; + } + } catch (err) { + console.error('Failed to load more images:', err); + } finally { + state.isLoading = false; + dom.loading.style.display = 'none'; + } + } + + // Append images to the gallery + function appendImages(images) { + // Group images by year_month + const groups = new Map(); + + images.forEach(img => { + const key = img.year_month; + if (!groups.has(key)) { + groups.set(key, { + key: key, + label: img.year_month_label, + images: [] + }); + } + groups.get(key).images.push(img); + }); + + // Append to existing sections or create new ones + groups.forEach((group, key) => { + let section = dom.container.querySelector(`.date-section[data-year-month="${key}"]`); + + if (section) { + // Append to existing section + const grid = section.querySelector('.gallery-grid'); + group.images.forEach(img => { + grid.appendChild(createImageElement(img)); + }); + // Update count + updateSectionCount(section); + } else { + // Create new section + section = createDateSection(group.key, group.label, group.images); + dom.container.appendChild(section); + } + + state.loadedSections.add(key); + }); + + // Update timeline highlights + updateTimelineLoaded(); + } + + // Create a date section element + function createDateSection(yearMonth, label, images) { + const section = document.createElement('section'); + section.className = 'date-section'; + section.dataset.yearMonth = yearMonth; + + const divider = document.createElement('h2'); + divider.className = 'date-divider'; + divider.id = `section-${yearMonth}`; + divider.innerHTML = ` + ${label} + ${images.length} images + `; + + const grid = document.createElement('div'); + grid.className = 'gallery-grid'; + + images.forEach(img => { + grid.appendChild(createImageElement(img)); + }); + + section.appendChild(divider); + section.appendChild(grid); + + return section; + } + + // Create a single image element + function createImageElement(img) { + const item = document.createElement('div'); + item.className = 'gallery-item img-clickable'; + item.dataset.id = img.id; + item.dataset.full = img.full_url; + item.dataset.type = img.is_video ? 'video' : 'image'; + item.dataset.filename = img.filename; + + // Build tag overlay HTML + let tagsHtml = ''; + if (img.tags && img.tags.length > 0) { + const tagChips = img.tags.map(t => { + const displayName = t.name.includes(':') ? t.name.split(':')[1] : t.name; + const prefix = t.kind === 'user' ? '#' : ''; + return `${prefix}${displayName}`; + }).join(''); + tagsHtml = `
${tagChips}
`; + } + + // Play overlay for videos + const playOverlay = img.is_video ? '
' : ''; + + item.innerHTML = ` + + ${tagsHtml} + ${playOverlay} + ${img.date} + `; + + return item; + } + + // Update section image count + function updateSectionCount(section) { + const grid = section.querySelector('.gallery-grid'); + const count = grid.querySelectorAll('.gallery-item').length; + const countEl = section.querySelector('.date-divider-count'); + if (countEl) { + countEl.textContent = `${count} images`; + } + } + + // Scroll handling + function setupScrollListener() { + let scrollTimeout = null; + + window.addEventListener('scroll', () => { + if (scrollTimeout) return; + scrollTimeout = setTimeout(() => { + scrollTimeout = null; + checkScrollPosition(); + }, CONFIG.DEBOUNCE_SCROLL); + }, { passive: true }); + } + + function checkScrollPosition() { + if (state.isLoading || !state.hasMore || state.isJumping) return; + + const scrollBottom = window.innerHeight + window.scrollY; + const docHeight = document.documentElement.scrollHeight; + + if (docHeight - scrollBottom < CONFIG.SCROLL_THRESHOLD) { + loadMore(); + } + } + + // Intersection observer for tracking visible section + function setupIntersectionObserver() { + const observer = new IntersectionObserver((entries) => { + entries.forEach(entry => { + if (entry.isIntersecting) { + const yearMonth = entry.target.dataset.yearMonth; + if (yearMonth && yearMonth !== state.currentVisibleSection) { + state.currentVisibleSection = yearMonth; + updateTimelineActive(yearMonth); + } + } + }); + }, { + rootMargin: '-100px 0px -60% 0px', + threshold: 0 + }); + + // Observe all date sections + const observeSections = () => { + dom.container.querySelectorAll('.date-section').forEach(section => { + observer.observe(section); + }); + }; + + observeSections(); + + // Re-observe when new sections are added + const mutationObserver = new MutationObserver(() => { + observeSections(); + }); + mutationObserver.observe(dom.container, { childList: true }); + } + + // Timeline functions + async function loadTimeline() { + try { + const params = state.activeTag ? `?tag=${encodeURIComponent(state.activeTag)}` : ''; + const response = await fetch(`/api/gallery/timeline${params}`); + const data = await response.json(); + + if (data.groups) { + state.timelineData = data.groups; + renderTimeline(); + } + } catch (err) { + console.error('Failed to load timeline:', err); + dom.timelineTrack.innerHTML = '
Failed to load
'; + } + } + + function renderTimeline() { + if (!dom.timelineTrack || state.timelineData.length === 0) { + dom.timelineTrack.innerHTML = '
No dates
'; + return; + } + + // Group by year + const yearGroups = new Map(); + state.timelineData.forEach(item => { + if (!yearGroups.has(item.year)) { + yearGroups.set(item.year, []); + } + yearGroups.get(item.year).push(item); + }); + + let html = ''; + + yearGroups.forEach((months, year) => { + html += `
`; + html += `
${year}
`; + html += '
'; + + months.forEach(month => { + const isLoaded = state.loadedSections.has(month.key); + const loadedClass = isLoaded ? ' loaded' : ''; + + html += ` + + ${month.short_label} + ${month.count} + + `; + }); + + html += '
'; + }); + + dom.timelineTrack.innerHTML = html; + + // Add click handlers for timeline navigation + dom.timelineTrack.querySelectorAll('.timeline-month').forEach(el => { + el.addEventListener('click', handleTimelineClick); + }); + + // Update active state + if (state.currentVisibleSection) { + updateTimelineActive(state.currentVisibleSection); + } + } + + function handleTimelineClick(e) { + e.preventDefault(); + + const yearMonth = e.currentTarget.dataset.yearMonth; + if (!yearMonth) return; + + // Check if section is already loaded + const existingSection = document.getElementById(`section-${yearMonth}`); + + if (existingSection) { + // Scroll to existing section + existingSection.scrollIntoView({ behavior: 'smooth', block: 'start' }); + } else { + // Jump to this date - fetch and load + jumpToDate(yearMonth); + } + } + + async function jumpToDate(yearMonth) { + if (state.isJumping) return; + + state.isJumping = true; + dom.loading.style.display = 'flex'; + + try { + const params = new URLSearchParams({ + year_month: yearMonth, + limit: CONFIG.BATCH_SIZE + }); + + if (state.activeTag) { + params.set('tag', state.activeTag); + } + + const response = await fetch(`/api/gallery/jump?${params}`); + const data = await response.json(); + + if (data.images && data.images.length > 0) { + // Clear existing content and replace with jumped-to content + dom.container.innerHTML = ''; + state.loadedSections.clear(); + + // Add images + appendImages(data.images); + + // Update cursor for continued scrolling + state.cursor = data.next_cursor; + state.hasMore = data.has_more; + + // Reset end marker + if (data.has_more) { + dom.end.style.display = 'none'; + } else { + dom.end.style.display = 'block'; + } + + // Scroll to top of new content + window.scrollTo({ top: 0, behavior: 'instant' }); + + // Update timeline + updateTimelineLoaded(); + + // Notify modal system + dispatchGalleryUpdate(); + } + } catch (err) { + console.error('Failed to jump to date:', err); + } finally { + state.isJumping = false; + dom.loading.style.display = 'none'; + } + } + + function updateTimelineActive(yearMonth) { + // Remove active from all + dom.timelineTrack.querySelectorAll('.timeline-month.active').forEach(el => { + el.classList.remove('active'); + }); + + // Add active to current + const activeEl = dom.timelineTrack.querySelector(`.timeline-month[data-year-month="${yearMonth}"]`); + if (activeEl) { + activeEl.classList.add('active'); + + // Scroll timeline to keep active visible + const track = dom.timelineTrack; + const elRect = activeEl.getBoundingClientRect(); + const trackRect = track.getBoundingClientRect(); + + if (elRect.top < trackRect.top || elRect.bottom > trackRect.bottom) { + activeEl.scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + } + } + + function updateTimelineLoaded() { + dom.timelineTrack.querySelectorAll('.timeline-month').forEach(el => { + const yearMonth = el.dataset.yearMonth; + if (state.loadedSections.has(yearMonth)) { + el.classList.add('loaded'); + } + }); + } + + // Dispatch event for modal system integration + function dispatchGalleryUpdate() { + document.dispatchEvent(new CustomEvent('galleryUpdated')); + } + + // Initialize on DOM ready + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } +})(); diff --git a/app/static/js/modal-pagination.js b/app/static/js/modal-pagination.js index 2fba69b..5ed081d 100644 --- a/app/static/js/modal-pagination.js +++ b/app/static/js/modal-pagination.js @@ -28,11 +28,16 @@ document.addEventListener('DOMContentLoaded', () => { let dragStartY = 0; let dragMoved = false; + // Legacy pagination inputs (for non-infinite-scroll pages) const hasNextPageInput = document.getElementById('hasNextPage'); const nextPageUrlInput = document.getElementById('nextPageUrl'); const hasPrevPageInput = document.getElementById('hasPrevPage'); const prevPageUrlInput = document.getElementById('prevPageUrl'); + // Check if we're in infinite scroll mode + // Infinite scroll mode is when galleryState exists (set by gallery.html template) + const isInfiniteScrollMode = typeof window.galleryState !== 'undefined'; + // --------------------------- // Tag editor helpers // --------------------------- @@ -158,9 +163,11 @@ document.addEventListener('DOMContentLoaded', () => { async function fetchAutocomplete(query) { try { + // Exclude system-managed tag kinds from autocomplete (archive, post, source) + const excludeKinds = 'archive,post,source'; const url = query - ? `/api/tags/search?q=${encodeURIComponent(query)}&limit=8&exclude_kind=archive` - : `/api/tags/search?limit=8&exclude_kind=archive`; + ? `/api/tags/search?q=${encodeURIComponent(query)}&limit=8&exclude_kind=${excludeKinds}` + : `/api/tags/search?limit=8&exclude_kind=${excludeKinds}`; const r = await fetch(url); const j = await r.json(); renderAutocomplete(j.tags || []); @@ -316,7 +323,12 @@ document.addEventListener('DOMContentLoaded', () => { function showNext() { if (currentIndex < images.length - 1) { updateImage(currentIndex + 1); + } else if (isInfiniteScrollMode) { + // In infinite scroll mode, just stay at the last image + // User needs to scroll the gallery to load more + return; } else if (hasNextPageInput?.value === "true") { + // Legacy pagination mode - fetch next page fetch(nextPageUrlInput.value) .then(res => res.text()) .then(html => { @@ -349,7 +361,11 @@ document.addEventListener('DOMContentLoaded', () => { function showPrev() { if (currentIndex > 0) { updateImage(currentIndex - 1); + } else if (isInfiniteScrollMode) { + // In infinite scroll mode, just stay at the first image + return; } else if (hasPrevPageInput?.value === "true") { + // Legacy pagination mode - fetch previous page fetch(prevPageUrlInput.value) .then(res => res.text()) .then(html => { @@ -480,6 +496,11 @@ document.addEventListener('DOMContentLoaded', () => { images = Array.from(document.querySelectorAll('.img-clickable')); }); + // Listen for gallery infinite scroll updates to refresh the images array + document.addEventListener('galleryUpdated', () => { + images = Array.from(document.querySelectorAll('.img-clickable')); + }); + window.addEventListener('popstate', () => { if (modal.classList.contains('active')) { closeModal(); diff --git a/app/static/js/showcase.js b/app/static/js/showcase.js index 731d74d..e3ebc87 100644 --- a/app/static/js/showcase.js +++ b/app/static/js/showcase.js @@ -97,10 +97,28 @@ } function checkScrollPosition() { - const scrollBottom = window.innerHeight + window.scrollY; - const docHeight = document.documentElement.scrollHeight; + // For masonry layouts, we need to check if the SHORTEST column is near the viewport bottom, + // not the overall page bottom. Otherwise, a tall image in one column prevents loading + // even when other columns have empty space visible. - if (docHeight - scrollBottom < SCROLL_THRESHOLD) { + // First, update accurate column heights + updateColumnHeights(); + + // Find the shortest column's bottom position relative to viewport + const shortestColumnIndex = getShortestColumnIndex(); + const shortestColumn = columns[shortestColumnIndex]; + if (!shortestColumn) return; + + const columnRect = shortestColumn.getBoundingClientRect(); + const columnBottom = columnRect.bottom; // Distance from viewport top to column bottom + const viewportHeight = window.innerHeight; + + // If the shortest column's bottom is within SCROLL_THRESHOLD of the viewport bottom, load more + // columnBottom - viewportHeight = how far below the viewport the column extends + // If this is negative or small, the column end is visible or nearly visible + const distanceFromViewportBottom = columnBottom - viewportHeight; + + if (distanceFromViewportBottom < SCROLL_THRESHOLD) { loadMore(); } } diff --git a/app/static/style.css b/app/static/style.css index b8fcd12..3ae27d3 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -378,14 +378,25 @@ header { display: flex; align-items: center; justify-content: center; - max-width: 100%; + width: 100%; + max-width: calc(100% - 300px); /* Account for tag editor */ + height: calc(100vh - 140px); max-height: calc(100vh - 140px); overflow: hidden; cursor: default; } -.modal-image-wrapper.zoomed { overflow: auto; cursor: grab; } +.modal-image-wrapper.zoomed { + overflow: auto; + cursor: grab; + /* Enable scrolling by removing flex centering when zoomed */ + display: block; +} .modal-image-wrapper.zoomed img { - width: auto; height: auto; max-width: none; max-height: none; + width: auto; + height: auto; + max-width: none; + max-height: none; + cursor: grab; } .modal-image-wrapper img { max-width: 100%; @@ -404,6 +415,15 @@ header { width: 90vw; height: auto; max-height: none; } +/* On smaller screens, full width for image wrapper */ +@media (max-width: 900px) { + .modal-image-wrapper { + max-width: 100%; + height: auto; + max-height: 60vh; + } +} + /* Modal navigation buttons - cleaner pill style */ .modal-button { position: absolute; @@ -1191,3 +1211,258 @@ select.form-input optgroup { overflow-wrap: anywhere; /* allows breaking long tokens */ word-break: break-word; } + +/*------------------------------------------------------------------------------ + Gallery Infinite Scroll Layout +------------------------------------------------------------------------------*/ +.gallery-infinite-container { + display: flex; + min-height: 100vh; + max-width: 1800px; + margin: 0 auto; + position: relative; +} + +/* Timeline Sidebar - positioned relative to gallery container */ +.timeline-sidebar { + position: sticky; + top: 60px; + height: calc(100vh - 60px); + width: 70px; + min-width: 70px; + background: rgba(0,0,0,0.3); + border-right: 1px solid rgba(255,255,255,0.08); + z-index: 50; + display: flex; + flex-direction: column; + padding: 0.5rem 0; + overflow-y: auto; + scrollbar-width: thin; + scrollbar-color: rgba(255,255,255,0.2) transparent; +} +.timeline-sidebar::-webkit-scrollbar { + width: 4px; +} +.timeline-sidebar::-webkit-scrollbar-track { + background: transparent; +} +.timeline-sidebar::-webkit-scrollbar-thumb { + background: rgba(255,255,255,0.2); + border-radius: 2px; +} + +.timeline-track { + display: flex; + flex-direction: column; + padding: 0 6px; +} + +.timeline-loading { + font-size: 0.7rem; + color: var(--text-muted); + padding: 0.5rem; + text-align: center; +} + +.timeline-year-group { + margin-bottom: 0.5rem; +} + +.timeline-year { + font-size: 0.8rem; + font-weight: 600; + color: var(--text); + padding: 0.5rem 0.25rem 0.25rem; + text-align: center; + position: sticky; + top: 0; + background: linear-gradient(to bottom, rgba(10,10,10,0.95), rgba(10,10,10,0.8)); + z-index: 1; + border-bottom: 1px solid rgba(255,255,255,0.1); +} + +.timeline-months { + display: flex; + flex-direction: column; + gap: 2px; + padding-top: 4px; +} + +.timeline-month { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.35rem 0.5rem; + font-size: 0.7rem; + color: var(--text-muted); + border-radius: 4px; + cursor: pointer; + transition: all 0.15s ease; + text-decoration: none; +} +.timeline-month:hover { + background: rgba(255,255,255,0.12); + color: var(--text); +} +.timeline-month.active { + background: var(--btn-primary); + color: white; +} +.timeline-month.loaded { + color: var(--text-dim); +} +.timeline-month-label { + font-weight: 500; +} +.timeline-month-count { + font-size: 0.6rem; + opacity: 0.7; +} + +/* Main gallery area - flex child next to timeline */ +.gallery-main { + flex: 1; + padding: 0 0.5rem; + min-width: 0; +} + +.gallery-header { + display: flex; + flex-direction: column; + align-items: center; + padding: 0.5rem 0 1rem; + gap: 0.5rem; +} +.gallery-header h1 { + margin: 0; + font-size: 1.5rem; +} + +.active-filter { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.5rem 1rem; + background: rgba(255,255,255,0.08); + border-radius: 999px; + font-size: 0.9rem; + color: var(--text-dim); +} +.filter-tag { + color: var(--link); + font-weight: 500; +} +.clear-filter { + display: flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + border-radius: 50%; + background: rgba(255,255,255,0.1); + color: var(--text-muted); + text-decoration: none; + font-size: 0.9rem; + line-height: 1; + transition: all 0.15s ease; +} +.clear-filter:hover { + background: rgba(255,255,255,0.2); + color: var(--text); +} + +/* Gallery content area */ +.gallery-infinite { + min-height: 50vh; +} + +.gallery-empty { + text-align: center; + padding: 4rem 1rem; + color: var(--text-muted); +} + +/* Date sections */ +.date-section { + margin-bottom: 1.5rem; +} + +.date-divider { + display: flex; + align-items: center; + gap: 1rem; + padding: 0.75rem 0.5rem; + margin: 0 0 0.5rem 0; + border-bottom: 1px solid rgba(255,255,255,0.1); + position: sticky; + top: 50px; + background: linear-gradient(to bottom, var(--bg) 0%, var(--bg) 85%, transparent 100%); + z-index: 10; +} + +.date-divider-text { + font-size: 1.1rem; + font-weight: 500; + color: var(--text); +} + +.date-divider-count { + font-size: 0.8rem; + color: var(--text-muted); + background: rgba(255,255,255,0.08); + padding: 0.2rem 0.6rem; + border-radius: 999px; +} + +/* Loading indicator */ +.gallery-loading { + display: flex; + align-items: center; + justify-content: center; + gap: 0.75rem; + padding: 2rem; + color: var(--text-muted); +} + +.loading-spinner { + display: inline-block; + width: 20px; + height: 20px; + border: 2px solid rgba(255,255,255,0.2); + border-top-color: var(--btn-primary); + border-radius: 50%; + animation: spin 0.8s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.gallery-end { + text-align: center; + padding: 2rem; + color: var(--text-muted); + font-style: italic; + border-top: 1px solid rgba(255,255,255,0.05); +} + +/* Play overlay with triangle icon */ +.play-overlay::before { + content: ''; + width: 0; + height: 0; + border-style: solid; + border-width: 20px 0 20px 35px; + border-color: transparent transparent transparent rgba(255,255,255,0.9); + filter: drop-shadow(0 2px 4px rgba(0,0,0,0.5)); +} + +/* Mobile: hide timeline sidebar */ +@media (max-width: 768px) { + .timeline-sidebar { + display: none; + } + .date-divider { + top: 45px; + } +} diff --git a/app/templates/_gallery_item.html b/app/templates/_gallery_item.html new file mode 100644 index 0000000..fda0c31 --- /dev/null +++ b/app/templates/_gallery_item.html @@ -0,0 +1,40 @@ + + diff --git a/app/templates/_gallery_modal.html b/app/templates/_gallery_modal.html new file mode 100644 index 0000000..13663ff --- /dev/null +++ b/app/templates/_gallery_modal.html @@ -0,0 +1,26 @@ + + diff --git a/app/templates/gallery.html b/app/templates/gallery.html index 0ae9e75..ad32a5f 100644 --- a/app/templates/gallery.html +++ b/app/templates/gallery.html @@ -3,14 +3,73 @@ {% block title %}Gallery{% endblock %} {% block content %} -

Gallery

+ + + +{% include '_gallery_modal.html' %} + + + + + + {% endblock %} diff --git a/app/templates/layout.html b/app/templates/layout.html index b07c1de..f9147a0 100644 --- a/app/templates/layout.html +++ b/app/templates/layout.html @@ -37,12 +37,6 @@ -{% if request.endpoint == 'main.gallery' %} - - - - -{% endif %} diff --git a/app/utils/image_importer.py b/app/utils/image_importer.py index c1ff9fa..105483d 100644 --- a/app/utils/image_importer.py +++ b/app/utils/image_importer.py @@ -50,21 +50,43 @@ IMPORT_STATS_PATH = "/import/stats.json" def load_import_settings() -> dict: - """Load import filter settings from file or environment.""" + """ + Load import filter settings from database, with file and environment fallbacks. + Priority: DB settings > file settings > environment variables + """ + # Start with environment defaults defaults = { "min_width": int(os.environ.get("IMPORT_MIN_WIDTH", "0")), "min_height": int(os.environ.get("IMPORT_MIN_HEIGHT", "0")), "skip_transparent": os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true", "transparency_threshold": float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9")), - "phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "2")), + "skip_single_color": os.environ.get("IMPORT_SKIP_SINGLE_COLOR", "true").lower() == "true", + "single_color_threshold": float(os.environ.get("IMPORT_SINGLE_COLOR_THRESHOLD", "0.95")), + "single_color_tolerance": int(os.environ.get("IMPORT_SINGLE_COLOR_TOLERANCE", "30")), + "phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "10")), # Default for 256-bit + "supersede_smaller": os.environ.get("IMPORT_SUPERSEDE_SMALLER", "true").lower() == "true", } + + # Try to load from file (legacy support) try: if os.path.exists(IMPORT_SETTINGS_PATH): with open(IMPORT_SETTINGS_PATH, "r") as f: file_settings = json.load(f) defaults.update(file_settings) except Exception as e: - print(f"[WARN] Failed to load import settings: {e}") + print(f"[WARN] Failed to load import settings from file: {e}") + + # Try to load from database (preferred) + try: + from app.utils.version_migration import get_import_settings + db_settings = get_import_settings() + # Only override if DB has non-default values + for key, value in db_settings.items(): + if value is not None: + defaults[key] = value + except Exception as e: + print(f"[WARN] Failed to load import settings from DB: {e}") + return defaults @@ -106,6 +128,136 @@ def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool: print(f"[WARN] Failed to check transparency for {file_path}: {e}") return False + +def is_mostly_single_color(file_path: str, threshold: float = 0.95, tolerance: int = 30) -> bool: + """ + Check if an image is mostly a single color (> threshold % of pixels are similar). + Uses color distance tolerance to group similar colors. + + Args: + file_path: Path to the image file + threshold: Percentage of pixels that must be similar (0.0-1.0, default 0.95 = 95%) + tolerance: Maximum color distance to be considered "same" color (0-255 per channel, default 30) + + Returns: + True if the image is mostly a single solid color + """ + try: + with Image.open(file_path) as img: + # Convert to RGB for consistent color comparison + if img.mode in ("RGBA", "LA"): + # For images with alpha, only consider non-transparent pixels + img_rgba = img.convert("RGBA") if img.mode != "RGBA" else img + pixels = list(img_rgba.getdata()) + # Filter out mostly transparent pixels (alpha < 128) + opaque_pixels = [(r, g, b) for r, g, b, a in pixels if a >= 128] + if len(opaque_pixels) == 0: + return True # Fully transparent = effectively single color + pixels = opaque_pixels + elif img.mode == "P": + img = img.convert("RGB") + pixels = list(img.getdata()) + elif img.mode == "L": + # Grayscale - convert to RGB tuples + pixels = [(p, p, p) for p in img.getdata()] + else: + if img.mode != "RGB": + img = img.convert("RGB") + pixels = list(img.getdata()) + + if len(pixels) == 0: + return True + + # Sample pixels for performance on large images + total_pixels = len(pixels) + if total_pixels > 10000: + # Sample evenly distributed pixels + step = total_pixels // 10000 + pixels = pixels[::step] + + # Find the dominant color (most common pixel or average of sampled area) + # Use the pixel at center as reference + center_idx = len(pixels) // 2 + ref_color = pixels[center_idx] + + # Count pixels within tolerance of reference color + similar_count = 0 + for pixel in pixels: + if isinstance(pixel, int): + pixel = (pixel, pixel, pixel) + # Calculate color distance (simple RGB distance) + dist = abs(pixel[0] - ref_color[0]) + abs(pixel[1] - ref_color[1]) + abs(pixel[2] - ref_color[2]) + if dist <= tolerance * 3: # tolerance per channel * 3 channels + similar_count += 1 + + similarity_ratio = similar_count / len(pixels) + return similarity_ratio > threshold + + except Exception as e: + print(f"[WARN] Failed to check single color for {file_path}: {e}") + return False + + +def supersede_image(old_record: "ImageRecord", new_filepath: str, new_thumb_path: str, + new_hash: str, new_phash, new_metadata: dict, new_filename: str) -> "ImageRecord": + """ + Replace a smaller image with a larger version, preserving tags and metadata. + + Args: + old_record: The existing ImageRecord to supersede + new_filepath: Path to the new larger image file + new_thumb_path: Path to the new thumbnail + new_hash: SHA256 hash of the new file + new_phash: Perceptual hash of the new image + new_metadata: Metadata dict for the new image + new_filename: Original filename of the new image + + Returns: + The updated ImageRecord + """ + # Store old file paths for cleanup + old_filepath = old_record.filepath + old_thumb_path = old_record.thumb_path + + print(f"[SUPERSEDE] Replacing {old_record.filename} ({old_record.width}x{old_record.height}) " + f"with {new_filename} ({new_metadata['width']}x{new_metadata['height']})") + + # Update record with new file info + old_record.filename = new_filename + old_record.filepath = new_filepath + old_record.thumb_path = new_thumb_path + old_record.hash = new_hash + old_record.perceptual_hash = str(new_phash) if new_phash else None + old_record.file_size = new_metadata["file_size"] + old_record.width = new_metadata["width"] + old_record.height = new_metadata["height"] + old_record.format = new_metadata["format"] + # Keep the earlier taken_at date (prefer original date) + if new_metadata.get("taken_at") and old_record.taken_at: + if new_metadata["taken_at"] < old_record.taken_at: + old_record.taken_at = new_metadata["taken_at"] + elif new_metadata.get("taken_at") and not old_record.taken_at: + old_record.taken_at = new_metadata["taken_at"] + # Don't update imported_at - keep original import time + + # Delete old files + try: + if old_filepath and os.path.exists(old_filepath): + os.remove(old_filepath) + print(f"[SUPERSEDE] Deleted old file: {old_filepath}") + except Exception as e: + print(f"[WARN] Failed to delete old file {old_filepath}: {e}") + + try: + if old_thumb_path and os.path.exists(old_thumb_path): + os.remove(old_thumb_path) + print(f"[SUPERSEDE] Deleted old thumbnail: {old_thumb_path}") + except Exception as e: + print(f"[WARN] Failed to delete old thumbnail {old_thumb_path}: {e}") + + return old_record + + # Regex for multi-part detection RAR_PART_RE = re.compile(r"\.part(\d+)\.rar$", re.IGNORECASE) SEVENZ_PART_RE = re.compile(r"\.7z\.(\d{3})$", re.IGNORECASE) @@ -233,19 +385,25 @@ def import_images_task(source_dir: str, dest_dir: str) -> str: min_height = settings.get("min_height", 0) skip_transparent = settings.get("skip_transparent", False) transparency_threshold = settings.get("transparency_threshold", 0.9) - + skip_single_color = settings.get("skip_single_color", True) + single_color_threshold = settings.get("single_color_threshold", 0.95) + single_color_tolerance = settings.get("single_color_tolerance", 30) phash_threshold = settings.get("phash_threshold", 2) + supersede_smaller = settings.get("supersede_smaller", True) print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, " f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, " - f"phash_threshold={phash_threshold}") + f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, " + f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}") # Statistics tracking stats = { "total_processed": 0, "total_imported": 0, + "total_superseded": 0, "filtered_by_dimension": 0, "filtered_by_transparency": 0, + "filtered_by_single_color": 0, "filtered_by_duplicate": 0, } @@ -254,8 +412,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str: batch_counter = 0 # Preload existing pHashes to avoid N+1 lookups + # Format: (phash, width, height, image_id) for supersede functionality existing_phashes = [ - (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height) + (imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id) for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all() ] @@ -320,24 +479,34 @@ def import_images_task(source_dir: str, dest_dir: str) -> str: stats["filtered_by_transparency"] += 1 continue - added, phash, _rec = import_single_file( + # Apply single-color filter (skip images that are mostly one solid color) + if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")): + if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance): + print(f"[SKIP] Mostly single color: {file}") + stats["filtered_by_single_color"] += 1 + continue + + added, phash, _rec, superseded = import_single_file( src_path=full_path, dest_dir=dest_dir, artist=artist_dir, existing_phashes=existing_phashes, - phash_threshold=phash_threshold + phash_threshold=phash_threshold, + supersede_smaller=supersede_smaller ) if added: imported.append(file) - stats["total_imported"] += 1 + if superseded: + stats["total_superseded"] += 1 + else: + stats["total_imported"] += 1 elif _rec is None and phash is None: # Was a duplicate stats["filtered_by_duplicate"] += 1 - if phash: - md = extract_metadata(full_path) - if md.get("width") and md.get("height"): - existing_phashes.append((phash, md["width"], md["height"])) + # Add new image's phash to the tracking list (for fresh imports only, not supersedes) + if phash and _rec and not superseded: + existing_phashes.append((phash, _rec.width, _rec.height, _rec.id)) batch_counter += 1 if batch_counter >= batch_size: @@ -354,7 +523,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str: # Save stats save_import_stats(stats) - summary = f"Imported {len(imported)} items. Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency." + summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. " + f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, " + f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.") print(f"[INFO] {summary}") return summary @@ -368,14 +539,19 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N if settings is None: settings = load_import_settings() if stats is None: - stats = {"total_processed": 0, "total_imported": 0, "filtered_by_dimension": 0, - "filtered_by_transparency": 0, "filtered_by_duplicate": 0} + stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0, + "filtered_by_dimension": 0, "filtered_by_transparency": 0, + "filtered_by_single_color": 0, "filtered_by_duplicate": 0} min_width = settings.get("min_width", 0) min_height = settings.get("min_height", 0) skip_transparent = settings.get("skip_transparent", False) transparency_threshold = settings.get("transparency_threshold", 0.9) + skip_single_color = settings.get("skip_single_color", True) + single_color_threshold = settings.get("single_color_threshold", 0.95) + single_color_tolerance = settings.get("single_color_tolerance", 30) phash_threshold = settings.get("phash_threshold", 2) + supersede_smaller = settings.get("supersede_smaller", True) archive_size = os.path.getsize(archive_path) archive_hash = calculate_hash(archive_path) @@ -420,19 +596,30 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N stats["filtered_by_transparency"] += 1 continue - added, _phash, rec = import_single_file( + # Apply single-color filter + if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")): + if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance): + print(f"[SKIP] Mostly single color: {file}") + stats["filtered_by_single_color"] += 1 + continue + + added, _phash, rec, superseded = import_single_file( src_path=src_path, dest_dir=dest_dir, artist=artist, # still used for auto artist: tag existing_phashes=existing_phashes, commit=False, - phash_threshold=phash_threshold + phash_threshold=phash_threshold, + supersede_smaller=supersede_smaller ) if rec is not None: touched_records.append(rec) if added: imported_or_tagged += 1 - stats["total_imported"] += 1 + if superseded: + stats["total_superseded"] += 1 + else: + stats["total_imported"] += 1 elif rec is None and _phash is None: stats["filtered_by_duplicate"] += 1 @@ -495,11 +682,19 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st i += 1 -def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, commit: bool = True, phash_threshold: int = 2): +def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, + commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True): """ Import a single media file. If an equivalent file already exists, return that record so callers can tag it (no duplicate import). - Returns (added: bool, phash_or_None, record_or_None). + + If a smaller visually-similar image exists and supersede_smaller is True, + the smaller image will be replaced with the larger one (preserving tags). + + Also enriches metadata from Gallery-DL sidecar JSON files if present. + + Returns (added: bool, phash_or_None, record_or_None, superseded: bool). + The superseded flag indicates if an existing smaller image was replaced. """ original_name = os.path.basename(src_path) print(f"[INFO] Processing: {src_path}") @@ -510,7 +705,9 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash existing = ImageRecord.query.filter_by(hash=file_hash).first() if existing: print(f"[SKIP] Duplicate by hash: {original_name}") - return (False, None, existing) + # Even for duplicates, try to enrich with sidecar metadata + _enrich_existing_record(existing, src_path) + return (False, None, existing, False) # Optional info: same name+size (but different content) → we still import with hashed name existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first() @@ -521,12 +718,67 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash metadata = extract_metadata(src_path) if not metadata["width"] or not metadata["height"]: print(f"[SKIP] Missing dimension data for {original_name}") - return (False, None, None) + return (False, None, None, False) phash = calculate_perceptual_hash(src_path) - if phash and phash_threshold > 0 and is_similar_image(phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold): - print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).") - return (False, phash, None) + + # Check for similar images + superseded = False + if phash and phash_threshold > 0: + relationship, similar_id = find_similar_image( + phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold + ) + + if relationship == "larger_exists": + print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).") + return (False, phash, None, False) + + elif relationship == "smaller_exists" and supersede_smaller and similar_id: + # Found a smaller version - supersede it + old_record = ImageRecord.query.get(similar_id) + if old_record: + # Enrich metadata from Gallery-DL sidecar JSON if present + enriched = _get_enriched_metadata(src_path, metadata) + metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"] + + # Copy new file to destination + target_dir = os.path.join(dest_dir, artist) + os.makedirs(target_dir, exist_ok=True) + dest_path = build_hashed_dest_path(target_dir, original_name, file_hash) + shutil.copy2(src_path, dest_path) + + # Generate new thumbnail + try: + if dest_path.lower().endswith((".mp4", ".mov")): + thumb_path = generate_video_thumbnail_mirrored(dest_path) + else: + thumb_path = generate_thumbnail(dest_path) + except Exception as e: + print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}") + thumb_path = None + + # Supersede the old record (preserves tags) + record = supersede_image( + old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name + ) + + # Apply any new tags from sidecar + _apply_enriched_tags(record, enriched) + + if commit: + db.session.commit() + + # Update existing_phashes list with new dimensions + for i, (ph, w, h, img_id) in enumerate(existing_phashes): + if img_id == similar_id: + existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id) + break + + return (True, phash, record, True) # superseded=True + + # No similar image found (or supersede disabled) - normal import + # Enrich metadata from Gallery-DL sidecar JSON if present + enriched = _get_enriched_metadata(src_path, metadata) # Copy into /images//__ target_dir = os.path.join(dest_dir, artist) @@ -545,6 +797,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash thumb_path = None # Create DB record (keep display name as original; filepath is content-addressed) + # Use enriched taken_at (earliest date wins) record = ImageRecord( filename=original_name, filepath=dest_path, @@ -556,7 +809,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash height=metadata["height"], format=metadata["format"], camera_model=metadata["camera_model"], - taken_at=metadata["taken_at"], + taken_at=enriched.get("taken_at") or metadata["taken_at"], imported_at=datetime.utcnow() ) @@ -571,11 +824,81 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash if artist_tag not in record.tags: record.tags.append(artist_tag) + # Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id) + _apply_enriched_tags(record, enriched) db.session.add(record) if commit: db.session.commit() - return (True, phash, record) + return (True, phash, record, False) # superseded=False (new import) + + +def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict: + """ + Get enriched metadata from Gallery-DL sidecar JSON. + Returns empty dict if no sidecar found. + """ + try: + from app.utils.metadata_enrichment import enrich_from_sidecar + return enrich_from_sidecar(src_path, existing_metadata) + except ImportError: + return {} + except Exception as e: + print(f"[WARN] Failed to enrich metadata for {src_path}: {e}") + return {} + + +def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None: + """ + Apply tags from enriched metadata to an ImageRecord. + Creates tags if they don't exist. + """ + if not enriched or "tags" not in enriched: + return + + for tag_info in enriched.get("tags", []): + tag_name = tag_info.get("name") + tag_kind = tag_info.get("kind") + if not tag_name: + continue + + tag = Tag.query.filter_by(name=tag_name).first() + if not tag: + tag = Tag(name=tag_name, kind=tag_kind) + db.session.add(tag) + db.session.flush() + + if tag not in record.tags: + record.tags.append(tag) + print(f"[INFO] Added tag: {tag_name}") + + +def _enrich_existing_record(record: ImageRecord, src_path: str) -> None: + """ + Enrich an existing record with metadata from sidecar JSON. + Used when a duplicate is found - we still want to merge metadata. + + - Updates taken_at if sidecar date is earlier + - Adds source/post tags if not already present + """ + try: + from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict + + enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at}) + if not enriched.get("raw_metadata"): + return # No sidecar found + + # Update date if sidecar has an earlier date + new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at")) + if new_date != record.taken_at: + print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}") + record.taken_at = new_date + + # Add any new tags + _apply_enriched_tags(record, enriched) + + except Exception as e: + print(f"[WARN] Failed to enrich existing record: {e}") # ============================================================================= @@ -593,6 +916,10 @@ def generate_thumbnail( Save thumbnails mirrored under /images/thumbs/<...>. If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG. If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`. + + For images with extreme aspect ratios (very tall like comic strips, or very wide), + crops to the center region before thumbnailing to produce better quality previews. + Returns the absolute path to the thumbnail. """ images_root = Path("/images").resolve() @@ -609,7 +936,27 @@ def generate_thumbnail( # Decide output extension/format with Image.open(image_path) as img: img = ImageOps.exif_transpose(img) # honor camera rotation - img.thumbnail(size) + + # Handle extreme aspect ratios by cropping to a more balanced region + # This prevents very tall/wide images from becoming tiny slivers + w, h = img.size + aspect_ratio = w / h if h > 0 else 1 + + # If aspect ratio is extreme (< 0.5 means very tall, > 2.0 means very wide) + # crop to a more balanced region before thumbnailing + if aspect_ratio < 0.5: + # Very tall image (like comic strip) - crop from top portion + # Take a region that's closer to square, from the top + crop_height = min(h, w * 2) # At most 2:1 tall + img = img.crop((0, 0, w, crop_height)) + elif aspect_ratio > 2.0: + # Very wide image - crop from center + crop_width = min(w, h * 2) # At most 2:1 wide + left = (w - crop_width) // 2 + img = img.crop((left, 0, left + crop_width, h)) + + # Use LANCZOS resampling for high-quality downscaling + img.thumbnail(size, Image.Resampling.LANCZOS) has_alpha = _has_alpha(img) @@ -687,24 +1034,69 @@ def calculate_hash(file_path: str) -> str: return hash_func.hexdigest() -def calculate_perceptual_hash(file_path: str): +def calculate_perceptual_hash(file_path: str, hash_size: int = 16): + """ + Calculate a perceptual hash for an image. + + Args: + file_path: Path to the image file + hash_size: Size of the hash grid (default 16 for 256-bit hash) + hash_size=8 gives 64-bit, hash_size=16 gives 256-bit + + Returns: + ImageHash object or None if hashing fails (e.g., for videos) + """ try: with Image.open(file_path) as img: - return imagehash.phash(img) + return imagehash.phash(img, hash_size=hash_size) except Exception: # videos or unreadable images land here (phash not applicable) return None -def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 2) -> bool: - for existing, ew, eh in existing_phashes: +def find_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10): + """ + Find visually similar images and determine relationship. + + Args: + phash: Perceptual hash of the new image + width: Width of the new image + height: Height of the new image + existing_phashes: List of (phash, width, height, image_id) tuples for existing images + threshold: Maximum Hamming distance to consider images similar + Default is 10 for 256-bit hashes (was 2 for 64-bit) + + Returns: + tuple: (relationship, image_id) where relationship is one of: + - None: No similar image found + - "larger_exists": A larger or equal similar image exists (skip new image) + - "smaller_exists": A smaller similar image exists (supersede it) + image_id is the ID of the matching image, or None + """ + for existing_phash, ew, eh, image_id in existing_phashes: try: - distance = phash - existing - if distance <= threshold and ew >= width and eh >= height: - return True + distance = phash - existing_phash + if distance <= threshold: + # Similar image found - check size relationship + if ew >= width and eh >= height: + # Existing is larger or equal - skip new image + return ("larger_exists", image_id) + elif width > ew or height > eh: + # New image is larger - supersede the existing one + return ("smaller_exists", image_id) except Exception as e: print(f"[WARN] Failed pHash comparison: {e}") - return False + return (None, None) + + +def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10) -> bool: + """ + Check if an image is visually similar to any existing larger image. + + Legacy wrapper for find_similar_image - returns True only if a larger version exists. + """ + relationship, _ = find_similar_image(phash, width, height, existing_phashes, threshold) + return relationship == "larger_exists" def extract_metadata(file_path: str) -> dict: diff --git a/app/utils/metadata_enrichment.py b/app/utils/metadata_enrichment.py new file mode 100644 index 0000000..8d40ebd --- /dev/null +++ b/app/utils/metadata_enrichment.py @@ -0,0 +1,346 @@ +# app/utils/metadata_enrichment.py +""" +Metadata enrichment from Gallery-DL sidecar JSON files. + +This module handles: +- Detection and parsing of Gallery-DL sidecar .json files +- Extraction of metadata (dates, tags, post info) from various platforms +- Tag generation for source tracking (source:platform, post:platform:artist:id) +- Date conflict resolution (earliest date wins) +""" + +import json +import logging +import os +import re +from datetime import datetime +from pathlib import Path +from typing import Optional + +log = logging.getLogger("metadata-enrichment") + + +# ============================================================================= +# Sidecar File Detection +# ============================================================================= + +def find_sidecar_json(image_path: str) -> Optional[str]: + """ + Find the Gallery-DL sidecar JSON file for an image. + + Gallery-DL creates JSON files with the UUID portion of the filename. + For example: + Image: 01_BBEFD45F-8775-4DF9-8BCA-99E6063E3616.jpg + JSON: BBEFD45F-8775-4DF9-8BCA-99E6063E3616.json + + Also checks for direct {filename}.json pattern. + + Returns the path to the sidecar JSON if found, None otherwise. + """ + image_path = Path(image_path) + parent_dir = image_path.parent + stem = image_path.stem # filename without extension + + # Pattern 1: Direct {filename}.json (e.g., image.jpg -> image.jpg.json) + direct_json = image_path.with_suffix(image_path.suffix + ".json") + if direct_json.exists(): + return str(direct_json) + + # Pattern 2: Extract UUID from filename and look for {uuid}.json + # Common patterns: "01_UUID", "UUID", "{num}_{UUID}" + uuid_pattern = re.compile(r'([A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12})', re.IGNORECASE) + match = uuid_pattern.search(stem) + if match: + uuid = match.group(1) + uuid_json = parent_dir / f"{uuid}.json" + if uuid_json.exists(): + return str(uuid_json) + + # Pattern 3: Just the stem without prefix numbers + # e.g., "01_filename" -> look for "filename.json" + stem_no_prefix = re.sub(r'^\d+_', '', stem) + if stem_no_prefix != stem: + alt_json = parent_dir / f"{stem_no_prefix}.json" + if alt_json.exists(): + return str(alt_json) + + return None + + +def load_sidecar_metadata(json_path: str) -> Optional[dict]: + """Load and parse a Gallery-DL sidecar JSON file.""" + try: + with open(json_path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, IOError) as e: + log.warning(f"Failed to load sidecar JSON {json_path}: {e}") + return None + + +# ============================================================================= +# Metadata Extraction (Platform-specific) +# ============================================================================= + +def extract_platform(metadata: dict) -> Optional[str]: + """Extract the platform name from metadata.""" + # Check common fields + if "category" in metadata: + return metadata["category"].lower() + + # Infer from URL patterns + url = metadata.get("url", "") or metadata.get("post_url", "") + if "patreon.com" in url: + return "patreon" + if "hentai-foundry" in url or "hentaifoundry" in url.lower(): + return "hentaifoundry" + if "subscribestar" in url: + return "subscribestar" + if "discord.com" in url: + return "discord" + + return None + + +def extract_artist(metadata: dict) -> Optional[str]: + """Extract the artist/creator name from metadata.""" + # Try various common fields + if "creator" in metadata and isinstance(metadata["creator"], dict): + return metadata["creator"].get("vanity") or metadata["creator"].get("full_name") + + if "author_name" in metadata: + return metadata["author_name"] + + if "artist" in metadata: + return metadata["artist"] + + # For hentaifoundry + if "user" in metadata: + return metadata["user"] + + return None + + +def extract_post_id(metadata: dict) -> Optional[str]: + """Extract the post ID from metadata.""" + if "id" in metadata: + return str(metadata["id"]) + + if "post_id" in metadata: + return str(metadata["post_id"]) + + if "message_id" in metadata: # Discord + return str(metadata["message_id"]) + + return None + + +def extract_post_title(metadata: dict) -> Optional[str]: + """Extract the post title from metadata.""" + return metadata.get("title") + + +def extract_date(metadata: dict) -> Optional[datetime]: + """ + Extract the publication/post date from metadata. + Tries multiple common date fields and formats. + """ + date_fields = ["published_at", "date", "created_at", "timestamp"] + + for field in date_fields: + if field not in metadata: + continue + + value = metadata[field] + if not value: + continue + + # Handle various formats + parsed = _parse_date(value) + if parsed: + return parsed + + return None + + +def _parse_date(value) -> Optional[datetime]: + """Parse a date value that could be string, int, or already datetime.""" + if isinstance(value, datetime): + return value + + if isinstance(value, (int, float)): + # Unix timestamp + try: + return datetime.fromtimestamp(value) + except (ValueError, OSError): + pass + + if isinstance(value, str): + # Try common formats + formats = [ + "%Y-%m-%dT%H:%M:%S.%f%z", # ISO with microseconds and timezone + "%Y-%m-%dT%H:%M:%S%z", # ISO with timezone + "%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds + "%Y-%m-%dT%H:%M:%S", # ISO basic + "%Y-%m-%d %H:%M:%S", # Space-separated + "%Y-%m-%d", # Date only + ] + + # Handle timezone offset with colon (Python < 3.11 compat) + # e.g., "2023-08-01T04:20:02.000+00:00" -> "2023-08-01T04:20:02.000+0000" + normalized = re.sub(r'([+-]\d{2}):(\d{2})$', r'\1\2', value) + + for fmt in formats: + try: + return datetime.strptime(normalized, fmt) + except ValueError: + continue + + # Try without timezone info + # Strip timezone suffix for naive parsing + stripped = re.sub(r'[+-]\d{2}:?\d{2}$', '', value) + stripped = re.sub(r'Z$', '', stripped) + for fmt in formats: + try: + return datetime.strptime(stripped, fmt) + except ValueError: + continue + + return None + + +def extract_content_tags(metadata: dict) -> list[str]: + """ + Extract content tags from metadata (not to be confused with DB tags). + These are tags the creator applied to the post. + """ + tags = [] + + # Direct tags field + if "tags" in metadata and isinstance(metadata["tags"], list): + tags.extend(metadata["tags"]) + + return [str(t) for t in tags if t] + + +# ============================================================================= +# Tag Generation +# ============================================================================= + +def generate_source_tags(metadata: dict, artist_override: Optional[str] = None) -> list[dict]: + """ + Generate tags from Gallery-DL metadata. + + Returns a list of tag dictionaries with 'name' and 'kind' keys. + Tag format ensures uniqueness: + - source:{platform} (kind='source') + - post:{platform}:{artist}:{post_id} (kind='post') + """ + tags = [] + + platform = extract_platform(metadata) + artist = artist_override or extract_artist(metadata) + post_id = extract_post_id(metadata) + + # Source platform tag + if platform: + tags.append({ + "name": f"source:{platform}", + "kind": "source" + }) + + # Post tag (unique across platform + artist + post_id) + if platform and artist and post_id: + # Sanitize artist name for tag (lowercase, replace spaces) + safe_artist = artist.lower().replace(" ", "_") + tags.append({ + "name": f"post:{platform}:{safe_artist}:{post_id}", + "kind": "post" + }) + + return tags + + +# ============================================================================= +# Date Conflict Resolution +# ============================================================================= + +def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[datetime]) -> Optional[datetime]: + """ + Resolve date conflicts by keeping the earliest date. + + Args: + existing_date: The currently stored date (may be None) + new_date: The new date from metadata (may be None) + + Returns: + The earliest non-None date, or None if both are None + """ + if existing_date is None: + return new_date + if new_date is None: + return existing_date + + # Return the earlier date + return min(existing_date, new_date) + + +# ============================================================================= +# Main Enrichment Function +# ============================================================================= + +def enrich_from_sidecar(image_path: str, existing_metadata: Optional[dict] = None) -> dict: + """ + Enrich image metadata from Gallery-DL sidecar JSON. + + Args: + image_path: Path to the image file + existing_metadata: Optional dict with existing metadata (e.g., from EXIF) + + Returns: + Dictionary with enriched metadata: + { + "platform": str or None, + "artist": str or None, + "post_id": str or None, + "post_title": str or None, + "taken_at": datetime or None, + "tags": [{"name": str, "kind": str}, ...], + "raw_metadata": dict or None (the full sidecar content) + } + """ + result = { + "platform": None, + "artist": None, + "post_id": None, + "post_title": None, + "taken_at": None, + "tags": [], + "raw_metadata": None, + } + + # Find and load sidecar + sidecar_path = find_sidecar_json(image_path) + if not sidecar_path: + return result + + metadata = load_sidecar_metadata(sidecar_path) + if not metadata: + return result + + result["raw_metadata"] = metadata + + # Extract fields + result["platform"] = extract_platform(metadata) + result["artist"] = extract_artist(metadata) + result["post_id"] = extract_post_id(metadata) + result["post_title"] = extract_post_title(metadata) + + # Extract date and resolve conflicts with existing + sidecar_date = extract_date(metadata) + existing_date = existing_metadata.get("taken_at") if existing_metadata else None + result["taken_at"] = resolve_date_conflict(existing_date, sidecar_date) + + # Generate tags + result["tags"] = generate_source_tags(metadata, result["artist"]) + + return result diff --git a/app/utils/version_migration.py b/app/utils/version_migration.py new file mode 100644 index 0000000..0d794af --- /dev/null +++ b/app/utils/version_migration.py @@ -0,0 +1,288 @@ +# app/utils/version_migration.py +""" +Version tracking and data migration utilities. + +This module handles: +- Application version tracking via AppSettings +- Automatic phash recomputation when version changes +- Settings retrieval/storage from database +""" + +import logging +import os +import time +from typing import Optional + +from PIL import Image +import imagehash + +from app import db +from app.models import AppSettings, ImageRecord + +log = logging.getLogger("version-migration") + +# ============================================================================= +# Version and pHash Configuration +# ============================================================================= + +APP_VERSION = "26.01.19.1" # Format: YY.MM.DD.iteration + +# pHash configuration - 256-bit (hash_size=16 -> 16x16 grid) +PHASH_HASH_SIZE = 16 +PHASH_BITS = PHASH_HASH_SIZE * PHASH_HASH_SIZE # 256 +PHASH_THRESHOLD_DEFAULT = 10 # Scaled from 2 (64-bit) to 10 (256-bit) + +# Batch size for phash recomputation +PHASH_RECOMPUTE_BATCH_SIZE = int(os.environ.get("PHASH_RECOMPUTE_BATCH_SIZE", "100")) + + +# ============================================================================= +# Settings Access Functions +# ============================================================================= + +def get_setting(key: str, default: Optional[str] = None) -> Optional[str]: + """ + Retrieve a setting value from the database. + Returns default if the key doesn't exist. + """ + try: + setting = AppSettings.query.filter_by(key=key).first() + if setting is not None: + return setting.value + return default + except Exception as e: + log.warning(f"Failed to get setting '{key}': {e}") + return default + + +def set_setting(key: str, value: str) -> None: + """ + Store a setting value in the database. + Creates the key if it doesn't exist, updates if it does. + """ + try: + setting = AppSettings.query.filter_by(key=key).first() + if setting is None: + setting = AppSettings(key=key, value=value) + db.session.add(setting) + else: + setting.value = value + db.session.commit() + except Exception as e: + log.error(f"Failed to set setting '{key}': {e}") + db.session.rollback() + raise + + +def get_setting_int(key: str, default: int) -> int: + """Get a setting as an integer.""" + val = get_setting(key) + if val is None: + return default + try: + return int(val) + except ValueError: + return default + + +def get_setting_float(key: str, default: float) -> float: + """Get a setting as a float.""" + val = get_setting(key) + if val is None: + return default + try: + return float(val) + except ValueError: + return default + + +def get_setting_bool(key: str, default: bool) -> bool: + """Get a setting as a boolean.""" + val = get_setting(key) + if val is None: + return default + return val.lower() in ("true", "1", "yes") + + +# ============================================================================= +# Import Settings (DB-backed) +# ============================================================================= + +def get_import_settings() -> dict: + """ + Load import filter settings from database, with environment variable fallbacks. + This replaces the file-based load_import_settings() function. + """ + return { + "min_width": get_setting_int("import_min_width", int(os.environ.get("IMPORT_MIN_WIDTH", "0"))), + "min_height": get_setting_int("import_min_height", int(os.environ.get("IMPORT_MIN_HEIGHT", "0"))), + "skip_transparent": get_setting_bool("import_skip_transparent", os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true"), + "transparency_threshold": get_setting_float("import_transparency_threshold", float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9"))), + "phash_threshold": get_setting_int("phash_threshold", PHASH_THRESHOLD_DEFAULT), + } + + +# ============================================================================= +# pHash Computation (256-bit) +# ============================================================================= + +def calculate_phash_256(file_path: str) -> Optional[imagehash.ImageHash]: + """ + Calculate a 256-bit perceptual hash for an image. + Uses hash_size=16 for a 16x16 grid = 256 bits. + """ + try: + with Image.open(file_path) as img: + return imagehash.phash(img, hash_size=PHASH_HASH_SIZE) + except Exception as e: + log.debug(f"Could not compute phash for {file_path}: {e}") + return None + + +# ============================================================================= +# Version Migration Logic +# ============================================================================= + +def recompute_all_phashes() -> dict: + """ + Recompute perceptual hashes for all images using 256-bit hashing. + Returns statistics about the operation. + """ + stats = { + "total": 0, + "updated": 0, + "skipped": 0, + "failed": 0, + } + + log.info("Starting phash recomputation (256-bit)...") + t_start = time.time() + + batch_size = PHASH_RECOMPUTE_BATCH_SIZE + last_id = 0 + + while True: + # Fetch batch of images + images = ( + ImageRecord.query + .filter(ImageRecord.id > last_id) + .order_by(ImageRecord.id.asc()) + .limit(batch_size) + .all() + ) + + if not images: + break + + batch_updated = 0 + for img in images: + stats["total"] += 1 + + # Skip videos (no phash) + ext = (img.format or "").lower() + if ext in ("mp4", "mov") or (img.filepath and img.filepath.lower().endswith((".mp4", ".mov"))): + stats["skipped"] += 1 + continue + + # Skip if file doesn't exist + if not img.filepath or not os.path.exists(img.filepath): + log.warning(f"File not found for image id={img.id}: {img.filepath}") + stats["failed"] += 1 + continue + + # Compute new phash + try: + phash = calculate_phash_256(img.filepath) + if phash: + img.perceptual_hash = str(phash) + batch_updated += 1 + stats["updated"] += 1 + else: + stats["failed"] += 1 + except Exception as e: + log.warning(f"Failed to compute phash for id={img.id}: {e}") + stats["failed"] += 1 + + # Commit batch + db.session.commit() + last_id = images[-1].id + log.info(f"Processed batch up to id={last_id}, updated={batch_updated}") + + elapsed = time.time() - t_start + log.info( + f"pHash recomputation complete in {elapsed:.1f}s: " + f"total={stats['total']}, updated={stats['updated']}, " + f"skipped={stats['skipped']}, failed={stats['failed']}" + ) + + return stats + + +def check_and_run_migrations() -> None: + """ + Check the stored app version and run data migrations if needed. + + This function: + 1. Reads the stored app_version from the database + 2. Compares it with the current APP_VERSION + 3. If different, triggers necessary data migrations (e.g., phash recomputation) + 4. Updates the stored version + """ + log.info(f"Checking app version (current: {APP_VERSION})...") + + stored_version = get_setting("app_version") + phash_version = get_setting("phash_version") + + log.info(f"Stored version: {stored_version}, pHash version: {phash_version}") + + # Initialize settings if this is first run + if stored_version is None: + log.info("First run detected, initializing settings...") + _initialize_default_settings() + stored_version = "0.0.0.0" # Force migration on first run + + # Check if phash recomputation is needed + needs_phash_recompute = ( + phash_version is None or + phash_version != APP_VERSION or + get_setting_int("phash_bits", 64) != PHASH_BITS + ) + + if needs_phash_recompute: + log.info(f"pHash migration needed (stored bits: {get_setting('phash_bits')}, new: {PHASH_BITS})") + + # Update phash configuration + set_setting("phash_bits", str(PHASH_BITS)) + set_setting("phash_hash_size", str(PHASH_HASH_SIZE)) + + # Recompute all hashes + recompute_all_phashes() + + # Mark phash version as current + set_setting("phash_version", APP_VERSION) + + # Update app version + if stored_version != APP_VERSION: + log.info(f"Updating app version from {stored_version} to {APP_VERSION}") + set_setting("app_version", APP_VERSION) + + log.info("Version check complete.") + + +def _initialize_default_settings() -> None: + """Initialize default settings on first run.""" + defaults = { + "app_version": APP_VERSION, + "phash_version": None, # Will trigger recomputation + "phash_bits": str(PHASH_BITS), + "phash_hash_size": str(PHASH_HASH_SIZE), + "phash_threshold": str(PHASH_THRESHOLD_DEFAULT), + "import_min_width": "0", + "import_min_height": "0", + "import_skip_transparent": "false", + "import_transparency_threshold": "0.9", + } + + for key, value in defaults.items(): + if get_setting(key) is None and value is not None: + set_setting(key, value) diff --git a/entrypoint.sh b/entrypoint.sh index e9a9f6f..85e97bf 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -20,6 +20,10 @@ if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then sleep "$SLEEP_SECS" i=$((i+1)) done + + # Run version check and data migrations (e.g., phash recomputation) + echo "Checking app version and running data migrations..." + flask version-check fi # --- Start the image import worker exactly once per container --- diff --git a/migrations/versions/b26011901_add_app_settings_expand_phash.py b/migrations/versions/b26011901_add_app_settings_expand_phash.py new file mode 100644 index 0000000..98712a7 --- /dev/null +++ b/migrations/versions/b26011901_add_app_settings_expand_phash.py @@ -0,0 +1,132 @@ +"""Add app_settings table and expand perceptual_hash to 64 chars for 256-bit hashes + +Revision ID: b26011901 +Revises: a46e641430b8 +Create Date: 2026-01-19 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b26011901' +down_revision = 'a46e641430b8' +branch_labels = None +depends_on = None + + +def upgrade(): + # Create app_settings table + op.create_table( + 'app_settings', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('key', sa.String(length=64), nullable=False), + sa.Column('value', sa.Text(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('key') + ) + + # Expand perceptual_hash column from 16 to 64 chars + bind = op.get_bind() + dialect = bind.dialect.name + + if dialect == "sqlite": + # SQLite requires table recreation for column type changes. + # IMPORTANT: When recreating image_record, we must also recreate + # image_tags to preserve foreign key relationships. + + # Step 1: Backup image_tags data + op.execute("CREATE TABLE image_tags_backup AS SELECT * FROM image_tags") + + # Step 2: Drop image_tags (it references image_record) + op.drop_table("image_tags") + + # Step 3: Recreate image_record with new column size + with op.batch_alter_table("image_record", recreate="always") as batch_op: + batch_op.alter_column( + 'perceptual_hash', + existing_type=sa.String(length=16), + type_=sa.String(length=64), + existing_nullable=True + ) + + # Step 4: Recreate image_tags with proper FK constraints + op.create_table( + "image_tags", + sa.Column("image_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("image_id", "tag_id"), + sa.ForeignKeyConstraint(["image_id"], ["image_record.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), + sa.UniqueConstraint("image_id", "tag_id", name="uq_image_tag"), + ) + + # Step 5: Restore image_tags data + op.execute("INSERT INTO image_tags (image_id, tag_id) SELECT image_id, tag_id FROM image_tags_backup") + + # Step 6: Drop backup table + op.drop_table("image_tags_backup") + else: + # PostgreSQL and others can alter directly + op.alter_column( + 'image_record', + 'perceptual_hash', + existing_type=sa.String(length=16), + type_=sa.String(length=64), + existing_nullable=True + ) + + +def downgrade(): + bind = op.get_bind() + dialect = bind.dialect.name + + # Shrink perceptual_hash back to 16 chars (will truncate data!) + if dialect == "sqlite": + # Same approach as upgrade: backup image_tags, recreate image_record, + # then restore image_tags to preserve FK relationships + + # Step 1: Backup image_tags data + op.execute("CREATE TABLE image_tags_backup AS SELECT * FROM image_tags") + + # Step 2: Drop image_tags (it references image_record) + op.drop_table("image_tags") + + # Step 3: Recreate image_record with old column size + with op.batch_alter_table("image_record", recreate="always") as batch_op: + batch_op.alter_column( + 'perceptual_hash', + existing_type=sa.String(length=64), + type_=sa.String(length=16), + existing_nullable=True + ) + + # Step 4: Recreate image_tags with proper FK constraints + op.create_table( + "image_tags", + sa.Column("image_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("image_id", "tag_id"), + sa.ForeignKeyConstraint(["image_id"], ["image_record.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), + sa.UniqueConstraint("image_id", "tag_id", name="uq_image_tag"), + ) + + # Step 5: Restore image_tags data + op.execute("INSERT INTO image_tags (image_id, tag_id) SELECT image_id, tag_id FROM image_tags_backup") + + # Step 6: Drop backup table + op.drop_table("image_tags_backup") + else: + op.alter_column( + 'image_record', + 'perceptual_hash', + existing_type=sa.String(length=64), + type_=sa.String(length=16), + existing_nullable=True + ) + + # Drop app_settings table + op.drop_table('app_settings')