moving styling and views to be more consistent and for gallery view to be less cumbersome.
This commit is contained in:
Bryan Van Deusen
@@ -1,6 +1,8 @@
|
||||
# app/__init__.py
|
||||
|
||||
import click
|
||||
from flask import Flask
|
||||
from flask.cli import with_appcontext
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from flask_migrate import Migrate
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
@@ -11,6 +13,18 @@ import sqlite3
|
||||
db = SQLAlchemy()
|
||||
migrate = Migrate()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Commands
|
||||
# =============================================================================
|
||||
|
||||
@click.command("version-check")
|
||||
@with_appcontext
|
||||
def version_check_command():
|
||||
"""Check app version and run data migrations if needed (e.g., phash recomputation)."""
|
||||
from app.utils.version_migration import check_and_run_migrations
|
||||
check_and_run_migrations()
|
||||
|
||||
@event.listens_for(Engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_connection, connection_record):
|
||||
# Only apply to SQLite connections
|
||||
@@ -38,4 +52,7 @@ def create_app(config_class='config.Config'):
|
||||
def datetimeformat(value, format="%Y-%m-%d"):
|
||||
return value.strftime(format) if value else ""
|
||||
|
||||
# Register CLI commands
|
||||
app.cli.add_command(version_check_command)
|
||||
|
||||
return app
|
||||
|
||||
+289
-16
@@ -2,8 +2,10 @@
|
||||
|
||||
from flask import Blueprint, render_template, flash, redirect, url_for, abort, send_from_directory, request, jsonify
|
||||
|
||||
from sqlalchemy import desc, func, text
|
||||
from sqlalchemy import desc, func, text, extract
|
||||
from sqlalchemy.orm import joinedload
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
from app.models import ImageRecord, Tag, ArchiveRecord, image_tags
|
||||
from app import db
|
||||
@@ -78,35 +80,47 @@ def random_images_api():
|
||||
@main.route('/gallery')
|
||||
def gallery():
|
||||
"""
|
||||
Gallery with pagination and optional tag filter (?tag=artist:NAME or any tag name).
|
||||
Keeps your pagination shape so modal-pagination.js continues to work.
|
||||
Gallery with infinite scroll and year-month date dividers.
|
||||
Server renders initial batch, then JS takes over for infinite scroll.
|
||||
Optional tag filter via ?tag=artist:NAME or any tag name.
|
||||
"""
|
||||
page = request.args.get('page', 1, type=int)
|
||||
per_page = request.args.get('per_page', 35, type=int)
|
||||
initial_limit = 30
|
||||
tag_name = request.args.get('tag')
|
||||
|
||||
# Base query + eager-load tags (cheap and future-proofs tag chips)
|
||||
# Base query + eager-load tags
|
||||
q = ImageRecord.query.options(joinedload(ImageRecord.tags))
|
||||
|
||||
# Optional tag filter
|
||||
if tag_name:
|
||||
q = q.join(ImageRecord.tags).filter(Tag.name == tag_name)
|
||||
|
||||
images = q.order_by(desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at))) \
|
||||
.paginate(page=page, per_page=per_page)
|
||||
# Fetch initial batch ordered by date descending
|
||||
images = q.order_by(
|
||||
desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at))
|
||||
).limit(initial_limit + 1).all()
|
||||
|
||||
# Preserve next/prev URLs (carry tag= so modal pagination fetches stay filtered)
|
||||
next_url = url_for('main.gallery', page=images.next_num, per_page=per_page, tag=tag_name) if images.has_next else ''
|
||||
prev_url = url_for('main.gallery', page=images.prev_num, per_page=per_page, tag=tag_name) if images.has_prev else ''
|
||||
# Check if there are more images
|
||||
has_more = len(images) > initial_limit
|
||||
if has_more:
|
||||
images = images[:initial_limit]
|
||||
|
||||
# Group by year-month for initial render
|
||||
grouped_images = _group_images_by_month(images)
|
||||
|
||||
# Compute initial cursor for JS to continue from
|
||||
initial_cursor = None
|
||||
if images and has_more:
|
||||
last_img = images[-1]
|
||||
last_dt = _get_image_date(last_img)
|
||||
if last_dt:
|
||||
initial_cursor = last_dt.isoformat()
|
||||
|
||||
return render_template(
|
||||
'gallery.html',
|
||||
images=images,
|
||||
grouped_images=grouped_images,
|
||||
active_tag=tag_name,
|
||||
has_next=images.has_next,
|
||||
next_page_url=next_url,
|
||||
has_prev=images.has_prev,
|
||||
prev_page_url=prev_url
|
||||
initial_cursor=initial_cursor,
|
||||
has_more=has_more
|
||||
)
|
||||
|
||||
|
||||
@@ -115,6 +129,265 @@ def serve_image(filename):
|
||||
return send_from_directory('/images', filename)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Gallery Infinite Scroll API
|
||||
# ----------------------------
|
||||
|
||||
def _get_image_date(img):
|
||||
"""Get the effective date for an image (taken_at or imported_at)."""
|
||||
return img.taken_at or img.imported_at
|
||||
|
||||
|
||||
def _format_year_month(dt):
|
||||
"""Format a datetime as year-month key and label."""
|
||||
if not dt:
|
||||
return 'unknown', 'Unknown Date'
|
||||
return dt.strftime('%Y-%m'), dt.strftime('%B %Y')
|
||||
|
||||
|
||||
def _group_images_by_month(images):
|
||||
"""
|
||||
Group images by year-month, preserving order.
|
||||
Returns list of (group_info, images) tuples.
|
||||
"""
|
||||
groups = OrderedDict()
|
||||
|
||||
for img in images:
|
||||
dt = _get_image_date(img)
|
||||
key, label = _format_year_month(dt)
|
||||
|
||||
if key not in groups:
|
||||
groups[key] = {'key': key, 'label': label, 'images': []}
|
||||
|
||||
groups[key]['images'].append(img)
|
||||
|
||||
return [(info['key'], info['label'], info['images']) for info in groups.values()]
|
||||
|
||||
|
||||
def _serialize_image(img):
|
||||
"""Serialize an ImageRecord for JSON response."""
|
||||
is_video = img.filename.lower().endswith(('.mp4', '.mov'))
|
||||
thumb_path = (img.thumb_path or img.filepath).replace('/images/', '')
|
||||
full_path = img.filepath.replace('/images/', '')
|
||||
dt = _get_image_date(img)
|
||||
key, label = _format_year_month(dt)
|
||||
|
||||
return {
|
||||
'id': img.id,
|
||||
'filename': img.filename,
|
||||
'thumb_url': url_for('main.serve_image', filename=thumb_path),
|
||||
'full_url': url_for('main.serve_image', filename=full_path),
|
||||
'is_video': is_video,
|
||||
'date': dt.strftime('%Y-%m-%d') if dt else '',
|
||||
'year_month': key,
|
||||
'year_month_label': label,
|
||||
'tags': [{'name': t.name, 'kind': t.kind} for t in img.tags if t.kind != 'archive']
|
||||
}
|
||||
|
||||
|
||||
@main.route('/api/gallery/scroll')
|
||||
def gallery_scroll():
|
||||
"""
|
||||
Cursor-based infinite scroll endpoint with date grouping.
|
||||
|
||||
Query params:
|
||||
- cursor: ISO datetime string for pagination (fetch images older than this)
|
||||
- limit: number of images per batch (default 30)
|
||||
- tag: optional tag filter
|
||||
|
||||
Returns:
|
||||
- images: list of image objects with year_month grouping info
|
||||
- next_cursor: datetime string for next batch (null if no more)
|
||||
- date_groups: summary of year-month groups in this batch
|
||||
- has_more: boolean indicating if more images exist
|
||||
"""
|
||||
cursor_str = request.args.get('cursor')
|
||||
limit = request.args.get('limit', 30, type=int)
|
||||
tag_name = request.args.get('tag')
|
||||
|
||||
# Build base query
|
||||
q = ImageRecord.query.options(joinedload(ImageRecord.tags))
|
||||
|
||||
# Optional tag filter
|
||||
if tag_name:
|
||||
q = q.join(ImageRecord.tags).filter(Tag.name == tag_name)
|
||||
|
||||
# Apply cursor filter (fetch images older than cursor)
|
||||
if cursor_str:
|
||||
try:
|
||||
cursor_dt = datetime.fromisoformat(cursor_str.replace('Z', '+00:00'))
|
||||
q = q.filter(
|
||||
func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at) < cursor_dt
|
||||
)
|
||||
except ValueError:
|
||||
pass # Invalid cursor, ignore
|
||||
|
||||
# Order by date descending and fetch with limit + 1 to check has_more
|
||||
images = q.order_by(
|
||||
desc(func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at))
|
||||
).limit(limit + 1).all()
|
||||
|
||||
# Check if there are more images
|
||||
has_more = len(images) > limit
|
||||
if has_more:
|
||||
images = images[:limit]
|
||||
|
||||
# Serialize images
|
||||
serialized = [_serialize_image(img) for img in images]
|
||||
|
||||
# Build date_groups summary
|
||||
groups_seen = OrderedDict()
|
||||
for s in serialized:
|
||||
key = s['year_month']
|
||||
if key not in groups_seen:
|
||||
groups_seen[key] = {'key': key, 'label': s['year_month_label'], 'count': 0}
|
||||
groups_seen[key]['count'] += 1
|
||||
|
||||
# Compute next cursor
|
||||
next_cursor = None
|
||||
if images and has_more:
|
||||
last_img = images[-1]
|
||||
last_dt = _get_image_date(last_img)
|
||||
if last_dt:
|
||||
next_cursor = last_dt.isoformat()
|
||||
|
||||
return jsonify(
|
||||
images=serialized,
|
||||
next_cursor=next_cursor,
|
||||
has_more=has_more,
|
||||
date_groups=list(groups_seen.values())
|
||||
)
|
||||
|
||||
|
||||
@main.route('/api/gallery/timeline')
|
||||
def gallery_timeline():
|
||||
"""
|
||||
Returns all year-month groups with counts for timeline navigation.
|
||||
|
||||
Query params:
|
||||
- tag: optional tag filter (same as gallery)
|
||||
|
||||
Returns list of {key, label, count, year} ordered newest to oldest.
|
||||
"""
|
||||
tag_name = request.args.get('tag')
|
||||
|
||||
# Use SQL to aggregate by year-month
|
||||
date_expr = func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at)
|
||||
|
||||
q = db.session.query(
|
||||
extract('year', date_expr).label('year'),
|
||||
extract('month', date_expr).label('month'),
|
||||
func.count(ImageRecord.id).label('count')
|
||||
)
|
||||
|
||||
if tag_name:
|
||||
q = q.join(ImageRecord.tags).filter(Tag.name == tag_name)
|
||||
|
||||
q = q.group_by(
|
||||
extract('year', date_expr),
|
||||
extract('month', date_expr)
|
||||
).order_by(
|
||||
extract('year', date_expr).desc(),
|
||||
extract('month', date_expr).desc()
|
||||
)
|
||||
|
||||
results = q.all()
|
||||
|
||||
# Format results
|
||||
groups = []
|
||||
for row in results:
|
||||
if row.year and row.month:
|
||||
year = int(row.year)
|
||||
month = int(row.month)
|
||||
key = f"{year:04d}-{month:02d}"
|
||||
# Create a date object to format the month name
|
||||
dt = datetime(year, month, 1)
|
||||
label = dt.strftime('%B %Y')
|
||||
short_label = dt.strftime('%b') # Short month name for sidebar
|
||||
groups.append({
|
||||
'key': key,
|
||||
'label': label,
|
||||
'short_label': short_label,
|
||||
'year': year,
|
||||
'month': month,
|
||||
'count': row.count
|
||||
})
|
||||
|
||||
return jsonify(groups=groups)
|
||||
|
||||
|
||||
@main.route('/api/gallery/jump')
|
||||
def gallery_jump():
|
||||
"""
|
||||
Fetch images starting from a specific year-month for timeline jump navigation.
|
||||
|
||||
Query params:
|
||||
- year_month: target year-month (e.g., '2024-03')
|
||||
- limit: number of images (default 30)
|
||||
- tag: optional tag filter
|
||||
|
||||
Returns same format as /api/gallery/scroll but starting from the target month.
|
||||
"""
|
||||
year_month = request.args.get('year_month', '')
|
||||
limit = request.args.get('limit', 30, type=int)
|
||||
tag_name = request.args.get('tag')
|
||||
|
||||
if not year_month:
|
||||
return jsonify(error='year_month is required'), 400
|
||||
|
||||
try:
|
||||
# Parse year-month to get start of that month
|
||||
target_dt = datetime.strptime(year_month + '-01', '%Y-%m-%d')
|
||||
# Get end of month (start of next month)
|
||||
if target_dt.month == 12:
|
||||
next_month = datetime(target_dt.year + 1, 1, 1)
|
||||
else:
|
||||
next_month = datetime(target_dt.year, target_dt.month + 1, 1)
|
||||
except ValueError:
|
||||
return jsonify(error='Invalid year_month format'), 400
|
||||
|
||||
# Build query for images in that month and earlier
|
||||
q = ImageRecord.query.options(joinedload(ImageRecord.tags))
|
||||
|
||||
if tag_name:
|
||||
q = q.join(ImageRecord.tags).filter(Tag.name == tag_name)
|
||||
|
||||
# Get images from start of target month onwards (but ordered descending)
|
||||
date_expr = func.coalesce(ImageRecord.taken_at, ImageRecord.imported_at)
|
||||
q = q.filter(date_expr < next_month)
|
||||
|
||||
images = q.order_by(desc(date_expr)).limit(limit + 1).all()
|
||||
|
||||
has_more = len(images) > limit
|
||||
if has_more:
|
||||
images = images[:limit]
|
||||
|
||||
serialized = [_serialize_image(img) for img in images]
|
||||
|
||||
# Build date_groups summary
|
||||
groups_seen = OrderedDict()
|
||||
for s in serialized:
|
||||
key = s['year_month']
|
||||
if key not in groups_seen:
|
||||
groups_seen[key] = {'key': key, 'label': s['year_month_label'], 'count': 0}
|
||||
groups_seen[key]['count'] += 1
|
||||
|
||||
next_cursor = None
|
||||
if images and has_more:
|
||||
last_img = images[-1]
|
||||
last_dt = _get_image_date(last_img)
|
||||
if last_dt:
|
||||
next_cursor = last_dt.isoformat()
|
||||
|
||||
return jsonify(
|
||||
images=serialized,
|
||||
next_cursor=next_cursor,
|
||||
has_more=has_more,
|
||||
date_groups=list(groups_seen.values()),
|
||||
target_year_month=year_month
|
||||
)
|
||||
|
||||
|
||||
@main.route('/tags')
|
||||
def tag_list():
|
||||
"""
|
||||
|
||||
+13
-1
@@ -23,7 +23,7 @@ class ImageRecord(db.Model):
|
||||
filepath = db.Column(db.String(512), nullable=False)
|
||||
thumb_path = db.Column(db.String(512))
|
||||
hash = db.Column(db.String(64), unique=True, nullable=False) # SHA256
|
||||
perceptual_hash = db.Column(db.String(16), nullable=True) # hex of 64-bit hash
|
||||
perceptual_hash = db.Column(db.String(64), nullable=True) # hex of 256-bit hash
|
||||
file_size = db.Column(db.Integer, nullable=True)
|
||||
width = db.Column(db.Integer, nullable=True)
|
||||
height = db.Column(db.Integer, nullable=True)
|
||||
@@ -64,3 +64,15 @@ class ArchiveRecord(db.Model):
|
||||
artist = db.Column(db.String(255), nullable=True)
|
||||
tag_id = db.Column(db.Integer, db.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True)
|
||||
tag = db.relationship("Tag")
|
||||
|
||||
|
||||
class AppSettings(db.Model):
|
||||
"""
|
||||
Key-value store for application settings and version tracking.
|
||||
Used to persist configuration and trigger data migrations on version changes.
|
||||
"""
|
||||
__tablename__ = "app_settings"
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
key = db.Column(db.String(64), unique=True, nullable=False)
|
||||
value = db.Column(db.Text, nullable=True)
|
||||
updated_at = db.Column(db.DateTime, default=db.func.now(), onupdate=db.func.now())
|
||||
|
||||
@@ -0,0 +1,475 @@
|
||||
// /app/static/js/gallery-infinite.js
|
||||
// Infinite scroll gallery with year-month date dividers and timeline navigation
|
||||
|
||||
(function() {
|
||||
'use strict';
|
||||
|
||||
// Configuration
|
||||
const CONFIG = {
|
||||
SCROLL_THRESHOLD: 1200, // px from bottom to trigger load
|
||||
BATCH_SIZE: 30, // images per API request
|
||||
DEBOUNCE_SCROLL: 100, // ms debounce for scroll handler
|
||||
DEBOUNCE_RESIZE: 200 // ms debounce for resize handler
|
||||
};
|
||||
|
||||
// State
|
||||
const state = {
|
||||
isLoading: false,
|
||||
hasMore: true,
|
||||
cursor: null,
|
||||
activeTag: null,
|
||||
loadedSections: new Set(), // year_month keys of loaded sections
|
||||
timelineData: [],
|
||||
currentVisibleSection: null,
|
||||
isJumping: false // true during jump navigation
|
||||
};
|
||||
|
||||
// DOM references
|
||||
const dom = {
|
||||
container: null,
|
||||
loading: null,
|
||||
end: null,
|
||||
timeline: null,
|
||||
timelineTrack: null
|
||||
};
|
||||
|
||||
// Initialize
|
||||
function init() {
|
||||
dom.container = document.getElementById('galleryInfinite');
|
||||
dom.loading = document.getElementById('galleryLoading');
|
||||
dom.end = document.getElementById('galleryEnd');
|
||||
dom.timeline = document.getElementById('timelineSidebar');
|
||||
dom.timelineTrack = document.getElementById('timelineTrack');
|
||||
|
||||
if (!dom.container) return;
|
||||
|
||||
// Parse initial state from window.galleryState (set by template)
|
||||
if (window.galleryState) {
|
||||
state.cursor = window.galleryState.cursor;
|
||||
state.hasMore = window.galleryState.hasMore;
|
||||
state.activeTag = window.galleryState.activeTag;
|
||||
}
|
||||
|
||||
// Track initially loaded sections
|
||||
parseInitialSections();
|
||||
|
||||
// Load timeline data
|
||||
loadTimeline();
|
||||
|
||||
// Setup event listeners
|
||||
setupScrollListener();
|
||||
setupIntersectionObserver();
|
||||
|
||||
// Check if we need to load more on init (short content)
|
||||
setTimeout(checkScrollPosition, 100);
|
||||
}
|
||||
|
||||
// Parse initially rendered sections
|
||||
function parseInitialSections() {
|
||||
const sections = dom.container.querySelectorAll('.date-section');
|
||||
sections.forEach(section => {
|
||||
const yearMonth = section.dataset.yearMonth;
|
||||
if (yearMonth) {
|
||||
state.loadedSections.add(yearMonth);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Fetch more images
|
||||
async function loadMore() {
|
||||
if (state.isLoading || !state.hasMore) return;
|
||||
|
||||
state.isLoading = true;
|
||||
dom.loading.style.display = 'flex';
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
limit: CONFIG.BATCH_SIZE
|
||||
});
|
||||
|
||||
if (state.cursor) {
|
||||
params.set('cursor', state.cursor);
|
||||
}
|
||||
if (state.activeTag) {
|
||||
params.set('tag', state.activeTag);
|
||||
}
|
||||
|
||||
const response = await fetch(`/api/gallery/scroll?${params}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.images && data.images.length > 0) {
|
||||
appendImages(data.images);
|
||||
state.cursor = data.next_cursor;
|
||||
state.hasMore = data.has_more;
|
||||
|
||||
// Notify modal system
|
||||
dispatchGalleryUpdate();
|
||||
}
|
||||
|
||||
if (!data.has_more) {
|
||||
state.hasMore = false;
|
||||
dom.end.style.display = 'block';
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load more images:', err);
|
||||
} finally {
|
||||
state.isLoading = false;
|
||||
dom.loading.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
// Append images to the gallery
|
||||
function appendImages(images) {
|
||||
// Group images by year_month
|
||||
const groups = new Map();
|
||||
|
||||
images.forEach(img => {
|
||||
const key = img.year_month;
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
key: key,
|
||||
label: img.year_month_label,
|
||||
images: []
|
||||
});
|
||||
}
|
||||
groups.get(key).images.push(img);
|
||||
});
|
||||
|
||||
// Append to existing sections or create new ones
|
||||
groups.forEach((group, key) => {
|
||||
let section = dom.container.querySelector(`.date-section[data-year-month="${key}"]`);
|
||||
|
||||
if (section) {
|
||||
// Append to existing section
|
||||
const grid = section.querySelector('.gallery-grid');
|
||||
group.images.forEach(img => {
|
||||
grid.appendChild(createImageElement(img));
|
||||
});
|
||||
// Update count
|
||||
updateSectionCount(section);
|
||||
} else {
|
||||
// Create new section
|
||||
section = createDateSection(group.key, group.label, group.images);
|
||||
dom.container.appendChild(section);
|
||||
}
|
||||
|
||||
state.loadedSections.add(key);
|
||||
});
|
||||
|
||||
// Update timeline highlights
|
||||
updateTimelineLoaded();
|
||||
}
|
||||
|
||||
// Create a date section element
|
||||
function createDateSection(yearMonth, label, images) {
|
||||
const section = document.createElement('section');
|
||||
section.className = 'date-section';
|
||||
section.dataset.yearMonth = yearMonth;
|
||||
|
||||
const divider = document.createElement('h2');
|
||||
divider.className = 'date-divider';
|
||||
divider.id = `section-${yearMonth}`;
|
||||
divider.innerHTML = `
|
||||
<span class="date-divider-text">${label}</span>
|
||||
<span class="date-divider-count">${images.length} images</span>
|
||||
`;
|
||||
|
||||
const grid = document.createElement('div');
|
||||
grid.className = 'gallery-grid';
|
||||
|
||||
images.forEach(img => {
|
||||
grid.appendChild(createImageElement(img));
|
||||
});
|
||||
|
||||
section.appendChild(divider);
|
||||
section.appendChild(grid);
|
||||
|
||||
return section;
|
||||
}
|
||||
|
||||
// Create a single image element
|
||||
function createImageElement(img) {
|
||||
const item = document.createElement('div');
|
||||
item.className = 'gallery-item img-clickable';
|
||||
item.dataset.id = img.id;
|
||||
item.dataset.full = img.full_url;
|
||||
item.dataset.type = img.is_video ? 'video' : 'image';
|
||||
item.dataset.filename = img.filename;
|
||||
|
||||
// Build tag overlay HTML
|
||||
let tagsHtml = '';
|
||||
if (img.tags && img.tags.length > 0) {
|
||||
const tagChips = img.tags.map(t => {
|
||||
const displayName = t.name.includes(':') ? t.name.split(':')[1] : t.name;
|
||||
const prefix = t.kind === 'user' ? '#' : '';
|
||||
return `<a class="tag-chip" href="/gallery?tag=${encodeURIComponent(t.name)}" title="Filter by ${t.name}" onclick="event.stopPropagation()">${prefix}${displayName}</a>`;
|
||||
}).join('');
|
||||
tagsHtml = `<div class="tag-overlay">${tagChips}</div>`;
|
||||
}
|
||||
|
||||
// Play overlay for videos
|
||||
const playOverlay = img.is_video ? '<div class="play-overlay"></div>' : '';
|
||||
|
||||
item.innerHTML = `
|
||||
<div class="gallery-thumb" style="background-image: url('${img.thumb_url}');"></div>
|
||||
${tagsHtml}
|
||||
${playOverlay}
|
||||
<span class="image-date">${img.date}</span>
|
||||
`;
|
||||
|
||||
return item;
|
||||
}
|
||||
|
||||
// Update section image count
|
||||
function updateSectionCount(section) {
|
||||
const grid = section.querySelector('.gallery-grid');
|
||||
const count = grid.querySelectorAll('.gallery-item').length;
|
||||
const countEl = section.querySelector('.date-divider-count');
|
||||
if (countEl) {
|
||||
countEl.textContent = `${count} images`;
|
||||
}
|
||||
}
|
||||
|
||||
// Scroll handling
|
||||
function setupScrollListener() {
|
||||
let scrollTimeout = null;
|
||||
|
||||
window.addEventListener('scroll', () => {
|
||||
if (scrollTimeout) return;
|
||||
scrollTimeout = setTimeout(() => {
|
||||
scrollTimeout = null;
|
||||
checkScrollPosition();
|
||||
}, CONFIG.DEBOUNCE_SCROLL);
|
||||
}, { passive: true });
|
||||
}
|
||||
|
||||
function checkScrollPosition() {
|
||||
if (state.isLoading || !state.hasMore || state.isJumping) return;
|
||||
|
||||
const scrollBottom = window.innerHeight + window.scrollY;
|
||||
const docHeight = document.documentElement.scrollHeight;
|
||||
|
||||
if (docHeight - scrollBottom < CONFIG.SCROLL_THRESHOLD) {
|
||||
loadMore();
|
||||
}
|
||||
}
|
||||
|
||||
// Intersection observer for tracking visible section
|
||||
function setupIntersectionObserver() {
|
||||
const observer = new IntersectionObserver((entries) => {
|
||||
entries.forEach(entry => {
|
||||
if (entry.isIntersecting) {
|
||||
const yearMonth = entry.target.dataset.yearMonth;
|
||||
if (yearMonth && yearMonth !== state.currentVisibleSection) {
|
||||
state.currentVisibleSection = yearMonth;
|
||||
updateTimelineActive(yearMonth);
|
||||
}
|
||||
}
|
||||
});
|
||||
}, {
|
||||
rootMargin: '-100px 0px -60% 0px',
|
||||
threshold: 0
|
||||
});
|
||||
|
||||
// Observe all date sections
|
||||
const observeSections = () => {
|
||||
dom.container.querySelectorAll('.date-section').forEach(section => {
|
||||
observer.observe(section);
|
||||
});
|
||||
};
|
||||
|
||||
observeSections();
|
||||
|
||||
// Re-observe when new sections are added
|
||||
const mutationObserver = new MutationObserver(() => {
|
||||
observeSections();
|
||||
});
|
||||
mutationObserver.observe(dom.container, { childList: true });
|
||||
}
|
||||
|
||||
// Timeline functions
|
||||
async function loadTimeline() {
|
||||
try {
|
||||
const params = state.activeTag ? `?tag=${encodeURIComponent(state.activeTag)}` : '';
|
||||
const response = await fetch(`/api/gallery/timeline${params}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.groups) {
|
||||
state.timelineData = data.groups;
|
||||
renderTimeline();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to load timeline:', err);
|
||||
dom.timelineTrack.innerHTML = '<div class="timeline-loading">Failed to load</div>';
|
||||
}
|
||||
}
|
||||
|
||||
function renderTimeline() {
|
||||
if (!dom.timelineTrack || state.timelineData.length === 0) {
|
||||
dom.timelineTrack.innerHTML = '<div class="timeline-loading">No dates</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
// Group by year
|
||||
const yearGroups = new Map();
|
||||
state.timelineData.forEach(item => {
|
||||
if (!yearGroups.has(item.year)) {
|
||||
yearGroups.set(item.year, []);
|
||||
}
|
||||
yearGroups.get(item.year).push(item);
|
||||
});
|
||||
|
||||
let html = '';
|
||||
|
||||
yearGroups.forEach((months, year) => {
|
||||
html += `<div class="timeline-year-group" data-year="${year}">`;
|
||||
html += `<div class="timeline-year">${year}</div>`;
|
||||
html += '<div class="timeline-months">';
|
||||
|
||||
months.forEach(month => {
|
||||
const isLoaded = state.loadedSections.has(month.key);
|
||||
const loadedClass = isLoaded ? ' loaded' : '';
|
||||
|
||||
html += `
|
||||
<a class="timeline-month${loadedClass}"
|
||||
data-year-month="${month.key}"
|
||||
href="#section-${month.key}"
|
||||
title="${month.label}: ${month.count} images">
|
||||
<span class="timeline-month-label">${month.short_label}</span>
|
||||
<span class="timeline-month-count">${month.count}</span>
|
||||
</a>
|
||||
`;
|
||||
});
|
||||
|
||||
html += '</div></div>';
|
||||
});
|
||||
|
||||
dom.timelineTrack.innerHTML = html;
|
||||
|
||||
// Add click handlers for timeline navigation
|
||||
dom.timelineTrack.querySelectorAll('.timeline-month').forEach(el => {
|
||||
el.addEventListener('click', handleTimelineClick);
|
||||
});
|
||||
|
||||
// Update active state
|
||||
if (state.currentVisibleSection) {
|
||||
updateTimelineActive(state.currentVisibleSection);
|
||||
}
|
||||
}
|
||||
|
||||
function handleTimelineClick(e) {
|
||||
e.preventDefault();
|
||||
|
||||
const yearMonth = e.currentTarget.dataset.yearMonth;
|
||||
if (!yearMonth) return;
|
||||
|
||||
// Check if section is already loaded
|
||||
const existingSection = document.getElementById(`section-${yearMonth}`);
|
||||
|
||||
if (existingSection) {
|
||||
// Scroll to existing section
|
||||
existingSection.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
||||
} else {
|
||||
// Jump to this date - fetch and load
|
||||
jumpToDate(yearMonth);
|
||||
}
|
||||
}
|
||||
|
||||
async function jumpToDate(yearMonth) {
|
||||
if (state.isJumping) return;
|
||||
|
||||
state.isJumping = true;
|
||||
dom.loading.style.display = 'flex';
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
year_month: yearMonth,
|
||||
limit: CONFIG.BATCH_SIZE
|
||||
});
|
||||
|
||||
if (state.activeTag) {
|
||||
params.set('tag', state.activeTag);
|
||||
}
|
||||
|
||||
const response = await fetch(`/api/gallery/jump?${params}`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.images && data.images.length > 0) {
|
||||
// Clear existing content and replace with jumped-to content
|
||||
dom.container.innerHTML = '';
|
||||
state.loadedSections.clear();
|
||||
|
||||
// Add images
|
||||
appendImages(data.images);
|
||||
|
||||
// Update cursor for continued scrolling
|
||||
state.cursor = data.next_cursor;
|
||||
state.hasMore = data.has_more;
|
||||
|
||||
// Reset end marker
|
||||
if (data.has_more) {
|
||||
dom.end.style.display = 'none';
|
||||
} else {
|
||||
dom.end.style.display = 'block';
|
||||
}
|
||||
|
||||
// Scroll to top of new content
|
||||
window.scrollTo({ top: 0, behavior: 'instant' });
|
||||
|
||||
// Update timeline
|
||||
updateTimelineLoaded();
|
||||
|
||||
// Notify modal system
|
||||
dispatchGalleryUpdate();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to jump to date:', err);
|
||||
} finally {
|
||||
state.isJumping = false;
|
||||
dom.loading.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
function updateTimelineActive(yearMonth) {
|
||||
// Remove active from all
|
||||
dom.timelineTrack.querySelectorAll('.timeline-month.active').forEach(el => {
|
||||
el.classList.remove('active');
|
||||
});
|
||||
|
||||
// Add active to current
|
||||
const activeEl = dom.timelineTrack.querySelector(`.timeline-month[data-year-month="${yearMonth}"]`);
|
||||
if (activeEl) {
|
||||
activeEl.classList.add('active');
|
||||
|
||||
// Scroll timeline to keep active visible
|
||||
const track = dom.timelineTrack;
|
||||
const elRect = activeEl.getBoundingClientRect();
|
||||
const trackRect = track.getBoundingClientRect();
|
||||
|
||||
if (elRect.top < trackRect.top || elRect.bottom > trackRect.bottom) {
|
||||
activeEl.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function updateTimelineLoaded() {
|
||||
dom.timelineTrack.querySelectorAll('.timeline-month').forEach(el => {
|
||||
const yearMonth = el.dataset.yearMonth;
|
||||
if (state.loadedSections.has(yearMonth)) {
|
||||
el.classList.add('loaded');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Dispatch event for modal system integration
|
||||
function dispatchGalleryUpdate() {
|
||||
document.dispatchEvent(new CustomEvent('galleryUpdated'));
|
||||
}
|
||||
|
||||
// Initialize on DOM ready
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', init);
|
||||
} else {
|
||||
init();
|
||||
}
|
||||
})();
|
||||
@@ -28,11 +28,16 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
let dragStartY = 0;
|
||||
let dragMoved = false;
|
||||
|
||||
// Legacy pagination inputs (for non-infinite-scroll pages)
|
||||
const hasNextPageInput = document.getElementById('hasNextPage');
|
||||
const nextPageUrlInput = document.getElementById('nextPageUrl');
|
||||
const hasPrevPageInput = document.getElementById('hasPrevPage');
|
||||
const prevPageUrlInput = document.getElementById('prevPageUrl');
|
||||
|
||||
// Check if we're in infinite scroll mode
|
||||
// Infinite scroll mode is when galleryState exists (set by gallery.html template)
|
||||
const isInfiniteScrollMode = typeof window.galleryState !== 'undefined';
|
||||
|
||||
// ---------------------------
|
||||
// Tag editor helpers
|
||||
// ---------------------------
|
||||
@@ -158,9 +163,11 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
|
||||
async function fetchAutocomplete(query) {
|
||||
try {
|
||||
// Exclude system-managed tag kinds from autocomplete (archive, post, source)
|
||||
const excludeKinds = 'archive,post,source';
|
||||
const url = query
|
||||
? `/api/tags/search?q=${encodeURIComponent(query)}&limit=8&exclude_kind=archive`
|
||||
: `/api/tags/search?limit=8&exclude_kind=archive`;
|
||||
? `/api/tags/search?q=${encodeURIComponent(query)}&limit=8&exclude_kind=${excludeKinds}`
|
||||
: `/api/tags/search?limit=8&exclude_kind=${excludeKinds}`;
|
||||
const r = await fetch(url);
|
||||
const j = await r.json();
|
||||
renderAutocomplete(j.tags || []);
|
||||
@@ -316,7 +323,12 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
function showNext() {
|
||||
if (currentIndex < images.length - 1) {
|
||||
updateImage(currentIndex + 1);
|
||||
} else if (isInfiniteScrollMode) {
|
||||
// In infinite scroll mode, just stay at the last image
|
||||
// User needs to scroll the gallery to load more
|
||||
return;
|
||||
} else if (hasNextPageInput?.value === "true") {
|
||||
// Legacy pagination mode - fetch next page
|
||||
fetch(nextPageUrlInput.value)
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
@@ -349,7 +361,11 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
function showPrev() {
|
||||
if (currentIndex > 0) {
|
||||
updateImage(currentIndex - 1);
|
||||
} else if (isInfiniteScrollMode) {
|
||||
// In infinite scroll mode, just stay at the first image
|
||||
return;
|
||||
} else if (hasPrevPageInput?.value === "true") {
|
||||
// Legacy pagination mode - fetch previous page
|
||||
fetch(prevPageUrlInput.value)
|
||||
.then(res => res.text())
|
||||
.then(html => {
|
||||
@@ -480,6 +496,11 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
images = Array.from(document.querySelectorAll('.img-clickable'));
|
||||
});
|
||||
|
||||
// Listen for gallery infinite scroll updates to refresh the images array
|
||||
document.addEventListener('galleryUpdated', () => {
|
||||
images = Array.from(document.querySelectorAll('.img-clickable'));
|
||||
});
|
||||
|
||||
window.addEventListener('popstate', () => {
|
||||
if (modal.classList.contains('active')) {
|
||||
closeModal();
|
||||
|
||||
@@ -97,10 +97,28 @@
|
||||
}
|
||||
|
||||
function checkScrollPosition() {
|
||||
const scrollBottom = window.innerHeight + window.scrollY;
|
||||
const docHeight = document.documentElement.scrollHeight;
|
||||
// For masonry layouts, we need to check if the SHORTEST column is near the viewport bottom,
|
||||
// not the overall page bottom. Otherwise, a tall image in one column prevents loading
|
||||
// even when other columns have empty space visible.
|
||||
|
||||
if (docHeight - scrollBottom < SCROLL_THRESHOLD) {
|
||||
// First, update accurate column heights
|
||||
updateColumnHeights();
|
||||
|
||||
// Find the shortest column's bottom position relative to viewport
|
||||
const shortestColumnIndex = getShortestColumnIndex();
|
||||
const shortestColumn = columns[shortestColumnIndex];
|
||||
if (!shortestColumn) return;
|
||||
|
||||
const columnRect = shortestColumn.getBoundingClientRect();
|
||||
const columnBottom = columnRect.bottom; // Distance from viewport top to column bottom
|
||||
const viewportHeight = window.innerHeight;
|
||||
|
||||
// If the shortest column's bottom is within SCROLL_THRESHOLD of the viewport bottom, load more
|
||||
// columnBottom - viewportHeight = how far below the viewport the column extends
|
||||
// If this is negative or small, the column end is visible or nearly visible
|
||||
const distanceFromViewportBottom = columnBottom - viewportHeight;
|
||||
|
||||
if (distanceFromViewportBottom < SCROLL_THRESHOLD) {
|
||||
loadMore();
|
||||
}
|
||||
}
|
||||
|
||||
+278
-3
@@ -378,14 +378,25 @@ header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
max-width: 100%;
|
||||
width: 100%;
|
||||
max-width: calc(100% - 300px); /* Account for tag editor */
|
||||
height: calc(100vh - 140px);
|
||||
max-height: calc(100vh - 140px);
|
||||
overflow: hidden;
|
||||
cursor: default;
|
||||
}
|
||||
.modal-image-wrapper.zoomed { overflow: auto; cursor: grab; }
|
||||
.modal-image-wrapper.zoomed {
|
||||
overflow: auto;
|
||||
cursor: grab;
|
||||
/* Enable scrolling by removing flex centering when zoomed */
|
||||
display: block;
|
||||
}
|
||||
.modal-image-wrapper.zoomed img {
|
||||
width: auto; height: auto; max-width: none; max-height: none;
|
||||
width: auto;
|
||||
height: auto;
|
||||
max-width: none;
|
||||
max-height: none;
|
||||
cursor: grab;
|
||||
}
|
||||
.modal-image-wrapper img {
|
||||
max-width: 100%;
|
||||
@@ -404,6 +415,15 @@ header {
|
||||
width: 90vw; height: auto; max-height: none;
|
||||
}
|
||||
|
||||
/* On smaller screens, full width for image wrapper */
|
||||
@media (max-width: 900px) {
|
||||
.modal-image-wrapper {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
max-height: 60vh;
|
||||
}
|
||||
}
|
||||
|
||||
/* Modal navigation buttons - cleaner pill style */
|
||||
.modal-button {
|
||||
position: absolute;
|
||||
@@ -1191,3 +1211,258 @@ select.form-input optgroup {
|
||||
overflow-wrap: anywhere; /* allows breaking long tokens */
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
/*------------------------------------------------------------------------------
|
||||
Gallery Infinite Scroll Layout
|
||||
------------------------------------------------------------------------------*/
|
||||
.gallery-infinite-container {
|
||||
display: flex;
|
||||
min-height: 100vh;
|
||||
max-width: 1800px;
|
||||
margin: 0 auto;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
/* Timeline Sidebar - positioned relative to gallery container */
|
||||
.timeline-sidebar {
|
||||
position: sticky;
|
||||
top: 60px;
|
||||
height: calc(100vh - 60px);
|
||||
width: 70px;
|
||||
min-width: 70px;
|
||||
background: rgba(0,0,0,0.3);
|
||||
border-right: 1px solid rgba(255,255,255,0.08);
|
||||
z-index: 50;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 0.5rem 0;
|
||||
overflow-y: auto;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgba(255,255,255,0.2) transparent;
|
||||
}
|
||||
.timeline-sidebar::-webkit-scrollbar {
|
||||
width: 4px;
|
||||
}
|
||||
.timeline-sidebar::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
.timeline-sidebar::-webkit-scrollbar-thumb {
|
||||
background: rgba(255,255,255,0.2);
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
.timeline-track {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
padding: 0 6px;
|
||||
}
|
||||
|
||||
.timeline-loading {
|
||||
font-size: 0.7rem;
|
||||
color: var(--text-muted);
|
||||
padding: 0.5rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.timeline-year-group {
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.timeline-year {
|
||||
font-size: 0.8rem;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
padding: 0.5rem 0.25rem 0.25rem;
|
||||
text-align: center;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: linear-gradient(to bottom, rgba(10,10,10,0.95), rgba(10,10,10,0.8));
|
||||
z-index: 1;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.1);
|
||||
}
|
||||
|
||||
.timeline-months {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
padding-top: 4px;
|
||||
}
|
||||
|
||||
.timeline-month {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.35rem 0.5rem;
|
||||
font-size: 0.7rem;
|
||||
color: var(--text-muted);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
text-decoration: none;
|
||||
}
|
||||
.timeline-month:hover {
|
||||
background: rgba(255,255,255,0.12);
|
||||
color: var(--text);
|
||||
}
|
||||
.timeline-month.active {
|
||||
background: var(--btn-primary);
|
||||
color: white;
|
||||
}
|
||||
.timeline-month.loaded {
|
||||
color: var(--text-dim);
|
||||
}
|
||||
.timeline-month-label {
|
||||
font-weight: 500;
|
||||
}
|
||||
.timeline-month-count {
|
||||
font-size: 0.6rem;
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
/* Main gallery area - flex child next to timeline */
|
||||
.gallery-main {
|
||||
flex: 1;
|
||||
padding: 0 0.5rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.gallery-header {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
padding: 0.5rem 0 1rem;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
.gallery-header h1 {
|
||||
margin: 0;
|
||||
font-size: 1.5rem;
|
||||
}
|
||||
|
||||
.active-filter {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: rgba(255,255,255,0.08);
|
||||
border-radius: 999px;
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-dim);
|
||||
}
|
||||
.filter-tag {
|
||||
color: var(--link);
|
||||
font-weight: 500;
|
||||
}
|
||||
.clear-filter {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 50%;
|
||||
background: rgba(255,255,255,0.1);
|
||||
color: var(--text-muted);
|
||||
text-decoration: none;
|
||||
font-size: 0.9rem;
|
||||
line-height: 1;
|
||||
transition: all 0.15s ease;
|
||||
}
|
||||
.clear-filter:hover {
|
||||
background: rgba(255,255,255,0.2);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
/* Gallery content area */
|
||||
.gallery-infinite {
|
||||
min-height: 50vh;
|
||||
}
|
||||
|
||||
.gallery-empty {
|
||||
text-align: center;
|
||||
padding: 4rem 1rem;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
/* Date sections */
|
||||
.date-section {
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.date-divider {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 0.75rem 0.5rem;
|
||||
margin: 0 0 0.5rem 0;
|
||||
border-bottom: 1px solid rgba(255,255,255,0.1);
|
||||
position: sticky;
|
||||
top: 50px;
|
||||
background: linear-gradient(to bottom, var(--bg) 0%, var(--bg) 85%, transparent 100%);
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.date-divider-text {
|
||||
font-size: 1.1rem;
|
||||
font-weight: 500;
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.date-divider-count {
|
||||
font-size: 0.8rem;
|
||||
color: var(--text-muted);
|
||||
background: rgba(255,255,255,0.08);
|
||||
padding: 0.2rem 0.6rem;
|
||||
border-radius: 999px;
|
||||
}
|
||||
|
||||
/* Loading indicator */
|
||||
.gallery-loading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.75rem;
|
||||
padding: 2rem;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.loading-spinner {
|
||||
display: inline-block;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border: 2px solid rgba(255,255,255,0.2);
|
||||
border-top-color: var(--btn-primary);
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.gallery-end {
|
||||
text-align: center;
|
||||
padding: 2rem;
|
||||
color: var(--text-muted);
|
||||
font-style: italic;
|
||||
border-top: 1px solid rgba(255,255,255,0.05);
|
||||
}
|
||||
|
||||
/* Play overlay with triangle icon */
|
||||
.play-overlay::before {
|
||||
content: '';
|
||||
width: 0;
|
||||
height: 0;
|
||||
border-style: solid;
|
||||
border-width: 20px 0 20px 35px;
|
||||
border-color: transparent transparent transparent rgba(255,255,255,0.9);
|
||||
filter: drop-shadow(0 2px 4px rgba(0,0,0,0.5));
|
||||
}
|
||||
|
||||
/* Mobile: hide timeline sidebar */
|
||||
@media (max-width: 768px) {
|
||||
.timeline-sidebar {
|
||||
display: none;
|
||||
}
|
||||
.date-divider {
|
||||
top: 45px;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
<!-- /app/templates/_gallery_item.html -->
|
||||
<div class="gallery-item img-clickable"
|
||||
data-id="{{ image.id }}"
|
||||
data-full="{{ url_for('main.serve_image', filename=image.filepath | replace('/images/', '') ) }}"
|
||||
data-type="{{ 'video' if image.filename.lower().endswith(('.mp4', '.mov')) else 'image' }}"
|
||||
data-filename="{{ image.filename }}">
|
||||
|
||||
<div class="gallery-thumb"
|
||||
style="background-image: url('{{ url_for('main.serve_image', filename=(image.thumb_path or image.filepath) | replace('/images/', '') ) }}');"></div>
|
||||
|
||||
{% set visible_tags = image.tags | selectattr('kind', 'ne', 'archive') | list %}
|
||||
{% if visible_tags %}
|
||||
<div class="tag-overlay">
|
||||
{% for t in visible_tags %}
|
||||
<a class="tag-chip"
|
||||
href="{{ url_for('main.gallery', tag=t.name) }}"
|
||||
title="Filter by {{ t.name }}"
|
||||
onclick="event.stopPropagation()">
|
||||
{% if t.kind == 'artist' %}
|
||||
{{ t.name.split(':', 1)[1] if ':' in t.name else t.name }}
|
||||
{% elif t.kind == 'character' %}
|
||||
{{ t.name.split(':', 1)[1] if ':' in t.name else t.name }}
|
||||
{% elif t.kind == 'series' %}
|
||||
{{ t.name.split(':', 1)[1] if ':' in t.name else t.name }}
|
||||
{% elif t.kind == 'rating' %}
|
||||
{{ t.name.split(':', 1)[1] if ':' in t.name else t.name }}
|
||||
{% else %}
|
||||
#{{ t.name }}
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if image.filename.lower().endswith(('.mp4', '.mov')) %}
|
||||
<div class="play-overlay"></div>
|
||||
{% endif %}
|
||||
|
||||
<span class="image-date">{{ image.taken_at or image.imported_at | datetimeformat }}</span>
|
||||
</div>
|
||||
@@ -0,0 +1,26 @@
|
||||
<!-- /app/templates/_gallery_modal.html -->
|
||||
<div id="imageModal" class="modal">
|
||||
<div class="modal-content">
|
||||
<button id="modalClose" class="modal-close-btn" title="Close (ESC)">×</button>
|
||||
<button id="modalPrev" class="modal-button modal-prev">‹</button>
|
||||
<button id="modalNext" class="modal-button modal-next">›</button>
|
||||
<span class="modal-close-hint">ESC to close</span>
|
||||
|
||||
<div class="modal-body">
|
||||
<div id="modalImageWrapper" class="modal-image-wrapper">
|
||||
<img id="modalImage" src="" alt="Full View">
|
||||
</div>
|
||||
|
||||
<div id="modalTagEditor" class="tag-editor" data-image-id="">
|
||||
<div id="modalTagList" class="tags"></div>
|
||||
<form id="modalTagForm" class="tag-form" autocomplete="off">
|
||||
<div class="tag-form-wrapper">
|
||||
<input type="text" id="tagInput" name="name" placeholder="Add tag..." autocomplete="off">
|
||||
<div id="tagAutocomplete" class="tag-autocomplete"></div>
|
||||
</div>
|
||||
<button type="submit">Add</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -3,14 +3,73 @@
|
||||
{% block title %}Gallery{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1 style="text-align:center; margin-top: 1rem;">Gallery</h1>
|
||||
<div class="gallery-infinite-container">
|
||||
<!-- Timeline Scrollbar (left sidebar) -->
|
||||
<aside id="timelineSidebar" class="timeline-sidebar">
|
||||
<div id="timelineTrack" class="timeline-track">
|
||||
<!-- Populated by JS with year-month markers -->
|
||||
<div class="timeline-loading">Loading...</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<!-- Gallery Grid -->
|
||||
{% include '_gallery_grid.html' %}
|
||||
<!-- Main Gallery Content -->
|
||||
<main class="gallery-main">
|
||||
<div class="gallery-header">
|
||||
<h1>Gallery</h1>
|
||||
{% if active_tag %}
|
||||
<div class="active-filter">
|
||||
Filtered by: <span class="filter-tag">{{ active_tag }}</span>
|
||||
<a href="{{ url_for('main.gallery') }}" class="clear-filter" title="Clear filter">×</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Floating Pagination Bar -->
|
||||
{% if images.pages > 1 %}
|
||||
{% include '_pagination_floating.html' %}
|
||||
{% endif %}
|
||||
<!-- Gallery Grid with Date Sections -->
|
||||
<div id="galleryInfinite" class="gallery-infinite">
|
||||
{% for year_month_key, year_month_label, images in grouped_images %}
|
||||
<section class="date-section" data-year-month="{{ year_month_key }}">
|
||||
<h2 class="date-divider" id="section-{{ year_month_key }}">
|
||||
<span class="date-divider-text">{{ year_month_label }}</span>
|
||||
<span class="date-divider-count">{{ images|length }} images</span>
|
||||
</h2>
|
||||
<div class="gallery-grid">
|
||||
{% for image in images %}
|
||||
{% include '_gallery_item.html' %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</section>
|
||||
{% else %}
|
||||
<div class="gallery-empty">
|
||||
<p>No images found{% if active_tag %} for tag "{{ active_tag }}"{% endif %}.</p>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Loading indicator -->
|
||||
<div id="galleryLoading" class="gallery-loading" style="display:none;">
|
||||
<span class="loading-spinner"></span>
|
||||
<span>Loading more images...</span>
|
||||
</div>
|
||||
|
||||
<!-- End of content marker -->
|
||||
<div id="galleryEnd" class="gallery-end" style="display:none;">
|
||||
No more images
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<!-- Modal -->
|
||||
{% include '_gallery_modal.html' %}
|
||||
|
||||
<!-- Pass initial state to JS -->
|
||||
<script>
|
||||
window.galleryState = {
|
||||
cursor: {{ initial_cursor | tojson | safe if initial_cursor else 'null' }},
|
||||
hasMore: {{ 'true' if has_more else 'false' }},
|
||||
activeTag: {{ active_tag | tojson | safe if active_tag else 'null' }}
|
||||
};
|
||||
</script>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/gallery-infinite.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/modal-pagination.js') }}"></script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -37,12 +37,6 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if request.endpoint == 'main.gallery' %}
|
||||
<input type="hidden" id="hasNextPage" value="{{ has_next|lower }}">
|
||||
<input type="hidden" id="nextPageUrl" value="{{ next_page_url }}">
|
||||
<input type="hidden" id="hasPrevPage" value="{{ has_prev|lower }}">
|
||||
<input type="hidden" id="prevPageUrl" value="{{ prev_page_url }}">
|
||||
{% endif %}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+429
-37
@@ -50,21 +50,43 @@ IMPORT_STATS_PATH = "/import/stats.json"
|
||||
|
||||
|
||||
def load_import_settings() -> dict:
|
||||
"""Load import filter settings from file or environment."""
|
||||
"""
|
||||
Load import filter settings from database, with file and environment fallbacks.
|
||||
Priority: DB settings > file settings > environment variables
|
||||
"""
|
||||
# Start with environment defaults
|
||||
defaults = {
|
||||
"min_width": int(os.environ.get("IMPORT_MIN_WIDTH", "0")),
|
||||
"min_height": int(os.environ.get("IMPORT_MIN_HEIGHT", "0")),
|
||||
"skip_transparent": os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true",
|
||||
"transparency_threshold": float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9")),
|
||||
"phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "2")),
|
||||
"skip_single_color": os.environ.get("IMPORT_SKIP_SINGLE_COLOR", "true").lower() == "true",
|
||||
"single_color_threshold": float(os.environ.get("IMPORT_SINGLE_COLOR_THRESHOLD", "0.95")),
|
||||
"single_color_tolerance": int(os.environ.get("IMPORT_SINGLE_COLOR_TOLERANCE", "30")),
|
||||
"phash_threshold": int(os.environ.get("IMPORT_PHASH_THRESHOLD", "10")), # Default for 256-bit
|
||||
"supersede_smaller": os.environ.get("IMPORT_SUPERSEDE_SMALLER", "true").lower() == "true",
|
||||
}
|
||||
|
||||
# Try to load from file (legacy support)
|
||||
try:
|
||||
if os.path.exists(IMPORT_SETTINGS_PATH):
|
||||
with open(IMPORT_SETTINGS_PATH, "r") as f:
|
||||
file_settings = json.load(f)
|
||||
defaults.update(file_settings)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to load import settings: {e}")
|
||||
print(f"[WARN] Failed to load import settings from file: {e}")
|
||||
|
||||
# Try to load from database (preferred)
|
||||
try:
|
||||
from app.utils.version_migration import get_import_settings
|
||||
db_settings = get_import_settings()
|
||||
# Only override if DB has non-default values
|
||||
for key, value in db_settings.items():
|
||||
if value is not None:
|
||||
defaults[key] = value
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to load import settings from DB: {e}")
|
||||
|
||||
return defaults
|
||||
|
||||
|
||||
@@ -106,6 +128,136 @@ def is_mostly_transparent(file_path: str, threshold: float = 0.9) -> bool:
|
||||
print(f"[WARN] Failed to check transparency for {file_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def is_mostly_single_color(file_path: str, threshold: float = 0.95, tolerance: int = 30) -> bool:
|
||||
"""
|
||||
Check if an image is mostly a single color (> threshold % of pixels are similar).
|
||||
Uses color distance tolerance to group similar colors.
|
||||
|
||||
Args:
|
||||
file_path: Path to the image file
|
||||
threshold: Percentage of pixels that must be similar (0.0-1.0, default 0.95 = 95%)
|
||||
tolerance: Maximum color distance to be considered "same" color (0-255 per channel, default 30)
|
||||
|
||||
Returns:
|
||||
True if the image is mostly a single solid color
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
# Convert to RGB for consistent color comparison
|
||||
if img.mode in ("RGBA", "LA"):
|
||||
# For images with alpha, only consider non-transparent pixels
|
||||
img_rgba = img.convert("RGBA") if img.mode != "RGBA" else img
|
||||
pixels = list(img_rgba.getdata())
|
||||
# Filter out mostly transparent pixels (alpha < 128)
|
||||
opaque_pixels = [(r, g, b) for r, g, b, a in pixels if a >= 128]
|
||||
if len(opaque_pixels) == 0:
|
||||
return True # Fully transparent = effectively single color
|
||||
pixels = opaque_pixels
|
||||
elif img.mode == "P":
|
||||
img = img.convert("RGB")
|
||||
pixels = list(img.getdata())
|
||||
elif img.mode == "L":
|
||||
# Grayscale - convert to RGB tuples
|
||||
pixels = [(p, p, p) for p in img.getdata()]
|
||||
else:
|
||||
if img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
pixels = list(img.getdata())
|
||||
|
||||
if len(pixels) == 0:
|
||||
return True
|
||||
|
||||
# Sample pixels for performance on large images
|
||||
total_pixels = len(pixels)
|
||||
if total_pixels > 10000:
|
||||
# Sample evenly distributed pixels
|
||||
step = total_pixels // 10000
|
||||
pixels = pixels[::step]
|
||||
|
||||
# Find the dominant color (most common pixel or average of sampled area)
|
||||
# Use the pixel at center as reference
|
||||
center_idx = len(pixels) // 2
|
||||
ref_color = pixels[center_idx]
|
||||
|
||||
# Count pixels within tolerance of reference color
|
||||
similar_count = 0
|
||||
for pixel in pixels:
|
||||
if isinstance(pixel, int):
|
||||
pixel = (pixel, pixel, pixel)
|
||||
# Calculate color distance (simple RGB distance)
|
||||
dist = abs(pixel[0] - ref_color[0]) + abs(pixel[1] - ref_color[1]) + abs(pixel[2] - ref_color[2])
|
||||
if dist <= tolerance * 3: # tolerance per channel * 3 channels
|
||||
similar_count += 1
|
||||
|
||||
similarity_ratio = similar_count / len(pixels)
|
||||
return similarity_ratio > threshold
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to check single color for {file_path}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def supersede_image(old_record: "ImageRecord", new_filepath: str, new_thumb_path: str,
|
||||
new_hash: str, new_phash, new_metadata: dict, new_filename: str) -> "ImageRecord":
|
||||
"""
|
||||
Replace a smaller image with a larger version, preserving tags and metadata.
|
||||
|
||||
Args:
|
||||
old_record: The existing ImageRecord to supersede
|
||||
new_filepath: Path to the new larger image file
|
||||
new_thumb_path: Path to the new thumbnail
|
||||
new_hash: SHA256 hash of the new file
|
||||
new_phash: Perceptual hash of the new image
|
||||
new_metadata: Metadata dict for the new image
|
||||
new_filename: Original filename of the new image
|
||||
|
||||
Returns:
|
||||
The updated ImageRecord
|
||||
"""
|
||||
# Store old file paths for cleanup
|
||||
old_filepath = old_record.filepath
|
||||
old_thumb_path = old_record.thumb_path
|
||||
|
||||
print(f"[SUPERSEDE] Replacing {old_record.filename} ({old_record.width}x{old_record.height}) "
|
||||
f"with {new_filename} ({new_metadata['width']}x{new_metadata['height']})")
|
||||
|
||||
# Update record with new file info
|
||||
old_record.filename = new_filename
|
||||
old_record.filepath = new_filepath
|
||||
old_record.thumb_path = new_thumb_path
|
||||
old_record.hash = new_hash
|
||||
old_record.perceptual_hash = str(new_phash) if new_phash else None
|
||||
old_record.file_size = new_metadata["file_size"]
|
||||
old_record.width = new_metadata["width"]
|
||||
old_record.height = new_metadata["height"]
|
||||
old_record.format = new_metadata["format"]
|
||||
# Keep the earlier taken_at date (prefer original date)
|
||||
if new_metadata.get("taken_at") and old_record.taken_at:
|
||||
if new_metadata["taken_at"] < old_record.taken_at:
|
||||
old_record.taken_at = new_metadata["taken_at"]
|
||||
elif new_metadata.get("taken_at") and not old_record.taken_at:
|
||||
old_record.taken_at = new_metadata["taken_at"]
|
||||
# Don't update imported_at - keep original import time
|
||||
|
||||
# Delete old files
|
||||
try:
|
||||
if old_filepath and os.path.exists(old_filepath):
|
||||
os.remove(old_filepath)
|
||||
print(f"[SUPERSEDE] Deleted old file: {old_filepath}")
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to delete old file {old_filepath}: {e}")
|
||||
|
||||
try:
|
||||
if old_thumb_path and os.path.exists(old_thumb_path):
|
||||
os.remove(old_thumb_path)
|
||||
print(f"[SUPERSEDE] Deleted old thumbnail: {old_thumb_path}")
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to delete old thumbnail {old_thumb_path}: {e}")
|
||||
|
||||
return old_record
|
||||
|
||||
|
||||
# Regex for multi-part detection
|
||||
RAR_PART_RE = re.compile(r"\.part(\d+)\.rar$", re.IGNORECASE)
|
||||
SEVENZ_PART_RE = re.compile(r"\.7z\.(\d{3})$", re.IGNORECASE)
|
||||
@@ -233,19 +385,25 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
min_height = settings.get("min_height", 0)
|
||||
skip_transparent = settings.get("skip_transparent", False)
|
||||
transparency_threshold = settings.get("transparency_threshold", 0.9)
|
||||
|
||||
skip_single_color = settings.get("skip_single_color", True)
|
||||
single_color_threshold = settings.get("single_color_threshold", 0.95)
|
||||
single_color_tolerance = settings.get("single_color_tolerance", 30)
|
||||
phash_threshold = settings.get("phash_threshold", 2)
|
||||
supersede_smaller = settings.get("supersede_smaller", True)
|
||||
|
||||
print(f"[INFO] Import settings: min_width={min_width}, min_height={min_height}, "
|
||||
f"skip_transparent={skip_transparent}, transparency_threshold={transparency_threshold}, "
|
||||
f"phash_threshold={phash_threshold}")
|
||||
f"skip_single_color={skip_single_color}, single_color_threshold={single_color_threshold}, "
|
||||
f"phash_threshold={phash_threshold}, supersede_smaller={supersede_smaller}")
|
||||
|
||||
# Statistics tracking
|
||||
stats = {
|
||||
"total_processed": 0,
|
||||
"total_imported": 0,
|
||||
"total_superseded": 0,
|
||||
"filtered_by_dimension": 0,
|
||||
"filtered_by_transparency": 0,
|
||||
"filtered_by_single_color": 0,
|
||||
"filtered_by_duplicate": 0,
|
||||
}
|
||||
|
||||
@@ -254,8 +412,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
batch_counter = 0
|
||||
|
||||
# Preload existing pHashes to avoid N+1 lookups
|
||||
# Format: (phash, width, height, image_id) for supersede functionality
|
||||
existing_phashes = [
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height)
|
||||
(imagehash.hex_to_hash(img.perceptual_hash), img.width, img.height, img.id)
|
||||
for img in ImageRecord.query.filter(ImageRecord.perceptual_hash != None).all()
|
||||
]
|
||||
|
||||
@@ -320,24 +479,34 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
stats["filtered_by_transparency"] += 1
|
||||
continue
|
||||
|
||||
added, phash, _rec = import_single_file(
|
||||
# Apply single-color filter (skip images that are mostly one solid color)
|
||||
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
|
||||
if is_mostly_single_color(full_path, single_color_threshold, single_color_tolerance):
|
||||
print(f"[SKIP] Mostly single color: {file}")
|
||||
stats["filtered_by_single_color"] += 1
|
||||
continue
|
||||
|
||||
added, phash, _rec, superseded = import_single_file(
|
||||
src_path=full_path,
|
||||
dest_dir=dest_dir,
|
||||
artist=artist_dir,
|
||||
existing_phashes=existing_phashes,
|
||||
phash_threshold=phash_threshold
|
||||
phash_threshold=phash_threshold,
|
||||
supersede_smaller=supersede_smaller
|
||||
)
|
||||
if added:
|
||||
imported.append(file)
|
||||
stats["total_imported"] += 1
|
||||
if superseded:
|
||||
stats["total_superseded"] += 1
|
||||
else:
|
||||
stats["total_imported"] += 1
|
||||
elif _rec is None and phash is None:
|
||||
# Was a duplicate
|
||||
stats["filtered_by_duplicate"] += 1
|
||||
|
||||
if phash:
|
||||
md = extract_metadata(full_path)
|
||||
if md.get("width") and md.get("height"):
|
||||
existing_phashes.append((phash, md["width"], md["height"]))
|
||||
# Add new image's phash to the tracking list (for fresh imports only, not supersedes)
|
||||
if phash and _rec and not superseded:
|
||||
existing_phashes.append((phash, _rec.width, _rec.height, _rec.id))
|
||||
|
||||
batch_counter += 1
|
||||
if batch_counter >= batch_size:
|
||||
@@ -354,7 +523,9 @@ def import_images_task(source_dir: str, dest_dir: str) -> str:
|
||||
# Save stats
|
||||
save_import_stats(stats)
|
||||
|
||||
summary = f"Imported {len(imported)} items. Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency."
|
||||
summary = (f"Imported {stats['total_imported']} items, superseded {stats['total_superseded']} smaller versions. "
|
||||
f"Filtered: {stats['filtered_by_dimension']} by size, {stats['filtered_by_transparency']} by transparency, "
|
||||
f"{stats['filtered_by_single_color']} by single color, {stats['filtered_by_duplicate']} duplicates.")
|
||||
print(f"[INFO] {summary}")
|
||||
return summary
|
||||
|
||||
@@ -368,14 +539,19 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N
|
||||
if settings is None:
|
||||
settings = load_import_settings()
|
||||
if stats is None:
|
||||
stats = {"total_processed": 0, "total_imported": 0, "filtered_by_dimension": 0,
|
||||
"filtered_by_transparency": 0, "filtered_by_duplicate": 0}
|
||||
stats = {"total_processed": 0, "total_imported": 0, "total_superseded": 0,
|
||||
"filtered_by_dimension": 0, "filtered_by_transparency": 0,
|
||||
"filtered_by_single_color": 0, "filtered_by_duplicate": 0}
|
||||
|
||||
min_width = settings.get("min_width", 0)
|
||||
min_height = settings.get("min_height", 0)
|
||||
skip_transparent = settings.get("skip_transparent", False)
|
||||
transparency_threshold = settings.get("transparency_threshold", 0.9)
|
||||
skip_single_color = settings.get("skip_single_color", True)
|
||||
single_color_threshold = settings.get("single_color_threshold", 0.95)
|
||||
single_color_tolerance = settings.get("single_color_tolerance", 30)
|
||||
phash_threshold = settings.get("phash_threshold", 2)
|
||||
supersede_smaller = settings.get("supersede_smaller", True)
|
||||
|
||||
archive_size = os.path.getsize(archive_path)
|
||||
archive_hash = calculate_hash(archive_path)
|
||||
@@ -420,19 +596,30 @@ def process_archive(archive_path, dest_dir, artist, existing_phashes, settings=N
|
||||
stats["filtered_by_transparency"] += 1
|
||||
continue
|
||||
|
||||
added, _phash, rec = import_single_file(
|
||||
# Apply single-color filter
|
||||
if skip_single_color and lower.endswith((".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp")):
|
||||
if is_mostly_single_color(src_path, single_color_threshold, single_color_tolerance):
|
||||
print(f"[SKIP] Mostly single color: {file}")
|
||||
stats["filtered_by_single_color"] += 1
|
||||
continue
|
||||
|
||||
added, _phash, rec, superseded = import_single_file(
|
||||
src_path=src_path,
|
||||
dest_dir=dest_dir,
|
||||
artist=artist, # still used for auto artist:<name> tag
|
||||
existing_phashes=existing_phashes,
|
||||
commit=False,
|
||||
phash_threshold=phash_threshold
|
||||
phash_threshold=phash_threshold,
|
||||
supersede_smaller=supersede_smaller
|
||||
)
|
||||
if rec is not None:
|
||||
touched_records.append(rec)
|
||||
if added:
|
||||
imported_or_tagged += 1
|
||||
stats["total_imported"] += 1
|
||||
if superseded:
|
||||
stats["total_superseded"] += 1
|
||||
else:
|
||||
stats["total_imported"] += 1
|
||||
elif rec is None and _phash is None:
|
||||
stats["filtered_by_duplicate"] += 1
|
||||
|
||||
@@ -495,11 +682,19 @@ def build_hashed_dest_path(target_dir: str, original_name: str, content_hash: st
|
||||
i += 1
|
||||
|
||||
|
||||
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes, commit: bool = True, phash_threshold: int = 2):
|
||||
def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phashes,
|
||||
commit: bool = True, phash_threshold: int = 2, supersede_smaller: bool = True):
|
||||
"""
|
||||
Import a single media file. If an equivalent file already exists, return that
|
||||
record so callers can tag it (no duplicate import).
|
||||
Returns (added: bool, phash_or_None, record_or_None).
|
||||
|
||||
If a smaller visually-similar image exists and supersede_smaller is True,
|
||||
the smaller image will be replaced with the larger one (preserving tags).
|
||||
|
||||
Also enriches metadata from Gallery-DL sidecar JSON files if present.
|
||||
|
||||
Returns (added: bool, phash_or_None, record_or_None, superseded: bool).
|
||||
The superseded flag indicates if an existing smaller image was replaced.
|
||||
"""
|
||||
original_name = os.path.basename(src_path)
|
||||
print(f"[INFO] Processing: {src_path}")
|
||||
@@ -510,7 +705,9 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
existing = ImageRecord.query.filter_by(hash=file_hash).first()
|
||||
if existing:
|
||||
print(f"[SKIP] Duplicate by hash: {original_name}")
|
||||
return (False, None, existing)
|
||||
# Even for duplicates, try to enrich with sidecar metadata
|
||||
_enrich_existing_record(existing, src_path)
|
||||
return (False, None, existing, False)
|
||||
|
||||
# Optional info: same name+size (but different content) → we still import with hashed name
|
||||
existing_ns = ImageRecord.query.filter_by(filename=original_name, file_size=file_size).first()
|
||||
@@ -521,12 +718,67 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
metadata = extract_metadata(src_path)
|
||||
if not metadata["width"] or not metadata["height"]:
|
||||
print(f"[SKIP] Missing dimension data for {original_name}")
|
||||
return (False, None, None)
|
||||
return (False, None, None, False)
|
||||
|
||||
phash = calculate_perceptual_hash(src_path)
|
||||
if phash and phash_threshold > 0 and is_similar_image(phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold):
|
||||
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
|
||||
return (False, phash, None)
|
||||
|
||||
# Check for similar images
|
||||
superseded = False
|
||||
if phash and phash_threshold > 0:
|
||||
relationship, similar_id = find_similar_image(
|
||||
phash, metadata["width"], metadata["height"], existing_phashes, threshold=phash_threshold
|
||||
)
|
||||
|
||||
if relationship == "larger_exists":
|
||||
print(f"[SKIP] {original_name} is visually similar to an existing larger image (threshold={phash_threshold}).")
|
||||
return (False, phash, None, False)
|
||||
|
||||
elif relationship == "smaller_exists" and supersede_smaller and similar_id:
|
||||
# Found a smaller version - supersede it
|
||||
old_record = ImageRecord.query.get(similar_id)
|
||||
if old_record:
|
||||
# Enrich metadata from Gallery-DL sidecar JSON if present
|
||||
enriched = _get_enriched_metadata(src_path, metadata)
|
||||
metadata["taken_at"] = enriched.get("taken_at") or metadata["taken_at"]
|
||||
|
||||
# Copy new file to destination
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
dest_path = build_hashed_dest_path(target_dir, original_name, file_hash)
|
||||
shutil.copy2(src_path, dest_path)
|
||||
|
||||
# Generate new thumbnail
|
||||
try:
|
||||
if dest_path.lower().endswith((".mp4", ".mov")):
|
||||
thumb_path = generate_video_thumbnail_mirrored(dest_path)
|
||||
else:
|
||||
thumb_path = generate_thumbnail(dest_path)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to generate thumbnail for {original_name}: {e}")
|
||||
thumb_path = None
|
||||
|
||||
# Supersede the old record (preserves tags)
|
||||
record = supersede_image(
|
||||
old_record, dest_path, thumb_path, file_hash, phash, metadata, original_name
|
||||
)
|
||||
|
||||
# Apply any new tags from sidecar
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
if commit:
|
||||
db.session.commit()
|
||||
|
||||
# Update existing_phashes list with new dimensions
|
||||
for i, (ph, w, h, img_id) in enumerate(existing_phashes):
|
||||
if img_id == similar_id:
|
||||
existing_phashes[i] = (phash, metadata["width"], metadata["height"], img_id)
|
||||
break
|
||||
|
||||
return (True, phash, record, True) # superseded=True
|
||||
|
||||
# No similar image found (or supersede disabled) - normal import
|
||||
# Enrich metadata from Gallery-DL sidecar JSON if present
|
||||
enriched = _get_enriched_metadata(src_path, metadata)
|
||||
|
||||
# Copy into /images/<artist>/<base>__<hash[:10]><ext>
|
||||
target_dir = os.path.join(dest_dir, artist)
|
||||
@@ -545,6 +797,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
thumb_path = None
|
||||
|
||||
# Create DB record (keep display name as original; filepath is content-addressed)
|
||||
# Use enriched taken_at (earliest date wins)
|
||||
record = ImageRecord(
|
||||
filename=original_name,
|
||||
filepath=dest_path,
|
||||
@@ -556,7 +809,7 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
height=metadata["height"],
|
||||
format=metadata["format"],
|
||||
camera_model=metadata["camera_model"],
|
||||
taken_at=metadata["taken_at"],
|
||||
taken_at=enriched.get("taken_at") or metadata["taken_at"],
|
||||
imported_at=datetime.utcnow()
|
||||
)
|
||||
|
||||
@@ -571,11 +824,81 @@ def import_single_file(src_path: str, dest_dir: str, artist: str, existing_phash
|
||||
if artist_tag not in record.tags:
|
||||
record.tags.append(artist_tag)
|
||||
|
||||
# Add tags from Gallery-DL metadata (source:platform, post:platform:artist:id)
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
db.session.add(record)
|
||||
if commit:
|
||||
db.session.commit()
|
||||
return (True, phash, record)
|
||||
return (True, phash, record, False) # superseded=False (new import)
|
||||
|
||||
|
||||
def _get_enriched_metadata(src_path: str, existing_metadata: dict) -> dict:
|
||||
"""
|
||||
Get enriched metadata from Gallery-DL sidecar JSON.
|
||||
Returns empty dict if no sidecar found.
|
||||
"""
|
||||
try:
|
||||
from app.utils.metadata_enrichment import enrich_from_sidecar
|
||||
return enrich_from_sidecar(src_path, existing_metadata)
|
||||
except ImportError:
|
||||
return {}
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to enrich metadata for {src_path}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def _apply_enriched_tags(record: ImageRecord, enriched: dict) -> None:
|
||||
"""
|
||||
Apply tags from enriched metadata to an ImageRecord.
|
||||
Creates tags if they don't exist.
|
||||
"""
|
||||
if not enriched or "tags" not in enriched:
|
||||
return
|
||||
|
||||
for tag_info in enriched.get("tags", []):
|
||||
tag_name = tag_info.get("name")
|
||||
tag_kind = tag_info.get("kind")
|
||||
if not tag_name:
|
||||
continue
|
||||
|
||||
tag = Tag.query.filter_by(name=tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name, kind=tag_kind)
|
||||
db.session.add(tag)
|
||||
db.session.flush()
|
||||
|
||||
if tag not in record.tags:
|
||||
record.tags.append(tag)
|
||||
print(f"[INFO] Added tag: {tag_name}")
|
||||
|
||||
|
||||
def _enrich_existing_record(record: ImageRecord, src_path: str) -> None:
|
||||
"""
|
||||
Enrich an existing record with metadata from sidecar JSON.
|
||||
Used when a duplicate is found - we still want to merge metadata.
|
||||
|
||||
- Updates taken_at if sidecar date is earlier
|
||||
- Adds source/post tags if not already present
|
||||
"""
|
||||
try:
|
||||
from app.utils.metadata_enrichment import enrich_from_sidecar, resolve_date_conflict
|
||||
|
||||
enriched = enrich_from_sidecar(src_path, {"taken_at": record.taken_at})
|
||||
if not enriched.get("raw_metadata"):
|
||||
return # No sidecar found
|
||||
|
||||
# Update date if sidecar has an earlier date
|
||||
new_date = resolve_date_conflict(record.taken_at, enriched.get("taken_at"))
|
||||
if new_date != record.taken_at:
|
||||
print(f"[INFO] Updating taken_at for {record.filename}: {record.taken_at} -> {new_date}")
|
||||
record.taken_at = new_date
|
||||
|
||||
# Add any new tags
|
||||
_apply_enriched_tags(record, enriched)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed to enrich existing record: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -593,6 +916,10 @@ def generate_thumbnail(
|
||||
Save thumbnails mirrored under /images/thumbs/<...>.
|
||||
If prefer_jpeg is False (default), transparent images save as PNG, others as JPEG.
|
||||
If prefer_jpeg is True, all thumbs are JPEG with transparency flattened to `jpeg_bg`.
|
||||
|
||||
For images with extreme aspect ratios (very tall like comic strips, or very wide),
|
||||
crops to the center region before thumbnailing to produce better quality previews.
|
||||
|
||||
Returns the absolute path to the thumbnail.
|
||||
"""
|
||||
images_root = Path("/images").resolve()
|
||||
@@ -609,7 +936,27 @@ def generate_thumbnail(
|
||||
# Decide output extension/format
|
||||
with Image.open(image_path) as img:
|
||||
img = ImageOps.exif_transpose(img) # honor camera rotation
|
||||
img.thumbnail(size)
|
||||
|
||||
# Handle extreme aspect ratios by cropping to a more balanced region
|
||||
# This prevents very tall/wide images from becoming tiny slivers
|
||||
w, h = img.size
|
||||
aspect_ratio = w / h if h > 0 else 1
|
||||
|
||||
# If aspect ratio is extreme (< 0.5 means very tall, > 2.0 means very wide)
|
||||
# crop to a more balanced region before thumbnailing
|
||||
if aspect_ratio < 0.5:
|
||||
# Very tall image (like comic strip) - crop from top portion
|
||||
# Take a region that's closer to square, from the top
|
||||
crop_height = min(h, w * 2) # At most 2:1 tall
|
||||
img = img.crop((0, 0, w, crop_height))
|
||||
elif aspect_ratio > 2.0:
|
||||
# Very wide image - crop from center
|
||||
crop_width = min(w, h * 2) # At most 2:1 wide
|
||||
left = (w - crop_width) // 2
|
||||
img = img.crop((left, 0, left + crop_width, h))
|
||||
|
||||
# Use LANCZOS resampling for high-quality downscaling
|
||||
img.thumbnail(size, Image.Resampling.LANCZOS)
|
||||
|
||||
has_alpha = _has_alpha(img)
|
||||
|
||||
@@ -687,24 +1034,69 @@ def calculate_hash(file_path: str) -> str:
|
||||
return hash_func.hexdigest()
|
||||
|
||||
|
||||
def calculate_perceptual_hash(file_path: str):
|
||||
def calculate_perceptual_hash(file_path: str, hash_size: int = 16):
|
||||
"""
|
||||
Calculate a perceptual hash for an image.
|
||||
|
||||
Args:
|
||||
file_path: Path to the image file
|
||||
hash_size: Size of the hash grid (default 16 for 256-bit hash)
|
||||
hash_size=8 gives 64-bit, hash_size=16 gives 256-bit
|
||||
|
||||
Returns:
|
||||
ImageHash object or None if hashing fails (e.g., for videos)
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
return imagehash.phash(img)
|
||||
return imagehash.phash(img, hash_size=hash_size)
|
||||
except Exception:
|
||||
# videos or unreadable images land here (phash not applicable)
|
||||
return None
|
||||
|
||||
|
||||
def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 2) -> bool:
|
||||
for existing, ew, eh in existing_phashes:
|
||||
def find_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10):
|
||||
"""
|
||||
Find visually similar images and determine relationship.
|
||||
|
||||
Args:
|
||||
phash: Perceptual hash of the new image
|
||||
width: Width of the new image
|
||||
height: Height of the new image
|
||||
existing_phashes: List of (phash, width, height, image_id) tuples for existing images
|
||||
threshold: Maximum Hamming distance to consider images similar
|
||||
Default is 10 for 256-bit hashes (was 2 for 64-bit)
|
||||
|
||||
Returns:
|
||||
tuple: (relationship, image_id) where relationship is one of:
|
||||
- None: No similar image found
|
||||
- "larger_exists": A larger or equal similar image exists (skip new image)
|
||||
- "smaller_exists": A smaller similar image exists (supersede it)
|
||||
image_id is the ID of the matching image, or None
|
||||
"""
|
||||
for existing_phash, ew, eh, image_id in existing_phashes:
|
||||
try:
|
||||
distance = phash - existing
|
||||
if distance <= threshold and ew >= width and eh >= height:
|
||||
return True
|
||||
distance = phash - existing_phash
|
||||
if distance <= threshold:
|
||||
# Similar image found - check size relationship
|
||||
if ew >= width and eh >= height:
|
||||
# Existing is larger or equal - skip new image
|
||||
return ("larger_exists", image_id)
|
||||
elif width > ew or height > eh:
|
||||
# New image is larger - supersede the existing one
|
||||
return ("smaller_exists", image_id)
|
||||
except Exception as e:
|
||||
print(f"[WARN] Failed pHash comparison: {e}")
|
||||
return False
|
||||
return (None, None)
|
||||
|
||||
|
||||
def is_similar_image(phash, width: int, height: int, existing_phashes, threshold: int = 10) -> bool:
|
||||
"""
|
||||
Check if an image is visually similar to any existing larger image.
|
||||
|
||||
Legacy wrapper for find_similar_image - returns True only if a larger version exists.
|
||||
"""
|
||||
relationship, _ = find_similar_image(phash, width, height, existing_phashes, threshold)
|
||||
return relationship == "larger_exists"
|
||||
|
||||
|
||||
def extract_metadata(file_path: str) -> dict:
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
# app/utils/metadata_enrichment.py
|
||||
"""
|
||||
Metadata enrichment from Gallery-DL sidecar JSON files.
|
||||
|
||||
This module handles:
|
||||
- Detection and parsing of Gallery-DL sidecar .json files
|
||||
- Extraction of metadata (dates, tags, post info) from various platforms
|
||||
- Tag generation for source tracking (source:platform, post:platform:artist:id)
|
||||
- Date conflict resolution (earliest date wins)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
log = logging.getLogger("metadata-enrichment")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Sidecar File Detection
|
||||
# =============================================================================
|
||||
|
||||
def find_sidecar_json(image_path: str) -> Optional[str]:
|
||||
"""
|
||||
Find the Gallery-DL sidecar JSON file for an image.
|
||||
|
||||
Gallery-DL creates JSON files with the UUID portion of the filename.
|
||||
For example:
|
||||
Image: 01_BBEFD45F-8775-4DF9-8BCA-99E6063E3616.jpg
|
||||
JSON: BBEFD45F-8775-4DF9-8BCA-99E6063E3616.json
|
||||
|
||||
Also checks for direct {filename}.json pattern.
|
||||
|
||||
Returns the path to the sidecar JSON if found, None otherwise.
|
||||
"""
|
||||
image_path = Path(image_path)
|
||||
parent_dir = image_path.parent
|
||||
stem = image_path.stem # filename without extension
|
||||
|
||||
# Pattern 1: Direct {filename}.json (e.g., image.jpg -> image.jpg.json)
|
||||
direct_json = image_path.with_suffix(image_path.suffix + ".json")
|
||||
if direct_json.exists():
|
||||
return str(direct_json)
|
||||
|
||||
# Pattern 2: Extract UUID from filename and look for {uuid}.json
|
||||
# Common patterns: "01_UUID", "UUID", "{num}_{UUID}"
|
||||
uuid_pattern = re.compile(r'([A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12})', re.IGNORECASE)
|
||||
match = uuid_pattern.search(stem)
|
||||
if match:
|
||||
uuid = match.group(1)
|
||||
uuid_json = parent_dir / f"{uuid}.json"
|
||||
if uuid_json.exists():
|
||||
return str(uuid_json)
|
||||
|
||||
# Pattern 3: Just the stem without prefix numbers
|
||||
# e.g., "01_filename" -> look for "filename.json"
|
||||
stem_no_prefix = re.sub(r'^\d+_', '', stem)
|
||||
if stem_no_prefix != stem:
|
||||
alt_json = parent_dir / f"{stem_no_prefix}.json"
|
||||
if alt_json.exists():
|
||||
return str(alt_json)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def load_sidecar_metadata(json_path: str) -> Optional[dict]:
|
||||
"""Load and parse a Gallery-DL sidecar JSON file."""
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
log.warning(f"Failed to load sidecar JSON {json_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Metadata Extraction (Platform-specific)
|
||||
# =============================================================================
|
||||
|
||||
def extract_platform(metadata: dict) -> Optional[str]:
|
||||
"""Extract the platform name from metadata."""
|
||||
# Check common fields
|
||||
if "category" in metadata:
|
||||
return metadata["category"].lower()
|
||||
|
||||
# Infer from URL patterns
|
||||
url = metadata.get("url", "") or metadata.get("post_url", "")
|
||||
if "patreon.com" in url:
|
||||
return "patreon"
|
||||
if "hentai-foundry" in url or "hentaifoundry" in url.lower():
|
||||
return "hentaifoundry"
|
||||
if "subscribestar" in url:
|
||||
return "subscribestar"
|
||||
if "discord.com" in url:
|
||||
return "discord"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_artist(metadata: dict) -> Optional[str]:
|
||||
"""Extract the artist/creator name from metadata."""
|
||||
# Try various common fields
|
||||
if "creator" in metadata and isinstance(metadata["creator"], dict):
|
||||
return metadata["creator"].get("vanity") or metadata["creator"].get("full_name")
|
||||
|
||||
if "author_name" in metadata:
|
||||
return metadata["author_name"]
|
||||
|
||||
if "artist" in metadata:
|
||||
return metadata["artist"]
|
||||
|
||||
# For hentaifoundry
|
||||
if "user" in metadata:
|
||||
return metadata["user"]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_post_id(metadata: dict) -> Optional[str]:
|
||||
"""Extract the post ID from metadata."""
|
||||
if "id" in metadata:
|
||||
return str(metadata["id"])
|
||||
|
||||
if "post_id" in metadata:
|
||||
return str(metadata["post_id"])
|
||||
|
||||
if "message_id" in metadata: # Discord
|
||||
return str(metadata["message_id"])
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_post_title(metadata: dict) -> Optional[str]:
|
||||
"""Extract the post title from metadata."""
|
||||
return metadata.get("title")
|
||||
|
||||
|
||||
def extract_date(metadata: dict) -> Optional[datetime]:
|
||||
"""
|
||||
Extract the publication/post date from metadata.
|
||||
Tries multiple common date fields and formats.
|
||||
"""
|
||||
date_fields = ["published_at", "date", "created_at", "timestamp"]
|
||||
|
||||
for field in date_fields:
|
||||
if field not in metadata:
|
||||
continue
|
||||
|
||||
value = metadata[field]
|
||||
if not value:
|
||||
continue
|
||||
|
||||
# Handle various formats
|
||||
parsed = _parse_date(value)
|
||||
if parsed:
|
||||
return parsed
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _parse_date(value) -> Optional[datetime]:
|
||||
"""Parse a date value that could be string, int, or already datetime."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
|
||||
if isinstance(value, (int, float)):
|
||||
# Unix timestamp
|
||||
try:
|
||||
return datetime.fromtimestamp(value)
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
if isinstance(value, str):
|
||||
# Try common formats
|
||||
formats = [
|
||||
"%Y-%m-%dT%H:%M:%S.%f%z", # ISO with microseconds and timezone
|
||||
"%Y-%m-%dT%H:%M:%S%z", # ISO with timezone
|
||||
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
|
||||
"%Y-%m-%dT%H:%M:%S", # ISO basic
|
||||
"%Y-%m-%d %H:%M:%S", # Space-separated
|
||||
"%Y-%m-%d", # Date only
|
||||
]
|
||||
|
||||
# Handle timezone offset with colon (Python < 3.11 compat)
|
||||
# e.g., "2023-08-01T04:20:02.000+00:00" -> "2023-08-01T04:20:02.000+0000"
|
||||
normalized = re.sub(r'([+-]\d{2}):(\d{2})$', r'\1\2', value)
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(normalized, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Try without timezone info
|
||||
# Strip timezone suffix for naive parsing
|
||||
stripped = re.sub(r'[+-]\d{2}:?\d{2}$', '', value)
|
||||
stripped = re.sub(r'Z$', '', stripped)
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(stripped, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_content_tags(metadata: dict) -> list[str]:
|
||||
"""
|
||||
Extract content tags from metadata (not to be confused with DB tags).
|
||||
These are tags the creator applied to the post.
|
||||
"""
|
||||
tags = []
|
||||
|
||||
# Direct tags field
|
||||
if "tags" in metadata and isinstance(metadata["tags"], list):
|
||||
tags.extend(metadata["tags"])
|
||||
|
||||
return [str(t) for t in tags if t]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Tag Generation
|
||||
# =============================================================================
|
||||
|
||||
def generate_source_tags(metadata: dict, artist_override: Optional[str] = None) -> list[dict]:
|
||||
"""
|
||||
Generate tags from Gallery-DL metadata.
|
||||
|
||||
Returns a list of tag dictionaries with 'name' and 'kind' keys.
|
||||
Tag format ensures uniqueness:
|
||||
- source:{platform} (kind='source')
|
||||
- post:{platform}:{artist}:{post_id} (kind='post')
|
||||
"""
|
||||
tags = []
|
||||
|
||||
platform = extract_platform(metadata)
|
||||
artist = artist_override or extract_artist(metadata)
|
||||
post_id = extract_post_id(metadata)
|
||||
|
||||
# Source platform tag
|
||||
if platform:
|
||||
tags.append({
|
||||
"name": f"source:{platform}",
|
||||
"kind": "source"
|
||||
})
|
||||
|
||||
# Post tag (unique across platform + artist + post_id)
|
||||
if platform and artist and post_id:
|
||||
# Sanitize artist name for tag (lowercase, replace spaces)
|
||||
safe_artist = artist.lower().replace(" ", "_")
|
||||
tags.append({
|
||||
"name": f"post:{platform}:{safe_artist}:{post_id}",
|
||||
"kind": "post"
|
||||
})
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Date Conflict Resolution
|
||||
# =============================================================================
|
||||
|
||||
def resolve_date_conflict(existing_date: Optional[datetime], new_date: Optional[datetime]) -> Optional[datetime]:
|
||||
"""
|
||||
Resolve date conflicts by keeping the earliest date.
|
||||
|
||||
Args:
|
||||
existing_date: The currently stored date (may be None)
|
||||
new_date: The new date from metadata (may be None)
|
||||
|
||||
Returns:
|
||||
The earliest non-None date, or None if both are None
|
||||
"""
|
||||
if existing_date is None:
|
||||
return new_date
|
||||
if new_date is None:
|
||||
return existing_date
|
||||
|
||||
# Return the earlier date
|
||||
return min(existing_date, new_date)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main Enrichment Function
|
||||
# =============================================================================
|
||||
|
||||
def enrich_from_sidecar(image_path: str, existing_metadata: Optional[dict] = None) -> dict:
|
||||
"""
|
||||
Enrich image metadata from Gallery-DL sidecar JSON.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file
|
||||
existing_metadata: Optional dict with existing metadata (e.g., from EXIF)
|
||||
|
||||
Returns:
|
||||
Dictionary with enriched metadata:
|
||||
{
|
||||
"platform": str or None,
|
||||
"artist": str or None,
|
||||
"post_id": str or None,
|
||||
"post_title": str or None,
|
||||
"taken_at": datetime or None,
|
||||
"tags": [{"name": str, "kind": str}, ...],
|
||||
"raw_metadata": dict or None (the full sidecar content)
|
||||
}
|
||||
"""
|
||||
result = {
|
||||
"platform": None,
|
||||
"artist": None,
|
||||
"post_id": None,
|
||||
"post_title": None,
|
||||
"taken_at": None,
|
||||
"tags": [],
|
||||
"raw_metadata": None,
|
||||
}
|
||||
|
||||
# Find and load sidecar
|
||||
sidecar_path = find_sidecar_json(image_path)
|
||||
if not sidecar_path:
|
||||
return result
|
||||
|
||||
metadata = load_sidecar_metadata(sidecar_path)
|
||||
if not metadata:
|
||||
return result
|
||||
|
||||
result["raw_metadata"] = metadata
|
||||
|
||||
# Extract fields
|
||||
result["platform"] = extract_platform(metadata)
|
||||
result["artist"] = extract_artist(metadata)
|
||||
result["post_id"] = extract_post_id(metadata)
|
||||
result["post_title"] = extract_post_title(metadata)
|
||||
|
||||
# Extract date and resolve conflicts with existing
|
||||
sidecar_date = extract_date(metadata)
|
||||
existing_date = existing_metadata.get("taken_at") if existing_metadata else None
|
||||
result["taken_at"] = resolve_date_conflict(existing_date, sidecar_date)
|
||||
|
||||
# Generate tags
|
||||
result["tags"] = generate_source_tags(metadata, result["artist"])
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,288 @@
|
||||
# app/utils/version_migration.py
|
||||
"""
|
||||
Version tracking and data migration utilities.
|
||||
|
||||
This module handles:
|
||||
- Application version tracking via AppSettings
|
||||
- Automatic phash recomputation when version changes
|
||||
- Settings retrieval/storage from database
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from PIL import Image
|
||||
import imagehash
|
||||
|
||||
from app import db
|
||||
from app.models import AppSettings, ImageRecord
|
||||
|
||||
log = logging.getLogger("version-migration")
|
||||
|
||||
# =============================================================================
|
||||
# Version and pHash Configuration
|
||||
# =============================================================================
|
||||
|
||||
APP_VERSION = "26.01.19.1" # Format: YY.MM.DD.iteration
|
||||
|
||||
# pHash configuration - 256-bit (hash_size=16 -> 16x16 grid)
|
||||
PHASH_HASH_SIZE = 16
|
||||
PHASH_BITS = PHASH_HASH_SIZE * PHASH_HASH_SIZE # 256
|
||||
PHASH_THRESHOLD_DEFAULT = 10 # Scaled from 2 (64-bit) to 10 (256-bit)
|
||||
|
||||
# Batch size for phash recomputation
|
||||
PHASH_RECOMPUTE_BATCH_SIZE = int(os.environ.get("PHASH_RECOMPUTE_BATCH_SIZE", "100"))
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Settings Access Functions
|
||||
# =============================================================================
|
||||
|
||||
def get_setting(key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
Retrieve a setting value from the database.
|
||||
Returns default if the key doesn't exist.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is not None:
|
||||
return setting.value
|
||||
return default
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to get setting '{key}': {e}")
|
||||
return default
|
||||
|
||||
|
||||
def set_setting(key: str, value: str) -> None:
|
||||
"""
|
||||
Store a setting value in the database.
|
||||
Creates the key if it doesn't exist, updates if it does.
|
||||
"""
|
||||
try:
|
||||
setting = AppSettings.query.filter_by(key=key).first()
|
||||
if setting is None:
|
||||
setting = AppSettings(key=key, value=value)
|
||||
db.session.add(setting)
|
||||
else:
|
||||
setting.value = value
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
log.error(f"Failed to set setting '{key}': {e}")
|
||||
db.session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def get_setting_int(key: str, default: int) -> int:
|
||||
"""Get a setting as an integer."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return int(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_float(key: str, default: float) -> float:
|
||||
"""Get a setting as a float."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def get_setting_bool(key: str, default: bool) -> bool:
|
||||
"""Get a setting as a boolean."""
|
||||
val = get_setting(key)
|
||||
if val is None:
|
||||
return default
|
||||
return val.lower() in ("true", "1", "yes")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Import Settings (DB-backed)
|
||||
# =============================================================================
|
||||
|
||||
def get_import_settings() -> dict:
|
||||
"""
|
||||
Load import filter settings from database, with environment variable fallbacks.
|
||||
This replaces the file-based load_import_settings() function.
|
||||
"""
|
||||
return {
|
||||
"min_width": get_setting_int("import_min_width", int(os.environ.get("IMPORT_MIN_WIDTH", "0"))),
|
||||
"min_height": get_setting_int("import_min_height", int(os.environ.get("IMPORT_MIN_HEIGHT", "0"))),
|
||||
"skip_transparent": get_setting_bool("import_skip_transparent", os.environ.get("IMPORT_SKIP_TRANSPARENT", "false").lower() == "true"),
|
||||
"transparency_threshold": get_setting_float("import_transparency_threshold", float(os.environ.get("IMPORT_TRANSPARENCY_THRESHOLD", "0.9"))),
|
||||
"phash_threshold": get_setting_int("phash_threshold", PHASH_THRESHOLD_DEFAULT),
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# pHash Computation (256-bit)
|
||||
# =============================================================================
|
||||
|
||||
def calculate_phash_256(file_path: str) -> Optional[imagehash.ImageHash]:
|
||||
"""
|
||||
Calculate a 256-bit perceptual hash for an image.
|
||||
Uses hash_size=16 for a 16x16 grid = 256 bits.
|
||||
"""
|
||||
try:
|
||||
with Image.open(file_path) as img:
|
||||
return imagehash.phash(img, hash_size=PHASH_HASH_SIZE)
|
||||
except Exception as e:
|
||||
log.debug(f"Could not compute phash for {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Version Migration Logic
|
||||
# =============================================================================
|
||||
|
||||
def recompute_all_phashes() -> dict:
|
||||
"""
|
||||
Recompute perceptual hashes for all images using 256-bit hashing.
|
||||
Returns statistics about the operation.
|
||||
"""
|
||||
stats = {
|
||||
"total": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
}
|
||||
|
||||
log.info("Starting phash recomputation (256-bit)...")
|
||||
t_start = time.time()
|
||||
|
||||
batch_size = PHASH_RECOMPUTE_BATCH_SIZE
|
||||
last_id = 0
|
||||
|
||||
while True:
|
||||
# Fetch batch of images
|
||||
images = (
|
||||
ImageRecord.query
|
||||
.filter(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(batch_size)
|
||||
.all()
|
||||
)
|
||||
|
||||
if not images:
|
||||
break
|
||||
|
||||
batch_updated = 0
|
||||
for img in images:
|
||||
stats["total"] += 1
|
||||
|
||||
# Skip videos (no phash)
|
||||
ext = (img.format or "").lower()
|
||||
if ext in ("mp4", "mov") or (img.filepath and img.filepath.lower().endswith((".mp4", ".mov"))):
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
# Skip if file doesn't exist
|
||||
if not img.filepath or not os.path.exists(img.filepath):
|
||||
log.warning(f"File not found for image id={img.id}: {img.filepath}")
|
||||
stats["failed"] += 1
|
||||
continue
|
||||
|
||||
# Compute new phash
|
||||
try:
|
||||
phash = calculate_phash_256(img.filepath)
|
||||
if phash:
|
||||
img.perceptual_hash = str(phash)
|
||||
batch_updated += 1
|
||||
stats["updated"] += 1
|
||||
else:
|
||||
stats["failed"] += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to compute phash for id={img.id}: {e}")
|
||||
stats["failed"] += 1
|
||||
|
||||
# Commit batch
|
||||
db.session.commit()
|
||||
last_id = images[-1].id
|
||||
log.info(f"Processed batch up to id={last_id}, updated={batch_updated}")
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
log.info(
|
||||
f"pHash recomputation complete in {elapsed:.1f}s: "
|
||||
f"total={stats['total']}, updated={stats['updated']}, "
|
||||
f"skipped={stats['skipped']}, failed={stats['failed']}"
|
||||
)
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def check_and_run_migrations() -> None:
|
||||
"""
|
||||
Check the stored app version and run data migrations if needed.
|
||||
|
||||
This function:
|
||||
1. Reads the stored app_version from the database
|
||||
2. Compares it with the current APP_VERSION
|
||||
3. If different, triggers necessary data migrations (e.g., phash recomputation)
|
||||
4. Updates the stored version
|
||||
"""
|
||||
log.info(f"Checking app version (current: {APP_VERSION})...")
|
||||
|
||||
stored_version = get_setting("app_version")
|
||||
phash_version = get_setting("phash_version")
|
||||
|
||||
log.info(f"Stored version: {stored_version}, pHash version: {phash_version}")
|
||||
|
||||
# Initialize settings if this is first run
|
||||
if stored_version is None:
|
||||
log.info("First run detected, initializing settings...")
|
||||
_initialize_default_settings()
|
||||
stored_version = "0.0.0.0" # Force migration on first run
|
||||
|
||||
# Check if phash recomputation is needed
|
||||
needs_phash_recompute = (
|
||||
phash_version is None or
|
||||
phash_version != APP_VERSION or
|
||||
get_setting_int("phash_bits", 64) != PHASH_BITS
|
||||
)
|
||||
|
||||
if needs_phash_recompute:
|
||||
log.info(f"pHash migration needed (stored bits: {get_setting('phash_bits')}, new: {PHASH_BITS})")
|
||||
|
||||
# Update phash configuration
|
||||
set_setting("phash_bits", str(PHASH_BITS))
|
||||
set_setting("phash_hash_size", str(PHASH_HASH_SIZE))
|
||||
|
||||
# Recompute all hashes
|
||||
recompute_all_phashes()
|
||||
|
||||
# Mark phash version as current
|
||||
set_setting("phash_version", APP_VERSION)
|
||||
|
||||
# Update app version
|
||||
if stored_version != APP_VERSION:
|
||||
log.info(f"Updating app version from {stored_version} to {APP_VERSION}")
|
||||
set_setting("app_version", APP_VERSION)
|
||||
|
||||
log.info("Version check complete.")
|
||||
|
||||
|
||||
def _initialize_default_settings() -> None:
|
||||
"""Initialize default settings on first run."""
|
||||
defaults = {
|
||||
"app_version": APP_VERSION,
|
||||
"phash_version": None, # Will trigger recomputation
|
||||
"phash_bits": str(PHASH_BITS),
|
||||
"phash_hash_size": str(PHASH_HASH_SIZE),
|
||||
"phash_threshold": str(PHASH_THRESHOLD_DEFAULT),
|
||||
"import_min_width": "0",
|
||||
"import_min_height": "0",
|
||||
"import_skip_transparent": "false",
|
||||
"import_transparency_threshold": "0.9",
|
||||
}
|
||||
|
||||
for key, value in defaults.items():
|
||||
if get_setting(key) is None and value is not None:
|
||||
set_setting(key, value)
|
||||
@@ -20,6 +20,10 @@ if [ "${RUN_DB_MIGRATIONS:-1}" = "1" ]; then
|
||||
sleep "$SLEEP_SECS"
|
||||
i=$((i+1))
|
||||
done
|
||||
|
||||
# Run version check and data migrations (e.g., phash recomputation)
|
||||
echo "Checking app version and running data migrations..."
|
||||
flask version-check
|
||||
fi
|
||||
|
||||
# --- Start the image import worker exactly once per container ---
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Add app_settings table and expand perceptual_hash to 64 chars for 256-bit hashes
|
||||
|
||||
Revision ID: b26011901
|
||||
Revises: a46e641430b8
|
||||
Create Date: 2026-01-19
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'b26011901'
|
||||
down_revision = 'a46e641430b8'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
# Create app_settings table
|
||||
op.create_table(
|
||||
'app_settings',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('key', sa.String(length=64), nullable=False),
|
||||
sa.Column('value', sa.Text(), nullable=True),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('key')
|
||||
)
|
||||
|
||||
# Expand perceptual_hash column from 16 to 64 chars
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
if dialect == "sqlite":
|
||||
# SQLite requires table recreation for column type changes.
|
||||
# IMPORTANT: When recreating image_record, we must also recreate
|
||||
# image_tags to preserve foreign key relationships.
|
||||
|
||||
# Step 1: Backup image_tags data
|
||||
op.execute("CREATE TABLE image_tags_backup AS SELECT * FROM image_tags")
|
||||
|
||||
# Step 2: Drop image_tags (it references image_record)
|
||||
op.drop_table("image_tags")
|
||||
|
||||
# Step 3: Recreate image_record with new column size
|
||||
with op.batch_alter_table("image_record", recreate="always") as batch_op:
|
||||
batch_op.alter_column(
|
||||
'perceptual_hash',
|
||||
existing_type=sa.String(length=16),
|
||||
type_=sa.String(length=64),
|
||||
existing_nullable=True
|
||||
)
|
||||
|
||||
# Step 4: Recreate image_tags with proper FK constraints
|
||||
op.create_table(
|
||||
"image_tags",
|
||||
sa.Column("image_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("image_id", "tag_id"),
|
||||
sa.ForeignKeyConstraint(["image_id"], ["image_record.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
|
||||
sa.UniqueConstraint("image_id", "tag_id", name="uq_image_tag"),
|
||||
)
|
||||
|
||||
# Step 5: Restore image_tags data
|
||||
op.execute("INSERT INTO image_tags (image_id, tag_id) SELECT image_id, tag_id FROM image_tags_backup")
|
||||
|
||||
# Step 6: Drop backup table
|
||||
op.drop_table("image_tags_backup")
|
||||
else:
|
||||
# PostgreSQL and others can alter directly
|
||||
op.alter_column(
|
||||
'image_record',
|
||||
'perceptual_hash',
|
||||
existing_type=sa.String(length=16),
|
||||
type_=sa.String(length=64),
|
||||
existing_nullable=True
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
dialect = bind.dialect.name
|
||||
|
||||
# Shrink perceptual_hash back to 16 chars (will truncate data!)
|
||||
if dialect == "sqlite":
|
||||
# Same approach as upgrade: backup image_tags, recreate image_record,
|
||||
# then restore image_tags to preserve FK relationships
|
||||
|
||||
# Step 1: Backup image_tags data
|
||||
op.execute("CREATE TABLE image_tags_backup AS SELECT * FROM image_tags")
|
||||
|
||||
# Step 2: Drop image_tags (it references image_record)
|
||||
op.drop_table("image_tags")
|
||||
|
||||
# Step 3: Recreate image_record with old column size
|
||||
with op.batch_alter_table("image_record", recreate="always") as batch_op:
|
||||
batch_op.alter_column(
|
||||
'perceptual_hash',
|
||||
existing_type=sa.String(length=64),
|
||||
type_=sa.String(length=16),
|
||||
existing_nullable=True
|
||||
)
|
||||
|
||||
# Step 4: Recreate image_tags with proper FK constraints
|
||||
op.create_table(
|
||||
"image_tags",
|
||||
sa.Column("image_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("image_id", "tag_id"),
|
||||
sa.ForeignKeyConstraint(["image_id"], ["image_record.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"),
|
||||
sa.UniqueConstraint("image_id", "tag_id", name="uq_image_tag"),
|
||||
)
|
||||
|
||||
# Step 5: Restore image_tags data
|
||||
op.execute("INSERT INTO image_tags (image_id, tag_id) SELECT image_id, tag_id FROM image_tags_backup")
|
||||
|
||||
# Step 6: Drop backup table
|
||||
op.drop_table("image_tags_backup")
|
||||
else:
|
||||
op.alter_column(
|
||||
'image_record',
|
||||
'perceptual_hash',
|
||||
existing_type=sa.String(length=64),
|
||||
type_=sa.String(length=16),
|
||||
existing_nullable=True
|
||||
)
|
||||
|
||||
# Drop app_settings table
|
||||
op.drop_table('app_settings')
|
||||
Reference in New Issue
Block a user