diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 6ad49b8..15c52e9 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -6,7 +6,7 @@ from quart import Blueprint, jsonify, request from sqlalchemy import func, select from ..extensions import get_session -from ..models import AppSetting, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag +from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag settings_bp = Blueprint("settings", __name__, url_prefix="/api") @@ -118,6 +118,9 @@ async def system_stats(): storage_bytes = ( (await session.execute(select(func.coalesce(func.sum(ImageRecord.size_bytes), 0)))).scalar_one() ) + subscription_count = (await session.execute( + select(func.count(Artist.id)).where(Artist.is_subscription.is_(True)) + )).scalar_one() # Task counts grouped by status status_rows = ( @@ -163,6 +166,7 @@ async def system_stats(): "total_images": total_images, "total_tags": total_tags, "storage_bytes": storage_bytes, + "subscription_count": int(subscription_count), "tasks": { "pending": status_counts.get("pending", 0), "queued": status_counts.get("queued", 0), diff --git a/backend/app/services/migrators/backup.py b/backend/app/services/migrators/backup.py index 88040fd..4d3cb96 100644 --- a/backend/app/services/migrators/backup.py +++ b/backend/app/services/migrators/backup.py @@ -16,6 +16,19 @@ from typing import Any _BACKUPS_DIRNAME = "_backups" +def _libpq_url(sa_url: str) -> str: + """Strip SQLAlchemy driver suffix so pg_dump/psql accept the URL. + + SQLAlchemy uses URLs like `postgresql+psycopg://...` or + `postgresql+asyncpg://...`. libpq tools (pg_dump, psql) only know + the plain `postgresql://` scheme. + """ + for driver in ("postgresql+psycopg", "postgresql+asyncpg", "postgresql+psycopg2"): + if sa_url.startswith(driver + "://"): + return "postgresql://" + sa_url[len(driver) + 3:] + return sa_url + + def _backups_dir(images_root: Path | None = None) -> Path: # Overridable for tests via monkeypatch. root = images_root if images_root is not None else Path("/images") @@ -49,7 +62,7 @@ def create_backup( manifest_path = out_dir / f"fc_{ts}.json" _run_subprocess( - ["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), db_url], + ["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), _libpq_url(db_url)], _test_ts=ts, ) _run_subprocess( @@ -104,7 +117,7 @@ def restore_backup( tar_path = Path(manifest["tar_path"]) _run_subprocess( - ["psql", "-d", db_url, "-f", str(sql_path)], + ["psql", "-d", _libpq_url(db_url), "-f", str(sql_path)], ) # Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup diff --git a/backend/app/tasks/_sync_engine.py b/backend/app/tasks/_sync_engine.py new file mode 100644 index 0000000..508e490 --- /dev/null +++ b/backend/app/tasks/_sync_engine.py @@ -0,0 +1,36 @@ +"""Process-level sync engine for Celery task modules. + +Each task module used to call ``create_engine(...)`` on every invocation, +which leaked engines (and Postgres connections) — high-fire-rate tasks +like ``import_media_file`` would exhaust ``max_connections`` within +minutes during a bulk migration. + +This module owns one engine per process. Celery prefork forks before any +task runs, so each worker process lazily initializes its own engine on +the first task and reuses it for the rest of its life. +""" + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ..config import get_config + +_ENGINE = None +_SESSIONMAKER = None + + +def sync_session_factory(): + """Return a process-wide ``sessionmaker`` bound to a single engine.""" + global _ENGINE, _SESSIONMAKER + if _SESSIONMAKER is None: + cfg = get_config() + _ENGINE = create_engine( + cfg.database_url_sync, + future=True, + pool_pre_ping=True, + pool_size=5, + max_overflow=5, + pool_recycle=300, + ) + _SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False) + return _SESSIONMAKER diff --git a/backend/app/tasks/import_file.py b/backend/app/tasks/import_file.py index d53de74..562e7e7 100644 --- a/backend/app/tasks/import_file.py +++ b/backend/app/tasks/import_file.py @@ -5,21 +5,13 @@ updates the ImportTask state machine + ImportBatch counters atomically. from datetime import UTC, datetime from pathlib import Path -from sqlalchemy import create_engine, select, update -from sqlalchemy.orm import sessionmaker +from sqlalchemy import select, update from ..celery_app import celery -from ..config import get_config from ..models import ImportBatch, ImportSettings, ImportTask from ..services.importer import Importer from ..services.thumbnailer import Thumbnailer - - -def _sync_session_factory(): - cfg = get_config() - engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) - return sessionmaker(engine, expire_on_commit=False) - +from ._sync_engine import sync_session_factory as _sync_session_factory IMAGES_ROOT = Path("/images") diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index ced907f..4b23ac0 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -6,13 +6,12 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from PIL import Image -from sqlalchemy import create_engine, delete, select, update -from sqlalchemy.orm import sessionmaker +from sqlalchemy import delete, select, update from ..celery_app import celery -from ..config import get_config from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask from ..utils.phash import compute_phash +from ._sync_engine import sync_session_factory as _sync_session_factory log = logging.getLogger(__name__) @@ -23,12 +22,6 @@ VERIFY_PAGE = 200 FFPROBE_TIMEOUT_SECONDS = 10 -def _sync_session_factory(): - cfg = get_config() - engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) - return sessionmaker(engine, expire_on_commit=False) - - @celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks") def recover_interrupted_tasks() -> int: """Find ImportTask rows stuck in 'processing' for >30 min and re-queue them. diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index 13d4394..a208376 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -8,23 +8,16 @@ apply_allowlist_tags sweeps which are 'maintenance' lane. Sync sessions from pathlib import Path -from sqlalchemy import create_engine, select -from sqlalchemy.orm import sessionmaker +from sqlalchemy import select from ..celery_app import celery -from ..config import get_config from ..models import ImageRecord, MLSettings +from ._sync_engine import sync_session_factory as _sync_session_factory IMAGES_ROOT = Path("/images") VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"} -def _sync_session_factory(): - cfg = get_config() - engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) - return sessionmaker(engine, expire_on_commit=False) - - def _is_video(path: Path) -> bool: return path.suffix.lower() in VIDEO_EXTS diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py index 8c7a400..c4581f7 100644 --- a/backend/app/tasks/scan.py +++ b/backend/app/tasks/scan.py @@ -11,14 +11,14 @@ import asyncio from datetime import UTC, datetime from pathlib import Path -from sqlalchemy import create_engine, select +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from sqlalchemy.orm import sessionmaker from ..celery_app import celery from ..config import get_config from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask from ..services.scheduler_service import select_due_sources +from ._sync_engine import sync_session_factory as _sync_session_factory def _iter_import_files(import_root: Path): @@ -35,12 +35,6 @@ def _iter_import_files(import_root: Path): yield entry -def _sync_session_factory(): - cfg = get_config() - engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) - return sessionmaker(engine, expire_on_commit=False) - - @celery.task(name="backend.app.tasks.scan.scan_directory", bind=True) def scan_directory(self, triggered_by: str = "manual", mode: str = "quick") -> int: @@ -65,16 +59,34 @@ def scan_directory(self, triggered_by: str = "manual", session.flush() batch_id = batch.id + # Skip-set: any source_path that already has a non-failed ImportTask + # row. Re-running scan_directory must not re-enqueue files the + # importer has already handled (or is currently handling); doing so + # creates duplicate work and inflates the queue. Failed prior tasks + # are eligible for retry. + non_failed_existing = set(session.execute( + select(ImportTask.source_path).where( + ImportTask.status.in_( + ["pending", "queued", "processing", "complete", "skipped"] + ), + ) + ).scalars().all()) + # Walk and enumerate. files_seen = 0 + files_skipped_existing = 0 for entry in _iter_import_files(import_root): + entry_str = str(entry) + if entry_str in non_failed_existing: + files_skipped_existing += 1 + continue try: size = entry.stat().st_size except OSError: size = None task = ImportTask( batch_id=batch_id, - source_path=str(entry), + source_path=entry_str, task_type="media", status="pending", size_bytes=size, diff --git a/backend/app/tasks/thumbnail.py b/backend/app/tasks/thumbnail.py index 68fa665..61bed80 100644 --- a/backend/app/tasks/thumbnail.py +++ b/backend/app/tasks/thumbnail.py @@ -7,24 +7,15 @@ so they deserve their own queue lane. from pathlib import Path -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - from ..celery_app import celery -from ..config import get_config from ..models import ImageRecord from ..services.importer import is_video from ..services.thumbnailer import Thumbnailer +from ._sync_engine import sync_session_factory as _sync_session_factory IMAGES_ROOT = Path("/images") -def _sync_session_factory(): - cfg = get_config() - engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) - return sessionmaker(engine, expire_on_commit=False) - - @celery.task(name="backend.app.tasks.thumbnail.generate_thumbnail", bind=True) def generate_thumbnail(self, image_id: int) -> dict: SessionLocal = _sync_session_factory() diff --git a/frontend/src/App.vue b/frontend/src/App.vue index a4b0086..5cfbdda 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -3,6 +3,7 @@ + @@ -11,7 +12,10 @@ import { onMounted, ref } from 'vue' import AppShell from './components/AppShell.vue' import AppSnackbar from './components/AppSnackbar.vue' +import ImageViewer from './components/modal/ImageViewer.vue' +import { useModalStore } from './stores/modal.js' +const modal = useModalStore() const snackbar = ref(null) onMounted(() => { diff --git a/frontend/src/components/gallery/GalleryGrid.vue b/frontend/src/components/gallery/GalleryGrid.vue index 111cf61..8552c01 100644 --- a/frontend/src/components/gallery/GalleryGrid.vue +++ b/frontend/src/components/gallery/GalleryGrid.vue @@ -86,7 +86,7 @@ function imagesForGroup(group) { } .fc-gallery-grid__items { display: grid; - grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 8px; } .fc-gallery-grid__sentinel { @@ -101,7 +101,7 @@ function imagesForGroup(group) { } .fc-gallery-grid__skeleton { display: grid; - grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 8px; } .fc-gallery-grid__skeleton-item { @@ -124,7 +124,7 @@ function imagesForGroup(group) { @media (max-width: 600px) { .fc-gallery-grid__items, .fc-gallery-grid__skeleton { - grid-template-columns: repeat(auto-fill, minmax(120px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 4px; } } diff --git a/frontend/src/components/settings/SystemStatsCards.vue b/frontend/src/components/settings/SystemStatsCards.vue index 92d9ba7..fda022f 100644 --- a/frontend/src/components/settings/SystemStatsCards.vue +++ b/frontend/src/components/settings/SystemStatsCards.vue @@ -1,6 +1,6 @@ @@ -35,7 +31,6 @@ import GalleryGrid from '../components/gallery/GalleryGrid.vue' import TimelineSidebar from '../components/gallery/TimelineSidebar.vue' import EmptyState from '../components/gallery/EmptyState.vue' import PostInfoHeader from '../components/gallery/PostInfoHeader.vue' -import ImageViewer from '../components/modal/ImageViewer.vue' import BulkEditorPanel from '../components/gallery/BulkEditorPanel.vue' import { useGallerySelectionStore } from '../stores/gallerySelection.js' diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index a9ccab1..b15b946 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -1,5 +1,5 @@