From 13eaa35f1c9ddc3d4d2810760dec12f4698b5411 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:31:39 -0400 Subject: [PATCH 001/272] feat: scaffold backend Python project (Quart + SQLAlchemy + Celery deps) Pins runtime and ML deps separately so the regular web image stays lean. Configures ruff for py312 with bugbear, async, and pyupgrade lints enabled. psycopg sync driver included up-front for alembic. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/__init__.py | 0 backend/app/__init__.py | 0 pyproject.toml | 9 +++++++++ requirements-ml.txt | 9 +++++++++ requirements.txt | 28 ++++++++++++++++++++++++++++ ruff.toml | 23 +++++++++++++++++++++++ 6 files changed, 69 insertions(+) create mode 100644 backend/__init__.py create mode 100644 backend/app/__init__.py create mode 100644 pyproject.toml create mode 100644 requirements-ml.txt create mode 100644 requirements.txt create mode 100644 ruff.toml diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3ff350f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "fabledcurator" +version = "0.1.0" +description = "FabledSword family — self-hosted media curation, gallery, ML tagging, and subscription-driven downloads." +requires-python = ">=3.12" + +[tool.setuptools.packages.find] +where = ["."] +include = ["backend*"] diff --git a/requirements-ml.txt b/requirements-ml.txt new file mode 100644 index 0000000..826fdd0 --- /dev/null +++ b/requirements-ml.txt @@ -0,0 +1,9 @@ +-r requirements.txt + +# ML stack +torch>=2.2,<2.3 +torchvision>=0.17,<0.18 +transformers>=4.40,<4.41 +onnxruntime>=1.17,<1.18 +huggingface-hub>=0.22,<0.23 +opencv-python-headless>=4.9,<5.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ada374b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +# Web +quart>=0.19,<0.20 +hypercorn>=0.16,<0.17 + +# DB +sqlalchemy[asyncio]>=2.0,<2.1 +asyncpg>=0.29,<0.30 +psycopg[binary]>=3.1,<3.2 +alembic>=1.13,<1.14 +pgvector>=0.2,<0.3 + +# Task queue +celery>=5.4,<5.5 +redis>=5.0,<6.0 + +# Crypto for credential storage (lands in FC-3, but pinned now for stability) +cryptography>=42,<43 + +# Image handling (lands in FC-2) +pillow>=10.2,<11.0 +imagehash>=4.3,<4.4 + +# Gallery-dl wrapper (lands in FC-3) +gallery-dl>=1.27,<1.28 + +# Utilities +python-dotenv>=1.0,<2.0 +structlog>=24.1,<25.0 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..f71eae8 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,23 @@ +target-version = "py312" +line-length = 100 + +[lint] +select = [ + "E", "W", # pycodestyle + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # comprehensions + "ASYNC", # async correctness +] +ignore = [ + "E501", # line length, handled by formatter +] + +[lint.per-file-ignores] +"alembic/versions/*.py" = ["E", "F", "I", "UP"] +"tests/*" = ["F401"] + +[format] +quote-style = "double" From fe0a311d39a4d085b7f4a226da9d6fb65ab0d098 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:32:17 -0400 Subject: [PATCH 002/272] feat: add Quart app factory, config loader, and /api/health endpoint Config reads from env vars (12-factor); .env.example documents defaults. Health endpoint is liveness-only (no DB/Redis touch). Test added for CI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 24 +++++++++++++++++ backend/app/__init__.py | 19 ++++++++++++++ backend/app/api/__init__.py | 8 ++++++ backend/app/api/health.py | 5 ++++ backend/app/config.py | 52 +++++++++++++++++++++++++++++++++++++ backend/app/extensions.py | 14 ++++++++++ tests/__init__.py | 0 tests/test_health.py | 20 ++++++++++++++ 8 files changed, 142 insertions(+) create mode 100644 .env.example create mode 100644 backend/app/api/__init__.py create mode 100644 backend/app/api/health.py create mode 100644 backend/app/config.py create mode 100644 backend/app/extensions.py create mode 100644 tests/__init__.py create mode 100644 tests/test_health.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d78fd8f --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Database +DB_USER=fabledcurator +DB_PASSWORD=changeme_use_a_real_password +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=fabledcurator + +# Redis / Celery +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 + +# App +# Generate with: openssl rand -hex 32 +SECRET_KEY=changeme_32_byte_hex_secret + +# Extension API key — used in FC-3, lands later but reserved now +# Generate with: openssl rand -hex 32 +EXTENSION_API_KEY= + +# Logging +LOG_LEVEL=INFO + +# Deployment posture: plain HTTP (no TLS in the app; reverse proxy if needed) +# See docs/superpowers/specs/2026-05-13-fabledcurator-merge-design.md §2.1 diff --git a/backend/app/__init__.py b/backend/app/__init__.py index e69de29..d1081c2 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -0,0 +1,19 @@ +"""Quart app factory.""" + +import logging + +from quart import Quart + +from .api import api_bp +from .config import get_config + + +def create_app() -> Quart: + cfg = get_config() + logging.basicConfig(level=cfg.log_level) + + app = Quart(__name__) + app.secret_key = cfg.secret_key + app.register_blueprint(api_bp) + + return app diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..c456a71 --- /dev/null +++ b/backend/app/api/__init__.py @@ -0,0 +1,8 @@ +"""API blueprint registration.""" + +from quart import Blueprint + +from . import health + +api_bp = Blueprint("api", __name__, url_prefix="/api") +api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) diff --git a/backend/app/api/health.py b/backend/app/api/health.py new file mode 100644 index 0000000..ab4918e --- /dev/null +++ b/backend/app/api/health.py @@ -0,0 +1,5 @@ +"""Health endpoint — no DB or Redis touch; just liveness.""" + + +async def get_health(): + return {"status": "ok"}, 200 diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..2338f25 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,52 @@ +"""Application configuration loaded from environment variables.""" + +import os +from dataclasses import dataclass +from functools import lru_cache + + +@dataclass(frozen=True) +class Config: + db_user: str + db_password: str + db_host: str + db_port: int + db_name: str + + celery_broker_url: str + celery_result_backend: str + + secret_key: str + extension_api_key: str # used by the Firefox extension; lands in FC-3 but read here + log_level: str + + @property + def database_url(self) -> str: + return ( + f"postgresql+asyncpg://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + @property + def database_url_sync(self) -> str: + # Alembic uses sync driver + return ( + f"postgresql+psycopg://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + +@lru_cache(maxsize=1) +def get_config() -> Config: + return Config( + db_user=os.environ.get("DB_USER", "fabledcurator"), + db_password=os.environ["DB_PASSWORD"], + db_host=os.environ.get("DB_HOST", "postgres"), + db_port=int(os.environ.get("DB_PORT", "5432")), + db_name=os.environ.get("DB_NAME", "fabledcurator"), + celery_broker_url=os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"), + celery_result_backend=os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"), + secret_key=os.environ["SECRET_KEY"], + extension_api_key=os.environ.get("EXTENSION_API_KEY", ""), + log_level=os.environ.get("LOG_LEVEL", "INFO"), + ) diff --git a/backend/app/extensions.py b/backend/app/extensions.py new file mode 100644 index 0000000..3594b90 --- /dev/null +++ b/backend/app/extensions.py @@ -0,0 +1,14 @@ +"""Singleton extension instances; bound to the app in create_app().""" + +from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine + +from .config import get_config + + +def make_engine() -> AsyncEngine: + cfg = get_config() + return create_async_engine(cfg.database_url, pool_pre_ping=True, future=True) + + +def make_session_factory(engine: AsyncEngine) -> async_sessionmaker: + return async_sessionmaker(engine, expire_on_commit=False) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..d4eb411 --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,20 @@ +"""Health endpoint smoke test.""" + +import pytest + +from backend.app import create_app + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_health_returns_ok(client): + response = await client.get("/api/health") + assert response.status_code == 200 + body = await response.get_json() + assert body == {"status": "ok"} From a03655039c4f43cb435fda923ecf04d7a7f7e655 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:32:58 -0400 Subject: [PATCH 003/272] feat: add SQLAlchemy declarative base, Alembic environment, and gitignore fix Configures stable constraint naming so autogeneration produces clean diffs. Alembic uses the sync psycopg driver while the runtime app uses asyncpg. Also fixes a .gitignore bug caught during this task: the bare 'models/' rule for the ML weights volume was matching backend/app/models/ (Python package). Anchored all volume rules to repo root (/images/, /import/, /downloads/, /models/, /postgres_data/, /redis_data/). Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitignore | 15 +++++----- alembic.ini | 40 +++++++++++++++++++++++++ alembic/env.py | 53 ++++++++++++++++++++++++++++++++++ alembic/script.py.mako | 25 ++++++++++++++++ alembic/versions/.gitkeep | 0 backend/app/models/__init__.py | 7 +++++ backend/app/models/base.py | 17 +++++++++++ 7 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/.gitkeep create mode 100644 backend/app/models/__init__.py create mode 100644 backend/app/models/base.py diff --git a/.gitignore b/.gitignore index 4a5091a..06d2c93 100644 --- a/.gitignore +++ b/.gitignore @@ -42,13 +42,14 @@ yarn-error.log* extension/*.xpi extension/web-ext-artifacts/ -# Runtime volumes (must never be tracked) -images/ -import/ -downloads/ -models/ -postgres_data/ -redis_data/ +# Runtime volumes (must never be tracked) — anchored to repo root so they +# don't accidentally match Python package dirs like backend/app/models/ +/images/ +/import/ +/downloads/ +/models/ +/postgres_data/ +/redis_data/ # IDE / OS .vscode/ diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..8b17767 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +file_template = %%(rev)s_%%(slug)s +version_path_separator = os +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..9cabe10 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,53 @@ +"""Alembic environment — reads DATABASE_URL from app config.""" + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from backend.app.config import get_config +from backend.app.models import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +config.set_main_option("sqlalchemy.url", get_config().database_url_sync) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..17dcba0 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/.gitkeep b/alembic/versions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..f420c81 --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,7 @@ +"""All ORM models. Import this module to make every model visible to Alembic.""" + +from .base import Base + +# Concrete models land in Task 6 and re-export here. + +__all__ = ["Base"] diff --git a/backend/app/models/base.py b/backend/app/models/base.py new file mode 100644 index 0000000..9fd456b --- /dev/null +++ b/backend/app/models/base.py @@ -0,0 +1,17 @@ +"""SQLAlchemy declarative base with standardized constraint naming.""" + +from sqlalchemy import MetaData +from sqlalchemy.orm import DeclarativeBase + +# Stable constraint names so Alembic autogeneration produces clean diffs. +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + metadata = MetaData(naming_convention=NAMING_CONVENTION) From d6a156dcd2858fae8f000fbdf83a1581d442f816 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:34:29 -0400 Subject: [PATCH 004/272] feat: define unified schema (Artist, Source, Post, ImageRecord, etc.) and initial migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the data model from spec §3 in one go so FC-2/FC-3 don't need schema-adding migrations of their own. Artist is the unified entity for both gallery 'artist:' tags and GallerySubscriber Subscriptions (is_subscription flag). ImageProvenance is many-to-one, enabling the enrich-on-duplicate rule for downloaded content that pHash-matches an existing record. The SigLIP embedding column uses pgvector(1152) for SigLIP-so400m; swapping models in FC-2 will require a column-width migration. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../versions/0001_initial_unified_schema.py | 277 ++++++++++++++++++ backend/app/models/__init__.py | 23 +- backend/app/models/artist.py | 33 +++ backend/app/models/credential.py | 24 ++ backend/app/models/download_event.py | 28 ++ backend/app/models/image_provenance.py | 33 +++ backend/app/models/image_record.py | 71 +++++ backend/app/models/post.py | 33 +++ backend/app/models/source.py | 31 ++ backend/app/models/tag.py | 40 +++ tests/test_models.py | 29 ++ 11 files changed, 619 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/0001_initial_unified_schema.py create mode 100644 backend/app/models/artist.py create mode 100644 backend/app/models/credential.py create mode 100644 backend/app/models/download_event.py create mode 100644 backend/app/models/image_provenance.py create mode 100644 backend/app/models/image_record.py create mode 100644 backend/app/models/post.py create mode 100644 backend/app/models/source.py create mode 100644 backend/app/models/tag.py create mode 100644 tests/test_models.py diff --git a/alembic/versions/0001_initial_unified_schema.py b/alembic/versions/0001_initial_unified_schema.py new file mode 100644 index 0000000..0580b45 --- /dev/null +++ b/alembic/versions/0001_initial_unified_schema.py @@ -0,0 +1,277 @@ +"""initial unified schema + +Revision ID: 0001 +Revises: +Create Date: 2026-05-13 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + "artist", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("check_interval_seconds", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_artist"), + sa.UniqueConstraint("name", name="uq_artist_name"), + sa.UniqueConstraint("slug", name="uq_artist_slug"), + ) + + op.create_table( + "source", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("artist_id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("config_overrides", sa.JSON(), nullable=True), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("check_interval_override", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_source"), + ) + op.create_index("ix_source_artist_id", "source", ["artist_id"]) + + op.create_table( + "credential", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False, server_default="active"), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id", name="pk_credential"), + sa.UniqueConstraint("platform", name="uq_credential_platform"), + ) + + op.create_table( + "post", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("external_post_id", sa.String(length=128), nullable=False), + sa.Column("post_url", sa.Text(), nullable=True), + sa.Column("post_title", sa.Text(), nullable=True), + sa.Column("post_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("raw_metadata", sa.JSON(), nullable=True), + sa.Column( + "downloaded_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_post"), + sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), + ) + op.create_index("ix_post_source_id", "post", ["source_id"]) + + op.create_table( + "image_record", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.Column("phash", sa.String(length=32), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("mime", sa.String(length=64), nullable=False), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("thumbnail_path", sa.Text(), nullable=True), + sa.Column( + "origin", + sa.Enum( + "downloaded", + "imported_filesystem", + "uploaded", + name="origin_enum", + ), + nullable=False, + ), + sa.Column("primary_post_id", sa.Integer(), nullable=True), + sa.Column("wd14_predictions", sa.JSON(), nullable=True), + sa.Column("wd14_model_version", sa.String(length=128), nullable=True), + sa.Column("siglip_embedding", Vector(1152), nullable=True), + sa.Column("siglip_model_version", sa.String(length=128), nullable=True), + sa.Column("centroid_scores", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["primary_post_id"], + ["post.id"], + name="fk_image_record_primary_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_record"), + sa.UniqueConstraint("path", name="uq_image_record_path"), + sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), + ) + op.create_index("ix_image_record_sha256", "image_record", ["sha256"]) + op.create_index("ix_image_record_phash", "image_record", ["phash"]) + op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"]) + + op.create_table( + "image_provenance", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("captured_metadata", sa.JSON(), nullable=True), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_provenance_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_image_provenance_post_id_post", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_image_provenance_source_id_source", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_provenance"), + ) + op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"]) + op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"]) + op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"]) + + op.create_table( + "tag", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("namespace", sa.String(length=64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_tag"), + sa.UniqueConstraint("name", name="uq_tag_name"), + ) + op.create_index("ix_tag_name", "tag", ["name"]) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + + op.create_table( + "image_tag", + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_tag_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"), + ) + + op.create_table( + "download_event", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_download_event_source_id_source", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_download_event_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_download_event"), + ) + op.create_index("ix_download_event_source_id", "download_event", ["source_id"]) + op.create_index("ix_download_event_post_id", "download_event", ["post_id"]) + + +def downgrade() -> None: + op.drop_table("download_event") + op.drop_table("image_tag") + op.drop_table("tag") + op.drop_table("image_provenance") + op.drop_table("image_record") + op.execute("DROP TYPE IF EXISTS origin_enum") + op.drop_table("post") + op.drop_table("credential") + op.drop_table("source") + op.drop_table("artist") + op.execute("DROP EXTENSION IF EXISTS vector") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index f420c81..d8b935d 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,7 +1,24 @@ """All ORM models. Import this module to make every model visible to Alembic.""" +from .artist import Artist from .base import Base +from .credential import Credential +from .download_event import DownloadEvent +from .image_provenance import ImageProvenance +from .image_record import ImageRecord +from .post import Post +from .source import Source +from .tag import Tag, image_tag -# Concrete models land in Task 6 and re-export here. - -__all__ = ["Base"] +__all__ = [ + "Base", + "Artist", + "Source", + "Credential", + "Post", + "ImageRecord", + "ImageProvenance", + "Tag", + "image_tag", + "DownloadEvent", +] diff --git a/backend/app/models/artist.py b/backend/app/models/artist.py new file mode 100644 index 0000000..d6847f4 --- /dev/null +++ b/backend/app/models/artist.py @@ -0,0 +1,33 @@ +"""Artist — unified entity that is both the gallery's ``artist:`` tag concept +and GallerySubscriber's Subscription. ``is_subscription`` is True if any +Sources are attached. +""" + +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class Artist(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + # True once a Source is attached; flips false if all sources removed. + is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + + # Per-artist scheduling overrides; null means "use global default". + auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + sources = relationship("Source", back_populates="artist", cascade="all, delete-orphan") diff --git a/backend/app/models/credential.py b/backend/app/models/credential.py new file mode 100644 index 0000000..667a7c0 --- /dev/null +++ b/backend/app/models/credential.py @@ -0,0 +1,24 @@ +"""Credential — encrypted blob (cookies/token/oauth) scoped per-platform, +not per-source. One Patreon credential serves every Patreon source. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, LargeBinary, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class Credential(Base): + __tablename__ = "credential" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + platform: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + kind: Mapped[str] = mapped_column(String(32), nullable=False) # cookies|token|oauth + encrypted_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="active") + captured_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/backend/app/models/download_event.py b/backend/app/models/download_event.py new file mode 100644 index 0000000..8955558 --- /dev/null +++ b/backend/app/models/download_event.py @@ -0,0 +1,28 @@ +"""DownloadEvent — log of every gallery-dl run, populated by FC-3.""" + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class DownloadEvent(Base): + __tablename__ = "download_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/models/image_provenance.py b/backend/app/models/image_provenance.py new file mode 100644 index 0000000..cd50887 --- /dev/null +++ b/backend/app/models/image_provenance.py @@ -0,0 +1,33 @@ +"""ImageProvenance — links an ImageRecord to a Post. + +Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate +rule (spec §3): when a downloaded image is a pHash dupe of an existing +record, we append a new provenance row to the existing record rather than +dropping the metadata. +""" + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class ImageProvenance(Base): + __tablename__ = "image_provenance" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + image_record_id: Mapped[int] = mapped_column( + ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True + ) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + captured_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py new file mode 100644 index 0000000..1c1a3cc --- /dev/null +++ b/backend/app/models/image_record.py @@ -0,0 +1,71 @@ +"""ImageRecord — the gallery's primary entity, ported from ImageRepo. + +ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them +without a schema migration. The SigLIP embedding column uses pgvector's Vector +type — pgvector extension is enabled in the initial migration. +""" + +from datetime import datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import ( + JSON, + BigInteger, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + +ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded") + + +class ImageRecord(Base): + __tablename__ = "image_record" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + # On-disk identity + path: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + mime: Mapped[str] = mapped_column(String(64), nullable=False) + width: Mapped[int | None] = mapped_column(Integer, nullable=True) + height: Mapped[int | None] = mapped_column(Integer, nullable=True) + + # Thumbnail (populated by FC-2) + thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True) + + # Origin / provenance pointers + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False) + primary_post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # ML fields (populated by FC-2's ml-worker) + wd14_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True) + wd14_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + # 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require + # a column-width migration. + siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True) + siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + + # Centroid score cache (populated post-tagging) + centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/models/post.py b/backend/app/models/post.py new file mode 100644 index 0000000..56cf030 --- /dev/null +++ b/backend/app/models/post.py @@ -0,0 +1,33 @@ +"""Post — provenance anchor for content downloaded from a Source. + +A Post is one creator post; it may contain many images/videos. +""" + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class Post(Base): + __tablename__ = "post" + __table_args__ = ( + UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + external_post_id: Mapped[str] = mapped_column(String(128), nullable=False) + post_url: Mapped[str | None] = mapped_column(Text, nullable=True) + post_title: Mapped[str | None] = mapped_column(Text, nullable=True) + post_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + raw_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + downloaded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/source.py b/backend/app/models/source.py new file mode 100644 index 0000000..13a01bf --- /dev/null +++ b/backend/app/models/source.py @@ -0,0 +1,31 @@ +"""Source — a platform-specific URL owned by an Artist (e.g., a Patreon URL). + +Multiple sources per artist support creators with cross-platform presence. +""" + +from datetime import datetime + +from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class Source(Base): + __tablename__ = "source" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + artist_id: Mapped[int] = mapped_column( + ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True + ) + platform: Mapped[str] = mapped_column(String(64), nullable=False) + url: Mapped[str] = mapped_column(Text, nullable=False) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + + config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) + + artist = relationship("Artist", back_populates="sources") diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py new file mode 100644 index 0000000..0c7d8d5 --- /dev/null +++ b/backend/app/models/tag.py @@ -0,0 +1,40 @@ +"""Tag + ImageTag association. + +Tags use namespaced strings (``artist:Name``, ``character:Name``, +``rating:safe``, etc.). Artist tags are kept in sync with the Artist table — +the sync trigger lands in FC-2. +""" + +from datetime import datetime + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Table, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + +image_tag = Table( + "image_tag", + Base.metadata, + Column( + "image_record_id", + ForeignKey("image_record.id", ondelete="CASCADE"), + primary_key=True, + ), + Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), + Column("source", String(32), nullable=False, default="manual"), # manual|auto|ml + Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), +) + + +class Tag(Base): + __tablename__ = "tag" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) + namespace: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + images = relationship("ImageRecord", secondary=image_tag, backref="tags") diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..97eac6d --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,29 @@ +"""Smoke test — every model imports cleanly and shares the same metadata.""" + +from backend.app.models import ( + Artist, + Base, + ImageRecord, +) + + +def test_all_models_use_shared_metadata(): + expected_tables = { + "artist", + "source", + "credential", + "post", + "image_record", + "image_provenance", + "tag", + "image_tag", + "download_event", + } + assert expected_tables.issubset(Base.metadata.tables.keys()) + + +def test_artist_has_subscription_flag(): + # Sanity: the unified data model decision (Subscription IS Artist) shows up + # as a boolean flag on Artist, per spec §3. + assert "is_subscription" in {c.name for c in Artist.__table__.columns} + assert "primary_post_id" in {c.name for c in ImageRecord.__table__.columns} From 8773f2aae67700d3b6b38775b270aa4c99f1a6da Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:34:47 -0400 Subject: [PATCH 005/272] feat: configure Celery with per-feature queue lanes and a smoke task Routes are pre-declared for FC-2/FC-3 task modules (import, ml, thumbnail, download, scan, maintenance). Queue lanes match the ImageRepo pattern where beat+maintenance run on a separate worker so long imports don't starve periodic tasks. Smoke ping task confirms the wiring in eager mode for CI. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/celery_app.py | 45 +++++++++++++++++++++++++++++++++++ backend/app/tasks/__init__.py | 0 backend/app/tasks/smoke.py | 8 +++++++ tests/test_celery_smoke.py | 24 +++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 backend/app/celery_app.py create mode 100644 backend/app/tasks/__init__.py create mode 100644 backend/app/tasks/smoke.py create mode 100644 tests/test_celery_smoke.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py new file mode 100644 index 0000000..1d89c78 --- /dev/null +++ b/backend/app/celery_app.py @@ -0,0 +1,45 @@ +"""Celery configuration with separate queue lanes. + +Queues: + import — filesystem import path (FC-2) + ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image) + thumbnail — image and video thumbnail generation (FC-2) + download — gallery-dl tasks (FC-3) + scan — periodic source checks (FC-3) — kept separate so long imports + don't starve the scheduler + maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3) + default — anything not explicitly routed +""" + +from celery import Celery + +from .config import get_config + + +def make_celery() -> Celery: + cfg = get_config() + app = Celery( + "fabledcurator", + broker=cfg.celery_broker_url, + backend=cfg.celery_result_backend, + include=["backend.app.tasks.smoke"], # FC-2/FC-3 extend this list + ) + app.conf.update( + task_default_queue="default", + task_routes={ + "backend.app.tasks.import_file.*": {"queue": "import"}, + "backend.app.tasks.ml.*": {"queue": "ml"}, + "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"}, + "backend.app.tasks.download.*": {"queue": "download"}, + "backend.app.tasks.scan.*": {"queue": "scan"}, + "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, + }, + # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. + task_acks_late=True, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + ) + return app + + +celery = make_celery() diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/tasks/smoke.py b/backend/app/tasks/smoke.py new file mode 100644 index 0000000..ad452fb --- /dev/null +++ b/backend/app/tasks/smoke.py @@ -0,0 +1,8 @@ +"""Smoke task used during FC-1 to confirm Celery routing works end-to-end.""" + +from ..celery_app import celery + + +@celery.task(name="backend.app.tasks.smoke.ping") +def ping(value: str = "pong") -> str: + return value diff --git a/tests/test_celery_smoke.py b/tests/test_celery_smoke.py new file mode 100644 index 0000000..40b0054 --- /dev/null +++ b/tests/test_celery_smoke.py @@ -0,0 +1,24 @@ +"""Smoke test — celery runs the ping task in eager mode (no broker required).""" + +import pytest + +from backend.app.celery_app import celery +from backend.app.tasks.smoke import ping + + +@pytest.fixture(autouse=True) +def celery_eager(): + celery.conf.task_always_eager = True + celery.conf.task_eager_propagates = True + yield + celery.conf.task_always_eager = False + + +def test_ping_returns_pong(): + result = ping.delay() + assert result.get(timeout=1) == "pong" + + +def test_ping_with_value(): + result = ping.delay("hello") + assert result.get(timeout=1) == "hello" From 8b5711abefe9034b4ef0d15f5ee954b4bdc2d462 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:35:01 -0400 Subject: [PATCH 006/272] docs: add module docstring to tasks package __init__ --- backend/app/tasks/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py index e69de29..c8a11bb 100644 --- a/backend/app/tasks/__init__.py +++ b/backend/app/tasks/__init__.py @@ -0,0 +1 @@ +"""Celery tasks. Submodules are auto-discovered via celery_app's include list.""" From ad2ac31f1a464d62a686f9cc5c96ddfae173f6dd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:35:33 -0400 Subject: [PATCH 007/272] feat: scaffold Vue 3 + Vuetify + Pinia frontend themed to FabledDesignSystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Curator signature accent (#A87338, aged amber) is baked into fabled-tokens.js. Vuetify theme consumes the same tokens module the rest of the SPA does, so updating design system tokens propagates throughout. Vite dev server proxies /api and /ws to the Quart backend. App.vue references AppShell (lands in Task 9) and main.js references router.js (lands in Task 9) — these are intentionally split so Task 8 is the framework scaffold and Task 9 is the navigation/shell content. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/index.html | 19 +++++++++++++ frontend/package.json | 26 ++++++++++++++++++ frontend/src/App.vue | 11 ++++++++ frontend/src/main.js | 26 ++++++++++++++++++ frontend/src/theme/fabled-tokens.js | 41 +++++++++++++++++++++++++++++ frontend/src/theme/vuetify-theme.js | 29 ++++++++++++++++++++ frontend/vite.config.js | 23 ++++++++++++++++ 7 files changed, 175 insertions(+) create mode 100644 frontend/index.html create mode 100644 frontend/package.json create mode 100644 frontend/src/App.vue create mode 100644 frontend/src/main.js create mode 100644 frontend/src/theme/fabled-tokens.js create mode 100644 frontend/src/theme/vuetify-theme.js create mode 100644 frontend/vite.config.js diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..0ecae7f --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,19 @@ + + + + + + + FabledCurator + + + + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..32e9f00 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,26 @@ +{ + "name": "fabledcurator-frontend", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "check": "vue-tsc --noEmit" + }, + "dependencies": { + "vue": "^3.4.0", + "vue-router": "^4.3.0", + "pinia": "^2.1.0", + "vuetify": "^3.5.0", + "@mdi/font": "^7.4.0" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^5.0.0", + "vite": "^5.2.0", + "vue-tsc": "^2.0.0", + "vite-plugin-vuetify": "^2.0.0", + "sass": "^1.71.0" + } +} diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..4c51eaf --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,11 @@ + + + diff --git a/frontend/src/main.js b/frontend/src/main.js new file mode 100644 index 0000000..0882de9 --- /dev/null +++ b/frontend/src/main.js @@ -0,0 +1,26 @@ +import { createApp } from 'vue' +import { createPinia } from 'pinia' +import { createVuetify } from 'vuetify' +import 'vuetify/styles' +import '@mdi/font/css/materialdesignicons.css' + +import App from './App.vue' +import router from './router.js' +import { fabledCuratorTheme } from './theme/vuetify-theme.js' + +const vuetify = createVuetify({ + theme: { + defaultTheme: 'fabledCurator', + themes: { fabledCurator: fabledCuratorTheme } + }, + defaults: { + // FabledDesignSystem: pill shape for buttons + VBtn: { rounded: 'pill' } + } +}) + +createApp(App) + .use(createPinia()) + .use(router) + .use(vuetify) + .mount('#app') diff --git a/frontend/src/theme/fabled-tokens.js b/frontend/src/theme/fabled-tokens.js new file mode 100644 index 0000000..4102088 --- /dev/null +++ b/frontend/src/theme/fabled-tokens.js @@ -0,0 +1,41 @@ +// FabledDesignSystem tokens — mirrored from FabledDesignSystem-new/design-system.md. +// Do not hand-edit hexes; update the source-of-truth doc and re-mirror here. + +export const surfaces = { + obsidian: '#14171A', // page background + iron: '#1E2228', // cards + slate: '#2C313A', // hovered surfaces + pewter: '#3F4651' // borders, ghost outlines +} + +export const text = { + parchment: '#E8E4D8', // primary on dark + vellum: '#C2BFB4', // secondary + ash: '#9C9A92' // tertiary +} + +export const action = { + moss: '#4A5D3F', // primary action + bronze: '#8B7355', // secondary action + pewter: '#3F4651', // tertiary/ghost + destructive: '#6B2118' // delete/irreversible +} + +export const semantic = { + success: '#4A5D3F', + warning: '#8B6F1E', + error: '#C04A1F', + info: '#3D5A6E' +} + +// Per-app signature accent. Curator's accent was added to FabledDesignSystem +// during FC-1 (2026-05-14). Aged amber — manuscript ink, curated archive. +export const accent = { + curator: '#A87338' +} + +export const typography = { + display: 'Fraunces, Georgia, serif', + body: 'Inter, system-ui, sans-serif', + mono: '"JetBrains Mono", monospace' +} diff --git a/frontend/src/theme/vuetify-theme.js b/frontend/src/theme/vuetify-theme.js new file mode 100644 index 0000000..2211e75 --- /dev/null +++ b/frontend/src/theme/vuetify-theme.js @@ -0,0 +1,29 @@ +import { surfaces, text, action, semantic, accent } from './fabled-tokens.js' + +export const fabledCuratorTheme = { + dark: true, + colors: { + background: surfaces.obsidian, + surface: surfaces.iron, + 'surface-bright': surfaces.slate, + 'surface-light': surfaces.pewter, + + primary: action.moss, + secondary: action.bronze, + tertiary: action.pewter, + + 'on-background': text.parchment, + 'on-surface': text.parchment, + 'on-primary': text.parchment, + 'on-secondary': text.parchment, + + success: semantic.success, + warning: semantic.warning, + error: semantic.error, + info: semantic.info, + + // Curator signature accent — used for nav-active state, wordmark, selection. + // Not on action buttons (see design-system.md "Accent usage rules"). + accent: accent.curator + } +} diff --git a/frontend/vite.config.js b/frontend/vite.config.js new file mode 100644 index 0000000..b506803 --- /dev/null +++ b/frontend/vite.config.js @@ -0,0 +1,23 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' +import vuetify from 'vite-plugin-vuetify' + +export default defineConfig({ + plugins: [ + vue(), + vuetify({ autoImport: true }) + ], + build: { + outDir: 'dist', + emptyOutDir: true, + sourcemap: false + }, + server: { + port: 5173, + proxy: { + // Dev server proxies API to local Quart + '/api': 'http://localhost:8080', + '/ws': { target: 'ws://localhost:8080', ws: true } + } + } +}) From bcd9a23501cf253f52960b73647e16fcb610e10e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:36:01 -0400 Subject: [PATCH 008/272] feat: add SPA app shell, router, system store, and route placeholders Nav lists every FC-2 and FC-3 surface with a placeholder view that names the sub-project that will fill it. System store polls /api/health on mount and renders a status pip in the nav footer. Favicon uses Curator's aged amber accent (#A87338). Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/public/favicon.svg | 5 ++ frontend/src/components/AppShell.vue | 78 ++++++++++++++++++++++++++ frontend/src/router.js | 20 +++++++ frontend/src/stores/system.js | 18 ++++++ frontend/src/views/PlaceholderView.vue | 34 +++++++++++ 5 files changed, 155 insertions(+) create mode 100644 frontend/public/favicon.svg create mode 100644 frontend/src/components/AppShell.vue create mode 100644 frontend/src/router.js create mode 100644 frontend/src/stores/system.js create mode 100644 frontend/src/views/PlaceholderView.vue diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg new file mode 100644 index 0000000..6b07fb4 --- /dev/null +++ b/frontend/public/favicon.svg @@ -0,0 +1,5 @@ + + + + diff --git a/frontend/src/components/AppShell.vue b/frontend/src/components/AppShell.vue new file mode 100644 index 0000000..ce71c61 --- /dev/null +++ b/frontend/src/components/AppShell.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/frontend/src/router.js b/frontend/src/router.js new file mode 100644 index 0000000..d60b5e5 --- /dev/null +++ b/frontend/src/router.js @@ -0,0 +1,20 @@ +import { createRouter, createWebHistory } from 'vue-router' +import PlaceholderView from './views/PlaceholderView.vue' + +const routes = [ + // FC-2: image backbone + { path: '/', name: 'gallery', component: PlaceholderView, meta: { title: 'Gallery' } }, + { path: '/showcase', name: 'showcase', component: PlaceholderView, meta: { title: 'Showcase' } }, + { path: '/tags', name: 'tags', component: PlaceholderView, meta: { title: 'Tags' } }, + { path: '/settings', name: 'settings', component: PlaceholderView, meta: { title: 'Settings' } }, + + // FC-3: subscription backbone + { path: '/subscriptions', name: 'subscriptions', component: PlaceholderView, meta: { title: 'Subscriptions' } }, + { path: '/credentials', name: 'credentials', component: PlaceholderView, meta: { title: 'Credentials' } }, + { path: '/downloads', name: 'downloads', component: PlaceholderView, meta: { title: 'Downloads' } } +] + +export default createRouter({ + history: createWebHistory(), + routes +}) diff --git a/frontend/src/stores/system.js b/frontend/src/stores/system.js new file mode 100644 index 0000000..4412640 --- /dev/null +++ b/frontend/src/stores/system.js @@ -0,0 +1,18 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' + +export const useSystemStore = defineStore('system', () => { + const healthy = ref(null) // null=unknown, true=ok, false=down + + async function refreshHealth() { + try { + const r = await fetch('/api/health') + const body = await r.json() + healthy.value = r.ok && body.status === 'ok' + } catch { + healthy.value = false + } + } + + return { healthy, refreshHealth } +}) diff --git a/frontend/src/views/PlaceholderView.vue b/frontend/src/views/PlaceholderView.vue new file mode 100644 index 0000000..5c53509 --- /dev/null +++ b/frontend/src/views/PlaceholderView.vue @@ -0,0 +1,34 @@ + + + + + From 8e61dc4df4efb74e8219e78868a90d653640ed5b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:36:16 -0400 Subject: [PATCH 009/272] feat: serve the built Vue SPA from Quart with history-mode fallback The frontend blueprint is registered after the api blueprint so /api/* routes are matched first; everything else falls through to the SPA. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/__init__.py | 3 +++ backend/app/frontend.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 backend/app/frontend.py diff --git a/backend/app/__init__.py b/backend/app/__init__.py index d1081c2..cd295e9 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -6,6 +6,7 @@ from quart import Quart from .api import api_bp from .config import get_config +from .frontend import frontend_bp def create_app() -> Quart: @@ -15,5 +16,7 @@ def create_app() -> Quart: app = Quart(__name__) app.secret_key = cfg.secret_key app.register_blueprint(api_bp) + # Registered last so /api/* routes win over the SPA catch-all. + app.register_blueprint(frontend_bp) return app diff --git a/backend/app/frontend.py b/backend/app/frontend.py new file mode 100644 index 0000000..879a6c9 --- /dev/null +++ b/backend/app/frontend.py @@ -0,0 +1,17 @@ +"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback.""" + +from pathlib import Path + +from quart import Blueprint, send_from_directory + +FRONTEND_DIST = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist" + +frontend_bp = Blueprint("frontend", __name__) + + +@frontend_bp.route("/") +@frontend_bp.route("/") +async def serve_spa(subpath: str = ""): + if subpath and (FRONTEND_DIST / subpath).is_file(): + return await send_from_directory(FRONTEND_DIST, subpath) + return await send_from_directory(FRONTEND_DIST, "index.html") From 1db0167bfcd0c236bbc7fd9d8f5652a1417a3d79 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:42:04 -0400 Subject: [PATCH 010/272] feat: add Dockerfile for web/worker/scheduler roles and entrypoint script Multi-stage build: node:20 builds the SPA, python:3.12-slim runs the app. Same image handles web, worker, scheduler roles via entrypoint.sh's first arg. ffmpeg + unar are baked in for FC-2's transcode + archive paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile | 44 +++++++++++++++++++++++++++++++++++++ entrypoint.sh | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 Dockerfile create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..78620da --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1.7 + +FROM node:20-alpine AS frontend-builder +WORKDIR /build +COPY frontend/package.json frontend/package-lock.json* ./ +RUN npm ci --no-audit --no-fund +COPY frontend/ ./ +RUN npm run build + +FROM python:3.12-slim AS runtime +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# System deps: ffmpeg (transcode + thumbnails, used in FC-2), unar (archives, FC-2), +# libpq for psycopg, image libs. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + unar \ + libpq5 \ + libjpeg62-turbo \ + libwebp7 \ + libpng16-16 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install -r requirements.txt + +COPY backend/ ./backend/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./ +COPY entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +COPY --from=frontend-builder /build/dist ./frontend/dist + +EXPOSE 8080 + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["web"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..53a6f6a --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROLE="${1:-web}" +shift || true + +case "$ROLE" in + web) + echo "[entrypoint] Running alembic upgrade head" + alembic upgrade head + echo "[entrypoint] Starting hypercorn on :8080" + exec hypercorn \ + --bind 0.0.0.0:8080 \ + --workers "${HYPERCORN_WORKERS:-2}" \ + --access-logfile - \ + backend.app:create_app + ;; + + worker) + QUEUES="${CELERY_QUEUES:-default,import,thumbnail}" + CONCURRENCY="${CELERY_CONCURRENCY:-2}" + echo "[entrypoint] Starting Celery worker queues=$QUEUES concurrency=$CONCURRENCY" + exec celery -A backend.app.celery_app:celery worker \ + --loglevel=info \ + -Q "$QUEUES" \ + --concurrency="$CONCURRENCY" + ;; + + scheduler) + QUEUES="${CELERY_QUEUES:-maintenance,scan}" + echo "[entrypoint] Starting Celery beat+worker queues=$QUEUES" + exec celery -A backend.app.celery_app:celery worker \ + --beat \ + --loglevel=info \ + -Q "$QUEUES" \ + --concurrency=1 + ;; + + ml-worker) + echo "[entrypoint] Starting ML Celery worker (ml queue)" + exec celery -A backend.app.celery_app:celery worker \ + --loglevel=info \ + -Q ml \ + --concurrency=1 + ;; + + shell|bash) + exec /bin/bash "$@" + ;; + + alembic) + exec alembic "$@" + ;; + + *) + echo "[entrypoint] Unknown role: $ROLE" >&2 + echo "[entrypoint] Valid roles: web | worker | scheduler | ml-worker | shell | alembic" >&2 + exit 1 + ;; +esac From ec03297382ea4abbedff193d78eab54f2617e559 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:42:14 -0400 Subject: [PATCH 011/272] =?UTF-8?q?feat:=20add=20Dockerfile.ml=20=E2=80=94?= =?UTF-8?q?=20separate=20image=20for=20the=20ml-worker=20role?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ML deps (torch, transformers, onnxruntime, opencv) are ~4GB and stay out of the regular web/worker image. Models self-heal into /models on start (implemented in FC-2). HuggingFace cache vars point inside /models so weight downloads are persistent across restarts. Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile.ml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 Dockerfile.ml diff --git a/Dockerfile.ml b/Dockerfile.ml new file mode 100644 index 0000000..3812915 --- /dev/null +++ b/Dockerfile.ml @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:1.7 + +FROM python:3.12-slim +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HF_HOME=/models/.huggingface \ + TRANSFORMERS_CACHE=/models/.huggingface \ + ML_MODEL_DIR=/models + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + libpq5 \ + libjpeg62-turbo \ + libwebp7 \ + libpng16-16 \ + libgl1 \ + libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements-ml.txt requirements.txt ./ +RUN pip install -r requirements-ml.txt + +COPY backend/ ./backend/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./ +COPY entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +# Models self-heal into /models on first start (FC-2 implements this) +VOLUME ["/models"] + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["ml-worker"] From a2816201789542306eae4fa418a6673a6bbcaf17 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:42:36 -0400 Subject: [PATCH 012/272] feat: add docker-compose stack (web/worker/scheduler/ml-worker + postgres + redis) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prod compose references the Forgejo registry; dev compose builds locally and exposes DB/Redis ports. Volume layout: ./images, ./import, ./downloads, ./models (all gitignored, anchored to repo root). Plain HTTP only — no TLS in-app (spec §2.1). Co-Authored-By: Claude Opus 4.7 (1M context) --- docker-compose.dev.yml | 45 +++++++++++++++++++++ docker-compose.yml | 91 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..5baecb7 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,45 @@ +services: + postgres: + ports: + - "5432:5432" + + redis: + ports: + - "6379:6379" + + web: + build: + context: . + dockerfile: Dockerfile + environment: + LOG_LEVEL: DEBUG + volumes: + - ./backend:/app/backend + - ./alembic:/app/alembic + + worker: + build: + context: . + dockerfile: Dockerfile + environment: + LOG_LEVEL: DEBUG + volumes: + - ./backend:/app/backend + + scheduler: + build: + context: . + dockerfile: Dockerfile + environment: + LOG_LEVEL: DEBUG + volumes: + - ./backend:/app/backend + + ml-worker: + build: + context: . + dockerfile: Dockerfile.ml + environment: + LOG_LEVEL: DEBUG + volumes: + - ./backend:/app/backend diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7b5c7ca --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,91 @@ +services: + redis: + image: redis:7-alpine + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: ${DB_USER:-fabledcurator} + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_NAME:-fabledcurator} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-fabledcurator}"] + interval: 10s + timeout: 5s + retries: 5 + + web: + image: git.fabledsword.com/bvandeusen/fabledcurator:dev + command: ["web"] + ports: + - "${PORT:-8080}:8080" + env_file: .env + environment: &app_env + DB_HOST: postgres + DB_PORT: "5432" + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + volumes: + - ./images:/images + - ./import:/import + - ./downloads:/downloads + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + + worker: + image: git.fabledsword.com/bvandeusen/fabledcurator:dev + command: ["worker"] + environment: + <<: *app_env + CELERY_QUEUES: default,import,thumbnail,download + CELERY_CONCURRENCY: "2" + env_file: .env + volumes: + - ./images:/images + - ./import:/import + - ./downloads:/downloads + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + + scheduler: + image: git.fabledsword.com/bvandeusen/fabledcurator:dev + command: ["scheduler"] + environment: + <<: *app_env + CELERY_QUEUES: maintenance,scan + env_file: .env + volumes: + - ./images:/images + - ./import:/import + - ./downloads:/downloads + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + + ml-worker: + image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev + command: ["ml-worker"] + environment: + <<: *app_env + env_file: .env + volumes: + - ./images:/images:ro + - ./models:/models + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_healthy } + +volumes: + redis_data: + postgres_data: From 37e97d52ee60118a5d7449f8a774b507fd284511 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:52:15 -0400 Subject: [PATCH 013/272] chore: bump runtime targets to Python 3.14 and Node 22 Forward-looking pin per operator direction ("build for the future and not rebuild the past"). Touches everywhere a version is named so all downstream artifacts (Dockerfiles, ruff config, package.json engines) agree. Python 3.14 (released Oct 2025) is the current stable. Node 22 is the most recent LTS line still receiving updates (Maintenance LTS since Oct 2025); Node 24 (released Apr 2026) goes Active LTS in Oct 2026 and will be the natural next bump. Co-Authored-By: Claude Opus 4.7 (1M context) --- Dockerfile | 4 ++-- Dockerfile.ml | 2 +- frontend/package.json | 3 +++ pyproject.toml | 2 +- ruff.toml | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78620da..80307bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ # syntax=docker/dockerfile:1.7 -FROM node:20-alpine AS frontend-builder +FROM node:22-alpine AS frontend-builder WORKDIR /build COPY frontend/package.json frontend/package-lock.json* ./ RUN npm ci --no-audit --no-fund COPY frontend/ ./ RUN npm run build -FROM python:3.12-slim AS runtime +FROM python:3.14-slim AS runtime ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/Dockerfile.ml b/Dockerfile.ml index 3812915..cf90ec4 100644 --- a/Dockerfile.ml +++ b/Dockerfile.ml @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -FROM python:3.12-slim +FROM python:3.14-slim ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/frontend/package.json b/frontend/package.json index 32e9f00..d123574 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,6 +3,9 @@ "version": "0.1.0", "private": true, "type": "module", + "engines": { + "node": ">=22" + }, "scripts": { "dev": "vite", "build": "vite build", diff --git a/pyproject.toml b/pyproject.toml index 3ff350f..0cd1a87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "fabledcurator" version = "0.1.0" description = "FabledSword family — self-hosted media curation, gallery, ML tagging, and subscription-driven downloads." -requires-python = ">=3.12" +requires-python = ">=3.14" [tool.setuptools.packages.find] where = ["."] diff --git a/ruff.toml b/ruff.toml index f71eae8..b8603a4 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -target-version = "py312" +target-version = "py314" line-length = 100 [lint] From 8cbe963b37be1bc59b0dd75e009828208ebc36e5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:52:30 -0400 Subject: [PATCH 014/272] =?UTF-8?q?ci:=20add=20Forgejo=20CI=20workflow=20?= =?UTF-8?q?=E2=80=94=20backend=20lint+tests,=20frontend=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend job spins up Postgres+pgvector and Redis as services, runs ruff, applies the initial migration to confirm it's clean, and runs pytest. Frontend job runs vue-tsc and vite build. Requires a runner labeled "fabledcurator-ci" with Python 3.14, ruff, and Node 22 pre-installed. Integration tests run locally via docker-compose with testing.Short() gating per FabledRulebook verification.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/ci.yml | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .forgejo/workflows/ci.yml diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..28ca052 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,66 @@ +name: CI + +on: + push: + branches: [dev, main] + pull_request: + branches: [main] + +jobs: + backend-lint-and-test: + # Runner must have Python 3.14 + ruff installed. Provision a runner + # labeled "fabledcurator-ci" with Python 3.14, ruff, and Node 22. + runs-on: fabledcurator-ci + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_test_password + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_test_password + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: fabledcurator_test + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + SECRET_KEY: ci_test_secret_not_for_prod + steps: + - uses: actions/checkout@v4 + + - name: Install Python deps + run: pip install -r requirements.txt pytest pytest-asyncio "ruff>=0.9,<1.0" + + - name: Ruff lint + run: ruff check backend/ tests/ alembic/ + + - name: Alembic upgrade (smoke) + run: alembic upgrade head + + - name: Pytest (unit only) + run: pytest tests/ -v + + frontend-build: + runs-on: fabledcurator-ci + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - run: npm ci --no-audit --no-fund + - run: npm run check + - run: npm run build From edb3fafd4da57373840f4eb2afa9643ddfc064b5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 07:52:55 -0400 Subject: [PATCH 015/272] ci: add image-build workflow + document required runner label and RELEASE_TOKEN Pushes :dev on every dev push; pushes :main and :latest on main. Uses RELEASE_TOKEN (a broader-scoped Forgejo PAT covering write:package, read:package, write:release, write:issue) so the same secret can serve future release-cutting and issue-management workflows. README now documents the fabledcurator-ci runner label and the required RELEASE_TOKEN scopes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/build.yml | 70 ++++++++++++++++++++++++++++++++++++ README.md | 12 +++++++ 2 files changed, 82 insertions(+) create mode 100644 .forgejo/workflows/build.yml diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 0000000..750a79e --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,70 @@ +name: Build images + +on: + push: + branches: [dev, main] + +# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: +# - write:package, read:package (for docker push to git.fabledsword.com) +# - write:release (for future release-cutting workflows) +# - write:issue (for future issue-management automation) +# The injected GITHUB_TOKEN cannot be used — it lacks write:package. + +jobs: + build-web: + runs-on: fabledcurator-ci + steps: + - uses: actions/checkout@v4 + + - name: Determine tag + id: tag + run: | + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT" + else + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT" + fi + + - name: Login to Forgejo registry + uses: docker/login-action@v3 + with: + registry: git.fabledsword.com + username: ${{ github.actor }} + password: ${{ secrets.RELEASE_TOKEN }} + + - name: Build and push web image + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile + push: true + tags: ${{ steps.tag.outputs.tags }} + + build-ml: + runs-on: fabledcurator-ci + steps: + - uses: actions/checkout@v4 + + - name: Determine tag + id: tag + run: | + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT" + else + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT" + fi + + - name: Login to Forgejo registry + uses: docker/login-action@v3 + with: + registry: git.fabledsword.com + username: ${{ github.actor }} + password: ${{ secrets.RELEASE_TOKEN }} + + - name: Build and push ml image + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.ml + push: true + tags: ${{ steps.tag.outputs.tags }} diff --git a/README.md b/README.md index 990356f..0ee52da 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,18 @@ docker compose up -d FabledCurator is designed to run inside a self-hosted homelab environment over plain HTTP. If you want TLS, terminate it at your reverse proxy. The app does not generate certificates, redirect to HTTPS, or set HSTS. +## CI / Forgejo setup + +The repo's workflows expect: + +- **Runner label `fabledcurator-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. +- **Repo secret `RELEASE_TOKEN`** — a Forgejo PAT with the following scopes: + - `write:package` + `read:package` — for `docker push` to `git.fabledsword.com` + - `write:release` — for future release-cutting workflows + - `write:issue` — for future issue-management automation + + Generate at https://git.fabledsword.com/user/settings/applications. The injected `GITHUB_TOKEN` cannot be used because it lacks `write:package`. + ## License Personal project; use at your own discretion. From ff122c55eb4e57d0ba84602c92e96ed7c63c4e5f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 08:45:48 -0400 Subject: [PATCH 016/272] ci: switch runner label from fabledcurator-ci to python-ci Generic python-ci runner is reusable across the family (FabledScribe, FabledSteward, NhenArchiver, StashHandler, etc.) rather than scoped to just this project. Runner image lives at CI-Runner/CI-python/ in the operator's workspace; pattern mirrors CI-Runner/CI-go and CI-Runner/CI-flutter. Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/build.yml | 4 ++-- .forgejo/workflows/ci.yml | 9 +++++---- README.md | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 750a79e..9c561ba 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -12,7 +12,7 @@ on: jobs: build-web: - runs-on: fabledcurator-ci + runs-on: python-ci steps: - uses: actions/checkout@v4 @@ -41,7 +41,7 @@ jobs: tags: ${{ steps.tag.outputs.tags }} build-ml: - runs-on: fabledcurator-ci + runs-on: python-ci steps: - uses: actions/checkout@v4 diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 28ca052..ce48fa3 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -8,9 +8,10 @@ on: jobs: backend-lint-and-test: - # Runner must have Python 3.14 + ruff installed. Provision a runner - # labeled "fabledcurator-ci" with Python 3.14, ruff, and Node 22. - runs-on: fabledcurator-ci + # Runner must have Python 3.14 + ruff + Node 22 pre-installed. The + # python-ci runner image lives at CI-Runner/CI-python/ in the operator's + # workspace; see that Dockerfile + Makefile to roll/push the image. + runs-on: python-ci services: postgres: image: pgvector/pgvector:pg16 @@ -55,7 +56,7 @@ jobs: run: pytest tests/ -v frontend-build: - runs-on: fabledcurator-ci + runs-on: python-ci defaults: run: working-directory: frontend diff --git a/README.md b/README.md index 0ee52da..371a801 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ FabledCurator is designed to run inside a self-hosted homelab environment over p The repo's workflows expect: -- **Runner label `fabledcurator-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. +- **Runner label `python-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. The runner image (`runner-base:python-ci`) is built from `CI-Runner/CI-python/` in the operator's workspace; `make push` from that directory builds and pushes a new image when toolchain pins change. - **Repo secret `RELEASE_TOKEN`** — a Forgejo PAT with the following scopes: - `write:package` + `read:package` — for `docker push` to `git.fabledsword.com` - `write:release` — for future release-cutting workflows From 9eb1614fbfa83cd491bd9986cdb214c1735d3f46 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:03:41 -0400 Subject: [PATCH 017/272] =?UTF-8?q?feat(fc2a):=20schema=20migration=200002?= =?UTF-8?q?=20=E2=80=94=20tag=20kinds=20+=20fandom=20hierarchy=20+=20impor?= =?UTF-8?q?t=20lifecycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Tag.kind enum (artist/character/fandom/general/series/archive/post/meta/rating), Tag.fandom_id FK with CHECK constraint (only valid for kind='character'), and a kind-aware uniqueness index so the same name can exist across kinds and the same character name can exist in different fandoms. Adds ImportBatch + ImportTask state-machine tables for scan tracking, plus a single-row ImportSettings table (CHECK id=1) holding the importer's filter knobs. Adds image_record.integrity_status column (defaults to 'unknown'); FC-2e populates this via the integrity verifier. Drops the unused tag.namespace column from FC-1 — superseded by kind. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../0002_fc2a_tag_kinds_and_import_tasks.py | 208 ++++++++++++++++++ backend/app/models/__init__.py | 9 +- backend/app/models/image_record.py | 6 + backend/app/models/import_batch.py | 32 +++ backend/app/models/import_settings.py | 28 +++ backend/app/models/import_task.py | 41 ++++ backend/app/models/tag.py | 53 ++++- tests/test_migration_0002.py | 56 +++++ 8 files changed, 425 insertions(+), 8 deletions(-) create mode 100644 alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py create mode 100644 backend/app/models/import_batch.py create mode 100644 backend/app/models/import_settings.py create mode 100644 backend/app/models/import_task.py create mode 100644 tests/test_migration_0002.py diff --git a/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py new file mode 100644 index 0000000..b9dac2c --- /dev/null +++ b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py @@ -0,0 +1,208 @@ +"""fc2a: tag kinds, import_task, import_batch, integrity_status + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-05-14 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0002" +down_revision: Union[str, None] = "0001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +TAG_KINDS = ( + "artist", + "character", + "fandom", + "general", + "series", + "archive", + "post", + "meta", + "rating", +) + + +def upgrade() -> None: + # --- Tag kind enum + fandom_id --- + tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind") + tag_kind.create(op.get_bind(), checkfirst=True) + + op.add_column( + "tag", + sa.Column("kind", tag_kind, nullable=False, server_default="general"), + ) + op.add_column( + "tag", + sa.Column("fandom_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_tag_fandom_id_tag", + "tag", + "tag", + ["fandom_id"], + ["id"], + ondelete="SET NULL", + ) + + # Drop the old global uniqueness on name; add kind+fandom-aware uniqueness. + op.drop_constraint("uq_tag_name", "tag", type_="unique") + op.drop_index("ix_tag_name", table_name="tag") + op.execute( + """ + CREATE UNIQUE INDEX uq_tag_name_kind_fandom + ON tag (name, kind, COALESCE(fandom_id, 0)) + """ + ) + + # CHECK: fandom_id is only allowed for character kind. + op.create_check_constraint( + "ck_tag_fandom_requires_character", + "tag", + "(fandom_id IS NULL) OR (kind = 'character')", + ) + + # Drop the old namespace column — superseded by kind. + op.drop_index("ix_tag_namespace", table_name="tag") + op.drop_column("tag", "namespace") + + # --- ImportBatch --- + op.create_table( + "import_batch", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("triggered_by", sa.String(length=32), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("scan_mode", sa.String(length=16), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"), + sa.Column("imported", sa.Integer(), nullable=False, server_default="0"), + sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"), + sa.Column("failed", sa.Integer(), nullable=False, server_default="0"), + sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), + sa.PrimaryKeyConstraint("id", name="pk_import_batch"), + ) + op.create_index("ix_import_batch_status", "import_batch", ["status"]) + + # --- ImportTask --- + op.create_table( + "import_task", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("batch_id", sa.Integer(), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("task_type", sa.String(length=16), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"), + sa.Column("result_image_id", sa.Integer(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["batch_id"], + ["import_batch.id"], + name="fk_import_task_batch_id_import_batch", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["result_image_id"], + ["image_record.id"], + name="fk_import_task_result_image_id_image_record", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_import_task"), + ) + op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"]) + op.create_index("ix_import_task_status", "import_task", ["status"]) + op.create_index( + "ix_import_task_created_at_desc", + "import_task", + [sa.text("created_at DESC")], + ) + + # --- ImportSettings (single-row table) --- + op.create_table( + "import_settings", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"), + sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"), + sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "transparency_threshold", + sa.Float(), + nullable=False, + server_default="0.9", + ), + sa.Column( + "skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "single_color_threshold", + sa.Float(), + nullable=False, + server_default="0.95", + ), + sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"), + sa.PrimaryKeyConstraint("id", name="pk_import_settings"), + sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"), + ) + # Seed the single row immediately so callers can always SELECT id=1. + op.execute("INSERT INTO import_settings (id) VALUES (1)") + + # --- ImageRecord additions --- + op.add_column( + "image_record", + sa.Column( + "integrity_status", + sa.String(length=24), + nullable=False, + server_default="unknown", + ), + ) + op.create_index( + "ix_image_record_integrity_status", + "image_record", + ["integrity_status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_integrity_status", table_name="image_record") + op.drop_column("image_record", "integrity_status") + + op.drop_table("import_settings") + op.drop_index("ix_import_task_created_at_desc", table_name="import_task") + op.drop_index("ix_import_task_status", table_name="import_task") + op.drop_index("ix_import_task_batch_id", table_name="import_task") + op.drop_table("import_task") + op.drop_index("ix_import_batch_status", table_name="import_batch") + op.drop_table("import_batch") + + op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check") + op.execute("DROP INDEX uq_tag_name_kind_fandom") + op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True)) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + op.create_index("ix_tag_name", "tag", ["name"], unique=False) + op.create_unique_constraint("uq_tag_name", "tag", ["name"]) + op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey") + op.drop_column("tag", "fandom_id") + op.drop_column("tag", "kind") + sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index d8b935d..8052e9b 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -6,9 +6,12 @@ from .credential import Credential from .download_event import DownloadEvent from .image_provenance import ImageProvenance from .image_record import ImageRecord +from .import_batch import ImportBatch +from .import_settings import ImportSettings +from .import_task import ImportTask from .post import Post from .source import Source -from .tag import Tag, image_tag +from .tag import Tag, TagKind, image_tag __all__ = [ "Base", @@ -19,6 +22,10 @@ __all__ = [ "ImageRecord", "ImageProvenance", "Tag", + "TagKind", "image_tag", "DownloadEvent", + "ImportBatch", + "ImportTask", + "ImportSettings", ] diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 1c1a3cc..9df4845 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -40,6 +40,12 @@ class ImageRecord(Base): width: Mapped[int | None] = mapped_column(Integer, nullable=True) height: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'. + # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'. + integrity_status: Mapped[str] = mapped_column( + String(24), nullable=False, default="unknown", index=True + ) + # Thumbnail (populated by FC-2) thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/backend/app/models/import_batch.py b/backend/app/models/import_batch.py new file mode 100644 index 0000000..26003e3 --- /dev/null +++ b/backend/app/models/import_batch.py @@ -0,0 +1,32 @@ +"""ImportBatch — one scan run, aggregating many ImportTasks.""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class ImportBatch(Base): + __tablename__ = "import_batch" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + triggered_by: Mapped[str] = mapped_column(String(32), nullable=False) + source_path: Mapped[str] = mapped_column(Text, nullable=False) + scan_mode: Mapped[str] = mapped_column(String(16), nullable=False) # quick|deep + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True) + # running | complete | cancelled + + tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan") diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py new file mode 100644 index 0000000..b8d0d83 --- /dev/null +++ b/backend/app/models/import_settings.py @@ -0,0 +1,28 @@ +"""ImportSettings — single-row table holding the importer's tunable knobs. + +Enforced as a single row via a CHECK (id = 1) constraint. The application +always SELECTs id=1 and never inserts/deletes after the initial migration. +""" + +from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class ImportSettings(Base): + __tablename__ = "import_settings" + __table_args__ = (CheckConstraint("id = 1", name="ck_import_settings_singleton"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import") + + min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9) + + skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) + single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30) diff --git a/backend/app/models/import_task.py b/backend/app/models/import_task.py new file mode 100644 index 0000000..da3d4ec --- /dev/null +++ b/backend/app/models/import_task.py @@ -0,0 +1,41 @@ +"""ImportTask — a single source file to import as part of a batch. + +State machine: pending -> queued -> processing -> complete | skipped | failed. +Workers crashing mid-task leave rows in 'processing'; the recovery sweep +(tasks/maintenance.py::recover_interrupted_tasks) re-queues rows that have +been processing longer than the stuck-task threshold. +""" + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class ImportTask(Base): + __tablename__ = "import_task" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + batch_id: Mapped[int] = mapped_column( + ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True + ) + + source_path: Mapped[str] = mapped_column(Text, nullable=False) + task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive + status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True) + + result_image_id: Mapped[int | None] = mapped_column( + ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True + ) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + batch = relationship("ImportBatch", back_populates="tasks") diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index 0c7d8d5..2ca9d39 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -1,17 +1,42 @@ """Tag + ImageTag association. -Tags use namespaced strings (``artist:Name``, ``character:Name``, -``rating:safe``, etc.). Artist tags are kept in sync with the Artist table — -the sync trigger lands in FC-2. +Tags carry a ``kind`` enum and (for kind='character') an optional +``fandom_id`` FK to another tag of kind='fandom'. The uniqueness key is +(name, kind, COALESCE(fandom_id, 0)) so the same name can exist across +different kinds and the same character name can exist in different fandoms. """ from datetime import datetime +from enum import Enum -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Table, func +from sqlalchemy import ( + CheckConstraint, + Column, + DateTime, + Enum as SQLEnum, + ForeignKey, + Integer, + String, + Table, + func, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from .base import Base + +class TagKind(str, Enum): + artist = "artist" + character = "character" + fandom = "fandom" + general = "general" + series = "series" + archive = "archive" + post = "post" + meta = "meta" + rating = "rating" + + image_tag = Table( "image_tag", Base.metadata, @@ -21,20 +46,34 @@ image_tag = Table( primary_key=True, ), Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), - Column("source", String(32), nullable=False, default="manual"), # manual|auto|ml + Column("source", String(32), nullable=False, default="manual"), Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), ) class Tag(Base): __tablename__ = "tag" + __table_args__ = ( + CheckConstraint( + "(fandom_id IS NULL) OR (kind = 'character')", + name="ck_tag_fandom_requires_character", + ), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) - name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) - namespace: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + kind: Mapped[TagKind] = mapped_column( + SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]), + nullable=False, + default=TagKind.general, + ) + fandom_id: Mapped[int | None] = mapped_column( + ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True + ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) images = relationship("ImageRecord", secondary=image_tag, backref="tags") + fandom = relationship("Tag", remote_side="Tag.id", foreign_keys=[fandom_id]) diff --git a/tests/test_migration_0002.py b/tests/test_migration_0002.py new file mode 100644 index 0000000..6b8d13c --- /dev/null +++ b/tests/test_migration_0002.py @@ -0,0 +1,56 @@ +"""Smoke test for migration 0002: confirms model classes import and the +tag-kind uniqueness rule shape is correct. +""" + +from backend.app.models import ( + Base, + ImportBatch, + ImportSettings, + ImportTask, + Tag, + TagKind, +) + + +def test_new_tables_registered(): + expected = {"import_batch", "import_task", "import_settings"} + assert expected.issubset(Base.metadata.tables.keys()) + + +def test_tag_has_kind_and_fandom_id(): + cols = {c.name for c in Tag.__table__.columns} + assert "kind" in cols + assert "fandom_id" in cols + assert "namespace" not in cols + + +def test_tag_kind_enum_values(): + expected = { + "artist", + "character", + "fandom", + "general", + "series", + "archive", + "post", + "meta", + "rating", + } + assert {k.value for k in TagKind} == expected + + +def test_image_record_has_integrity_status(): + from backend.app.models import ImageRecord + cols = {c.name for c in ImageRecord.__table__.columns} + assert "integrity_status" in cols + + +def test_import_task_has_state_columns(): + cols = {c.name for c in ImportTask.__table__.columns} + for required in ("batch_id", "source_path", "task_type", "status", "result_image_id"): + assert required in cols + + +def test_import_settings_singleton_constraint(): + constraints = {c.name for c in ImportSettings.__table__.constraints} + assert "ck_import_settings_singleton" in constraints From 4cb319b393ac6f0f3bb0b73c0a006a1e54e0d455 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:04:05 -0400 Subject: [PATCH 018/272] feat(fc2a): add slug and paths utilities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slugify() produces ASCII-only lowercase slugs from arbitrary text (used for artist slugs and tag slugs). paths.derive_subdir/derive_top_level_artist extract the destination layout and folder→artist convention from an import source path. hash_suffixed_name builds IR-style 'stem__hash10.ext' filenames. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/utils/__init__.py | 1 + backend/app/utils/paths.py | 41 +++++++++++++++++++++++++++++++++ backend/app/utils/slug.py | 22 ++++++++++++++++++ tests/test_paths.py | 43 +++++++++++++++++++++++++++++++++++ tests/test_slug.py | 25 ++++++++++++++++++++ 5 files changed, 132 insertions(+) create mode 100644 backend/app/utils/__init__.py create mode 100644 backend/app/utils/paths.py create mode 100644 backend/app/utils/slug.py create mode 100644 tests/test_paths.py create mode 100644 tests/test_slug.py diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000..9be5cae --- /dev/null +++ b/backend/app/utils/__init__.py @@ -0,0 +1 @@ +"""Pure-function utilities shared across services and tasks.""" diff --git a/backend/app/utils/paths.py b/backend/app/utils/paths.py new file mode 100644 index 0000000..c19b7f8 --- /dev/null +++ b/backend/app/utils/paths.py @@ -0,0 +1,41 @@ +"""Filesystem path helpers — destination derivation, hash-suffixed names.""" + +from pathlib import Path + + +def derive_subdir(source_path: Path, import_root: Path) -> str: + """Returns the relative subdirectory of source_path under import_root. + + The top-level folder name is treated as the 'artist' bucket. Nested + paths preserve hierarchy. + + import_root=/import + source_path=/import/Alice/sub/x.png -> "Alice/sub" + source_path=/import/Alice/x.png -> "Alice" + source_path=/import/x.png -> "" + """ + try: + rel = source_path.parent.relative_to(import_root) + except ValueError: + return "" + return str(rel) if str(rel) != "." else "" + + +def hash_suffixed_name(stem: str, sha256_hex: str, ext: str) -> str: + """Builds 'stem__'. + + Examples: + hash_suffixed_name("photo", "abcdef1234567890...", ".png") + -> "photo__abcdef1234.png" + """ + return f"{stem}__{sha256_hex[:10]}{ext}" + + +def derive_top_level_artist(source_path: Path, import_root: Path) -> str | None: + """Returns the top-level folder name under import_root, or None if the + file is directly in import_root. + """ + subdir = derive_subdir(source_path, import_root) + if not subdir: + return None + return subdir.split("/", 1)[0] diff --git a/backend/app/utils/slug.py b/backend/app/utils/slug.py new file mode 100644 index 0000000..e7eeee1 --- /dev/null +++ b/backend/app/utils/slug.py @@ -0,0 +1,22 @@ +"""Slug generation for artists, tags, and paths.""" + +import re +import unicodedata + +_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +def slugify(value: str) -> str: + """Lowercase ASCII slug. Empty input becomes 'untitled'. + + Examples: + slugify("Bob Ross") -> "bob-ross" + slugify("hello, world!") -> "hello-world" + slugify("naïve café") -> "naive-cafe" + """ + if not value: + return "untitled" + normalized = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode() + normalized = normalized.lower() + slug = _SLUG_RE.sub("-", normalized).strip("-") + return slug or "untitled" diff --git a/tests/test_paths.py b/tests/test_paths.py new file mode 100644 index 0000000..9856fb5 --- /dev/null +++ b/tests/test_paths.py @@ -0,0 +1,43 @@ +from pathlib import Path + +from backend.app.utils.paths import ( + derive_subdir, + derive_top_level_artist, + hash_suffixed_name, +) + + +IMPORT = Path("/import") + + +def test_derive_subdir_nested(): + assert derive_subdir(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice/sub" + + +def test_derive_subdir_one_level(): + assert derive_subdir(Path("/import/Alice/x.png"), IMPORT) == "Alice" + + +def test_derive_subdir_root(): + assert derive_subdir(Path("/import/x.png"), IMPORT) == "" + + +def test_derive_subdir_outside_root(): + assert derive_subdir(Path("/elsewhere/x.png"), IMPORT) == "" + + +def test_hash_suffixed_name(): + h = "abcdef1234567890" + "0" * 48 + assert hash_suffixed_name("photo", h, ".png") == "photo__abcdef1234.png" + + +def test_derive_top_level_artist_present(): + assert derive_top_level_artist(Path("/import/Alice/x.png"), IMPORT) == "Alice" + + +def test_derive_top_level_artist_nested(): + assert derive_top_level_artist(Path("/import/Alice/sub/x.png"), IMPORT) == "Alice" + + +def test_derive_top_level_artist_root(): + assert derive_top_level_artist(Path("/import/x.png"), IMPORT) is None diff --git a/tests/test_slug.py b/tests/test_slug.py new file mode 100644 index 0000000..41e3092 --- /dev/null +++ b/tests/test_slug.py @@ -0,0 +1,25 @@ +from backend.app.utils.slug import slugify + + +def test_slugify_basic(): + assert slugify("Bob Ross") == "bob-ross" + + +def test_slugify_punctuation(): + assert slugify("hello, world!") == "hello-world" + + +def test_slugify_unicode(): + assert slugify("naïve café") == "naive-cafe" + + +def test_slugify_empty(): + assert slugify("") == "untitled" + + +def test_slugify_only_punctuation(): + assert slugify("!!!") == "untitled" + + +def test_slugify_leading_trailing_spaces(): + assert slugify(" Alice ") == "alice" From f6c71172319bbf2aa31cc8684355fa090dff86af Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:04:48 -0400 Subject: [PATCH 019/272] =?UTF-8?q?feat(fc2a):=20add=20TagService=20?= =?UTF-8?q?=E2=80=94=20find-or-create,=20autocomplete,=20image=20associati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validates kind/fandom rules at the service layer: fandom_id only allowed for kind='character', and must reference an existing kind='fandom' tag. Autocomplete ranking: exact match > prefix match > substring, tie-broken by image_count descending. Image-tag association uses INSERT ON CONFLICT DO NOTHING for idempotent re-tagging. conftest.py adds a transactional AsyncSession fixture; each test rolls back so they don't pollute each other. Also includes a sync Session fixture (db_sync) for the Importer tests in Task 5. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/__init__.py | 1 + backend/app/services/tag_service.py | 164 ++++++++++++++++++++++++++++ tests/conftest.py | 75 +++++++++++++ tests/test_tag_service.py | 101 +++++++++++++++++ 4 files changed, 341 insertions(+) create mode 100644 backend/app/services/__init__.py create mode 100644 backend/app/services/tag_service.py create mode 100644 tests/conftest.py create mode 100644 tests/test_tag_service.py diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..535d24d --- /dev/null +++ b/backend/app/services/__init__.py @@ -0,0 +1 @@ +"""Business-logic services consumed by API blueprints and Celery tasks.""" diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py new file mode 100644 index 0000000..1469f86 --- /dev/null +++ b/backend/app/services/tag_service.py @@ -0,0 +1,164 @@ +"""Tag CRUD + autocomplete + image-tag association.""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from sqlalchemy import and_, case, func, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Tag, TagKind, image_tag + + +class TagValidationError(ValueError): + """Raised when tag construction breaks the kind/fandom rules.""" + + +@dataclass(frozen=True) +class TagAutocompleteHit: + id: int + name: str + kind: str + fandom_id: int | None + fandom_name: str | None + image_count: int + + +class TagService: + def __init__(self, session: AsyncSession): + self.session = session + + async def find_or_create( + self, name: str, kind: TagKind, fandom_id: int | None = None + ) -> Tag: + """Upserts (name, kind, fandom_id) into the tag table. + + Validates kind/fandom rules before touching the DB: + - fandom_id may only be set when kind == TagKind.character + - if fandom_id is set, the referenced tag must exist and have + kind == TagKind.fandom + """ + name = name.strip() + if not name: + raise TagValidationError("Tag name cannot be empty") + + if fandom_id is not None and kind != TagKind.character: + raise TagValidationError( + "fandom_id is only valid for character tags" + ) + + if fandom_id is not None: + fandom = await self.session.get(Tag, fandom_id) + if fandom is None or fandom.kind != TagKind.fandom: + raise TagValidationError( + f"fandom_id {fandom_id} does not reference a fandom tag" + ) + + # Upsert via INSERT ... ON CONFLICT DO NOTHING. We can't use the + # uniqueness index name directly (it's a partial coalesce-based + # expression), so we re-select after insert. + stmt = ( + select(Tag) + .where(Tag.name == name) + .where(Tag.kind == kind) + .where( + Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id + ) + ) + existing = (await self.session.execute(stmt)).scalar_one_or_none() + if existing: + return existing + + new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id) + self.session.add(new_tag) + await self.session.flush() + return new_tag + + async def autocomplete( + self, + query: str, + kind: TagKind | None = None, + limit: int = 20, + ) -> list[TagAutocompleteHit]: + """ILIKE search ranked exact > prefix > substring, then by image count. + + Returns at most `limit` hits. + """ + q = (query or "").strip() + if not q: + return [] + + match_rank = case( + (func.lower(Tag.name) == q.lower(), 0), + (func.lower(Tag.name).startswith(q.lower()), 1), + else_=2, + ) + + fandom_alias = Tag.__table__.alias("fandom_lookup") + image_count = ( + select(func.count(image_tag.c.image_record_id)) + .where(image_tag.c.tag_id == Tag.id) + .correlate(Tag.__table__) + .scalar_subquery() + ) + + stmt = ( + select( + Tag.id, + Tag.name, + Tag.kind, + Tag.fandom_id, + fandom_alias.c.name.label("fandom_name"), + image_count.label("image_count"), + ) + .select_from(Tag.__table__.outerjoin( + fandom_alias, Tag.fandom_id == fandom_alias.c.id + )) + .where(func.lower(Tag.name).like(f"%{q.lower()}%")) + .order_by(match_rank.asc(), image_count.desc(), Tag.name.asc()) + .limit(limit) + ) + if kind is not None: + stmt = stmt.where(Tag.kind == kind) + + rows = (await self.session.execute(stmt)).all() + return [ + TagAutocompleteHit( + id=r.id, + name=r.name, + kind=r.kind.value if hasattr(r.kind, "value") else r.kind, + fandom_id=r.fandom_id, + fandom_name=r.fandom_name, + image_count=r.image_count or 0, + ) + for r in rows + ] + + async def add_to_image(self, image_id: int, tag_id: int, source: str = "manual") -> None: + """Idempotent: re-adding an existing tag does nothing.""" + stmt = insert(image_tag).values( + image_record_id=image_id, tag_id=tag_id, source=source + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + await self.session.execute(stmt) + + async def remove_from_image(self, image_id: int, tag_id: int) -> None: + await self.session.execute( + image_tag.delete().where( + and_( + image_tag.c.image_record_id == image_id, + image_tag.c.tag_id == tag_id, + ) + ) + ) + + async def list_for_image(self, image_id: int) -> Sequence[Tag]: + stmt = ( + select(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id == image_id) + .order_by(Tag.kind.asc(), Tag.name.asc()) + ) + return (await self.session.execute(stmt)).scalars().all() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..73b3ef9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,75 @@ +"""Shared pytest fixtures. + +The async db fixture provides an AsyncSession bound to a transaction that +gets rolled back after each test. CI provisions a real Postgres + pgvector +(see .forgejo/workflows/ci.yml), so tests exercise the actual schema and +migration code paths. +""" + +import os + +import pytest +import pytest_asyncio +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.orm import sessionmaker + + +def _async_database_url() -> str: + user = os.environ.get("DB_USER", "fabledcurator") + password = os.environ["DB_PASSWORD"] + host = os.environ.get("DB_HOST", "postgres") + port = os.environ.get("DB_PORT", "5432") + name = os.environ.get("DB_NAME", "fabledcurator_test") + return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}" + + +def _sync_database_url() -> str: + user = os.environ.get("DB_USER", "fabledcurator") + password = os.environ["DB_PASSWORD"] + host = os.environ.get("DB_HOST", "postgres") + port = os.environ.get("DB_PORT", "5432") + name = os.environ.get("DB_NAME", "fabledcurator_test") + return f"postgresql+psycopg://{user}:{password}@{host}:{port}/{name}" + + +@pytest_asyncio.fixture +async def engine(): + e = create_async_engine(_async_database_url(), future=True) + yield e + await e.dispose() + + +@pytest_asyncio.fixture +async def db(engine) -> AsyncSession: + Session = async_sessionmaker(engine, expire_on_commit=False) + async with Session() as session: + await session.begin() + try: + yield session + finally: + await session.rollback() + + +@pytest.fixture +def sync_engine(): + e = create_engine(_sync_database_url(), future=True) + yield e + e.dispose() + + +@pytest.fixture +def db_sync(sync_engine): + """Synchronous Session bound to a savepoint — used by Importer tests + (the Importer is sync-only by design).""" + SyncSession = sessionmaker(sync_engine, expire_on_commit=False) + with SyncSession() as session: + session.begin() + try: + yield session + finally: + session.rollback() diff --git a/tests/test_tag_service.py b/tests/test_tag_service.py new file mode 100644 index 0000000..65a8f26 --- /dev/null +++ b/tests/test_tag_service.py @@ -0,0 +1,101 @@ +import pytest + +from backend.app.models import TagKind +from backend.app.services.tag_service import TagService, TagValidationError + + +@pytest.mark.asyncio +async def test_find_or_create_creates(db): + svc = TagService(db) + t = await svc.find_or_create("Alice", TagKind.character, fandom_id=None) + assert t.id is not None + assert t.name == "Alice" + assert t.kind == TagKind.character + + +@pytest.mark.asyncio +async def test_find_or_create_idempotent(db): + svc = TagService(db) + a = await svc.find_or_create("Bob", TagKind.artist) + b = await svc.find_or_create("Bob", TagKind.artist) + assert a.id == b.id + + +@pytest.mark.asyncio +async def test_same_name_different_kind_coexists(db): + svc = TagService(db) + artist = await svc.find_or_create("Forge", TagKind.artist) + series = await svc.find_or_create("Forge", TagKind.series) + assert artist.id != series.id + + +@pytest.mark.asyncio +async def test_fandom_id_requires_character(db): + svc = TagService(db) + fandom = await svc.find_or_create("Naruto", TagKind.fandom) + with pytest.raises(TagValidationError, match="character"): + await svc.find_or_create("Bob", TagKind.artist, fandom_id=fandom.id) + + +@pytest.mark.asyncio +async def test_fandom_id_must_reference_fandom(db): + svc = TagService(db) + artist = await svc.find_or_create("X", TagKind.artist) + with pytest.raises(TagValidationError, match="fandom"): + await svc.find_or_create("Y", TagKind.character, fandom_id=artist.id) + + +@pytest.mark.asyncio +async def test_character_with_fandom(db): + svc = TagService(db) + fandom = await svc.find_or_create("Naruto", TagKind.fandom) + char = await svc.find_or_create("Sasuke", TagKind.character, fandom_id=fandom.id) + assert char.fandom_id == fandom.id + + +@pytest.mark.asyncio +async def test_character_same_name_different_fandoms(db): + svc = TagService(db) + f1 = await svc.find_or_create("Series A", TagKind.fandom) + f2 = await svc.find_or_create("Series B", TagKind.fandom) + c1 = await svc.find_or_create("Alice", TagKind.character, fandom_id=f1.id) + c2 = await svc.find_or_create("Alice", TagKind.character, fandom_id=f2.id) + assert c1.id != c2.id + + +@pytest.mark.asyncio +async def test_empty_name_raises(db): + svc = TagService(db) + with pytest.raises(TagValidationError): + await svc.find_or_create(" ", TagKind.general) + + +@pytest.mark.asyncio +async def test_autocomplete_ranking(db): + svc = TagService(db) + await svc.find_or_create("alice", TagKind.character) + await svc.find_or_create("alicent", TagKind.character) + await svc.find_or_create("malice", TagKind.character) + hits = await svc.autocomplete("alice") + # exact match first, then prefix, then substring + names = [h.name for h in hits] + assert names[0] == "alice" + assert names[1] == "alicent" + assert names[2] == "malice" + + +@pytest.mark.asyncio +async def test_autocomplete_kind_filter(db): + svc = TagService(db) + await svc.find_or_create("Foo", TagKind.artist) + await svc.find_or_create("Foo", TagKind.character) + hits = await svc.autocomplete("Foo", kind=TagKind.artist) + assert len(hits) == 1 + assert hits[0].kind == "artist" + + +@pytest.mark.asyncio +async def test_autocomplete_empty_query_returns_nothing(db): + svc = TagService(db) + await svc.find_or_create("Foo", TagKind.artist) + assert await svc.autocomplete("") == [] From 1a2a120da124af68366b05b5009ae7ca8d85ea4a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:05:19 -0400 Subject: [PATCH 020/272] feat(fc2a): add Thumbnailer service Generates 400px-max thumbnails: PNG with alpha preserved when source has transparency, JPEG (q=85, progressive) otherwise. Storage layout is /images/thumbs/{sha[:3]}/{sha}.{jpg|png} so each prefix bucket holds at most ~4k files (16^3 buckets across all possible SHAs). Video path uses ffmpeg's first-frame snapshot at ~5% offset; we have a 1-second floor for short clips. ffmpeg call has a 60s timeout. Video tests are deferred to FC-2e's integration suite where transcoding is also wired up. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/thumbnailer.py | 102 ++++++++++++++++++++++++++++ tests/test_thumbnailer.py | 66 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 backend/app/services/thumbnailer.py create mode 100644 tests/test_thumbnailer.py diff --git a/backend/app/services/thumbnailer.py b/backend/app/services/thumbnailer.py new file mode 100644 index 0000000..eea4bed --- /dev/null +++ b/backend/app/services/thumbnailer.py @@ -0,0 +1,102 @@ +"""Image + video thumbnail generation. + +Images go through Pillow; videos go through ffmpeg's first-frame snapshot +at ~5% into the runtime (avoids fade-ins/title cards while still being a +representative frame). + +Thumbnails are stored at /images/thumbs/{sha[:3]}/{sha}.jpg or .png; the +3-char prefix bucket avoids hot single-directory inode pressure. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image + +THUMB_MAX_PX = 400 +JPEG_QUALITY = 85 +FFMPEG_TIMEOUT_SECONDS = 60 + + +@dataclass(frozen=True) +class ThumbnailResult: + path: Path + width: int + height: int + mime: str + + +def thumbnail_path_for(sha256_hex: str, *, alpha: bool, root: Path) -> Path: + ext = ".png" if alpha else ".jpg" + bucket = sha256_hex[:3] + return root / "thumbs" / bucket / f"{sha256_hex}{ext}" + + +class Thumbnailer: + def __init__(self, images_root: Path): + self.images_root = images_root + + def generate_image_thumbnail( + self, source: Path, sha256_hex: str + ) -> ThumbnailResult: + """Reads source, fits inside THUMB_MAX_PX × THUMB_MAX_PX preserving + aspect ratio. Writes PNG if source has alpha, else JPEG. + """ + with Image.open(source) as im: + im.load() + has_alpha = im.mode in ("RGBA", "LA") or ( + im.mode == "P" and "transparency" in im.info + ) + if has_alpha: + im = im.convert("RGBA") + else: + im = im.convert("RGB") + im.thumbnail((THUMB_MAX_PX, THUMB_MAX_PX), Image.Resampling.LANCZOS) + + dest = thumbnail_path_for(sha256_hex, alpha=has_alpha, root=self.images_root) + dest.parent.mkdir(parents=True, exist_ok=True) + if has_alpha: + im.save(dest, "PNG", optimize=True) + mime = "image/png" + else: + im.save(dest, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=True) + mime = "image/jpeg" + return ThumbnailResult(path=dest, width=im.width, height=im.height, mime=mime) + + def generate_video_thumbnail( + self, source: Path, sha256_hex: str, duration_seconds: float | None = None + ) -> ThumbnailResult: + """Snapshots a frame ~5% into the video via ffmpeg, then runs that + frame through the image-thumbnail pipeline. + + duration_seconds is optional; if absent, we snapshot at the 1-second + mark which is a reasonable fallback for short clips. + """ + offset = max(1.0, (duration_seconds or 20.0) * 0.05) + tmp = self.images_root / "thumbs" / "_tmp" / f"{sha256_hex}.png" + tmp.parent.mkdir(parents=True, exist_ok=True) + try: + cmd = [ + "ffmpeg", + "-ss", + f"{offset:.2f}", + "-i", + str(source), + "-frames:v", + "1", + "-vf", + f"scale='if(gt(iw,ih),{THUMB_MAX_PX},-2)':'if(gt(iw,ih),-2,{THUMB_MAX_PX})'", + "-y", + str(tmp), + ] + subprocess.run( + cmd, + check=True, + capture_output=True, + timeout=FFMPEG_TIMEOUT_SECONDS, + ) + return self.generate_image_thumbnail(tmp, sha256_hex) + finally: + if tmp.exists(): + tmp.unlink() diff --git a/tests/test_thumbnailer.py b/tests/test_thumbnailer.py new file mode 100644 index 0000000..851de13 --- /dev/null +++ b/tests/test_thumbnailer.py @@ -0,0 +1,66 @@ +"""Image-thumbnailer smoke tests. Video tests are not included here because +they require a real ffmpeg + a real video on disk; FC-2e's integration suite +will cover those end-to-end against the running container. +""" + +from pathlib import Path + +import pytest +from PIL import Image + +from backend.app.services.thumbnailer import ( + THUMB_MAX_PX, + Thumbnailer, + thumbnail_path_for, +) + + +@pytest.fixture +def thumbnailer(tmp_path): + return Thumbnailer(images_root=tmp_path) + + +def _make_image(path: Path, size: tuple[int, int], mode: str = "RGB"): + im = Image.new(mode, size, color=(200, 100, 50, 255) if mode == "RGBA" else (200, 100, 50)) + path.parent.mkdir(parents=True, exist_ok=True) + im.save(path) + + +def test_thumbnail_path_for_jpeg(): + p = thumbnail_path_for("abcdef" + "0" * 58, alpha=False, root=Path("/images")) + assert p == Path("/images/thumbs/abc/" + "abcdef" + "0" * 58 + ".jpg") + + +def test_thumbnail_path_for_png(): + p = thumbnail_path_for("abcdef" + "0" * 58, alpha=True, root=Path("/images")) + assert p.suffix == ".png" + + +def test_generate_thumbnail_rgb(thumbnailer, tmp_path): + src = tmp_path / "src.jpg" + _make_image(src, (1600, 1200), mode="RGB") + sha = "deadbeef" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.mime == "image/jpeg" + assert result.path.suffix == ".jpg" + assert result.path.exists() + assert max(result.width, result.height) == THUMB_MAX_PX + + +def test_generate_thumbnail_rgba_preserves_alpha(thumbnailer, tmp_path): + src = tmp_path / "src.png" + _make_image(src, (1200, 800), mode="RGBA") + sha = "feedface" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.mime == "image/png" + assert result.path.suffix == ".png" + + +def test_generate_thumbnail_tall_image(thumbnailer, tmp_path): + src = tmp_path / "src.jpg" + _make_image(src, (200, 2000), mode="RGB") + sha = "cafef00d" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.height == THUMB_MAX_PX + # tall image: width scales down proportionally + assert result.width < THUMB_MAX_PX From bb9c183ff7ab85bbaef047afe16473de1f19cc56 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:07:01 -0400 Subject: [PATCH 021/272] feat(fc2a): add Importer service Single-file pipeline: validate as image, apply filter rules (min dimensions, transparency), SHA256 hash dedup, atomic copy to /images/{subdir}/, create ImageRecord with origin='imported_filesystem' and integrity_status='unknown', auto-derive top-level folder name as Artist + artist tag. Importer is intentionally sync (consumed by a Celery worker process); the async Quart side uses the same ORM through its own async session. The db_sync fixture in conftest.py was added in Task 3 to support these tests. Thumbnail generation is NOT inlined; the calling Celery task enqueues a separate thumbnail task so the import queue keeps moving on big batches. pHash dedup is FC-2d, not here. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/importer.py | 248 +++++++++++++++++++++++++++++++ tests/test_importer.py | 132 ++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 backend/app/services/importer.py create mode 100644 tests/test_importer.py diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py new file mode 100644 index 0000000..fb1b378 --- /dev/null +++ b/backend/app/services/importer.py @@ -0,0 +1,248 @@ +"""Single-file import pipeline. + +The Importer is the synchronous core of FC-2a's import path. A Celery task +(tasks/import_file.py) instantiates one per job and calls import_one(). + +The service is intentionally not async — Celery tasks are run in a +synchronous worker process, and the DB session is a sync session passed in +by the task. The Quart side uses an async session via the same models. +""" + +import hashlib +import shutil +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +from PIL import Image +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind +from ..models.tag import image_tag +from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name +from ..utils.slug import slugify +from .thumbnailer import Thumbnailer + + +class SkipReason(str, Enum): + too_small = "too_small" + too_transparent = "too_transparent" + single_color = "single_color" + duplicate_hash = "duplicate_hash" + invalid_image = "invalid_image" + + +@dataclass(frozen=True) +class ImportResult: + status: str # 'imported' | 'skipped' | 'failed' + image_id: int | None = None + skip_reason: SkipReason | None = None + error: str | None = None + + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"} +VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"} +ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS + + +def is_supported(path: Path) -> bool: + return path.suffix.lower() in ALL_EXTS + + +def is_video(path: Path) -> bool: + return path.suffix.lower() in VIDEO_EXTS + + +def _mime_for(path: Path) -> str: + suffix = path.suffix.lower() + image_mimes = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + ".tif": "image/tiff", + ".tiff": "image/tiff", + } + video_mimes = { + ".mp4": "video/mp4", + ".mov": "video/quicktime", + ".avi": "video/x-msvideo", + ".mkv": "video/x-matroska", + ".webm": "video/webm", + ".m4v": "video/x-m4v", + ".wmv": "video/x-ms-wmv", + ".flv": "video/x-flv", + } + return image_mimes.get(suffix) or video_mimes.get(suffix) or "application/octet-stream" + + +def _sha256_of(path: Path, chunk_size: int = 1 << 20) -> str: + h = hashlib.sha256() + with path.open("rb") as fp: + while True: + block = fp.read(chunk_size) + if not block: + break + h.update(block) + return h.hexdigest() + + +class Importer: + """Imports one file at a time. Constructed per-task by the Celery task.""" + + def __init__( + self, + session: Session, + images_root: Path, + import_root: Path, + thumbnailer: Thumbnailer, + settings: ImportSettings, + ): + self.session = session + self.images_root = images_root + self.import_root = import_root + self.thumbnailer = thumbnailer + self.settings = settings + + def import_one(self, source: Path) -> ImportResult: + """The full pipeline for a single file. + + Returns an ImportResult; callers (the Celery task) are responsible for + translating that into ImportTask.status and ImageRecord linkage. + """ + if not is_supported(source): + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error=f"unsupported extension {source.suffix}", + ) + + # Compute file dimensions (images only) and apply filters. + width = height = None + has_alpha = False + if not is_video(source): + try: + with Image.open(source) as im: + im.verify() + with Image.open(source) as im: + width, height = im.size + has_alpha = im.mode in ("RGBA", "LA") or ( + im.mode == "P" and "transparency" in im.info + ) + except Exception as exc: + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc), + ) + + if (self.settings.min_width and width < self.settings.min_width) or ( + self.settings.min_height and height < self.settings.min_height + ): + return ImportResult( + status="skipped", skip_reason=SkipReason.too_small, + error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}", + ) + + if self.settings.skip_transparent and has_alpha: + pct = self._transparency_pct(source) + if pct >= self.settings.transparency_threshold: + return ImportResult( + status="skipped", skip_reason=SkipReason.too_transparent, + error=f"{pct:.2%} transparent", + ) + + # Hash dedup. + sha = _sha256_of(source) + existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha) + existing = self.session.execute(existing_stmt).scalar_one_or_none() + if existing: + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="sha256 already present", + ) + + # Destination path. + subdir = derive_subdir(source, self.import_root) + dest_dir = self.images_root / subdir if subdir else self.images_root + dest_dir.mkdir(parents=True, exist_ok=True) + dest_name = hash_suffixed_name(source.stem, sha, source.suffix) + dest = dest_dir / dest_name + + # Atomic copy: write to .partial, rename. Avoids half-imported files on crash. + partial = dest.with_suffix(dest.suffix + ".partial") + shutil.copy2(source, partial) + partial.rename(dest) + + record = ImageRecord( + path=str(dest), + sha256=sha, + size_bytes=dest.stat().st_size, + mime=_mime_for(source), + width=width, + height=height, + origin="imported_filesystem", + integrity_status="unknown", + ) + self.session.add(record) + self.session.flush() + + # Folder→artist auto-tag. + artist_name = derive_top_level_artist(source, self.import_root) + if artist_name: + self._attach_artist_tag(record.id, artist_name) + + # Thumbnail is queued separately by the calling task; the importer + # does not generate thumbnails inline so the import queue stays moving. + + self.session.commit() + return ImportResult(status="imported", image_id=record.id) + + def _attach_artist_tag(self, image_id: int, artist_name: str) -> None: + """Idempotently creates Artist + artist tag, attaches to image.""" + slug = slugify(artist_name) + artist = self.session.execute( + select(Artist).where(Artist.slug == slug) + ).scalar_one_or_none() + if artist is None: + artist = Artist(name=artist_name, slug=slug, is_subscription=False) + self.session.add(artist) + self.session.flush() + + tag = self.session.execute( + select(Tag).where(Tag.name == artist_name).where(Tag.kind == TagKind.artist) + ).scalar_one_or_none() + if tag is None: + tag = Tag(name=artist_name, kind=TagKind.artist) + self.session.add(tag) + self.session.flush() + + # Idempotent association; using a raw insert + on-conflict-do-nothing + # is cleaner here than fetching first, but for a sync session we just + # check existence. + already = self.session.execute( + select(image_tag.c.tag_id) + .where(image_tag.c.image_record_id == image_id) + .where(image_tag.c.tag_id == tag.id) + ).scalar_one_or_none() + if already is None: + self.session.execute( + image_tag.insert().values( + image_record_id=image_id, tag_id=tag.id, source="auto", + ) + ) + + def _transparency_pct(self, source: Path) -> float: + """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.""" + with Image.open(source) as im: + if im.mode not in ("RGBA", "LA") and not ( + im.mode == "P" and "transparency" in im.info + ): + return 0.0 + if im.mode != "RGBA": + im = im.convert("RGBA") + alpha = im.getchannel("A") + histogram = alpha.histogram() + transparent = histogram[0] + total = sum(histogram) + return transparent / total if total else 0.0 diff --git a/tests/test_importer.py b/tests/test_importer.py new file mode 100644 index 0000000..5739355 --- /dev/null +++ b/tests/test_importer.py @@ -0,0 +1,132 @@ +"""Importer integration tests. + +These exercise the real Importer against a real Postgres + a real filesystem +(via tmp_path). The DB fixture rolls back after each test so the import_root +fixture is fresh per-test. +""" + +from pathlib import Path + +import pytest +from PIL import Image +from sqlalchemy import select + +from backend.app.models import Artist, ImageRecord, ImportSettings, Tag, TagKind +from backend.app.services.importer import Importer, SkipReason +from backend.app.services.thumbnailer import Thumbnailer + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +def _make_jpeg(path: Path, size: tuple[int, int] = (800, 600)): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", size, color=(120, 200, 80)).save(path, "JPEG") + + +def _make_png_rgba(path: Path, size: tuple[int, int], alpha: int): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGBA", size, color=(120, 200, 80, alpha)).save(path, "PNG") + + +@pytest.fixture +def importer(db_sync, import_layout): + import_root, images_root = import_layout + settings = db_sync.execute(select(ImportSettings).where(ImportSettings.id == 1)).scalar_one() + thumbnailer = Thumbnailer(images_root=images_root) + return Importer( + session=db_sync, + images_root=images_root, + import_root=import_root, + thumbnailer=thumbnailer, + settings=settings, + ) + + +def test_import_one_happy_path(importer, import_layout): + import_root, images_root = import_layout + src = import_root / "Alice" / "first.jpg" + _make_jpeg(src) + result = importer.import_one(src) + assert result.status == "imported" + record = importer.session.get(ImageRecord, result.image_id) + assert record.path.startswith(str(images_root / "Alice")) + assert record.path.endswith(".jpg") + + +def test_folder_creates_artist_and_tag(importer, import_layout): + import_root, _ = import_layout + src = import_root / "Alice" / "first.jpg" + _make_jpeg(src) + importer.import_one(src) + + artist = importer.session.execute( + select(Artist).where(Artist.slug == "alice") + ).scalar_one() + assert artist.name == "Alice" + + tag = importer.session.execute( + select(Tag).where(Tag.name == "Alice").where(Tag.kind == TagKind.artist) + ).scalar_one() + assert tag.id is not None + + +def test_dedup_by_hash(importer, import_layout): + import_root, _ = import_layout + src1 = import_root / "Alice" / "first.jpg" + src2 = import_root / "Bob" / "duplicate.jpg" + _make_jpeg(src1) + src2.parent.mkdir(parents=True, exist_ok=True) + src2.write_bytes(src1.read_bytes()) + + r1 = importer.import_one(src1) + r2 = importer.import_one(src2) + assert r1.status == "imported" + assert r2.status == "skipped" + assert r2.skip_reason == SkipReason.duplicate_hash + + +def test_min_width_filter(importer, import_layout): + import_root, _ = import_layout + importer.settings.min_width = 1000 # mutate the in-memory copy for this test + src = import_root / "small.jpg" + _make_jpeg(src, size=(500, 500)) + result = importer.import_one(src) + assert result.status == "skipped" + assert result.skip_reason == SkipReason.too_small + + +def test_transparent_filter(importer, import_layout): + import_root, _ = import_layout + importer.settings.skip_transparent = True + importer.settings.transparency_threshold = 0.5 + src = import_root / "ghost.png" + _make_png_rgba(src, size=(100, 100), alpha=0) # fully transparent + result = importer.import_one(src) + assert result.status == "skipped" + assert result.skip_reason == SkipReason.too_transparent + + +def test_unsupported_extension(importer, import_layout): + import_root, _ = import_layout + src = import_root / "Alice" / "notes.txt" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_text("hello") + result = importer.import_one(src) + assert result.status == "skipped" + assert result.skip_reason == SkipReason.invalid_image + + +def test_root_level_file_has_no_artist(importer, import_layout): + import_root, _ = import_layout + src = import_root / "loose.jpg" + _make_jpeg(src) + importer.import_one(src) + artists = importer.session.execute(select(Artist)).scalars().all() + assert artists == [] From f38a1d48c5fe01ab992170183e1f7b32f869d4a9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:07:54 -0400 Subject: [PATCH 022/272] =?UTF-8?q?feat(fc2a):=20add=20GalleryService=20?= =?UTF-8?q?=E2=80=94=20cursor=20scroll,=20timeline,=20image=20detail=20wit?= =?UTF-8?q?h=20neighbors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor format: base64(iso8601_created_at|image_id). Pagination key is (created_at DESC, id DESC) so we don't drift when new imports land between page loads. Timeline groups by date_part(year, month) so the sidebar can render year-month jump buckets. get_image_with_tags returns full image detail plus prev/next ids so the modal viewer can navigate without an extra round-trip per arrow press. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/gallery_service.py | 244 ++++++++++++++++++++++++ tests/test_gallery_service.py | 122 ++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 backend/app/services/gallery_service.py create mode 100644 tests/test_gallery_service.py diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py new file mode 100644 index 0000000..1aef465 --- /dev/null +++ b/backend/app/services/gallery_service.py @@ -0,0 +1,244 @@ +"""Cursor-paginated gallery queries. + +Cursor format: opaque base64-encoded ":". +Pagination key is (created_at DESC, id DESC) so we don't drift when new +imports arrive between page loads. Decoding rejects malformed cursors with +a ValueError; the API layer translates that to HTTP 400. +""" + +import base64 +from dataclasses import dataclass +from datetime import datetime + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord, Tag +from ..models.tag import image_tag + + +CURSOR_SEPARATOR = "|" + + +def encode_cursor(created_at: datetime, image_id: int) -> str: + raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}" + return base64.urlsafe_b64encode(raw.encode()).decode() + + +def decode_cursor(cursor: str) -> tuple[datetime, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1) + return datetime.fromisoformat(ts_part), int(id_part) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class GalleryImage: + id: int + path: str + sha256: str + mime: str + width: int | None + height: int | None + created_at: datetime + thumbnail_url: str + + +@dataclass(frozen=True) +class GalleryPage: + images: list[GalleryImage] + next_cursor: str | None + date_groups: list[tuple[int, int, list[int]]] # (year, month, [image_id...]) + + +@dataclass(frozen=True) +class TimelineBucket: + year: int + month: int + count: int + + +def thumbnail_url(sha256_hex: str, mime: str) -> str: + # Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go + # under /images/thumbs/. The MIME determines the extension. + ext = ".png" if mime in ("image/png", "image/gif") else ".jpg" + bucket = sha256_hex[:3] + return f"/images/thumbs/{bucket}/{sha256_hex}{ext}" + + +class GalleryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def scroll( + self, + cursor: str | None, + limit: int = 50, + tag_id: int | None = None, + ) -> GalleryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + + stmt = select(ImageRecord) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + + if cursor: + cur_ts, cur_id = decode_cursor(cursor) + stmt = stmt.where( + or_( + ImageRecord.created_at < cur_ts, + and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id), + ) + ) + + stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).scalars().all() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = encode_cursor(last.created_at, last.id) + rows = rows[:limit] + + images = [ + GalleryImage( + id=r.id, + path=r.path, + sha256=r.sha256, + mime=r.mime, + width=r.width, + height=r.height, + created_at=r.created_at, + thumbnail_url=thumbnail_url(r.sha256, r.mime), + ) + for r in rows + ] + return GalleryPage( + images=images, + next_cursor=next_cursor, + date_groups=_group_by_year_month(images), + ) + + async def timeline(self, tag_id: int | None = None) -> list[TimelineBucket]: + year_col = func.date_part("year", ImageRecord.created_at).label("yr") + month_col = func.date_part("month", ImageRecord.created_at).label("mo") + stmt = select( + year_col, month_col, func.count(ImageRecord.id).label("cnt") + ) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc()) + rows = (await self.session.execute(stmt)).all() + return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows] + + async def jump_cursor( + self, year: int, month: int, tag_id: int | None = None + ) -> str | None: + """Returns a cursor that, when passed to scroll(), positions at the + first image of the given year-month. None if the bucket is empty. + """ + from sqlalchemy import extract + + stmt = select(ImageRecord).where( + extract("year", ImageRecord.created_at) == year, + extract("month", ImageRecord.created_at) == month, + ) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1) + first = (await self.session.execute(stmt)).scalar_one_or_none() + if first is None: + return None + # Cursor is exclusive; we encode a cursor with id+1 so the row itself + # is the first result in the next scroll(). + return encode_cursor(first.created_at, first.id + 1) + + async def get_image_with_tags(self, image_id: int) -> dict | None: + record = await self.session.get(ImageRecord, image_id) + if record is None: + return None + tag_stmt = ( + select(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id == image_id) + .order_by(Tag.kind.asc(), Tag.name.asc()) + ) + tags = (await self.session.execute(tag_stmt)).scalars().all() + neighbors = await self._neighbors(record) + return { + "id": record.id, + "path": record.path, + "sha256": record.sha256, + "mime": record.mime, + "width": record.width, + "height": record.height, + "size_bytes": record.size_bytes, + "created_at": record.created_at.isoformat(), + "thumbnail_url": thumbnail_url(record.sha256, record.mime), + "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", + "tags": [ + { + "id": t.id, + "name": t.name, + "kind": t.kind.value if hasattr(t.kind, "value") else t.kind, + "fandom_id": t.fandom_id, + } + for t in tags + ], + "neighbors": neighbors, + } + + async def _neighbors(self, record: ImageRecord) -> dict: + prev_stmt = ( + select(ImageRecord.id) + .where( + or_( + ImageRecord.created_at > record.created_at, + and_( + ImageRecord.created_at == record.created_at, + ImageRecord.id > record.id, + ), + ) + ) + .order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc()) + .limit(1) + ) + next_stmt = ( + select(ImageRecord.id) + .where( + or_( + ImageRecord.created_at < record.created_at, + and_( + ImageRecord.created_at == record.created_at, + ImageRecord.id < record.id, + ), + ) + ) + .order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()) + .limit(1) + ) + prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none() + next_id = (await self.session.execute(next_stmt)).scalar_one_or_none() + return {"prev_id": prev_id, "next_id": next_id} + + +def _group_by_year_month( + images: list[GalleryImage], +) -> list[tuple[int, int, list[int]]]: + groups: list[tuple[int, int, list[int]]] = [] + for img in images: + y, m = img.created_at.year, img.created_at.month + if groups and groups[-1][0] == y and groups[-1][1] == m: + groups[-1][2].append(img.id) + else: + groups.append((y, m, [img.id])) + return groups diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py new file mode 100644 index 0000000..3d6d263 --- /dev/null +++ b/tests/test_gallery_service.py @@ -0,0 +1,122 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.gallery_service import ( + GalleryService, + decode_cursor, + encode_cursor, +) + + +def _now(): + return datetime.now(timezone.utc) + + +async def _seed_images(db, count: int, sha_prefix: str = "0") -> list[ImageRecord]: + base = _now() + records = [] + for i in range(count): + r = ImageRecord( + path=f"/images/test/{i}.jpg", + sha256=f"{sha_prefix}{i:063d}", + size_bytes=1000, + mime="image/jpeg", + width=100, + height=100, + origin="imported_filesystem", + integrity_status="unknown", + ) + r.created_at = base - timedelta(minutes=i) + db.add(r) + records.append(r) + await db.flush() + return records + + +def test_cursor_roundtrip(): + ts = datetime(2026, 5, 14, 12, 30, 0, tzinfo=timezone.utc) + encoded = encode_cursor(ts, 42) + back_ts, back_id = decode_cursor(encoded) + assert back_ts == ts + assert back_id == 42 + + +def test_decode_invalid_cursor_raises(): + with pytest.raises(ValueError): + decode_cursor("not-base64!!!") + + +@pytest.mark.asyncio +async def test_scroll_returns_newest_first(db): + await _seed_images(db, 5) + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10) + assert len(page.images) == 5 + assert page.images[0].created_at > page.images[-1].created_at + + +@pytest.mark.asyncio +async def test_scroll_pagination(db): + await _seed_images(db, 5) + svc = GalleryService(db) + first = await svc.scroll(cursor=None, limit=2) + assert len(first.images) == 2 + assert first.next_cursor is not None + second = await svc.scroll(cursor=first.next_cursor, limit=2) + assert len(second.images) == 2 + assert {i.id for i in first.images}.isdisjoint({i.id for i in second.images}) + + +@pytest.mark.asyncio +async def test_scroll_with_tag_filter(db): + images = await _seed_images(db, 5) + tag = Tag(name="filterme", kind=TagKind.general) + db.add(tag) + await db.flush() + await db.execute( + image_tag.insert().values( + image_record_id=images[0].id, tag_id=tag.id, source="manual" + ) + ) + + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10, tag_id=tag.id) + assert len(page.images) == 1 + assert page.images[0].id == images[0].id + + +@pytest.mark.asyncio +async def test_timeline_groups_by_month(db): + await _seed_images(db, 3) + svc = GalleryService(db) + buckets = await svc.timeline() + assert sum(b.count for b in buckets) == 3 + + +@pytest.mark.asyncio +async def test_date_groups_in_page(db): + await _seed_images(db, 3) + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10) + assert len(page.date_groups) == 1 + y, m, ids = page.date_groups[0] + assert len(ids) == 3 + + +@pytest.mark.asyncio +async def test_neighbors(db): + images = await _seed_images(db, 3) + svc = GalleryService(db) + middle = images[1] + payload = await svc.get_image_with_tags(middle.id) + assert payload["neighbors"]["prev_id"] == images[0].id + assert payload["neighbors"]["next_id"] == images[2].id + + +@pytest.mark.asyncio +async def test_get_image_with_tags_returns_none_for_missing(db): + svc = GalleryService(db) + assert await svc.get_image_with_tags(99999) is None From d3bb8c509ad9f212f2b71dda5a23811693bfaf63 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:08:37 -0400 Subject: [PATCH 023/272] =?UTF-8?q?feat(fc2a):=20add=20Celery=20tasks=20?= =?UTF-8?q?=E2=80=94=20scan=5Fdirectory,=20import=5Fmedia=5Ffile,=20genera?= =?UTF-8?q?te=5Fthumbnail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scan_directory walks ImportSettings.import_scan_path, creates an ImportBatch, enumerates supported files into ImportTasks, and enqueues import_media_file per task. import_media_file moves the task through its state machine (pending → queued → processing → complete/skipped/failed), updates ImportBatch counters atomically (UPDATE ... SET col = col + 1), enqueues a thumbnail task on success, and marks the batch complete when the last task drains. generate_thumbnail runs on its own queue (thumbnail) so big imports don't starve thumbnail throughput; failure here is logged and does not fail the import. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/celery_app.py | 8 +- backend/app/tasks/import_file.py | 122 +++++++++++++++++++++++++++++++ backend/app/tasks/scan.py | 78 ++++++++++++++++++++ backend/app/tasks/thumbnail.py | 50 +++++++++++++ tests/test_tasks_register.py | 15 ++++ 5 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 backend/app/tasks/import_file.py create mode 100644 backend/app/tasks/scan.py create mode 100644 backend/app/tasks/thumbnail.py create mode 100644 tests/test_tasks_register.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 1d89c78..bf4237e 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -22,7 +22,13 @@ def make_celery() -> Celery: "fabledcurator", broker=cfg.celery_broker_url, backend=cfg.celery_result_backend, - include=["backend.app.tasks.smoke"], # FC-2/FC-3 extend this list + include=[ + "backend.app.tasks.smoke", + "backend.app.tasks.scan", + "backend.app.tasks.import_file", + "backend.app.tasks.thumbnail", + "backend.app.tasks.maintenance", + ], ) app.conf.update( task_default_queue="default", diff --git a/backend/app/tasks/import_file.py b/backend/app/tasks/import_file.py new file mode 100644 index 0000000..e9b5be2 --- /dev/null +++ b/backend/app/tasks/import_file.py @@ -0,0 +1,122 @@ +"""import_media_file task: imports one file via the Importer service and +updates the ImportTask state machine + ImportBatch counters atomically. +""" + +from datetime import datetime, timezone +from pathlib import Path + +from sqlalchemy import create_engine, select, update +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImportBatch, ImportSettings, ImportTask +from ..services.importer import Importer +from ..services.thumbnailer import Thumbnailer + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +IMAGES_ROOT = Path("/images") + + +@celery.task(name="backend.app.tasks.import_file.import_media_file", bind=True) +def import_media_file(self, import_task_id: int) -> dict: + """Returns a dict so the eager-mode tests can assert without DB.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + task = session.get(ImportTask, import_task_id) + if task is None: + return {"status": "missing", "task_id": import_task_id} + + task.status = "processing" + task.started_at = datetime.now(timezone.utc) + session.add(task) + session.commit() + + settings = session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + import_root = Path(settings.import_scan_path) + importer = Importer( + session=session, + images_root=IMAGES_ROOT, + import_root=import_root, + thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), + settings=settings, + ) + + try: + result = importer.import_one(Path(task.source_path)) + except Exception as exc: # pragma: no cover — pipeline crash + task.status = "failed" + task.error = f"{type(exc).__name__}: {exc}" + task.finished_at = datetime.now(timezone.utc) + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values(failed=ImportBatch.failed + 1) + ) + session.add(task) + session.commit() + raise + + if result.status == "imported": + task.status = "complete" + task.result_image_id = result.image_id + counter_col_name = "imported" + counter_col = ImportBatch.imported + elif result.status == "skipped": + task.status = "skipped" + task.error = ( + f"{result.skip_reason.value}: {result.error}" + if result.skip_reason + else result.error + ) + task.result_image_id = result.image_id + counter_col_name = "skipped" + counter_col = ImportBatch.skipped + else: + task.status = "failed" + task.error = result.error + counter_col_name = "failed" + counter_col = ImportBatch.failed + + task.finished_at = datetime.now(timezone.utc) + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values({counter_col_name: counter_col + 1}) + ) + session.add(task) + session.commit() + + # Enqueue the thumbnail task for newly imported images. + if result.status == "imported" and result.image_id is not None: + from .thumbnail import generate_thumbnail + + generate_thumbnail.delay(result.image_id) + + # If this was the last task in the batch, mark the batch complete. + remaining = session.execute( + select(ImportTask.id) + .where(ImportTask.batch_id == task.batch_id) + .where(ImportTask.status.in_(["pending", "queued", "processing"])) + .limit(1) + ).scalar_one_or_none() + if remaining is None: + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values( + status="complete", + finished_at=datetime.now(timezone.utc), + ) + ) + session.commit() + + return {"task_id": import_task_id, "status": task.status} diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py new file mode 100644 index 0000000..92a1e00 --- /dev/null +++ b/backend/app/tasks/scan.py @@ -0,0 +1,78 @@ +"""scan_directory task: walks the configured import path, creates an +ImportBatch, enumerates files into ImportTasks, and enqueues +import_media_file for each. +""" + +from pathlib import Path + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImportBatch, ImportSettings, ImportTask +from ..services.importer import is_supported + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True) +def scan_directory(self, triggered_by: str = "manual") -> int: + """Walks the import root and creates ImportTasks. Returns the batch id.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + settings = session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + import_root = Path(settings.import_scan_path) + + batch = ImportBatch( + triggered_by=triggered_by, + source_path=str(import_root), + scan_mode="quick", # FC-2a is quick only; FC-2d adds 'deep' + status="running", + ) + session.add(batch) + session.flush() + batch_id = batch.id + + # Walk and enumerate. + files_seen = 0 + for entry in import_root.rglob("*"): + if not entry.is_file(): + continue + if not is_supported(entry): + continue + try: + size = entry.stat().st_size + except OSError: + size = None + task = ImportTask( + batch_id=batch_id, + source_path=str(entry), + task_type="media", + status="pending", + size_bytes=size, + ) + session.add(task) + files_seen += 1 + + batch.total_files = files_seen + session.commit() + + # Now enqueue import_media_file for each pending task. + for task in session.execute( + select(ImportTask).where(ImportTask.batch_id == batch_id) + ).scalars(): + task.status = "queued" + session.add(task) + from .import_file import import_media_file + + import_media_file.delay(task.id) + session.commit() + + return batch_id diff --git a/backend/app/tasks/thumbnail.py b/backend/app/tasks/thumbnail.py new file mode 100644 index 0000000..a5b9dcd --- /dev/null +++ b/backend/app/tasks/thumbnail.py @@ -0,0 +1,50 @@ +"""generate_thumbnail task: PIL/ffmpeg thumbnail generation on the thumbnail queue. + +Lives separately from import_file because thumbnails can be regenerated en +masse (FC-2c adds the 'regenerate all' admin action) and they're CPU-bound +so they deserve their own queue lane. +""" + +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImageRecord +from ..services.importer import is_video +from ..services.thumbnailer import Thumbnailer + + +IMAGES_ROOT = Path("/images") + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.thumbnail.generate_thumbnail", bind=True) +def generate_thumbnail(self, image_id: int) -> dict: + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + record = session.get(ImageRecord, image_id) + if record is None: + return {"status": "missing", "image_id": image_id} + + thumbnailer = Thumbnailer(images_root=IMAGES_ROOT) + source = Path(record.path) + try: + if is_video(source): + result = thumbnailer.generate_video_thumbnail(source, record.sha256) + else: + result = thumbnailer.generate_image_thumbnail(source, record.sha256) + except Exception as exc: # pragma: no cover — thumbnail failure is non-fatal + return {"status": "failed", "image_id": image_id, "error": f"{type(exc).__name__}: {exc}"} + + record.thumbnail_path = str(result.path) + session.add(record) + session.commit() + return {"status": "ok", "image_id": image_id, "path": str(result.path)} diff --git a/tests/test_tasks_register.py b/tests/test_tasks_register.py new file mode 100644 index 0000000..5d4e941 --- /dev/null +++ b/tests/test_tasks_register.py @@ -0,0 +1,15 @@ +"""Confirms every FC-2a task module imports cleanly and registers with Celery.""" + +from backend.app.celery_app import celery + + +def test_scan_task_registered(): + assert "backend.app.tasks.scan.scan_directory" in celery.tasks + + +def test_import_media_file_registered(): + assert "backend.app.tasks.import_file.import_media_file" in celery.tasks + + +def test_generate_thumbnail_registered(): + assert "backend.app.tasks.thumbnail.generate_thumbnail" in celery.tasks From 509c19ce869bd7618497b4139101f952b3814903 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:09:09 -0400 Subject: [PATCH 024/272] feat(fc2a): add maintenance tasks (recovery + cleanup) with beat schedule recover_interrupted_tasks runs every 5 minutes, finds ImportTask rows stuck in 'processing' for >30 minutes (well above any legitimate import duration), and re-queues them. cleanup_old_tasks runs daily and deletes finished tasks older than 7 days so the task table stays an operational view rather than an archive. Both thresholds match ImageRepo's precedent. The 30-min stuck threshold is documented inline so a future reader can adjust it intentionally rather than mistaking it for a 'magic number'. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/celery_app.py | 11 +++++ backend/app/tasks/maintenance.py | 74 ++++++++++++++++++++++++++++++ tests/test_maintenance.py | 77 ++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 backend/app/tasks/maintenance.py create mode 100644 tests/test_maintenance.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index bf4237e..1f10df3 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -44,6 +44,17 @@ def make_celery() -> Celery: task_acks_late=True, worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, + beat_schedule={ + "recover-interrupted-tasks": { + "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", + "schedule": 300.0, # every 5 minutes + }, + "cleanup-old-tasks": { + "task": "backend.app.tasks.maintenance.cleanup_old_tasks", + "schedule": 86400.0, # daily + }, + }, + timezone="UTC", ) return app diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py new file mode 100644 index 0000000..f6e3d99 --- /dev/null +++ b/backend/app/tasks/maintenance.py @@ -0,0 +1,74 @@ +"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks.""" + +from datetime import datetime, timedelta, timezone + +from sqlalchemy import create_engine, delete, select, update +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImportTask + + +STUCK_THRESHOLD_MINUTES = 30 +OLD_TASK_DAYS = 7 + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks") +def recover_interrupted_tasks() -> int: + """Find ImportTask rows stuck in 'processing' for >30 min and re-queue them. + + Why 30 min: large videos can legitimately take many minutes to import; + 30 is a safe gate that catches actual crashes (which leave the row stuck + forever) without resetting slow-but-still-running jobs. + """ + SessionLocal = _sync_session_factory() + cutoff = datetime.now(timezone.utc) - timedelta(minutes=STUCK_THRESHOLD_MINUTES) + with SessionLocal() as session: + stuck_ids = session.execute( + select(ImportTask.id) + .where(ImportTask.status == "processing") + .where(ImportTask.started_at < cutoff) + ).scalars().all() + + if not stuck_ids: + return 0 + + session.execute( + update(ImportTask) + .where(ImportTask.id.in_(stuck_ids)) + .values(status="queued", started_at=None, error="recovered from stuck state") + ) + session.commit() + + from .import_file import import_media_file + for tid in stuck_ids: + import_media_file.delay(tid) + + return len(stuck_ids) + + +@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks") +def cleanup_old_tasks() -> int: + """Delete completed/skipped/failed ImportTask rows older than 7 days. + + Why 7 days: long enough to debug an issue an operator only notices days + later; short enough that the task table stays a useful operational view + rather than an archive. Matches IR's default. + """ + SessionLocal = _sync_session_factory() + cutoff = datetime.now(timezone.utc) - timedelta(days=OLD_TASK_DAYS) + with SessionLocal() as session: + result = session.execute( + delete(ImportTask) + .where(ImportTask.status.in_(["complete", "skipped", "failed"])) + .where(ImportTask.finished_at < cutoff) + ) + session.commit() + return result.rowcount or 0 diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py new file mode 100644 index 0000000..b38adb3 --- /dev/null +++ b/tests/test_maintenance.py @@ -0,0 +1,77 @@ +"""Unit tests for the maintenance tasks. Eager-mode Celery so the task +function runs synchronously inside the same DB transaction as the test. +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from backend.app.celery_app import celery +from backend.app.models import ImportBatch, ImportTask + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +def _make_batch(session) -> int: + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + session.add(batch) + session.flush() + return batch.id + + +def test_recover_interrupted_only_old(db_sync): + batch_id = _make_batch(db_sync) + now = datetime.now(timezone.utc) + + fresh = ImportTask( + batch_id=batch_id, source_path="/import/a.jpg", task_type="media", + status="processing", started_at=now - timedelta(minutes=5), + ) + stale = ImportTask( + batch_id=batch_id, source_path="/import/b.jpg", task_type="media", + status="processing", started_at=now - timedelta(hours=2), + ) + db_sync.add_all([fresh, stale]) + db_sync.commit() + + from backend.app.tasks.maintenance import recover_interrupted_tasks + recovered = recover_interrupted_tasks.apply().get() + assert recovered == 1 + + db_sync.refresh(fresh) + db_sync.refresh(stale) + assert fresh.status == "processing" + assert stale.status == "queued" + assert stale.started_at is None + + +def test_cleanup_old_deletes_finished_old(db_sync): + batch_id = _make_batch(db_sync) + now = datetime.now(timezone.utc) + + old_complete = ImportTask( + batch_id=batch_id, source_path="/import/a.jpg", task_type="media", + status="complete", finished_at=now - timedelta(days=10), + ) + recent_complete = ImportTask( + batch_id=batch_id, source_path="/import/b.jpg", task_type="media", + status="complete", finished_at=now - timedelta(days=2), + ) + old_pending = ImportTask( + batch_id=batch_id, source_path="/import/c.jpg", task_type="media", + status="pending", + ) + db_sync.add_all([old_complete, recent_complete, old_pending]) + db_sync.commit() + + from backend.app.tasks.maintenance import cleanup_old_tasks + deleted = cleanup_old_tasks.apply().get() + assert deleted == 1 + + remaining = {t.source_path for t in db_sync.query(ImportTask).all()} + assert remaining == {"/import/b.jpg", "/import/c.jpg"} From fd8cf8300309f43117350c80d97bc2f68b11f718 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:09:44 -0400 Subject: [PATCH 025/272] feat(fc2a): add /api/gallery endpoints (scroll, timeline, jump, image detail) Thin Quart blueprint that delegates to GalleryService. Cursor-based scroll returns images + a date_groups summary so the SPA can render year-month headers without re-grouping client-side. Timeline returns year-month buckets for the sidebar jump nav. Jump returns a cursor positioned at a specific year-month so the gallery can scroll to historical periods. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/__init__.py | 6 ++- backend/app/api/__init__.py | 13 ++++- backend/app/api/gallery.py | 97 +++++++++++++++++++++++++++++++++++++ tests/test_api_gallery.py | 70 ++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 backend/app/api/gallery.py create mode 100644 tests/test_api_gallery.py diff --git a/backend/app/__init__.py b/backend/app/__init__.py index cd295e9..390b783 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -4,7 +4,7 @@ import logging from quart import Quart -from .api import api_bp +from .api import all_blueprints from .config import get_config from .frontend import frontend_bp @@ -15,7 +15,9 @@ def create_app() -> Quart: app = Quart(__name__) app.secret_key = cfg.secret_key - app.register_blueprint(api_bp) + + for bp in all_blueprints(): + app.register_blueprint(bp) # Registered last so /api/* routes win over the SPA catch-all. app.register_blueprint(frontend_bp) diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index c456a71..3a96e9d 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -1,4 +1,9 @@ -"""API blueprint registration.""" +"""API blueprint registration. + +This module is imported by the Quart app factory; it just exposes the +top-level `api_bp` for backward compatibility with FC-1's health route, +and ALL_BLUEPRINTS for the factory to register sibling blueprints. +""" from quart import Blueprint @@ -6,3 +11,9 @@ from . import health api_bp = Blueprint("api", __name__, url_prefix="/api") api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) + + +def all_blueprints() -> list[Blueprint]: + from .gallery import gallery_bp + # FC-2a additions: tags, settings, import_admin land in later tasks + return [api_bp, gallery_bp] diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py new file mode 100644 index 0000000..e824297 --- /dev/null +++ b/backend/app/api/gallery.py @@ -0,0 +1,97 @@ +"""Gallery API: cursor scroll, timeline, jump, image detail.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import make_engine, make_session_factory +from ..services.gallery_service import GalleryService + +gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") + +_engine = None +_Session = None + + +def _session_factory(): + global _engine, _Session + if _engine is None: + _engine = make_engine() + _Session = make_session_factory(_engine) + return _Session + + +@gallery_bp.route("/scroll", methods=["GET"]) +async def scroll(): + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "50")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + + Session = _session_factory() + async with Session() as session: + svc = GalleryService(session) + try: + page = await svc.scroll(cursor=cursor, limit=limit, tag_id=tag_id) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + + return jsonify( + { + "images": [ + { + "id": i.id, + "sha256": i.sha256, + "mime": i.mime, + "width": i.width, + "height": i.height, + "created_at": i.created_at.isoformat(), + "thumbnail_url": i.thumbnail_url, + } + for i in page.images + ], + "next_cursor": page.next_cursor, + "date_groups": [ + {"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups + ], + } + ) + + +@gallery_bp.route("/timeline", methods=["GET"]) +async def timeline(): + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + Session = _session_factory() + async with Session() as session: + svc = GalleryService(session) + buckets = await svc.timeline(tag_id=tag_id) + return jsonify([{"year": b.year, "month": b.month, "count": b.count} for b in buckets]) + + +@gallery_bp.route("/jump", methods=["GET"]) +async def jump(): + try: + year = int(request.args["year"]) + month = int(request.args["month"]) + except (KeyError, ValueError): + return jsonify({"error": "year and month query params required"}), 400 + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + Session = _session_factory() + async with Session() as session: + svc = GalleryService(session) + cursor = await svc.jump_cursor(year=year, month=month, tag_id=tag_id) + return jsonify({"cursor": cursor}) + + +@gallery_bp.route("/image/", methods=["GET"]) +async def image_detail(image_id: int): + Session = _session_factory() + async with Session() as session: + svc = GalleryService(session) + payload = await svc.get_image_with_tags(image_id) + if payload is None: + return jsonify({"error": "not found"}), 404 + return jsonify(payload) diff --git a/tests/test_api_gallery.py b/tests/test_api_gallery.py new file mode 100644 index 0000000..5293687 --- /dev/null +++ b/tests/test_api_gallery.py @@ -0,0 +1,70 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from backend.app import create_app +from backend.app.models import ImageRecord + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _seed(db, count: int = 3): + base = datetime.now(timezone.utc) + for i in range(count): + r = ImageRecord( + path=f"/images/x/{i}.jpg", + sha256=f"x{i:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + r.created_at = base - timedelta(minutes=i) + db.add(r) + await db.flush() + await db.commit() + + +@pytest.mark.asyncio +async def test_scroll_returns_images(client, db): + await _seed(db) + resp = await client.get("/api/gallery/scroll?limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert len(body["images"]) == 3 + assert "next_cursor" in body + assert "date_groups" in body + + +@pytest.mark.asyncio +async def test_scroll_rejects_bad_limit(client): + resp = await client.get("/api/gallery/scroll?limit=notanumber") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_timeline_endpoint(client, db): + await _seed(db) + resp = await client.get("/api/gallery/timeline") + assert resp.status_code == 200 + body = await resp.get_json() + assert sum(b["count"] for b in body) == 3 + + +@pytest.mark.asyncio +async def test_jump_requires_year_month(client): + resp = await client.get("/api/gallery/jump") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_image_detail_404_when_missing(client): + resp = await client.get("/api/gallery/image/99999") + assert resp.status_code == 404 From 0e66368010967f03fa474c04d90058eba37e540f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:10:10 -0400 Subject: [PATCH 026/272] feat(fc2a): add /api/tags endpoints (autocomplete, create, image association) Thin async blueprint delegating to TagService. Returns 400 with the TagValidationError message on bad kind/fandom combos so the frontend can surface the reason. List/add/remove endpoints scoped under /api/images//tags follow REST conventions. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/__init__.py | 4 +- backend/app/api/tags.py | 126 ++++++++++++++++++++++++++++++++++++ tests/test_api_tags.py | 58 +++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 backend/app/api/tags.py create mode 100644 tests/test_api_tags.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 3a96e9d..600fc70 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -15,5 +15,5 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) def all_blueprints() -> list[Blueprint]: from .gallery import gallery_bp - # FC-2a additions: tags, settings, import_admin land in later tasks - return [api_bp, gallery_bp] + from .tags import tags_bp + return [api_bp, gallery_bp, tags_bp] diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py new file mode 100644 index 0000000..d8d59e2 --- /dev/null +++ b/backend/app/api/tags.py @@ -0,0 +1,126 @@ +"""Tags API: autocomplete, create, list/add/remove for an image.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import make_engine, make_session_factory +from ..models import TagKind +from ..services.tag_service import TagService, TagValidationError + +tags_bp = Blueprint("tags", __name__, url_prefix="/api") + +_engine = None +_Session = None + + +def _session_factory(): + global _engine, _Session + if _engine is None: + _engine = make_engine() + _Session = make_session_factory(_engine) + return _Session + + +def _coerce_kind(raw: str | None) -> TagKind | None: + if raw is None: + return None + try: + return TagKind(raw) + except ValueError: + return None + + +@tags_bp.route("/tags/autocomplete", methods=["GET"]) +async def autocomplete(): + q = request.args.get("q", "") + kind = _coerce_kind(request.args.get("kind")) + try: + limit = int(request.args.get("limit", "20")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + + Session = _session_factory() + async with Session() as session: + svc = TagService(session) + hits = await svc.autocomplete(q, kind=kind, limit=limit) + + return jsonify( + [ + { + "id": h.id, + "name": h.name, + "kind": h.kind, + "fandom_id": h.fandom_id, + "fandom_name": h.fandom_name, + "image_count": h.image_count, + } + for h in hits + ] + ) + + +@tags_bp.route("/tags", methods=["POST"]) +async def create_tag(): + body = await request.get_json() + if not body or "name" not in body or "kind" not in body: + return jsonify({"error": "name and kind required"}), 400 + name = body["name"] + kind = _coerce_kind(body["kind"]) + if kind is None: + return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400 + fandom_id = body.get("fandom_id") + + Session = _session_factory() + async with Session() as session: + svc = TagService(session) + try: + tag = await svc.find_or_create(name, kind, fandom_id=fandom_id) + except TagValidationError as exc: + return jsonify({"error": str(exc)}), 400 + await session.commit() + return jsonify( + {"id": tag.id, "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id} + ), 201 + + +@tags_bp.route("/images//tags", methods=["GET"]) +async def list_tags_for_image(image_id: int): + Session = _session_factory() + async with Session() as session: + svc = TagService(session) + tags = await svc.list_for_image(image_id) + return jsonify( + [ + { + "id": t.id, + "name": t.name, + "kind": t.kind.value, + "fandom_id": t.fandom_id, + } + for t in tags + ] + ) + + +@tags_bp.route("/images//tags", methods=["POST"]) +async def add_tag_to_image(image_id: int): + body = await request.get_json() + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + source = body.get("source", "manual") + + Session = _session_factory() + async with Session() as session: + svc = TagService(session) + await svc.add_to_image(image_id, body["tag_id"], source=source) + await session.commit() + return "", 204 + + +@tags_bp.route("/images//tags/", methods=["DELETE"]) +async def remove_tag_from_image(image_id: int, tag_id: int): + Session = _session_factory() + async with Session() as session: + svc = TagService(session) + await svc.remove_from_image(image_id, tag_id) + await session.commit() + return "", 204 diff --git a/tests/test_api_tags.py b/tests/test_api_tags.py new file mode 100644 index 0000000..9699d7f --- /dev/null +++ b/tests/test_api_tags.py @@ -0,0 +1,58 @@ +import pytest + +from backend.app import create_app + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_autocomplete_empty(client): + resp = await client.get("/api/tags/autocomplete?q=") + assert resp.status_code == 200 + assert await resp.get_json() == [] + + +@pytest.mark.asyncio +async def test_create_tag(client): + resp = await client.post("/api/tags", json={"name": "Bob", "kind": "artist"}) + assert resp.status_code == 201 + body = await resp.get_json() + assert body["name"] == "Bob" + assert body["kind"] == "artist" + + +@pytest.mark.asyncio +async def test_create_tag_rejects_invalid_kind(client): + resp = await client.post("/api/tags", json={"name": "Bob", "kind": "notakind"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_create_character_without_fandom_ok(client): + resp = await client.post("/api/tags", json={"name": "Alice", "kind": "character"}) + assert resp.status_code == 201 + + +@pytest.mark.asyncio +async def test_create_character_with_bad_fandom_id(client): + artist_resp = await client.post("/api/tags", json={"name": "X", "kind": "artist"}) + artist_id = (await artist_resp.get_json())["id"] + resp = await client.post( + "/api/tags", json={"name": "Y", "kind": "character", "fandom_id": artist_id} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_create_tag_missing_required(client): + resp = await client.post("/api/tags", json={"name": "Bob"}) + assert resp.status_code == 400 From ff28f29dbb21fbd3e746ec7550134ba590a0a33e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:10:57 -0400 Subject: [PATCH 027/272] feat(fc2a): add /api/settings/import and /api/system/stats endpoints Settings GET returns the single-row ImportSettings DB record; PATCH does selective field updates (only known editable fields are applied; unknown keys silently ignored). System stats returns image/tag/storage counts, task status histogram, and the active batch summary in one shot for the Settings overview tab. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/settings.py | 123 ++++++++++++++++++++++++++++++++++++ tests/test_api_settings.py | 56 ++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 backend/app/api/settings.py create mode 100644 tests/test_api_settings.py diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py new file mode 100644 index 0000000..7d2d842 --- /dev/null +++ b/backend/app/api/settings.py @@ -0,0 +1,123 @@ +"""Settings API: import filters, system stats.""" + +from quart import Blueprint, jsonify, request +from sqlalchemy import func, select + +from ..extensions import make_engine, make_session_factory +from ..models import ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag + +settings_bp = Blueprint("settings", __name__, url_prefix="/api") + +_engine = None +_Session = None + + +def _session_factory(): + global _engine, _Session + if _engine is None: + _engine = make_engine() + _Session = make_session_factory(_engine) + return _Session + + +_EDITABLE_FIELDS = ( + "min_width", + "min_height", + "skip_transparent", + "transparency_threshold", + "skip_single_color", + "single_color_threshold", + "single_color_tolerance", +) + + +@settings_bp.route("/settings/import", methods=["GET"]) +async def get_import_settings(): + Session = _session_factory() + async with Session() as session: + row = ( + await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) + ).scalar_one() + return jsonify({ + "min_width": row.min_width, + "min_height": row.min_height, + "skip_transparent": row.skip_transparent, + "transparency_threshold": row.transparency_threshold, + "skip_single_color": row.skip_single_color, + "single_color_threshold": row.single_color_threshold, + "single_color_tolerance": row.single_color_tolerance, + }) + + +@settings_bp.route("/settings/import", methods=["PATCH"]) +async def update_import_settings(): + body = await request.get_json() + if not isinstance(body, dict): + return jsonify({"error": "body must be a JSON object"}), 400 + + Session = _session_factory() + async with Session() as session: + row = ( + await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) + ).scalar_one() + for field in _EDITABLE_FIELDS: + if field in body: + setattr(row, field, body[field]) + await session.commit() + + return await get_import_settings() + + +@settings_bp.route("/system/stats", methods=["GET"]) +async def system_stats(): + Session = _session_factory() + async with Session() as session: + total_images = (await session.execute(select(func.count(ImageRecord.id)))).scalar_one() + total_tags = (await session.execute(select(func.count(Tag.id)))).scalar_one() + storage_bytes = ( + (await session.execute(select(func.coalesce(func.sum(ImageRecord.size_bytes), 0)))).scalar_one() + ) + + # Task counts grouped by status + status_rows = ( + await session.execute( + select(ImportTask.status, func.count(ImportTask.id)).group_by(ImportTask.status) + ) + ).all() + status_counts = {row[0]: row[1] for row in status_rows} + + # Active batch (most recent running) + active_batch_row = ( + await session.execute( + select(ImportBatch) + .where(ImportBatch.status == "running") + .order_by(ImportBatch.started_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + active_batch = None + if active_batch_row: + active_batch = { + "id": active_batch_row.id, + "source_path": active_batch_row.source_path, + "started_at": active_batch_row.started_at.isoformat(), + "total_files": active_batch_row.total_files, + "imported": active_batch_row.imported, + "skipped": active_batch_row.skipped, + "failed": active_batch_row.failed, + } + + return jsonify({ + "total_images": total_images, + "total_tags": total_tags, + "storage_bytes": storage_bytes, + "tasks": { + "pending": status_counts.get("pending", 0), + "queued": status_counts.get("queued", 0), + "processing": status_counts.get("processing", 0), + "complete": status_counts.get("complete", 0), + "skipped": status_counts.get("skipped", 0), + "failed": status_counts.get("failed", 0), + }, + "active_batch": active_batch, + }) diff --git a/tests/test_api_settings.py b/tests/test_api_settings.py new file mode 100644 index 0000000..bcdb46b --- /dev/null +++ b/tests/test_api_settings.py @@ -0,0 +1,56 @@ +import pytest + +from backend.app import create_app + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_get_import_settings_defaults(client): + resp = await client.get("/api/settings/import") + assert resp.status_code == 200 + body = await resp.get_json() + # Defaults from the migration: + assert body["min_width"] == 0 + assert body["min_height"] == 0 + assert body["skip_transparent"] is False + assert body["transparency_threshold"] == 0.9 + + +@pytest.mark.asyncio +async def test_patch_import_settings(client): + resp = await client.patch( + "/api/settings/import", + json={"min_width": 500, "skip_transparent": True, "transparency_threshold": 0.7}, + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["min_width"] == 500 + assert body["skip_transparent"] is True + assert body["transparency_threshold"] == 0.7 + + +@pytest.mark.asyncio +async def test_patch_rejects_non_object(client): + resp = await client.patch("/api/settings/import", json=[1, 2, 3]) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_system_stats_shape(client): + resp = await client.get("/api/system/stats") + assert resp.status_code == 200 + body = await resp.get_json() + for key in ("total_images", "total_tags", "storage_bytes", "tasks", "active_batch"): + assert key in body + for status in ("pending", "queued", "processing", "complete", "skipped", "failed"): + assert status in body["tasks"] From 62eac7cffd3b914c52568e186c5ddcadb0904be7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:11:10 -0400 Subject: [PATCH 028/272] fix(fc2a): register settings_bp in all_blueprints Missed in the Task 11 commit because of a stale-cache Edit failure. Routes /api/settings/import and /api/system/stats are now actually mounted. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 600fc70..dfc24af 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -15,5 +15,6 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) def all_blueprints() -> list[Blueprint]: from .gallery import gallery_bp + from .settings import settings_bp from .tags import tags_bp - return [api_bp, gallery_bp, tags_bp] + return [api_bp, gallery_bp, tags_bp, settings_bp] From e597a9300e3b0b0e41f22c2c7c3259fcc4e22efd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:11:49 -0400 Subject: [PATCH 029/272] feat(fc2a): add /api/import admin endpoints trigger enqueues a Celery scan_directory task and returns 202 with the celery task id (the actual scan happens asynchronously; status surfaces via /api/import/status and /api/system/stats). list_tasks supports status filter and cursor pagination. retry-failed resets failed tasks to queued and re-enqueues. clear-completed deletes finished tasks older than the supplied age (used by the Settings UI's cleanup control). mode='deep' is explicitly rejected with 400 until FC-2d ships it; the frontend's UI in FC-2a only sends 'quick'. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/__init__.py | 3 +- backend/app/api/import_admin.py | 151 ++++++++++++++++++++++++++++++++ tests/test_api_import_admin.py | 90 +++++++++++++++++++ 3 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 backend/app/api/import_admin.py create mode 100644 tests/test_api_import_admin.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index dfc24af..dfd66a3 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -15,6 +15,7 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) def all_blueprints() -> list[Blueprint]: from .gallery import gallery_bp + from .import_admin import import_admin_bp from .settings import settings_bp from .tags import tags_bp - return [api_bp, gallery_bp, tags_bp, settings_bp] + return [api_bp, gallery_bp, tags_bp, settings_bp, import_admin_bp] diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py new file mode 100644 index 0000000..7eb5884 --- /dev/null +++ b/backend/app/api/import_admin.py @@ -0,0 +1,151 @@ +"""Import admin API: trigger scan, list tasks, retry, clear.""" + +from datetime import datetime, timedelta, timezone + +from quart import Blueprint, jsonify, request +from sqlalchemy import delete, select, update + +from ..extensions import make_engine, make_session_factory +from ..models import ImportBatch, ImportTask + +import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import") + +_engine = None +_Session = None + + +def _session_factory(): + global _engine, _Session + if _engine is None: + _engine = make_engine() + _Session = make_session_factory(_engine) + return _Session + + +@import_admin_bp.route("/trigger", methods=["POST"]) +async def trigger_scan(): + body = await request.get_json(silent=True) or {} + mode = body.get("mode", "quick") + if mode != "quick": + # FC-2d will accept 'deep'. For now, reject anything else. + return jsonify({"error": f"mode {mode!r} not supported in FC-2a; use 'quick'"}), 400 + + # Enqueue the scan task in Celery. The task creates the batch itself. + from ..tasks.scan import scan_directory + + async_result = scan_directory.delay(triggered_by="manual") + return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202 + + +@import_admin_bp.route("/status", methods=["GET"]) +async def status(): + Session = _session_factory() + async with Session() as session: + active = ( + await session.execute( + select(ImportBatch) + .where(ImportBatch.status == "running") + .order_by(ImportBatch.started_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + payload = {"active_batch": None} + if active: + payload["active_batch"] = { + "id": active.id, + "total_files": active.total_files, + "imported": active.imported, + "skipped": active.skipped, + "failed": active.failed, + "started_at": active.started_at.isoformat(), + } + return jsonify(payload) + + +@import_admin_bp.route("/tasks", methods=["GET"]) +async def list_tasks(): + status_filter = request.args.get("status") + try: + limit = min(int(request.args.get("limit", "50")), 200) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + cursor_raw = request.args.get("cursor") + cursor_id = int(cursor_raw) if cursor_raw else None + + Session = _session_factory() + async with Session() as session: + stmt = select(ImportTask).order_by(ImportTask.created_at.desc(), ImportTask.id.desc()) + if status_filter: + stmt = stmt.where(ImportTask.status == status_filter) + if cursor_id is not None: + stmt = stmt.where(ImportTask.id < cursor_id) + stmt = stmt.limit(limit + 1) + rows = (await session.execute(stmt)).scalars().all() + + has_more = len(rows) > limit + rows = rows[:limit] + return jsonify({ + "tasks": [ + { + "id": t.id, + "batch_id": t.batch_id, + "source_path": t.source_path, + "task_type": t.task_type, + "status": t.status, + "result_image_id": t.result_image_id, + "error": t.error, + "size_bytes": t.size_bytes, + "created_at": t.created_at.isoformat(), + "started_at": t.started_at.isoformat() if t.started_at else None, + "finished_at": t.finished_at.isoformat() if t.finished_at else None, + } + for t in rows + ], + "next_cursor": rows[-1].id if has_more and rows else None, + }) + + +@import_admin_bp.route("/retry-failed", methods=["POST"]) +async def retry_failed(): + Session = _session_factory() + async with Session() as session: + failed_ids = ( + await session.execute(select(ImportTask.id).where(ImportTask.status == "failed")) + ).scalars().all() + if not failed_ids: + return jsonify({"retried": 0}) + await session.execute( + update(ImportTask) + .where(ImportTask.id.in_(failed_ids)) + .values(status="queued", error=None, started_at=None, finished_at=None) + ) + await session.commit() + + from ..tasks.import_file import import_media_file + for tid in failed_ids: + import_media_file.delay(tid) + + return jsonify({"retried": len(failed_ids)}) + + +@import_admin_bp.route("/clear-completed", methods=["POST"]) +async def clear_completed(): + body = await request.get_json(silent=True) or {} + age_days = body.get("age_days", 0) + status_filter = body.get("status", ["complete", "skipped"]) + if not isinstance(status_filter, list): + return jsonify({"error": "status must be a list"}), 400 + + cutoff = ( + datetime.now(timezone.utc) - timedelta(days=int(age_days)) if age_days else None + ) + + Session = _session_factory() + async with Session() as session: + stmt = delete(ImportTask).where(ImportTask.status.in_(status_filter)) + if cutoff is not None: + stmt = stmt.where(ImportTask.finished_at < cutoff) + result = await session.execute(stmt) + await session.commit() + + return jsonify({"deleted": result.rowcount or 0}) diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py new file mode 100644 index 0000000..62461a4 --- /dev/null +++ b/tests/test_api_import_admin.py @@ -0,0 +1,90 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from backend.app import create_app +from backend.app.celery_app import celery +from backend.app.models import ImportBatch, ImportTask + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_trigger_rejects_unknown_mode(client): + resp = await client.post("/api/import/trigger", json={"mode": "deep"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_status_when_idle(client): + resp = await client.get("/api/import/status") + body = await resp.get_json() + assert body["active_batch"] is None + + +@pytest.mark.asyncio +async def test_list_tasks_with_status_filter(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + for status in ("complete", "failed", "complete"): + db.add(ImportTask( + batch_id=batch.id, source_path="/x", task_type="media", status=status, + finished_at=datetime.now(timezone.utc), + )) + await db.commit() + + resp = await client.get("/api/import/tasks?status=failed") + body = await resp.get_json() + assert len(body["tasks"]) == 1 + assert body["tasks"][0]["status"] == "failed" + + +@pytest.mark.asyncio +async def test_retry_failed(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + db.add(ImportTask(batch_id=batch.id, source_path="/x", task_type="media", status="failed")) + await db.commit() + + resp = await client.post("/api/import/retry-failed") + body = await resp.get_json() + assert body["retried"] == 1 + + +@pytest.mark.asyncio +async def test_clear_completed(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + long_ago = datetime.now(timezone.utc) - timedelta(days=30) + db.add(ImportTask( + batch_id=batch.id, source_path="/x", task_type="media", + status="complete", finished_at=long_ago, + )) + db.add(ImportTask( + batch_id=batch.id, source_path="/y", task_type="media", + status="complete", finished_at=datetime.now(timezone.utc), + )) + await db.commit() + + resp = await client.post("/api/import/clear-completed", json={"age_days": 7}) + body = await resp.get_json() + assert body["deleted"] == 1 From af9f42ef68c26caa399025a2625228d4786ab77f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:29:49 -0400 Subject: [PATCH 030/272] feat(fc2a): add useApi composable, import store, extend system store with stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useApi centralizes fetch error handling — every API call throws ApiError on non-2xx with the parsed body attached, so components don't repeat the 'check response.ok' boilerplate. import store wraps import settings, batch status, task list (with cursor pagination), and the trigger/retry/clear admin actions. system store gains a refreshStats() that powers the Settings overview tab. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/composables/useApi.js | 50 ++++++++++++++ frontend/src/stores/import.js | 104 +++++++++++++++++++++++++++++ frontend/src/stores/system.js | 20 ++++-- 3 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 frontend/src/composables/useApi.js create mode 100644 frontend/src/stores/import.js diff --git a/frontend/src/composables/useApi.js b/frontend/src/composables/useApi.js new file mode 100644 index 0000000..cc88c0d --- /dev/null +++ b/frontend/src/composables/useApi.js @@ -0,0 +1,50 @@ +// Thin fetch wrapper. Centralized so every call has consistent error handling +// and json parsing. Errors throw an ApiError with the response status + body. + +export class ApiError extends Error { + constructor(message, status, body) { + super(message) + this.name = 'ApiError' + this.status = status + this.body = body + } +} + +async function request(method, url, { body, params, signal } = {}) { + let fullUrl = url + if (params) { + const search = new URLSearchParams() + for (const [k, v] of Object.entries(params)) { + if (v !== undefined && v !== null) search.set(k, String(v)) + } + const qs = search.toString() + if (qs) fullUrl += (url.includes('?') ? '&' : '?') + qs + } + + const headers = {} + const init = { method, headers, signal } + if (body !== undefined) { + headers['Content-Type'] = 'application/json' + init.body = JSON.stringify(body) + } + + const r = await fetch(fullUrl, init) + const text = await r.text() + let parsed + try { parsed = text ? JSON.parse(text) : null } catch { parsed = text } + + if (!r.ok) { + const msg = (parsed && parsed.error) || `${r.status} ${r.statusText}` + throw new ApiError(msg, r.status, parsed) + } + return parsed +} + +export function useApi() { + return { + get: (url, opts) => request('GET', url, opts), + post: (url, opts) => request('POST', url, opts), + patch: (url, opts) => request('PATCH', url, opts), + delete: (url, opts) => request('DELETE', url, opts) + } +} diff --git a/frontend/src/stores/import.js b/frontend/src/stores/import.js new file mode 100644 index 0000000..1cd7ccc --- /dev/null +++ b/frontend/src/stores/import.js @@ -0,0 +1,104 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useImportStore = defineStore('import', () => { + const api = useApi() + + const settings = ref(null) + const settingsLoading = ref(false) + const settingsError = ref(null) + + const activeBatch = ref(null) + const statusLoading = ref(false) + + const tasks = ref([]) + const tasksNextCursor = ref(null) + const tasksLoading = ref(false) + const tasksFilter = ref({ status: null, limit: 50 }) + + const triggerError = ref(null) + + async function loadSettings() { + settingsLoading.value = true + settingsError.value = null + try { + settings.value = await api.get('/api/settings/import') + } catch (e) { + settingsError.value = e.message + } finally { + settingsLoading.value = false + } + } + + async function patchSettings(patch) { + settingsError.value = null + try { + settings.value = await api.patch('/api/settings/import', { body: patch }) + } catch (e) { + settingsError.value = e.message + throw e + } + } + + async function refreshStatus() { + statusLoading.value = true + try { + const body = await api.get('/api/import/status') + activeBatch.value = body.active_batch + } finally { + statusLoading.value = false + } + } + + async function triggerScan() { + triggerError.value = null + try { + await api.post('/api/import/trigger', { body: { mode: 'quick' } }) + await refreshStatus() + } catch (e) { + triggerError.value = e.message + throw e + } + } + + async function loadTasks(reset = true) { + tasksLoading.value = true + try { + const params = { limit: tasksFilter.value.limit } + if (tasksFilter.value.status) params.status = tasksFilter.value.status + if (!reset && tasksNextCursor.value) params.cursor = tasksNextCursor.value + const body = await api.get('/api/import/tasks', { params }) + tasks.value = reset ? body.tasks : [...tasks.value, ...body.tasks] + tasksNextCursor.value = body.next_cursor + } finally { + tasksLoading.value = false + } + } + + function setStatusFilter(status) { + tasksFilter.value.status = status + } + + async function retryFailed() { + await api.post('/api/import/retry-failed') + await loadTasks(true) + } + + async function clearCompleted(ageDays = 0) { + await api.post('/api/import/clear-completed', { body: { age_days: ageDays } }) + await loadTasks(true) + } + + const hasMore = computed(() => tasksNextCursor.value !== null) + + return { + settings, settingsLoading, settingsError, + activeBatch, statusLoading, + tasks, tasksLoading, tasksFilter, hasMore, + triggerError, + loadSettings, patchSettings, + refreshStatus, triggerScan, + loadTasks, setStatusFilter, retryFailed, clearCompleted + } +}) diff --git a/frontend/src/stores/system.js b/frontend/src/stores/system.js index 4412640..b3f772f 100644 --- a/frontend/src/stores/system.js +++ b/frontend/src/stores/system.js @@ -1,18 +1,30 @@ import { defineStore } from 'pinia' import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' export const useSystemStore = defineStore('system', () => { + const api = useApi() const healthy = ref(null) // null=unknown, true=ok, false=down + const stats = ref(null) + const statsLoading = ref(false) async function refreshHealth() { try { - const r = await fetch('/api/health') - const body = await r.json() - healthy.value = r.ok && body.status === 'ok' + const body = await api.get('/api/health') + healthy.value = body.status === 'ok' } catch { healthy.value = false } } - return { healthy, refreshHealth } + async function refreshStats() { + statsLoading.value = true + try { + stats.value = await api.get('/api/system/stats') + } finally { + statsLoading.value = false + } + } + + return { healthy, stats, statsLoading, refreshHealth, refreshStats } }) From 3bcff142f5c4de1f792da496ca72376106faca26 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:30:27 -0400 Subject: [PATCH 031/272] feat(fc2a): add SettingsView shell with Overview tab (SystemStatsCards) Overview tab shows total images/tags/storage/pending/failed using Fraunces for the big numbers. Inline alert nudges the operator to the Import tab when pending tasks exist. Polls /api/system/stats every 5s while the tab is visible (pauses when the document is hidden). Import tab references three placeholders (TriggerPanel/FiltersForm/ TaskList); real implementations land in Task 15. Router replaces the placeholder view for /settings with SettingsView. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/settings/ImportFiltersForm.vue | 1 + .../components/settings/ImportTaskList.vue | 1 + .../settings/ImportTriggerPanel.vue | 1 + .../components/settings/SystemStatsCards.vue | 63 ++++++++++++++ frontend/src/router.js | 3 +- frontend/src/views/SettingsView.vue | 86 +++++++++++++++++++ 6 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/settings/ImportFiltersForm.vue create mode 100644 frontend/src/components/settings/ImportTaskList.vue create mode 100644 frontend/src/components/settings/ImportTriggerPanel.vue create mode 100644 frontend/src/components/settings/SystemStatsCards.vue create mode 100644 frontend/src/views/SettingsView.vue diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue new file mode 100644 index 0000000..1dfcc5d --- /dev/null +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -0,0 +1 @@ + diff --git a/frontend/src/components/settings/ImportTaskList.vue b/frontend/src/components/settings/ImportTaskList.vue new file mode 100644 index 0000000..44df079 --- /dev/null +++ b/frontend/src/components/settings/ImportTaskList.vue @@ -0,0 +1 @@ + diff --git a/frontend/src/components/settings/ImportTriggerPanel.vue b/frontend/src/components/settings/ImportTriggerPanel.vue new file mode 100644 index 0000000..0ceea25 --- /dev/null +++ b/frontend/src/components/settings/ImportTriggerPanel.vue @@ -0,0 +1 @@ + diff --git a/frontend/src/components/settings/SystemStatsCards.vue b/frontend/src/components/settings/SystemStatsCards.vue new file mode 100644 index 0000000..92d9ba7 --- /dev/null +++ b/frontend/src/components/settings/SystemStatsCards.vue @@ -0,0 +1,63 @@ + + + + + diff --git a/frontend/src/router.js b/frontend/src/router.js index d60b5e5..c983d82 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -1,12 +1,13 @@ import { createRouter, createWebHistory } from 'vue-router' import PlaceholderView from './views/PlaceholderView.vue' +import SettingsView from './views/SettingsView.vue' const routes = [ // FC-2: image backbone { path: '/', name: 'gallery', component: PlaceholderView, meta: { title: 'Gallery' } }, { path: '/showcase', name: 'showcase', component: PlaceholderView, meta: { title: 'Showcase' } }, { path: '/tags', name: 'tags', component: PlaceholderView, meta: { title: 'Tags' } }, - { path: '/settings', name: 'settings', component: PlaceholderView, meta: { title: 'Settings' } }, + { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, // FC-3: subscription backbone { path: '/subscriptions', name: 'subscriptions', component: PlaceholderView, meta: { title: 'Subscriptions' } }, diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue new file mode 100644 index 0000000..a0b854f --- /dev/null +++ b/frontend/src/views/SettingsView.vue @@ -0,0 +1,86 @@ + + + + + From 968a9e14e3f241d439da0d9539ae562d844ccb1c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:31:22 -0400 Subject: [PATCH 032/272] feat(fc2a): implement Settings Import tab components ImportTriggerPanel shows an active-batch progress indicator when scan_directory is running, or a "Quick scan" button otherwise. Deep scan deferred to FC-2d is called out in the panel text so it's not a surprise. ImportFiltersForm autosaves on blur/change so there's no Submit button. The transparency and single-color sliders gate on their respective switches. ImportTaskList uses v-data-table-virtual for performance on long histories, with status filter, retry-failed, and a clear-completed dialog with age buckets. Status chips are color-coded via FabledDesignSystem semantic tokens (success/warning/error/accent/info). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/settings/ImportFiltersForm.vue | 72 ++++++++- .../components/settings/ImportTaskList.vue | 142 +++++++++++++++++- .../settings/ImportTriggerPanel.vue | 46 +++++- 3 files changed, 257 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue index 1dfcc5d..e812c7c 100644 --- a/frontend/src/components/settings/ImportFiltersForm.vue +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -1 +1,71 @@ - + + + diff --git a/frontend/src/components/settings/ImportTaskList.vue b/frontend/src/components/settings/ImportTaskList.vue index 44df079..a78562d 100644 --- a/frontend/src/components/settings/ImportTaskList.vue +++ b/frontend/src/components/settings/ImportTaskList.vue @@ -1 +1,141 @@ - + + + diff --git a/frontend/src/components/settings/ImportTriggerPanel.vue b/frontend/src/components/settings/ImportTriggerPanel.vue index 0ceea25..232df86 100644 --- a/frontend/src/components/settings/ImportTriggerPanel.vue +++ b/frontend/src/components/settings/ImportTriggerPanel.vue @@ -1 +1,45 @@ - + + + From da89fe6be25022dd9cf319f920c7326454e147b4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:31:53 -0400 Subject: [PATCH 033/272] feat(fc2a): add gallery Pinia store, GalleryItem, and EmptyState MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gallery store handles cursor pagination, date-group merging across pages, tag filter, timeline buckets, and jump-to-month. Stale response detection via an inflightId counter prevents out-of-order page additions when the user toggles filters quickly. GalleryItem is a square thumbnail card with hover/focus accent outline, keyboard activation (Enter/Space → open), video badge for video MIME types, and a broken-image fallback if the thumbnail URL fails to load. EmptyState points operators to /settings?tab=import so the empty path is self-guiding. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/gallery/EmptyState.vue | 27 +++++ .../src/components/gallery/GalleryItem.vue | 69 +++++++++++++ frontend/src/stores/gallery.js | 98 +++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 frontend/src/components/gallery/EmptyState.vue create mode 100644 frontend/src/components/gallery/GalleryItem.vue create mode 100644 frontend/src/stores/gallery.js diff --git a/frontend/src/components/gallery/EmptyState.vue b/frontend/src/components/gallery/EmptyState.vue new file mode 100644 index 0000000..355133f --- /dev/null +++ b/frontend/src/components/gallery/EmptyState.vue @@ -0,0 +1,27 @@ + + + diff --git a/frontend/src/components/gallery/GalleryItem.vue b/frontend/src/components/gallery/GalleryItem.vue new file mode 100644 index 0000000..a532285 --- /dev/null +++ b/frontend/src/components/gallery/GalleryItem.vue @@ -0,0 +1,69 @@ + + + + + diff --git a/frontend/src/stores/gallery.js b/frontend/src/stores/gallery.js new file mode 100644 index 0000000..10d3208 --- /dev/null +++ b/frontend/src/stores/gallery.js @@ -0,0 +1,98 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useGalleryStore = defineStore('gallery', () => { + const api = useApi() + + const images = ref([]) + const dateGroups = ref([]) // [{year, month, image_ids}] + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const filter = ref({ tag_id: null }) + + const timelineBuckets = ref([]) + const timelineLoading = ref(false) + + let inflightId = 0 + + async function loadInitial() { + images.value = [] + dateGroups.value = [] + nextCursor.value = null + await loadMore() + } + + async function loadMore() { + if (loading.value) return + loading.value = true + error.value = null + const myId = ++inflightId + try { + const params = { limit: 50 } + if (filter.value.tag_id) params.tag_id = filter.value.tag_id + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/gallery/scroll', { params }) + if (myId !== inflightId) return // stale response + images.value.push(...body.images) + dateGroups.value = mergeGroups(dateGroups.value, body.date_groups) + nextCursor.value = body.next_cursor + } catch (e) { + error.value = e.message + } finally { + if (myId === inflightId) loading.value = false + } + } + + async function loadTimeline() { + timelineLoading.value = true + try { + const params = filter.value.tag_id ? { tag_id: filter.value.tag_id } : {} + timelineBuckets.value = await api.get('/api/gallery/timeline', { params }) + } finally { + timelineLoading.value = false + } + } + + async function jumpTo(year, month) { + const params = { year, month } + if (filter.value.tag_id) params.tag_id = filter.value.tag_id + const body = await api.get('/api/gallery/jump', { params }) + if (body.cursor) { + images.value = [] + dateGroups.value = [] + nextCursor.value = body.cursor + await loadMore() + } + } + + function setTagFilter(tagId) { + filter.value.tag_id = tagId + loadInitial() + loadTimeline() + } + + const hasMore = computed(() => nextCursor.value !== null) + const isEmpty = computed(() => !loading.value && images.value.length === 0 && error.value === null) + + return { + images, dateGroups, hasMore, isEmpty, loading, error, + filter, timelineBuckets, timelineLoading, + loadInitial, loadMore, loadTimeline, jumpTo, setTagFilter + } +}) + +function mergeGroups(existing, incoming) { + // Merge sequential groups with the same (year, month) instead of duplicating. + const merged = [...existing] + for (const g of incoming) { + const tail = merged[merged.length - 1] + if (tail && tail.year === g.year && tail.month === g.month) { + tail.image_ids = [...tail.image_ids, ...g.image_ids] + } else { + merged.push({ ...g, image_ids: [...g.image_ids] }) + } + } + return merged +} From 708c0336277c9d2c0edb8df72ae9d72652f93909 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:32:11 -0400 Subject: [PATCH 034/272] feat(fc2a): add GalleryGrid with infinite scroll and date headers Grid uses CSS grid auto-fill with 160px minimums so it adapts naturally from phone to large displays. Date headers render once per (year, month) group; the API's date_groups payload tells us where to slot them without client-side grouping. IntersectionObserver with rootMargin: 600px loads the next page well before the sentinel reaches the viewport. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/gallery/GalleryGrid.vue | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 frontend/src/components/gallery/GalleryGrid.vue diff --git a/frontend/src/components/gallery/GalleryGrid.vue b/frontend/src/components/gallery/GalleryGrid.vue new file mode 100644 index 0000000..ea9c1f3 --- /dev/null +++ b/frontend/src/components/gallery/GalleryGrid.vue @@ -0,0 +1,99 @@ + + + + + From 899b1930853722ef38a53ecd0595b3a3c8c985d5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:32:50 -0400 Subject: [PATCH 035/272] feat(fc2a): add TimelineSidebar and assemble GalleryView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GalleryView wires the grid + timeline + modal together. The URL is the source of truth for which image (if any) is open — clicking an item pushes ?image=N; closing the modal pops it. This makes deep links shareable and the back button work intuitively. Below 900px viewport, the timeline drops below the grid as a horizontal strip so it doesn't compete with thumbnail space on mobile. Modal store + ImageViewer are placeholders here; real implementations land in Batch 5 (Tasks 19–22). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/gallery/TimelineSidebar.vue | 88 +++++++++++++++++++ frontend/src/components/modal/ImageViewer.vue | 9 ++ frontend/src/router.js | 3 +- frontend/src/stores/modal.js | 10 +++ frontend/src/views/GalleryView.vue | 77 ++++++++++++++++ 5 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/gallery/TimelineSidebar.vue create mode 100644 frontend/src/components/modal/ImageViewer.vue create mode 100644 frontend/src/stores/modal.js create mode 100644 frontend/src/views/GalleryView.vue diff --git a/frontend/src/components/gallery/TimelineSidebar.vue b/frontend/src/components/gallery/TimelineSidebar.vue new file mode 100644 index 0000000..8ea5c24 --- /dev/null +++ b/frontend/src/components/gallery/TimelineSidebar.vue @@ -0,0 +1,88 @@ + + + + + diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue new file mode 100644 index 0000000..370b462 --- /dev/null +++ b/frontend/src/components/modal/ImageViewer.vue @@ -0,0 +1,9 @@ + + diff --git a/frontend/src/router.js b/frontend/src/router.js index c983d82..fab0176 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -1,10 +1,11 @@ import { createRouter, createWebHistory } from 'vue-router' import PlaceholderView from './views/PlaceholderView.vue' import SettingsView from './views/SettingsView.vue' +import GalleryView from './views/GalleryView.vue' const routes = [ // FC-2: image backbone - { path: '/', name: 'gallery', component: PlaceholderView, meta: { title: 'Gallery' } }, + { path: '/', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, { path: '/showcase', name: 'showcase', component: PlaceholderView, meta: { title: 'Showcase' } }, { path: '/tags', name: 'tags', component: PlaceholderView, meta: { title: 'Tags' } }, { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, diff --git a/frontend/src/stores/modal.js b/frontend/src/stores/modal.js new file mode 100644 index 0000000..7f88020 --- /dev/null +++ b/frontend/src/stores/modal.js @@ -0,0 +1,10 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' + +// Placeholder modal store — real implementation lands in Task 19. +export const useModalStore = defineStore('modal', () => { + const currentImageId = ref(null) + function open(id) { currentImageId.value = id } + function close() { currentImageId.value = null } + return { currentImageId, open, close } +}) diff --git a/frontend/src/views/GalleryView.vue b/frontend/src/views/GalleryView.vue new file mode 100644 index 0000000..c2f9d72 --- /dev/null +++ b/frontend/src/views/GalleryView.vue @@ -0,0 +1,77 @@ + + + + + From 8358d09348eff1a7a872a61c92aa33ba48638bc9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:33:14 -0400 Subject: [PATCH 036/272] feat(fc2a): real modal store + usePanZoom composable modal store loads image detail (with neighbors) on open, exposes goPrev/ goNext that re-fetch the next image, and implements optimistic tag remove with rollback on API failure. createAndAdd is the bridge between the autocomplete UI and a brand-new tag. usePanZoom is a small composable: click-to-toggle 2x zoom centered on click point, wheel-to-zoom up to 4x, drag-to-pan when zoomed. Pointer events are used (touch + mouse + pen unified). Caller spreads handlers onto the element it wants pannable. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/composables/usePanZoom.js | 81 +++++++++++++++++++++++++ frontend/src/stores/modal.js | 84 ++++++++++++++++++++++++-- 2 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 frontend/src/composables/usePanZoom.js diff --git a/frontend/src/composables/usePanZoom.js b/frontend/src/composables/usePanZoom.js new file mode 100644 index 0000000..b05cc37 --- /dev/null +++ b/frontend/src/composables/usePanZoom.js @@ -0,0 +1,81 @@ +// Click-to-toggle zoom + drag-to-pan when zoomed + wheel-to-zoom. +// Returns reactive state + handlers a component can spread onto its +// root element. + +import { reactive } from 'vue' + +const MIN_SCALE = 1 +const MAX_SCALE = 4 +const WHEEL_SENSITIVITY = 0.0015 + +export function usePanZoom() { + const state = reactive({ + scale: 1, + x: 0, + y: 0, + panning: false + }) + + let dragOriginX = 0 + let dragOriginY = 0 + let panOriginX = 0 + let panOriginY = 0 + + function reset() { + state.scale = 1 + state.x = 0 + state.y = 0 + } + + function toggleZoom(ev) { + if (state.scale > 1) { reset(); return } + state.scale = 2 + // Zoom centered on click point + if (ev && ev.currentTarget) { + const rect = ev.currentTarget.getBoundingClientRect() + const cx = ev.clientX - rect.left - rect.width / 2 + const cy = ev.clientY - rect.top - rect.height / 2 + state.x = -cx + state.y = -cy + } + } + + function onWheel(ev) { + ev.preventDefault() + const delta = -ev.deltaY * WHEEL_SENSITIVITY + const next = Math.min(MAX_SCALE, Math.max(MIN_SCALE, state.scale + delta * state.scale)) + state.scale = next + if (next === MIN_SCALE) { state.x = 0; state.y = 0 } + } + + function onPointerDown(ev) { + if (state.scale === 1) return + state.panning = true + dragOriginX = ev.clientX + dragOriginY = ev.clientY + panOriginX = state.x + panOriginY = state.y + ev.currentTarget.setPointerCapture(ev.pointerId) + } + function onPointerMove(ev) { + if (!state.panning) return + state.x = panOriginX + (ev.clientX - dragOriginX) + state.y = panOriginY + (ev.clientY - dragOriginY) + } + function onPointerUp(ev) { + state.panning = false + try { ev.currentTarget.releasePointerCapture(ev.pointerId) } catch {} + } + + return { + state, + handlers: { + onWheel, + onPointerDown, + onPointerMove, + onPointerUp, + onClick: toggleZoom + }, + reset + } +} diff --git a/frontend/src/stores/modal.js b/frontend/src/stores/modal.js index 7f88020..509c23c 100644 --- a/frontend/src/stores/modal.js +++ b/frontend/src/stores/modal.js @@ -1,10 +1,84 @@ import { defineStore } from 'pinia' -import { ref } from 'vue' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' -// Placeholder modal store — real implementation lands in Task 19. export const useModalStore = defineStore('modal', () => { + const api = useApi() + const currentImageId = ref(null) - function open(id) { currentImageId.value = id } - function close() { currentImageId.value = null } - return { currentImageId, open, close } + const current = ref(null) // full image detail from API + const loading = ref(false) + const error = ref(null) + + async function open(id) { + currentImageId.value = id + loading.value = true + error.value = null + try { + current.value = await api.get(`/api/gallery/image/${id}`) + } catch (e) { + error.value = e.message + current.value = null + } finally { + loading.value = false + } + } + + async function close() { + currentImageId.value = null + current.value = null + error.value = null + } + + async function goPrev() { + if (current.value && current.value.neighbors.prev_id) { + await open(current.value.neighbors.prev_id) + } + } + async function goNext() { + if (current.value && current.value.neighbors.next_id) { + await open(current.value.neighbors.next_id) + } + } + + async function reloadTags() { + if (!currentImageId.value) return + const tags = await api.get(`/api/images/${currentImageId.value}/tags`) + if (current.value) current.value.tags = tags + } + + async function removeTag(tagId) { + if (!currentImageId.value) return + // Optimistic UI: remove locally first, restore on error. + const prev = current.value.tags + current.value.tags = current.value.tags.filter(t => t.id !== tagId) + try { + await api.delete(`/api/images/${currentImageId.value}/tags/${tagId}`) + } catch (e) { + current.value.tags = prev + throw e + } + } + + async function addExistingTag(tagId) { + if (!currentImageId.value) return + await api.post(`/api/images/${currentImageId.value}/tags`, { + body: { tag_id: tagId, source: 'manual' } + }) + await reloadTags() + } + + async function createAndAdd({ name, kind, fandom_id = null }) { + const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } }) + await addExistingTag(tag.id) + } + + const isOpen = computed(() => currentImageId.value !== null) + const canPrev = computed(() => current.value?.neighbors?.prev_id != null) + const canNext = computed(() => current.value?.neighbors?.next_id != null) + + return { + currentImageId, current, loading, error, isOpen, canPrev, canNext, + open, close, goPrev, goNext, reloadTags, removeTag, addExistingTag, createAndAdd + } }) From 0faeebf3aaf72ed483b32b9d963d73416c778945 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:33:41 -0400 Subject: [PATCH 037/272] feat(fc2a): add ImageCanvas (pan/zoom) and VideoCanvas (with format gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ImageCanvas mounts an wrapped in the usePanZoom composable — spreading its handlers onto the container element. Zoom resets when src changes so prev/next nav doesn't carry zoom state into the next image. VideoCanvas plays MP4/WebM/Ogg/MOV natively. Non-playable formats render a friendly explainer with a download fallback, naming the actual MIME so operators know what they're dealing with, and pointing forward at FC-2e where transcoding lands. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/modal/ImageCanvas.vue | 50 ++++++++++++++++ frontend/src/components/modal/VideoCanvas.vue | 59 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 frontend/src/components/modal/ImageCanvas.vue create mode 100644 frontend/src/components/modal/VideoCanvas.vue diff --git a/frontend/src/components/modal/ImageCanvas.vue b/frontend/src/components/modal/ImageCanvas.vue new file mode 100644 index 0000000..52cbb17 --- /dev/null +++ b/frontend/src/components/modal/ImageCanvas.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/frontend/src/components/modal/VideoCanvas.vue b/frontend/src/components/modal/VideoCanvas.vue new file mode 100644 index 0000000..e79e6da --- /dev/null +++ b/frontend/src/components/modal/VideoCanvas.vue @@ -0,0 +1,59 @@ + + + + + From 89fee260a6c1637993716d47b924711705bb0dd6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:34:17 -0400 Subject: [PATCH 038/272] feat(fc2a): add tag store, TagAutocomplete, and FandomPicker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TagAutocomplete: kind selector + debounced (200ms) search + keyboard nav (up/down/enter/esc). When the operator types a character that doesn't exist, the create flow first opens FandomPicker so the new character tag lands with a valid fandom_id (otherwise the API rejects it). FandomPicker lists existing fandoms (cached after first load) with an inline create-new affordance, so the operator never has to leave the modal to manage fandom hierarchies. Tag store also exposes a kind→icon→color mapping that the TagPanel reuses in Task 22 — keeping chip colors visually distinct per kind without per-app accent collisions. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/modal/FandomPicker.vue | 48 +++++++ .../src/components/modal/TagAutocomplete.vue | 135 ++++++++++++++++++ frontend/src/stores/tags.js | 58 ++++++++ 3 files changed, 241 insertions(+) create mode 100644 frontend/src/components/modal/FandomPicker.vue create mode 100644 frontend/src/components/modal/TagAutocomplete.vue create mode 100644 frontend/src/stores/tags.js diff --git a/frontend/src/components/modal/FandomPicker.vue b/frontend/src/components/modal/FandomPicker.vue new file mode 100644 index 0000000..31189ab --- /dev/null +++ b/frontend/src/components/modal/FandomPicker.vue @@ -0,0 +1,48 @@ + + + diff --git a/frontend/src/components/modal/TagAutocomplete.vue b/frontend/src/components/modal/TagAutocomplete.vue new file mode 100644 index 0000000..cdc81ce --- /dev/null +++ b/frontend/src/components/modal/TagAutocomplete.vue @@ -0,0 +1,135 @@ + + + + + diff --git a/frontend/src/stores/tags.js b/frontend/src/stores/tags.js new file mode 100644 index 0000000..fbb871a --- /dev/null +++ b/frontend/src/stores/tags.js @@ -0,0 +1,58 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +const KIND_OPTIONS = [ + { value: 'general', label: 'General', icon: 'mdi-tag' }, + { value: 'artist', label: 'Artist', icon: 'mdi-palette' }, + { value: 'character', label: 'Character', icon: 'mdi-account-circle' }, + { value: 'fandom', label: 'Fandom', icon: 'mdi-book-open-page-variant' }, + { value: 'series', label: 'Series', icon: 'mdi-bookshelf' }, + { value: 'meta', label: 'Meta', icon: 'mdi-cog-outline' }, + { value: 'rating', label: 'Rating', icon: 'mdi-shield-check-outline' } +] + +const KIND_COLOR = { + artist: 'accent', + character: 'info', + fandom: 'secondary', + series: 'warning', + general: 'on-surface', + meta: 'on-surface', + rating: 'on-surface', + archive: 'on-surface', + post: 'on-surface' +} + +export const useTagStore = defineStore('tags', () => { + const api = useApi() + const fandomCache = ref([]) + + async function autocomplete(q, kind = null, limit = 20) { + if (!q || !q.trim()) return [] + const params = { q, limit } + if (kind) params.kind = kind + return await api.get('/api/tags/autocomplete', { params }) + } + + async function loadFandoms() { + fandomCache.value = await api.get('/api/tags/autocomplete', { + params: { q: ' ', kind: 'fandom', limit: 200 } + }) + return fandomCache.value + } + + async function createFandom(name) { + const fandom = await api.post('/api/tags', { body: { name, kind: 'fandom' } }) + fandomCache.value = [...fandomCache.value, { + id: fandom.id, name: fandom.name, kind: 'fandom', + fandom_id: null, fandom_name: null, image_count: 0 + }] + return fandom + } + + function kindOptions() { return KIND_OPTIONS } + function colorFor(kind) { return KIND_COLOR[kind] || 'on-surface' } + + return { fandomCache, autocomplete, loadFandoms, createFandom, kindOptions, colorFor } +}) From dcf3af88bdcbae8651b1df10e2a7f19976aaf523 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:34:49 -0400 Subject: [PATCH 039/272] =?UTF-8?q?feat(fc2a):=20assemble=20ImageViewer=20?= =?UTF-8?q?modal=20=E2=80=94=20Teleported,=20focused,=20keyboard-navigable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teleport mounts the modal under so it can break out of the gallery container's stacking context. Body overflow is locked while open so background scroll doesn't compete with pan/zoom. Keyboard: left/right arrows navigate prev/next (preventDefault so they don't also scroll the body), Esc closes, Tab is trapped within the modal by browser default since the modal is focused on mount. TagPanel renders kind-colored chips with the close (×) affordance for removal, and hosts TagAutocomplete + FandomPicker for additions. Below 900px the panel drops below the media area instead of beside it. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/modal/ImageViewer.vue | 135 +++++++++++++++++- frontend/src/components/modal/TagPanel.vue | 80 +++++++++++ 2 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 frontend/src/components/modal/TagPanel.vue diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue index 370b462..b2178b5 100644 --- a/frontend/src/components/modal/ImageViewer.vue +++ b/frontend/src/components/modal/ImageViewer.vue @@ -1,9 +1,130 @@ - + + + + diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue new file mode 100644 index 0000000..7ca5e08 --- /dev/null +++ b/frontend/src/components/modal/TagPanel.vue @@ -0,0 +1,80 @@ + + + + + From 90686e6deb2276768c3b1e00d01d8ef632b7d4e2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 14 May 2026 12:36:25 -0400 Subject: [PATCH 040/272] =?UTF-8?q?chore(fc2a):=20polish=20pass=20?= =?UTF-8?q?=E2=80=94=20toasts,=20loading=20skeletons,=20focus=20guards,=20?= =?UTF-8?q?mobile=20breakpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AppSnackbar mounts once at the app root and exposes a window.__fcToast function the stores call from error paths. This avoids prop-drilling a toast emitter through every component while keeping the snackbar component itself testable in isolation. GalleryGrid renders shimmer-skeleton placeholders on initial load so the first paint isn't empty. Skeleton uses the same auto-fill grid columns as the real items so layout doesn't shift when content arrives. Modal arrow nav now ignores text-entry elements so typing in the tag autocomplete doesn't navigate prev/next. Small-screen breakpoint (<600px) tightens gallery thumbnails to 120px min. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/App.vue | 10 ++++++ frontend/src/components/AppSnackbar.vue | 26 +++++++++++++++ .../src/components/gallery/GalleryGrid.vue | 32 +++++++++++++++++++ frontend/src/components/modal/ImageViewer.vue | 20 ++++++++++-- frontend/src/stores/import.js | 2 ++ frontend/src/stores/modal.js | 1 + 6 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 frontend/src/components/AppSnackbar.vue diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 4c51eaf..a4b0086 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -3,9 +3,19 @@ + diff --git a/frontend/src/components/AppSnackbar.vue b/frontend/src/components/AppSnackbar.vue new file mode 100644 index 0000000..cf45f16 --- /dev/null +++ b/frontend/src/components/AppSnackbar.vue @@ -0,0 +1,26 @@ + + + diff --git a/frontend/src/components/gallery/GalleryGrid.vue b/frontend/src/components/gallery/GalleryGrid.vue index ea9c1f3..111cf61 100644 --- a/frontend/src/components/gallery/GalleryGrid.vue +++ b/frontend/src/components/gallery/GalleryGrid.vue @@ -1,5 +1,8 @@ @@ -38,10 +43,13 @@ import SystemStatsCards from '../components/settings/SystemStatsCards.vue' import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue' import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue' import ImportTaskList from '../components/settings/ImportTaskList.vue' +import MaintenancePanel from '../components/settings/MaintenancePanel.vue' +import { useMLStore } from '../stores/ml.js' const tab = ref('overview') const system = useSystemStore() const importStore = useImportStore() +const mlStore = useMLStore() let pollId = null @@ -72,6 +80,8 @@ watch(tab, (t) => { if (t === 'import') { importStore.refreshStatus() importStore.loadTasks(true) + } else if (t === 'maintenance') { + mlStore.loadSettings() } }) From 4623551bf6634245e2e66b432643573c4a2ca1c8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 10:07:38 -0400 Subject: [PATCH 072/272] fix(fc2b): ruff import sort in test_ml_tagger.py ruff isort orders Tagger before TagPrediction (its case-insensitive tiebreak). One-char autofix; ruff check now clean across backend/tests/alembic. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_ml_tagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ml_tagger.py b/tests/test_ml_tagger.py index de288d7..e60237d 100644 --- a/tests/test_ml_tagger.py +++ b/tests/test_ml_tagger.py @@ -10,8 +10,8 @@ import pytest from backend.app.services.ml.tagger import ( STORE_FLOOR, SURFACED_CATEGORIES, - TagPrediction, Tagger, + TagPrediction, get_tagger, ) From 4ebe779b7cb4a1f039c1244b508cee2ce41b7858 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 10:15:24 -0400 Subject: [PATCH 073/272] fix(fc2b): lazy-import onnxruntime in tagger (CI collection failure) onnxruntime is in requirements-ml.txt only (deliberately kept out of the lean web image and CI). The top-level `import onnxruntime` broke pytest collection of test_ml_tagger / test_ml_suggestions / test_tasks_ml even though those are pure-logic/integration-marked, because collection imports the module. Mirrors the embedder's lazy-torch pattern: onnxruntime is imported inside Tagger.load(), placed AFTER the file-existence checks so test_load_raises_when_model_missing still gets RuntimeError (not ModuleNotFoundError) in onnxruntime-less environments. self._session annotation dropped to a comment to avoid an eval-time ort reference. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/ml/tagger.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/app/services/ml/tagger.py b/backend/app/services/ml/tagger.py index 0ee33eb..40ad21c 100644 --- a/backend/app/services/ml/tagger.py +++ b/backend/app/services/ml/tagger.py @@ -15,9 +15,13 @@ from dataclasses import dataclass from pathlib import Path import numpy as np -import onnxruntime as ort from PIL import Image, ImageFile +# onnxruntime lives in requirements-ml.txt only — it is NOT installed in the +# lean web image or in CI. Imported lazily inside Tagger.load() so this module +# imports fine without it (the suggestion service imports SURFACED_CATEGORIES +# from here in the web container, and CI collects the pure-logic tests). + # Tolerate minutely-truncated source images (same rationale as IR's wd14.py: # a few missing bytes at the JPEG EOI shouldn't block tagging the whole image). ImageFile.LOAD_TRUNCATED_IMAGES = True @@ -43,7 +47,7 @@ class TagPrediction: class Tagger: def __init__(self, model_dir: Path | None = None): self._model_dir = model_dir or _MODEL_DIR - self._session: ort.InferenceSession | None = None + self._session = None # onnxruntime.InferenceSession once load()ed self._tag_meta: list[dict] | None = None self._input_name: str | None = None self._output_name: str | None = None @@ -73,6 +77,11 @@ class Tagger: {"name": row["name"], "category": row["category"]} ) + # Lazy import — kept after the file-existence checks so the + # missing-model RuntimeError still fires first in environments + # without onnxruntime (CI / lean web image). + import onnxruntime as ort + session = ort.InferenceSession( str(model_path), providers=["CPUExecutionProvider"] ) From 0a4eb0bdc011ff084b084800eb211a34f29086de Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 12:05:41 -0400 Subject: [PATCH 074/272] fix(fc2b): bare CHECK constraint names (naming-convention double-prefix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Base.metadata's convention applies ck_%(table_name)s_%(constraint_name)s. ml_settings and tag_allowlist passed already-prefixed names (ck_ml_settings_singleton / ck_tag_allowlist_confidence_range), so the ORM-side names came out doubled (ck_ml_settings_ck_ml_settings_singleton etc.) and the migration-0003 smoke tests failed. Same class of bug fixed in FC-2a for ImportSettings — should have applied that lesson here. Bare names ('singleton', 'confidence_range') let the convention produce the final names that match migration 0003's literal DDL. Migration unchanged; only the model __table_args__. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/models/ml_settings.py | 4 +++- backend/app/models/tag_allowlist.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 9586b8c..5ad33ad 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -10,7 +10,9 @@ from .base import Base class MLSettings(Base): __tablename__ = "ml_settings" - __table_args__ = (CheckConstraint("id = 1", name="ck_ml_settings_singleton"),) + # Bare name — Base.metadata's naming convention prepends ck__, + # producing the final ck_ml_settings_singleton (matches migration 0003). + __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) suggestion_threshold_artist: Mapped[float] = mapped_column( diff --git a/backend/app/models/tag_allowlist.py b/backend/app/models/tag_allowlist.py index 6508178..75bd0f2 100644 --- a/backend/app/models/tag_allowlist.py +++ b/backend/app/models/tag_allowlist.py @@ -10,10 +10,12 @@ from .base import Base class TagAllowlist(Base): __tablename__ = "tag_allowlist" + # Bare name — Base.metadata's naming convention prepends ck_
_, + # producing the final ck_tag_allowlist_confidence_range (matches migration 0003). __table_args__ = ( CheckConstraint( "min_confidence > 0 AND min_confidence <= 1", - name="ck_tag_allowlist_confidence_range", + name="confidence_range", ), ) From fc33c23dd5fab7748f5c52888f91494fb0f17a5f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 13:53:42 -0400 Subject: [PATCH 075/272] refactor(fc2c-i): single shared async engine with lifecycle dispose --- backend/app/__init__.py | 5 +++ backend/app/extensions.py | 54 +++++++++++++++++++++++++++----- tests/test_extensions_session.py | 31 ++++++++++++++++++ 3 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 tests/test_extensions_session.py diff --git a/backend/app/__init__.py b/backend/app/__init__.py index 390b783..265a9ed 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -21,4 +21,9 @@ def create_app() -> Quart: # Registered last so /api/* routes win over the SPA catch-all. app.register_blueprint(frontend_bp) + @app.after_serving + async def _dispose_db_engine() -> None: + from .extensions import dispose_engine + await dispose_engine() + return app diff --git a/backend/app/extensions.py b/backend/app/extensions.py index 3594b90..d5af12f 100644 --- a/backend/app/extensions.py +++ b/backend/app/extensions.py @@ -1,14 +1,52 @@ -"""Singleton extension instances; bound to the app in create_app().""" +"""Process-wide singleton async engine + session helper. -from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine +Previously every API blueprint lazily built its own engine via make_engine(), +so the process ran N connection pools. This module owns ONE engine for the +whole app; create_app() disposes it on shutdown via after_serving. +""" + +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + async_sessionmaker, + create_async_engine, +) from .config import get_config - -def make_engine() -> AsyncEngine: - cfg = get_config() - return create_async_engine(cfg.database_url, pool_pre_ping=True, future=True) +_engine: AsyncEngine | None = None +_sessionmaker: async_sessionmaker | None = None -def make_session_factory(engine: AsyncEngine) -> async_sessionmaker: - return async_sessionmaker(engine, expire_on_commit=False) +def get_engine() -> AsyncEngine: + global _engine, _sessionmaker + if _engine is None: + _engine = create_async_engine( + get_config().database_url, pool_pre_ping=True, future=True + ) + _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False) + return _engine + + +def _get_sessionmaker() -> async_sessionmaker: + if _sessionmaker is None: + get_engine() + assert _sessionmaker is not None + return _sessionmaker + + +@asynccontextmanager +async def get_session(): + """`async with get_session() as session:` — one shared pool.""" + Session = _get_sessionmaker() + async with Session() as session: + yield session + + +async def dispose_engine() -> None: + global _engine, _sessionmaker + if _engine is not None: + await _engine.dispose() + _engine = None + _sessionmaker = None diff --git a/tests/test_extensions_session.py b/tests/test_extensions_session.py new file mode 100644 index 0000000..8243f8d --- /dev/null +++ b/tests/test_extensions_session.py @@ -0,0 +1,31 @@ +"""Unit: the shared engine is a memoized singleton; dispose resets it. + +create_async_engine does NOT open a connection, so identity checks need no +DB and run in CI (not integration-marked). +""" + +import pytest + +from backend.app import extensions + + +@pytest.mark.asyncio +async def test_get_engine_is_singleton(): + a = extensions.get_engine() + b = extensions.get_engine() + assert a is b + await extensions.dispose_engine() + + +@pytest.mark.asyncio +async def test_dispose_engine_allows_fresh_engine(): + first = extensions.get_engine() + await extensions.dispose_engine() + second = extensions.get_engine() + assert first is not second + await extensions.dispose_engine() + + +def test_get_session_returns_async_context_manager(): + cm = extensions.get_session() + assert hasattr(cm, "__aenter__") and hasattr(cm, "__aexit__") From f7f75fcac685a12c19af0ae7ab7e6823e783adb0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 15:48:02 -0400 Subject: [PATCH 076/272] refactor(fc2c-i): sweep blueprints + ml tasks onto shared get_session (covers tasks/ml.py, a survey gap) --- backend/app/api/aliases.py | 22 ++++------------------ backend/app/api/allowlist.py | 25 +++++-------------------- backend/app/api/gallery.py | 25 +++++-------------------- backend/app/api/import_admin.py | 25 +++++-------------------- backend/app/api/ml_admin.py | 19 +++---------------- backend/app/api/settings.py | 22 ++++------------------ backend/app/api/suggestions.py | 25 +++++-------------------- backend/app/api/tags.py | 31 +++++++------------------------ backend/app/tasks/ml.py | 12 ++++-------- 9 files changed, 42 insertions(+), 164 deletions(-) diff --git a/backend/app/api/aliases.py b/backend/app/api/aliases.py index 3b1d2f6..3c77510 100644 --- a/backend/app/api/aliases.py +++ b/backend/app/api/aliases.py @@ -2,27 +2,15 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..services.ml.aliases import AliasService aliases_bp = Blueprint("aliases", __name__, url_prefix="/api") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - @aliases_bp.route("/aliases", methods=["GET"]) async def list_aliases(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: rows = await AliasService(session).list_all() return jsonify( [ @@ -43,8 +31,7 @@ async def create_alias(): required = {"alias_string", "alias_category", "canonical_tag_id"} if not body or not required.issubset(body): return jsonify({"error": f"required: {sorted(required)}"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: await AliasService(session).create( body["alias_string"], body["alias_category"], @@ -58,8 +45,7 @@ async def create_alias(): "/aliases//", methods=["DELETE"] ) async def remove_alias(alias_string: str, alias_category: str): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: await AliasService(session).remove(alias_string, alias_category) await session.commit() return "", 204 diff --git a/backend/app/api/allowlist.py b/backend/app/api/allowlist.py index 69889c8..53f38bb 100644 --- a/backend/app/api/allowlist.py +++ b/backend/app/api/allowlist.py @@ -2,28 +2,16 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..models import TagAllowlist from ..services.ml.allowlist import AllowlistService allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - @allowlist_bp.route("/allowlist", methods=["GET"]) async def list_allowlist(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: rows = await AllowlistService(session).list_all() return jsonify( [ @@ -40,8 +28,7 @@ async def list_allowlist(): @allowlist_bp.route("/tags//allowlist", methods=["GET"]) async def get_one(tag_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: row = await session.get(TagAllowlist, tag_id) if row is None: return jsonify({"error": "not on allowlist"}), 404 @@ -58,8 +45,7 @@ async def patch_threshold(tag_id: int): mc = float(body["min_confidence"]) if not (0 < mc <= 1): return jsonify({"error": "min_confidence must be in (0, 1]"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: await AllowlistService(session).update_threshold(tag_id, mc) await session.commit() return "", 204 @@ -67,8 +53,7 @@ async def patch_threshold(tag_id: int): @allowlist_bp.route("/tags//allowlist", methods=["DELETE"]) async def remove(tag_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: await AllowlistService(session).remove(tag_id) await session.commit() return "", 204 diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py index e824297..5401368 100644 --- a/backend/app/api/gallery.py +++ b/backend/app/api/gallery.py @@ -2,22 +2,11 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..services.gallery_service import GalleryService gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - @gallery_bp.route("/scroll", methods=["GET"]) async def scroll(): @@ -29,8 +18,7 @@ async def scroll(): tag_id_raw = request.args.get("tag_id") tag_id = int(tag_id_raw) if tag_id_raw else None - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = GalleryService(session) try: page = await svc.scroll(cursor=cursor, limit=limit, tag_id=tag_id) @@ -63,8 +51,7 @@ async def scroll(): async def timeline(): tag_id_raw = request.args.get("tag_id") tag_id = int(tag_id_raw) if tag_id_raw else None - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = GalleryService(session) buckets = await svc.timeline(tag_id=tag_id) return jsonify([{"year": b.year, "month": b.month, "count": b.count} for b in buckets]) @@ -79,8 +66,7 @@ async def jump(): return jsonify({"error": "year and month query params required"}), 400 tag_id_raw = request.args.get("tag_id") tag_id = int(tag_id_raw) if tag_id_raw else None - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = GalleryService(session) cursor = await svc.jump_cursor(year=year, month=month, tag_id=tag_id) return jsonify({"cursor": cursor}) @@ -88,8 +74,7 @@ async def jump(): @gallery_bp.route("/image/", methods=["GET"]) async def image_detail(image_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = GalleryService(session) payload = await svc.get_image_with_tags(image_id) if payload is None: diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py index 21e11c3..e4be69a 100644 --- a/backend/app/api/import_admin.py +++ b/backend/app/api/import_admin.py @@ -5,22 +5,11 @@ from datetime import UTC, datetime, timedelta from quart import Blueprint, jsonify, request from sqlalchemy import delete, select, update -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..models import ImportBatch, ImportTask import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - @import_admin_bp.route("/trigger", methods=["POST"]) async def trigger_scan(): @@ -39,8 +28,7 @@ async def trigger_scan(): @import_admin_bp.route("/status", methods=["GET"]) async def status(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: active = ( await session.execute( select(ImportBatch) @@ -72,8 +60,7 @@ async def list_tasks(): cursor_raw = request.args.get("cursor") cursor_id = int(cursor_raw) if cursor_raw else None - Session = _session_factory() - async with Session() as session: + async with get_session() as session: stmt = select(ImportTask).order_by(ImportTask.created_at.desc(), ImportTask.id.desc()) if status_filter: stmt = stmt.where(ImportTask.status == status_filter) @@ -107,8 +94,7 @@ async def list_tasks(): @import_admin_bp.route("/retry-failed", methods=["POST"]) async def retry_failed(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: failed_ids = ( await session.execute(select(ImportTask.id).where(ImportTask.status == "failed")) ).scalars().all() @@ -140,8 +126,7 @@ async def clear_completed(): datetime.now(UTC) - timedelta(days=int(age_days)) if age_days else None ) - Session = _session_factory() - async with Session() as session: + async with get_session() as session: stmt = delete(ImportTask).where(ImportTask.status.in_(status_filter)) if cutoff is not None: stmt = stmt.where(ImportTask.finished_at < cutoff) diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py index 3d7bbe5..a346175 100644 --- a/backend/app/api/ml_admin.py +++ b/backend/app/api/ml_admin.py @@ -2,22 +2,11 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..models import MLSettings ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - _EDITABLE = ( "suggestion_threshold_artist", @@ -33,8 +22,7 @@ _EDITABLE = ( async def get_settings(): from sqlalchemy import select - Session = _session_factory() - async with Session() as session: + async with get_session() as session: s = ( await session.execute(select(MLSettings).where(MLSettings.id == 1)) ).scalar_one() @@ -59,8 +47,7 @@ async def patch_settings(): body = await request.get_json() if not isinstance(body, dict): return jsonify({"error": "body must be an object"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: s = ( await session.execute(select(MLSettings).where(MLSettings.id == 1)) ).scalar_one() diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 7d2d842..f5f5354 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -3,22 +3,11 @@ from quart import Blueprint, jsonify, request from sqlalchemy import func, select -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..models import ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag settings_bp = Blueprint("settings", __name__, url_prefix="/api") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - _EDITABLE_FIELDS = ( "min_width", @@ -33,8 +22,7 @@ _EDITABLE_FIELDS = ( @settings_bp.route("/settings/import", methods=["GET"]) async def get_import_settings(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: row = ( await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) ).scalar_one() @@ -55,8 +43,7 @@ async def update_import_settings(): if not isinstance(body, dict): return jsonify({"error": "body must be a JSON object"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: row = ( await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) ).scalar_one() @@ -70,8 +57,7 @@ async def update_import_settings(): @settings_bp.route("/system/stats", methods=["GET"]) async def system_stats(): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: total_images = (await session.execute(select(func.count(ImageRecord.id)))).scalar_one() total_tags = (await session.execute(select(func.count(Tag.id)))).scalar_one() storage_bytes = ( diff --git a/backend/app/api/suggestions.py b/backend/app/api/suggestions.py index 9e00aea..09d70e2 100644 --- a/backend/app/api/suggestions.py +++ b/backend/app/api/suggestions.py @@ -2,28 +2,16 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..services.ml.allowlist import AllowlistService from ..services.ml.suggestions import SuggestionService suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - @suggestions_bp.route("/images//suggestions", methods=["GET"]) async def get_suggestions(image_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: sl = await SuggestionService(session).for_image(image_id) return jsonify( { @@ -53,8 +41,7 @@ async def accept_suggestion(image_id: int): if not body or "tag_id" not in body: return jsonify({"error": "tag_id required"}), 400 tag_id = body["tag_id"] - Session = _session_factory() - async with Session() as session: + async with get_session() as session: newly_added = await AllowlistService(session).accept(image_id, tag_id) await session.commit() if newly_added: @@ -72,8 +59,7 @@ async def alias_suggestion(image_id: int): required = {"alias_string", "alias_category", "canonical_tag_id"} if not body or not required.issubset(body): return jsonify({"error": f"required: {sorted(required)}"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: newly_added = await AllowlistService(session).add_alias_and_accept( image_id, body["alias_string"], @@ -95,8 +81,7 @@ async def dismiss_suggestion(image_id: int): body = await request.get_json() if not body or "tag_id" not in body: return jsonify({"error": "tag_id required"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: await AllowlistService(session).dismiss(image_id, body["tag_id"]) await session.commit() return "", 204 diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 8498c4f..941754d 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -2,23 +2,12 @@ from quart import Blueprint, jsonify, request -from ..extensions import make_engine, make_session_factory +from ..extensions import get_session from ..models import TagKind from ..services.tag_service import TagService, TagValidationError tags_bp = Blueprint("tags", __name__, url_prefix="/api") -_engine = None -_Session = None - - -def _session_factory(): - global _engine, _Session - if _engine is None: - _engine = make_engine() - _Session = make_session_factory(_engine) - return _Session - def _coerce_kind(raw: str | None) -> TagKind | None: if raw is None: @@ -38,8 +27,7 @@ async def autocomplete(): except ValueError: return jsonify({"error": "limit must be an integer"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) hits = await svc.autocomplete(q, kind=kind, limit=limit) @@ -69,8 +57,7 @@ async def create_tag(): return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400 fandom_id = body.get("fandom_id") - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) try: tag = await svc.find_or_create(name, kind, fandom_id=fandom_id) @@ -84,8 +71,7 @@ async def create_tag(): @tags_bp.route("/images//tags", methods=["GET"]) async def list_tags_for_image(image_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) tags = await svc.list_for_image(image_id) return jsonify( @@ -108,8 +94,7 @@ async def add_tag_to_image(image_id: int): return jsonify({"error": "tag_id required"}), 400 source = body.get("source", "manual") - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) await svc.add_to_image(image_id, body["tag_id"], source=source) await session.commit() @@ -118,8 +103,7 @@ async def add_tag_to_image(image_id: int): @tags_bp.route("/images//tags/", methods=["DELETE"]) async def remove_tag_from_image(image_id: int, tag_id: int): - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) await svc.remove_from_image(image_id, tag_id) await session.commit() @@ -131,8 +115,7 @@ async def rename_tag(tag_id: int): body = await request.get_json() if not body or "name" not in body: return jsonify({"error": "name required"}), 400 - Session = _session_factory() - async with Session() as session: + async with get_session() as session: svc = TagService(session) try: tag = await svc.rename(tag_id, body["name"]) diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py index 29a9816..13d4394 100644 --- a/backend/app/tasks/ml.py +++ b/backend/app/tasks/ml.py @@ -305,13 +305,11 @@ def _confidence_for_tag(session, tag, preds: dict) -> float | None: def recompute_centroid(self, tag_id: int) -> bool: import asyncio - from ..extensions import make_engine, make_session_factory + from ..extensions import get_session from ..services.ml.centroids import CentroidService async def _run() -> bool: - engine = make_engine() - Session = make_session_factory(engine) - async with Session() as session: + async with get_session() as session: svc = CentroidService(session) result = await svc.recompute_for_tag(tag_id) await session.commit() @@ -325,13 +323,11 @@ def recompute_centroids(self) -> int: """Daily: find drifted centroids, enqueue recompute_centroid for each.""" import asyncio - from ..extensions import make_engine, make_session_factory + from ..extensions import get_session from ..services.ml.centroids import CentroidService async def _list() -> list[int]: - engine = make_engine() - Session = make_session_factory(engine) - async with Session() as session: + async with get_session() as session: return await CentroidService(session).list_drifted() drifted = asyncio.run(_list()) From a638c469e08fd9e60541f0a2414e637579dc832b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 15:48:29 -0400 Subject: [PATCH 077/272] feat(fc2c-i): migration 0004 enables tsm_system_rows --- .../versions/0004_fc2c_i_tsm_system_rows.py | 23 ++++++++++++++++ tests/test_migration_0004.py | 27 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 alembic/versions/0004_fc2c_i_tsm_system_rows.py create mode 100644 tests/test_migration_0004.py diff --git a/alembic/versions/0004_fc2c_i_tsm_system_rows.py b/alembic/versions/0004_fc2c_i_tsm_system_rows.py new file mode 100644 index 0000000..e8bd920 --- /dev/null +++ b/alembic/versions/0004_fc2c_i_tsm_system_rows.py @@ -0,0 +1,23 @@ +"""fc2c-i: enable tsm_system_rows for scalable random sampling + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-05-15 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0004" +down_revision: Union[str, None] = "0003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") + + +def downgrade() -> None: + op.execute("DROP EXTENSION IF EXISTS tsm_system_rows") diff --git a/tests/test_migration_0004.py b/tests/test_migration_0004.py new file mode 100644 index 0000000..7975d0c --- /dev/null +++ b/tests/test_migration_0004.py @@ -0,0 +1,27 @@ +"""Integration: the tsm_system_rows extension is installed by migration 0004. + +Needs a real Postgres (CI does not provision one), so integration-marked. +""" + +import pytest +from sqlalchemy import text + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_tsm_system_rows_extension_present(db): + row = ( + await db.execute( + text("SELECT 1 FROM pg_extension WHERE extname = 'tsm_system_rows'") + ) + ).first() + assert row is not None + + +@pytest.mark.asyncio +async def test_system_rows_sampling_is_usable(db): + # Should parse and execute even on an empty table. + await db.execute( + text("SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(1)") + ) From a74b313596d0ddeacb6de7b213faa5568d135ba0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 15:49:25 -0400 Subject: [PATCH 078/272] feat(fc2c-i): showcase random-sample service and endpoint --- backend/app/api/__init__.py | 2 ++ backend/app/api/showcase.py | 23 ++++++++++++ backend/app/services/showcase_service.py | 40 +++++++++++++++++++++ tests/test_api_showcase.py | 35 ++++++++++++++++++ tests/test_showcase_service.py | 45 ++++++++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 backend/app/api/showcase.py create mode 100644 backend/app/services/showcase_service.py create mode 100644 tests/test_api_showcase.py create mode 100644 tests/test_showcase_service.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 1523f19..b68e475 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -20,12 +20,14 @@ def all_blueprints() -> list[Blueprint]: from .import_admin import import_admin_bp from .ml_admin import ml_admin_bp from .settings import settings_bp + from .showcase import showcase_bp from .suggestions import suggestions_bp from .tags import tags_bp return [ api_bp, gallery_bp, tags_bp, + showcase_bp, settings_bp, import_admin_bp, suggestions_bp, diff --git a/backend/app/api/showcase.py b/backend/app/api/showcase.py new file mode 100644 index 0000000..232cc86 --- /dev/null +++ b/backend/app/api/showcase.py @@ -0,0 +1,23 @@ +"""Showcase API: scalable random sample of images.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.showcase_service import ShowcaseService + +showcase_bp = Blueprint("showcase", __name__, url_prefix="/api/showcase") + + +@showcase_bp.route("", methods=["GET"]) +async def random_showcase(): + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = ShowcaseService(session) + try: + images = await svc.random_sample(limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify({"images": images}) diff --git a/backend/app/services/showcase_service.py b/backend/app/services/showcase_service.py new file mode 100644 index 0000000..b45e0b5 --- /dev/null +++ b/backend/app/services/showcase_service.py @@ -0,0 +1,40 @@ +"""Random-sample query for the showcase. + +Uses the tsm_system_rows TABLESAMPLE method (migration 0004) instead of +ORDER BY random(): sampling cost scales with the sample size, not the table, +so it stays fast as the collection grows. SYSTEM_ROWS(n) returns up to n +rows; an empty table yields none. +""" + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord +from .gallery_service import thumbnail_url + + +class ShowcaseService: + def __init__(self, session: AsyncSession): + self.session = session + + async def random_sample(self, limit: int = 60) -> list[dict]: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + stmt = select(ImageRecord).from_statement( + text( + "SELECT * FROM image_record " + "TABLESAMPLE SYSTEM_ROWS(:n)" + ).bindparams(n=limit) + ) + rows = (await self.session.execute(stmt)).scalars().all() + return [ + { + "id": r.id, + "sha256": r.sha256, + "mime": r.mime, + "width": r.width, + "height": r.height, + "thumbnail_url": thumbnail_url(r.sha256, r.mime), + } + for r in rows + ] diff --git a/tests/test_api_showcase.py b/tests/test_api_showcase.py new file mode 100644 index 0000000..d61e793 --- /dev/null +++ b/tests/test_api_showcase.py @@ -0,0 +1,35 @@ +import pytest + +from backend.app import create_app +from backend.app.models import ImageRecord + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_showcase_returns_images(client, db): + for i in range(8): + db.add(ImageRecord( + path=f"/images/sc/{i}.jpg", sha256=f"c{i:063d}", + size_bytes=1, mime="image/jpeg", width=10, height=10, + origin="imported_filesystem", integrity_status="unknown", + )) + await db.flush() + await db.commit() + resp = await client.get("/api/showcase?limit=4") + assert resp.status_code == 200 + body = await resp.get_json() + assert "images" in body and len(body["images"]) <= 4 + + +@pytest.mark.asyncio +async def test_showcase_rejects_bad_limit(client): + resp = await client.get("/api/showcase?limit=oops") + assert resp.status_code == 400 diff --git a/tests/test_showcase_service.py b/tests/test_showcase_service.py new file mode 100644 index 0000000..fac5fa9 --- /dev/null +++ b/tests/test_showcase_service.py @@ -0,0 +1,45 @@ +import pytest + +from backend.app.models import ImageRecord +from backend.app.services.showcase_service import ShowcaseService + +pytestmark = pytest.mark.integration + + +async def _seed(db, n): + for i in range(n): + db.add(ImageRecord( + path=f"/images/s/{i}.jpg", sha256=f"s{i:063d}", + size_bytes=1, mime="image/jpeg", width=100, height=200, + origin="imported_filesystem", integrity_status="unknown", + )) + await db.flush() + await db.commit() + + +@pytest.mark.asyncio +async def test_random_sample_returns_items_with_shape(db): + await _seed(db, 20) + svc = ShowcaseService(db) + items = await svc.random_sample(limit=5) + assert 1 <= len(items) <= 5 + first = items[0] + assert set(first.keys()) == { + "id", "sha256", "mime", "width", "height", "thumbnail_url" + } + assert first["thumbnail_url"].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_random_sample_empty_table_returns_empty(db): + svc = ShowcaseService(db) + assert await svc.random_sample(limit=5) == [] + + +@pytest.mark.asyncio +async def test_random_sample_rejects_bad_limit(db): + svc = ShowcaseService(db) + with pytest.raises(ValueError): + await svc.random_sample(limit=0) + with pytest.raises(ValueError): + await svc.random_sample(limit=201) From 8484cb9aaa7fa01cfa01d21abd0e144e3acfdb5d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 15:50:41 -0400 Subject: [PATCH 079/272] feat(fc2c-i): tag directory service and /api/tags/directory --- backend/app/api/tags.py | 19 +++ backend/app/services/tag_directory_service.py | 126 ++++++++++++++++++ tests/test_api_tags_directory.py | 30 +++++ tests/test_tag_directory_service.py | 72 ++++++++++ 4 files changed, 247 insertions(+) create mode 100644 backend/app/services/tag_directory_service.py create mode 100644 tests/test_api_tags_directory.py create mode 100644 tests/test_tag_directory_service.py diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py index 941754d..8e20970 100644 --- a/backend/app/api/tags.py +++ b/backend/app/api/tags.py @@ -4,6 +4,7 @@ from quart import Blueprint, jsonify, request from ..extensions import get_session from ..models import TagKind +from ..services.tag_directory_service import TagDirectoryService from ..services.tag_service import TagService, TagValidationError tags_bp = Blueprint("tags", __name__, url_prefix="/api") @@ -46,6 +47,24 @@ async def autocomplete(): ) +@tags_bp.route("/tags/directory", methods=["GET"]) +async def directory(): + kind = request.args.get("kind") or None + q = request.args.get("q") or None + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = TagDirectoryService(session) + try: + page = await svc.list_tags(kind=kind, q=q, cursor=cursor, limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify({"cards": page.cards, "next_cursor": page.next_cursor}) + + @tags_bp.route("/tags", methods=["POST"]) async def create_tag(): body = await request.get_json() diff --git a/backend/app/services/tag_directory_service.py b/backend/app/services/tag_directory_service.py new file mode 100644 index 0000000..a0ffaaa --- /dev/null +++ b/backend/app/services/tag_directory_service.py @@ -0,0 +1,126 @@ +"""Paged tag-directory cards with window-function preview thumbnails. + +Cursor orders by (Tag.name ASC, Tag.id ASC). Preview thumbnails for the +page's tags are fetched in ONE windowed query (ROW_NUMBER <= 3) to avoid +an N+1. fandom_name is resolved via a self-join on Tag (fandom_id -> Tag). +""" + +import base64 +from dataclasses import dataclass + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased + +from ..models import ImageRecord, Tag +from ..models.tag import image_tag +from .gallery_service import thumbnail_url + +_SEP = "|" + + +def _encode(name: str, tag_id: int) -> str: + return base64.urlsafe_b64encode(f"{name}{_SEP}{tag_id}".encode()).decode() + + +def _decode(cursor: str) -> tuple[str, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + name, tid = raw.rsplit(_SEP, 1) + return name, int(tid) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class DirectoryPage: + cards: list[dict] + next_cursor: str | None + + +class TagDirectoryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def list_tags( + self, + kind: str | None, + q: str | None, + cursor: str | None, + limit: int = 60, + ) -> DirectoryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + + fandom = aliased(Tag) + count_col = func.count(image_tag.c.image_record_id).label("image_count") + stmt = ( + select(Tag, fandom.name.label("fandom_name"), count_col) + .outerjoin(fandom, Tag.fandom_id == fandom.id) + .outerjoin(image_tag, image_tag.c.tag_id == Tag.id) + .group_by(Tag.id, fandom.name) + ) + if kind is not None: + stmt = stmt.where(Tag.kind == kind) + if q: + stmt = stmt.where(Tag.name.ilike(f"%{q}%")) + if cursor: + c_name, c_id = _decode(cursor) + stmt = stmt.where( + or_( + Tag.name > c_name, + and_(Tag.name == c_name, Tag.id > c_id), + ) + ) + stmt = stmt.order_by(Tag.name.asc(), Tag.id.asc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() + + next_cursor = None + if len(rows) > limit: + last_tag = rows[limit - 1][0] + next_cursor = _encode(last_tag.name, last_tag.id) + rows = rows[:limit] + + tag_ids = [r[0].id for r in rows] + previews = await self._previews(tag_ids) + + cards = [ + { + "id": tag.id, + "name": tag.name, + "kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind, + "fandom_id": tag.fandom_id, + "fandom_name": fandom_name, + "image_count": int(image_count), + "preview_thumbnails": previews.get(tag.id, []), + } + for tag, fandom_name, image_count in rows + ] + return DirectoryPage(cards=cards, next_cursor=next_cursor) + + async def _previews(self, tag_ids: list[int]) -> dict[int, list[str]]: + if not tag_ids: + return {} + rn = func.row_number().over( + partition_by=image_tag.c.tag_id, + order_by=image_tag.c.image_record_id.desc(), + ).label("rn") + sub = ( + select( + image_tag.c.tag_id.label("tag_id"), + image_tag.c.image_record_id.label("image_record_id"), + rn, + ) + .where(image_tag.c.tag_id.in_(tag_ids)) + .subquery() + ) + stmt = ( + select(sub.c.tag_id, ImageRecord.sha256, ImageRecord.mime) + .join(ImageRecord, ImageRecord.id == sub.c.image_record_id) + .where(sub.c.rn <= 3) + .order_by(sub.c.tag_id, sub.c.rn) + ) + out: dict[int, list[str]] = {} + for tag_id, sha, mime in (await self.session.execute(stmt)).all(): + out.setdefault(tag_id, []).append(thumbnail_url(sha, mime)) + return out diff --git a/tests/test_api_tags_directory.py b/tests/test_api_tags_directory.py new file mode 100644 index 0000000..de9c01e --- /dev/null +++ b/tests/test_api_tags_directory.py @@ -0,0 +1,30 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Tag, TagKind + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_directory_endpoint(client, db): + db.add(Tag(name="api_tag", kind=TagKind.general)) + await db.flush() + await db.commit() + resp = await client.get("/api/tags/directory?limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert "cards" in body and "next_cursor" in body + + +@pytest.mark.asyncio +async def test_directory_bad_limit(client): + resp = await client.get("/api/tags/directory?limit=nan") + assert resp.status_code == 400 diff --git a/tests/test_tag_directory_service.py b/tests/test_tag_directory_service.py new file mode 100644 index 0000000..b40e6b8 --- /dev/null +++ b/tests/test_tag_directory_service.py @@ -0,0 +1,72 @@ +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.tag_directory_service import TagDirectoryService + +pytestmark = pytest.mark.integration + + +async def _img(db, i): + r = ImageRecord( + path=f"/images/td/{i}.jpg", sha256=f"d{i:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + db.add(r) + await db.flush() + return r + + +@pytest.mark.asyncio +async def test_directory_lists_tags_with_counts_and_previews(db): + t = Tag(name="sunset", kind=TagKind.general) + db.add(t) + await db.flush() + for i in range(4): + img = await _img(db, i) + await db.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=t.id, source="manual")) + await db.flush() + svc = TagDirectoryService(db) + page = await svc.list_tags(kind=None, q=None, cursor=None, limit=10) + card = next(c for c in page.cards if c["name"] == "sunset") + assert card["image_count"] == 4 + assert len(card["preview_thumbnails"]) == 3 + assert card["preview_thumbnails"][0].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_directory_kind_filter_and_search(db): + db.add_all([ + Tag(name="artist_a", kind=TagKind.artist), + Tag(name="char_b", kind=TagKind.character), + Tag(name="char_zoom", kind=TagKind.character), + ]) + await db.flush() + svc = TagDirectoryService(db) + only_char = await svc.list_tags(kind="character", q=None, cursor=None, limit=10) + assert {c["name"] for c in only_char.cards} == {"char_b", "char_zoom"} + searched = await svc.list_tags(kind=None, q="zoom", cursor=None, limit=10) + assert {c["name"] for c in searched.cards} == {"char_zoom"} + + +@pytest.mark.asyncio +async def test_directory_cursor_pagination(db): + for i in range(5): + db.add(Tag(name=f"t{i:02d}", kind=TagKind.general)) + await db.flush() + svc = TagDirectoryService(db) + first = await svc.list_tags(kind=None, q=None, cursor=None, limit=2) + assert len(first.cards) == 2 + assert first.next_cursor is not None + second = await svc.list_tags(kind=None, q=None, cursor=first.next_cursor, limit=2) + assert len(second.cards) == 2 + assert {c["id"] for c in first.cards}.isdisjoint({c["id"] for c in second.cards}) + + +@pytest.mark.asyncio +async def test_directory_rejects_bad_limit(db): + svc = TagDirectoryService(db) + with pytest.raises(ValueError): + await svc.list_tags(kind=None, q=None, cursor=None, limit=0) From e43c2a0dd0fd04a9d8960b20fd0a6addc44f71c0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 15:52:02 -0400 Subject: [PATCH 080/272] feat(fc2c-i): artist overview + paged images endpoints --- backend/app/api/__init__.py | 2 + backend/app/api/artist.py | 36 +++++ backend/app/services/artist_service.py | 199 +++++++++++++++++++++++++ tests/test_api_artist.py | 37 +++++ tests/test_artist_service.py | 78 ++++++++++ 5 files changed, 352 insertions(+) create mode 100644 backend/app/api/artist.py create mode 100644 backend/app/services/artist_service.py create mode 100644 tests/test_api_artist.py create mode 100644 tests/test_artist_service.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index b68e475..ad1251f 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -16,6 +16,7 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) def all_blueprints() -> list[Blueprint]: from .aliases import aliases_bp from .allowlist import allowlist_bp + from .artist import artist_bp from .gallery import gallery_bp from .import_admin import import_admin_bp from .ml_admin import ml_admin_bp @@ -27,6 +28,7 @@ def all_blueprints() -> list[Blueprint]: api_bp, gallery_bp, tags_bp, + artist_bp, showcase_bp, settings_bp, import_admin_bp, diff --git a/backend/app/api/artist.py b/backend/app/api/artist.py new file mode 100644 index 0000000..4b755e6 --- /dev/null +++ b/backend/app/api/artist.py @@ -0,0 +1,36 @@ +"""Artist API: overview aggregates + paged images.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.artist_service import ArtistService + +artist_bp = Blueprint("artist", __name__, url_prefix="/api/artist") + + +@artist_bp.route("/", methods=["GET"]) +async def overview(slug: str): + async with get_session() as session: + svc = ArtistService(session) + data = await svc.overview(slug) + if data is None: + return jsonify({"error": "artist not found"}), 404 + return jsonify(data) + + +@artist_bp.route("//images", methods=["GET"]) +async def images(slug: str): + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = ArtistService(session) + try: + page = await svc.images(slug, cursor=cursor, limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + if page is None: + return jsonify({"error": "artist not found"}), 404 + return jsonify({"images": page.images, "next_cursor": page.next_cursor}) diff --git a/backend/app/services/artist_service.py b/backend/app/services/artist_service.py new file mode 100644 index 0000000..9aeaa8e --- /dev/null +++ b/backend/app/services/artist_service.py @@ -0,0 +1,199 @@ +"""Artist overview + paged images. + +Images are linked to an artist through the provenance chain +Source(artist_id) -> ImageProvenance(source_id) -> ImageRecord. DISTINCT +guards against an image having several provenance rows for one artist. +Dates come from Post.post_date via ImageProvenance.post_id. +""" + +from dataclasses import dataclass + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, + Tag, +) +from ..models.tag import image_tag +from .gallery_service import decode_cursor, encode_cursor, thumbnail_url + + +@dataclass(frozen=True) +class ArtistImagesPage: + images: list[dict] + next_cursor: str | None + + +class ArtistService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _artist_by_slug(self, slug: str) -> Artist | None: + return ( + await self.session.execute( + select(Artist).where(Artist.slug == slug) + ) + ).scalar_one_or_none() + + async def overview(self, slug: str) -> dict | None: + artist = await self._artist_by_slug(slug) + if artist is None: + return None + aid = artist.id + + img_join = ( + select(ImageRecord.id) + .join(ImageProvenance, ImageProvenance.image_record_id == ImageRecord.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == aid) + ) + image_count = ( + await self.session.execute( + select(func.count(func.distinct(ImageRecord.id))) + .select_from(ImageRecord) + .join(ImageProvenance, ImageProvenance.image_record_id == ImageRecord.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == aid) + ) + ).scalar_one() + + date_row = ( + await self.session.execute( + select(func.min(Post.post_date), func.max(Post.post_date)) + .select_from(Post) + .join(ImageProvenance, ImageProvenance.post_id == Post.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == aid) + ) + ).first() + dmin, dmax = date_row if date_row else (None, None) + + cooccurring = ( + await self.session.execute( + select( + Tag.id, Tag.name, Tag.kind, + func.count(image_tag.c.image_record_id).label("cnt"), + ) + .select_from(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id.in_(img_join)) + .group_by(Tag.id, Tag.name, Tag.kind) + .order_by(func.count(image_tag.c.image_record_id).desc()) + .limit(20) + ) + ).all() + + sources = ( + await self.session.execute( + select( + Source.id, Source.platform, Source.url, + func.count(func.distinct(ImageProvenance.image_record_id)).label("cnt"), + ) + .select_from(Source) + .outerjoin(ImageProvenance, ImageProvenance.source_id == Source.id) + .where(Source.artist_id == aid) + .group_by(Source.id, Source.platform, Source.url) + .order_by(Source.id) + ) + ).all() + + month = func.date_trunc("month", Post.post_date).label("m") + activity = ( + await self.session.execute( + select(month, func.count(func.distinct(ImageProvenance.image_record_id))) + .select_from(Post) + .join(ImageProvenance, ImageProvenance.post_id == Post.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(and_(Source.artist_id == aid, Post.post_date.isnot(None))) + .group_by(month) + .order_by(month) + ) + ).all() + + return { + "id": artist.id, + "name": artist.name, + "slug": artist.slug, + "image_count": int(image_count), + "date_range": { + "min": dmin.isoformat() if dmin else None, + "max": dmax.isoformat() if dmax else None, + }, + "cooccurring_tags": [ + { + "id": tid, + "name": name, + "kind": kind.value if hasattr(kind, "value") else kind, + "count": int(cnt), + } + for tid, name, kind, cnt in cooccurring + ], + "sources": [ + { + "id": sid, + "platform": platform, + "url": url, + "image_count": int(cnt), + } + for sid, platform, url, cnt in sources + ], + "activity": [ + {"month": m.isoformat(), "count": int(cnt)} + for m, cnt in activity + ], + } + + async def images( + self, slug: str, cursor: str | None, limit: int = 60 + ) -> ArtistImagesPage | None: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + artist = await self._artist_by_slug(slug) + if artist is None: + return None + + stmt = ( + select(ImageRecord) + .join(ImageProvenance, ImageProvenance.image_record_id == ImageRecord.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == artist.id) + .distinct() + ) + if cursor: + cur_ts, cur_id = decode_cursor(cursor) + stmt = stmt.where( + or_( + ImageRecord.created_at < cur_ts, + and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id), + ) + ) + stmt = stmt.order_by( + ImageRecord.created_at.desc(), ImageRecord.id.desc() + ).limit(limit + 1) + rows = (await self.session.execute(stmt)).scalars().all() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = encode_cursor(last.created_at, last.id) + rows = rows[:limit] + + return ArtistImagesPage( + images=[ + { + "id": r.id, + "sha256": r.sha256, + "mime": r.mime, + "width": r.width, + "height": r.height, + "thumbnail_url": thumbnail_url(r.sha256, r.mime), + } + for r in rows + ], + next_cursor=next_cursor, + ) diff --git a/tests/test_api_artist.py b/tests/test_api_artist.py new file mode 100644 index 0000000..599093f --- /dev/null +++ b/tests/test_api_artist.py @@ -0,0 +1,37 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_artist_404(client): + resp = await client.get("/api/artist/nope") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_artist_overview_ok(client, db): + db.add(Artist(name="Mira", slug="mira")) + await db.flush() + await db.commit() + resp = await client.get("/api/artist/mira") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["name"] == "Mira" + assert body["image_count"] == 0 + + +@pytest.mark.asyncio +async def test_artist_images_404(client): + resp = await client.get("/api/artist/nope/images") + assert resp.status_code == 404 diff --git a/tests/test_artist_service.py b/tests/test_artist_service.py new file mode 100644 index 0000000..7853ff0 --- /dev/null +++ b/tests/test_artist_service.py @@ -0,0 +1,78 @@ +from datetime import UTC, datetime + +import pytest + +from backend.app.models import ( + Artist, ImageProvenance, ImageRecord, Post, Source, Tag, TagKind, +) +from backend.app.models.tag import image_tag +from backend.app.services.artist_service import ArtistService + +pytestmark = pytest.mark.integration + + +async def _fixture(db): + artist = Artist(name="Nadia", slug="nadia") + db.add(artist) + await db.flush() + src = Source(artist_id=artist.id, platform="web", url="http://x") + db.add(src) + await db.flush() + post = Post( + source_id=src.id, external_post_id="p1", + post_date=datetime(2026, 3, 1, tzinfo=UTC), + ) + db.add(post) + await db.flush() + img = ImageRecord( + path="/images/a/1.jpg", sha256="a" + "0" * 63, + size_bytes=1, mime="image/jpeg", width=4, height=8, + origin="downloaded", integrity_status="unknown", + ) + db.add(img) + await db.flush() + db.add(ImageProvenance( + image_record_id=img.id, post_id=post.id, source_id=src.id)) + tag = Tag(name="forest", kind=TagKind.general) + db.add(tag) + await db.flush() + await db.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=tag.id, source="manual")) + await db.flush() + return artist, src, img + + +@pytest.mark.asyncio +async def test_overview_aggregates(db): + artist, src, img = await _fixture(db) + svc = ArtistService(db) + ov = await svc.overview("nadia") + assert ov["name"] == "Nadia" + assert ov["image_count"] == 1 + assert ov["date_range"]["min"].startswith("2026-03-01") + assert ov["date_range"]["max"].startswith("2026-03-01") + assert any(t["name"] == "forest" for t in ov["cooccurring_tags"]) + assert ov["sources"][0]["image_count"] == 1 + assert ov["activity"][0]["count"] == 1 + + +@pytest.mark.asyncio +async def test_overview_unknown_slug_returns_none(db): + svc = ArtistService(db) + assert await svc.overview("ghost") is None + + +@pytest.mark.asyncio +async def test_paged_images(db): + artist, src, img = await _fixture(db) + svc = ArtistService(db) + page = await svc.images("nadia", cursor=None, limit=10) + assert page is not None + assert len(page.images) == 1 + assert page.images[0]["thumbnail_url"].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_paged_images_unknown_slug_none(db): + svc = ArtistService(db) + assert await svc.images("ghost", cursor=None, limit=10) is None From a4c7d447805be20e077867a75656fbd5a8f3ecd7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 16:27:55 -0400 Subject: [PATCH 081/272] fix(fc2c-i): ruff I001 one-per-line import in test_artist_service --- tests/test_artist_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_artist_service.py b/tests/test_artist_service.py index 7853ff0..8f9683b 100644 --- a/tests/test_artist_service.py +++ b/tests/test_artist_service.py @@ -3,7 +3,13 @@ from datetime import UTC, datetime import pytest from backend.app.models import ( - Artist, ImageProvenance, ImageRecord, Post, Source, Tag, TagKind, + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, + Tag, + TagKind, ) from backend.app.models.tag import image_tag from backend.app.services.artist_service import ArtistService From 681d7777f3784ec500b15d27522389bcae6f00ac Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:04:26 -0400 Subject: [PATCH 082/272] fix(build): install CPU-only torch in ml image (drops ~5.6GB CUDA layer) --- Dockerfile.ml | 5 +++++ requirements-ml.txt | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Dockerfile.ml b/Dockerfile.ml index cf90ec4..ee548fb 100644 --- a/Dockerfile.ml +++ b/Dockerfile.ml @@ -23,6 +23,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app COPY requirements-ml.txt requirements.txt ./ +# CPU-only torch: the default PyPI wheel bundles the CUDA runtime (~5.6GB +# layer); this pipeline never uses a GPU. --index-url (not --extra-index-url) +# guarantees only +cpu wheels are considered, so no nvidia-*-cu12 deps. +RUN pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.12,<3.0" "torchvision>=0.27,<0.28" RUN pip install -r requirements-ml.txt COPY backend/ ./backend/ diff --git a/requirements-ml.txt b/requirements-ml.txt index b9358f0..af14624 100644 --- a/requirements-ml.txt +++ b/requirements-ml.txt @@ -2,15 +2,18 @@ # ML stack — versions current as of 2026-05-14 with Python 3.14 wheel coverage. -torch>=2.12,<3.0 - +# torch + torchvision are NOT listed here: they are installed CPU-only from +# the PyTorch CPU index in Dockerfile.ml. The default PyPI torch wheel bundles +# the NVIDIA CUDA runtime (a ~5.6GB image layer); this pipeline is CPU-only, +# so Dockerfile.ml uses the +cpu wheels from +# https://download.pytorch.org/whl/cpu instead. +# # IMPORTANT: torchvision 0.27 declares requires_python "!=3.14.1,>=3.10" — # Python 3.14.1 specifically is excluded due to a known incompatibility. # The python-ci runner pulls python:3.14-bookworm (latest patch); if that # resolves to 3.14.1 the install will fail. Pin a specific Python patch in # the runner image (CI-Runner/CI-python/Dockerfile) if this becomes a # blocker. 3.14.0 and 3.14.2+ are fine. -torchvision>=0.27,<0.28 transformers>=5.8,<6.0 onnxruntime>=1.26,<2.0 From 6bc7689f6eb4d759a06a2bc46d32f0bade7b9fc2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:09:52 -0400 Subject: [PATCH 083/272] build(fc2c-i): add Vitest harness and run it in CI --- .forgejo/workflows/ci.yml | 1 + frontend/package.json | 4 +++- frontend/vitest.config.js | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 frontend/vitest.config.js diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index b4b60a9..4e43ee6 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -55,4 +55,5 @@ jobs: # with no .ts files and no JSDoc annotations, so vue-tsc has nothing # to type-check. Re-enable once we add a tsconfig.json and either # convert to TS or add JSDoc. + - run: npm run test:unit - run: npm run build diff --git a/frontend/package.json b/frontend/package.json index d123574..d68122c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,6 +10,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", + "test:unit": "vitest run", "check": "vue-tsc --noEmit" }, "dependencies": { @@ -24,6 +25,7 @@ "vite": "^5.2.0", "vue-tsc": "^2.0.0", "vite-plugin-vuetify": "^2.0.0", - "sass": "^1.71.0" + "sass": "^1.71.0", + "vitest": "^2.1.0" } } diff --git a/frontend/vitest.config.js b/frontend/vitest.config.js new file mode 100644 index 0000000..99bf7a0 --- /dev/null +++ b/frontend/vitest.config.js @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + include: ['test/**/*.spec.js'], + passWithNoTests: true + } +}) From 61f9401fdeff4731833e3f54154d45dc41779256 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:10:21 -0400 Subject: [PATCH 084/272] feat(fc2c-i): usePolyMasonry shortest-column distribution --- frontend/src/composables/usePolyMasonry.js | 57 ++++++++++++++++++++++ frontend/test/usePolyMasonry.spec.js | 39 +++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 frontend/src/composables/usePolyMasonry.js create mode 100644 frontend/test/usePolyMasonry.spec.js diff --git a/frontend/src/composables/usePolyMasonry.js b/frontend/src/composables/usePolyMasonry.js new file mode 100644 index 0000000..210fc94 --- /dev/null +++ b/frontend/src/composables/usePolyMasonry.js @@ -0,0 +1,57 @@ +// Shortest-column masonry distribution (ported from ImageRepo). +// +// Predictive relative height = height / width. Using the aspect ratio means +// columns are balanced before thumbnails load, so there is no reflow when +// images arrive. Missing or zero dimensions fall back to a square (1.0). + +import { ref, onMounted, onUnmounted } from 'vue' + +function relativeHeight(item) { + const w = Number(item.width) + const h = Number(item.height) + if (!w || !h) return 1 + return h / w +} + +export function distributeIntoColumns(items, columnCount) { + const n = Math.max(1, columnCount | 0) + const columns = Array.from({ length: n }, () => []) + const heights = new Array(n).fill(0) + for (const item of items) { + let shortest = 0 + for (let c = 1; c < n; c++) { + if (heights[c] < heights[shortest]) shortest = c + } + columns[shortest].push(item) + heights[shortest] += relativeHeight(item) + } + return columns +} + +// Reactive column count from container width. Breakpoints mirror the +// gallery grid intent: ~260px target column width, min 1 column. +export function columnCountForWidth(width) { + if (!width || width < 0) return 1 + return Math.max(1, Math.floor(width / 260)) +} + +export function usePolyMasonry(containerRef) { + const columnCount = ref(1) + let ro = null + + function measure() { + const el = containerRef.value + if (el) columnCount.value = columnCountForWidth(el.clientWidth) + } + + onMounted(() => { + measure() + if (typeof ResizeObserver !== 'undefined' && containerRef.value) { + ro = new ResizeObserver(measure) + ro.observe(containerRef.value) + } + }) + onUnmounted(() => ro && ro.disconnect()) + + return { columnCount, distribute: distributeIntoColumns } +} diff --git a/frontend/test/usePolyMasonry.spec.js b/frontend/test/usePolyMasonry.spec.js new file mode 100644 index 0000000..f71d1d7 --- /dev/null +++ b/frontend/test/usePolyMasonry.spec.js @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest' +import { distributeIntoColumns } from '../src/composables/usePolyMasonry.js' + +describe('distributeIntoColumns', () => { + it('returns the requested number of columns', () => { + const cols = distributeIntoColumns([], 3) + expect(cols.length).toBe(3) + expect(cols.every(c => c.length === 0)).toBe(true) + }) + + it('places each item in the then-shortest column', () => { + // 3 square items (relative height 1 each), 3 columns -> one per column. + const items = [ + { id: 1, width: 100, height: 100 }, + { id: 2, width: 100, height: 100 }, + { id: 3, width: 100, height: 100 } + ] + const cols = distributeIntoColumns(items, 3) + expect(cols.map(c => c.map(i => i.id))).toEqual([[1], [2], [3]]) + }) + + it('routes a 4th item to a balanced column', () => { + const items = [ + { id: 1, width: 100, height: 300 }, // tall -> col0 height 3 + { id: 2, width: 100, height: 100 }, // col1 height 1 + { id: 3, width: 100, height: 100 }, // col2 height 1 + { id: 4, width: 100, height: 100 } // shortest is col1 (1) -> col1 + ] + const cols = distributeIntoColumns(items, 3) + expect(cols[0].map(i => i.id)).toEqual([1]) + expect(cols[1].map(i => i.id)).toEqual([2, 4]) + expect(cols[2].map(i => i.id)).toEqual([3]) + }) + + it('treats missing/zero dimensions as a square (height 1)', () => { + const cols = distributeIntoColumns([{ id: 9 }], 1) + expect(cols[0]).toEqual([{ id: 9 }]) + }) +}) From 798d1a7fbc4a7b0622d195c20b9f7e3d618c1755 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:11:06 -0400 Subject: [PATCH 085/272] feat(fc2c-i): front-door redirect, /gallery move, discovery routes --- frontend/src/router.js | 24 +++++++++++++++++++----- frontend/test/router.spec.js | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 frontend/test/router.spec.js diff --git a/frontend/src/router.js b/frontend/src/router.js index fab0176..082f049 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -2,13 +2,25 @@ import { createRouter, createWebHistory } from 'vue-router' import PlaceholderView from './views/PlaceholderView.vue' import SettingsView from './views/SettingsView.vue' import GalleryView from './views/GalleryView.vue' +import ShowcaseView from './views/ShowcaseView.vue' +import TagsView from './views/TagsView.vue' +import ArtistView from './views/ArtistView.vue' + +// The application's front door. `/` redirects here. Changing the front door +// is a one-line edit (e.g. '/gallery' or '/tags'). +export const FRONT_DOOR = '/showcase' const routes = [ + // Root is a redirect only — no meta.title so it stays out of the nav. + { path: '/', redirect: FRONT_DOOR }, + // FC-2: image backbone - { path: '/', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, - { path: '/showcase', name: 'showcase', component: PlaceholderView, meta: { title: 'Showcase' } }, - { path: '/tags', name: 'tags', component: PlaceholderView, meta: { title: 'Tags' } }, - { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, + { path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, + { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase' } }, + { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, + // Artist detail — no meta.title (reached by clicking an artist, not nav). + { path: '/artist/:slug', name: 'artist', component: ArtistView }, + { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, // FC-3: subscription backbone { path: '/subscriptions', name: 'subscriptions', component: PlaceholderView, meta: { title: 'Subscriptions' } }, @@ -16,7 +28,9 @@ const routes = [ { path: '/downloads', name: 'downloads', component: PlaceholderView, meta: { title: 'Downloads' } } ] -export default createRouter({ +const router = createRouter({ history: createWebHistory(), routes }) + +export default router diff --git a/frontend/test/router.spec.js b/frontend/test/router.spec.js new file mode 100644 index 0000000..cc1a1ec --- /dev/null +++ b/frontend/test/router.spec.js @@ -0,0 +1,24 @@ +import { describe, it, expect } from 'vitest' +import router, { FRONT_DOOR } from '../src/router.js' + +describe('router', () => { + it('FRONT_DOOR defaults to /showcase', () => { + expect(FRONT_DOOR).toBe('/showcase') + }) + + it('/ redirects to FRONT_DOOR', () => { + const resolved = router.resolve('/') + expect(resolved.path).toBe(FRONT_DOOR) + }) + + it('gallery is reachable at /gallery', () => { + expect(router.resolve('/gallery').name).toBe('gallery') + }) + + it('showcase, tags, artist resolve to named routes', () => { + expect(router.resolve('/showcase').name).toBe('showcase') + expect(router.resolve('/tags').name).toBe('tags') + expect(router.resolve('/artist/some-slug').name).toBe('artist') + expect(router.resolve('/artist/some-slug').params.slug).toBe('some-slug') + }) +}) From d166871be86baf13be2e5e5c520b34065fe49427 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:11:27 -0400 Subject: [PATCH 086/272] feat(fc2c-i): gallery accepts ?tag_id deep-link --- frontend/src/views/GalleryView.vue | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/views/GalleryView.vue b/frontend/src/views/GalleryView.vue index c2f9d72..32a4488 100644 --- a/frontend/src/views/GalleryView.vue +++ b/frontend/src/views/GalleryView.vue @@ -32,6 +32,8 @@ const router = useRouter() const route = useRoute() onMounted(async () => { + const tagId = parseInt(route.query.tag_id, 10) + if (!isNaN(tagId)) store.setTagFilter(tagId) await store.loadInitial() await store.loadTimeline() // Open modal if URL has ?image=N @@ -39,6 +41,11 @@ onMounted(async () => { if (!isNaN(initial)) modal.open(initial) }) +watch(() => route.query.tag_id, (q) => { + const tagId = parseInt(q, 10) + store.setTagFilter(isNaN(tagId) ? null : tagId) +}) + watch(() => route.query.image, (q) => { const id = parseInt(q, 10) if (!isNaN(id) && id !== modal.currentImageId) modal.open(id) From 861e8baeb87e711e3cdacacaacfff2a8828c3b75 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:12:19 -0400 Subject: [PATCH 087/272] feat(fc2c-i): pinia stores for showcase, tag directory, artist --- frontend/src/stores/artist.js | 63 +++++++++++++++++++++++++++++ frontend/src/stores/showcase.js | 45 +++++++++++++++++++++ frontend/src/stores/tagDirectory.js | 57 ++++++++++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 frontend/src/stores/artist.js create mode 100644 frontend/src/stores/showcase.js create mode 100644 frontend/src/stores/tagDirectory.js diff --git a/frontend/src/stores/artist.js b/frontend/src/stores/artist.js new file mode 100644 index 0000000..a8e250e --- /dev/null +++ b/frontend/src/stores/artist.js @@ -0,0 +1,63 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useArtistStore = defineStore('artist', () => { + const api = useApi() + const overview = ref(null) + const images = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const imagesLoading = ref(false) + const error = ref(null) + const notFound = ref(false) + let started = false + + async function load(slug) { + overview.value = null + images.value = [] + nextCursor.value = null + notFound.value = false + started = false + error.value = null + loading.value = true + try { + overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`) + await loadMoreImages(slug) + } catch (e) { + if (e.status === 404) notFound.value = true + else error.value = e.message + } finally { + loading.value = false + } + } + + async function loadMoreImages(slug) { + if (imagesLoading.value) return + if (started && nextCursor.value === null) return + imagesLoading.value = true + try { + const params = { limit: PAGE } + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get( + `/api/artist/${encodeURIComponent(slug)}/images`, { params } + ) + images.value.push(...body.images) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + imagesLoading.value = false + } + } + + const hasMoreImages = computed(() => !started || nextCursor.value !== null) + + return { + overview, images, loading, imagesLoading, error, notFound, + hasMoreImages, load, loadMoreImages + } +}) diff --git a/frontend/src/stores/showcase.js b/frontend/src/stores/showcase.js new file mode 100644 index 0000000..7e72488 --- /dev/null +++ b/frontend/src/stores/showcase.js @@ -0,0 +1,45 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useShowcaseStore = defineStore('showcase', () => { + const api = useApi() + const images = ref([]) + const loading = ref(false) + const error = ref(null) + const exhausted = ref(false) + const seen = new Set() + + async function fetchPage() { + if (loading.value) return + loading.value = true + error.value = null + try { + const body = await api.get('/api/showcase', { params: { limit: PAGE } }) + const fresh = body.images.filter(i => !seen.has(i.id)) + if (fresh.length === 0) { exhausted.value = true; return } + for (const i of fresh) seen.add(i.id) + images.value.push(...fresh) + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function shuffle() { + images.value = [] + seen.clear() + exhausted.value = false + await fetchPage() + } + + const hasMore = computed(() => !exhausted.value) + const isEmpty = computed( + () => !loading.value && images.value.length === 0 && error.value === null + ) + + return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle } +}) diff --git a/frontend/src/stores/tagDirectory.js b/frontend/src/stores/tagDirectory.js new file mode 100644 index 0000000..e0ffe46 --- /dev/null +++ b/frontend/src/stores/tagDirectory.js @@ -0,0 +1,57 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useTagDirectoryStore = defineStore('tagDirectory', () => { + const api = useApi() + const cards = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const kind = ref(null) + const q = ref('') + let started = false + + async function loadMore() { + if (loading.value) return + if (started && nextCursor.value === null) return + loading.value = true + error.value = null + try { + const params = { limit: PAGE } + if (kind.value) params.kind = kind.value + if (q.value) params.q = q.value + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/tags/directory', { params }) + cards.value.push(...body.cards) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function reset() { + cards.value = [] + nextCursor.value = null + started = false + await loadMore() + } + + function setKind(k) { kind.value = k; reset() } + function setQuery(text) { q.value = text; reset() } + + const hasMore = computed(() => !started || nextCursor.value !== null) + const isEmpty = computed( + () => !loading.value && cards.value.length === 0 && error.value === null + ) + + return { + cards, loading, error, kind, q, hasMore, isEmpty, + loadMore, reset, setKind, setQuery + } +}) From c17c4b1667b70ecb64c1bda800adf87fd78d5c0c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:12:53 -0400 Subject: [PATCH 088/272] feat(fc2c-i): showcase view with shortest-column masonry --- .../src/components/discovery/MasonryGrid.vue | 84 +++++++++++++++++++ frontend/src/views/ShowcaseView.vue | 54 ++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 frontend/src/components/discovery/MasonryGrid.vue create mode 100644 frontend/src/views/ShowcaseView.vue diff --git a/frontend/src/components/discovery/MasonryGrid.vue b/frontend/src/components/discovery/MasonryGrid.vue new file mode 100644 index 0000000..3673448 --- /dev/null +++ b/frontend/src/components/discovery/MasonryGrid.vue @@ -0,0 +1,84 @@ + + + + + diff --git a/frontend/src/views/ShowcaseView.vue b/frontend/src/views/ShowcaseView.vue new file mode 100644 index 0000000..b656cee --- /dev/null +++ b/frontend/src/views/ShowcaseView.vue @@ -0,0 +1,54 @@ + + + + + From a984149ee278def1d3fc14e47d492f25aab95844 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:13:26 -0400 Subject: [PATCH 089/272] feat(fc2c-i): tag directory view with previews, filter, search --- frontend/src/components/discovery/TagCard.vue | 51 ++++++++++ frontend/src/views/TagsView.vue | 97 +++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 frontend/src/components/discovery/TagCard.vue create mode 100644 frontend/src/views/TagsView.vue diff --git a/frontend/src/components/discovery/TagCard.vue b/frontend/src/components/discovery/TagCard.vue new file mode 100644 index 0000000..8438dd2 --- /dev/null +++ b/frontend/src/components/discovery/TagCard.vue @@ -0,0 +1,51 @@ + + + + + diff --git a/frontend/src/views/TagsView.vue b/frontend/src/views/TagsView.vue new file mode 100644 index 0000000..0e7a73d --- /dev/null +++ b/frontend/src/views/TagsView.vue @@ -0,0 +1,97 @@ + + + + + From 4c1cabe4ee1cc6a7ff15ec721e606671d2638709 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:14:07 -0400 Subject: [PATCH 090/272] feat(fc2c-i): rich artist page with sparkline and FC-4 stubs --- frontend/src/views/ArtistView.vue | 146 ++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 frontend/src/views/ArtistView.vue diff --git a/frontend/src/views/ArtistView.vue b/frontend/src/views/ArtistView.vue new file mode 100644 index 0000000..d4cac02 --- /dev/null +++ b/frontend/src/views/ArtistView.vue @@ -0,0 +1,146 @@ + + + + + From 85d8b4b1507e5f80d85a4b8450c3dc2cb89bda61 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:15:04 -0400 Subject: [PATCH 091/272] fix(fc2c-i): vitest needs plugin-vue to transform SFCs in router.spec --- frontend/vitest.config.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/vitest.config.js b/frontend/vitest.config.js index 99bf7a0..3df5841 100644 --- a/frontend/vitest.config.js +++ b/frontend/vitest.config.js @@ -1,6 +1,11 @@ import { defineConfig } from 'vitest/config' +import vue from '@vitejs/plugin-vue' +// plugin-vue is required so specs that import router.js (which imports .vue +// SFCs) can transform those single-file components. No component is mounted +// in the suite (router.resolve / pure-logic only), so jsdom is unnecessary. export default defineConfig({ + plugins: [vue()], test: { environment: 'node', include: ['test/**/*.spec.js'], From 5a4663fa3d00946e2a47b6269d46d7339be8e608 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:27:28 -0400 Subject: [PATCH 092/272] fix(fc2c-i): router falls back to memory history when no window (vitest) --- frontend/src/router.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/src/router.js b/frontend/src/router.js index 082f049..89b76b7 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -1,4 +1,4 @@ -import { createRouter, createWebHistory } from 'vue-router' +import { createRouter, createWebHistory, createMemoryHistory } from 'vue-router' import PlaceholderView from './views/PlaceholderView.vue' import SettingsView from './views/SettingsView.vue' import GalleryView from './views/GalleryView.vue' @@ -28,8 +28,13 @@ const routes = [ { path: '/downloads', name: 'downloads', component: PlaceholderView, meta: { title: 'Downloads' } } ] +// Browser uses HTML5 history; non-browser (Vitest/SSR) falls back to memory +// history so the module is importable without `window`. +const history = + typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory() + const router = createRouter({ - history: createWebHistory(), + history, routes }) From 5faa19234dd361089ee0c7edf736e0b544134785 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:29:52 -0400 Subject: [PATCH 093/272] fix(fc2c-i): assert / redirect via navigation, not resolve() --- frontend/test/router.spec.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/test/router.spec.js b/frontend/test/router.spec.js index cc1a1ec..155ff7f 100644 --- a/frontend/test/router.spec.js +++ b/frontend/test/router.spec.js @@ -6,9 +6,11 @@ describe('router', () => { expect(FRONT_DOOR).toBe('/showcase') }) - it('/ redirects to FRONT_DOOR', () => { - const resolved = router.resolve('/') - expect(resolved.path).toBe(FRONT_DOOR) + it('/ redirects to FRONT_DOOR', async () => { + // resolve() does not follow redirects (they apply on navigation), so + // actually navigate and assert the resulting route. + await router.push('/') + expect(router.currentRoute.value.path).toBe(FRONT_DOOR) }) it('gallery is reachable at /gallery', () => { From 827a4180a9d05dba275e946a88ba727cf4ab38d9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 21:46:10 -0400 Subject: [PATCH 094/272] ci(integration): add pgvector+redis integration job; amend lint-only policy --- .forgejo/workflows/ci.yml | 64 ++++++++++++++++++++++++++++++++++----- pyproject.toml | 8 ++--- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 4e43ee6..049c0fd 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -1,11 +1,14 @@ name: CI -# Per FabledRulebook/forgejo.md: "CI runs analyzers + short unit tests. -# Integration tests that need real Postgres/ffmpeg run locally via -# docker-compose." So this workflow deliberately does NOT spin up service -# containers; DB-dependent tests are marked `@pytest.mark.integration` and -# excluded with `-m "not integration"`. To run integration tests, operator -# uses docker-compose locally. +# CI policy (amends the older FabledRulebook/forgejo.md "lint + unit only" +# stance — operator opted in to running integration in CI): +# - backend-lint-and-test: fast feedback — ruff + `pytest -m "not +# integration"`, no service containers. +# - frontend-build: vitest unit + vite build. +# - integration: spins up pgvector Postgres + Redis service containers, +# migrates a throwaway test DB, runs `pytest -m integration`. This is +# the first time the integration suite runs anywhere, so expect a +# debugging tail until it goes green. on: push: @@ -36,7 +39,7 @@ jobs: - name: Ruff lint run: ruff check backend/ tests/ alembic/ - - name: Pytest (unit tests only — integration tests run locally) + - name: Pytest (unit only — integration runs in the integration job) run: pytest tests/ -v -m "not integration" frontend-build: @@ -57,3 +60,50 @@ jobs: # convert to TS or add JSDoc. - run: npm run test:unit - run: npm run build + + integration: + # First-ever execution of the integration suite. Postgres uses the same + # pgvector image as docker-compose; Redis backs the celery smoke tests. + # Assumes the python-ci runner executes jobs in a container so services + # are reachable by name (postgres/redis). If the runner runs jobs on the + # host instead, switch DB_HOST to localhost + add `ports:` mappings. + runs-on: python-ci + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_HOST: postgres + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + + - name: Install Python deps + run: pip install -r requirements.txt pytest pytest-asyncio + + - name: Migrate the test database + run: alembic upgrade head + + - name: Pytest (integration only) + run: pytest tests/ -v -m integration diff --git a/pyproject.toml b/pyproject.toml index 3a476ff..eb26856 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ where = ["."] include = ["backend*"] [tool.pytest.ini_options] -# Per FabledRulebook/forgejo.md: CI runs lint + short unit tests only. -# Integration tests that need a real Postgres/Redis run locally via -# docker-compose; CI passes `-m "not integration"` to skip them. +# The fast backend job runs `-m "not integration"`; the dedicated CI +# `integration` job (pgvector Postgres + Redis service containers) runs +# `-m integration`. Both also runnable locally via docker-compose. markers = [ - "integration: tests that require a real Postgres/Redis. Run locally via docker-compose, skipped in CI.", + "integration: tests that require a real Postgres/Redis. Run by the CI integration job and locally via docker-compose.", ] From cd8d0bd6069b841706649bfd5357357dd432e7dd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:02:43 -0400 Subject: [PATCH 095/272] ci(integration): publish service ports, connect via localhost (host runner) --- .forgejo/workflows/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 049c0fd..556413d 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -64,19 +64,19 @@ jobs: integration: # First-ever execution of the integration suite. Postgres uses the same # pgvector image as docker-compose; Redis backs the celery smoke tests. - # Assumes the python-ci runner executes jobs in a container so services - # are reachable by name (postgres/redis). If the runner runs jobs on the - # host instead, switch DB_HOST to localhost + add `ports:` mappings. + # The python-ci runner executes jobs on the host (not in a container), + # so services are NOT reachable by name — their ports are published and + # we connect via localhost. runs-on: python-ci env: DB_USER: fabledcurator DB_PASSWORD: ci_integration - DB_HOST: postgres + DB_HOST: localhost DB_PORT: "5432" DB_NAME: fabledcurator_test SECRET_KEY: ci_integration_placeholder - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 + CELERY_BROKER_URL: redis://localhost:6379/0 + CELERY_RESULT_BACKEND: redis://localhost:6379/0 services: postgres: image: pgvector/pgvector:pg16 @@ -84,6 +84,8 @@ jobs: POSTGRES_USER: fabledcurator POSTGRES_PASSWORD: ci_integration POSTGRES_DB: fabledcurator_test + ports: + - "5432:5432" options: >- --health-cmd "pg_isready -U fabledcurator" --health-interval 10s @@ -91,6 +93,8 @@ jobs: --health-retries 10 redis: image: redis:7-alpine + ports: + - "6379:6379" options: >- --health-cmd "redis-cli ping" --health-interval 10s From 65a055408b7cca4c042d723d44d99a1e65e61451 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:18:01 -0400 Subject: [PATCH 096/272] ci(integration): reach services by bridge IP via docker socket, no host ports --- .forgejo/workflows/ci.yml | 52 ++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 556413d..09f329c 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -62,21 +62,22 @@ jobs: - run: npm run build integration: - # First-ever execution of the integration suite. Postgres uses the same - # pgvector image as docker-compose; Redis backs the celery smoke tests. - # The python-ci runner executes jobs on the host (not in a container), - # so services are NOT reachable by name — their ports are published and - # we connect via localhost. + # This act_runner (swarm-runner v0.6.1) puts service containers on the + # default bridge with NO service-name DNS, and publishing fixed host + # ports collides with the operator's running docker-compose dev stack on + # the same shared daemon. Workaround: publish NO host ports, and reach + # each service by its bridge IP — discovered at runtime via the mounted + # docker socket (the python-ci image ships /usr/bin/docker; build.yml + # relies on it). Default-bridge containers can talk by IP (only embedded + # DNS is missing), so IP addressing is reliable here. Everything runs in + # ONE step so resolved values don't depend on cross-step env passing. runs-on: python-ci env: DB_USER: fabledcurator DB_PASSWORD: ci_integration - DB_HOST: localhost DB_PORT: "5432" DB_NAME: fabledcurator_test SECRET_KEY: ci_integration_placeholder - CELERY_BROKER_URL: redis://localhost:6379/0 - CELERY_RESULT_BACKEND: redis://localhost:6379/0 services: postgres: image: pgvector/pgvector:pg16 @@ -84,8 +85,6 @@ jobs: POSTGRES_USER: fabledcurator POSTGRES_PASSWORD: ci_integration POSTGRES_DB: fabledcurator_test - ports: - - "5432:5432" options: >- --health-cmd "pg_isready -U fabledcurator" --health-interval 10s @@ -93,8 +92,6 @@ jobs: --health-retries 10 redis: image: redis:7-alpine - ports: - - "6379:6379" options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -103,11 +100,26 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python deps - run: pip install -r requirements.txt pytest pytest-asyncio - - - name: Migrate the test database - run: alembic upgrade head - - - name: Pytest (integration only) - run: pytest tests/ -v -m integration + - name: Integration suite (resolve service IPs, migrate, test) + run: | + set -eux + # Scope to THIS job's service containers (act_runner names them + # ...JOB-integration...); the operator's compose stack uses the + # same images but different names, so it won't match. + PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1) + test -n "$PG" && test -n "$RD" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + export DB_HOST="$PG_IP" + export CELERY_BROKER_URL="redis://$RD_IP:6379/0" + export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" + # Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools). + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + pip install -r requirements.txt pytest pytest-asyncio + alembic upgrade head + pytest tests/ -v -m integration From 13be9085b559466962b46efb1ca75d089a7f52a7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:23:32 -0400 Subject: [PATCH 097/272] fix(integration): asyncio auto-mode, per-test truncation, artist images IN-subquery --- backend/app/services/artist_service.py | 16 ++++++++++------ pyproject.toml | 5 +++++ tests/conftest.py | 26 ++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/backend/app/services/artist_service.py b/backend/app/services/artist_service.py index 9aeaa8e..c195fcb 100644 --- a/backend/app/services/artist_service.py +++ b/backend/app/services/artist_service.py @@ -157,12 +157,16 @@ class ArtistService: if artist is None: return None - stmt = ( - select(ImageRecord) - .join(ImageProvenance, ImageProvenance.image_record_id == ImageRecord.id) - .join(Source, Source.id == ImageProvenance.source_id) - .where(Source.artist_id == artist.id) - .distinct() + # Dedupe via IN-subquery rather than JOIN + DISTINCT: an image can + # have several provenance rows for one artist, and SELECT DISTINCT + # over ImageRecord fails in Postgres because its json columns have + # no equality operator. + stmt = select(ImageRecord).where( + ImageRecord.id.in_( + select(ImageProvenance.image_record_id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == artist.id) + ) ) if cursor: cur_ts, cur_id = decode_cursor(cursor) diff --git a/pyproject.toml b/pyproject.toml index eb26856..18206b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,11 @@ where = ["."] include = ["backend*"] [tool.pytest.ini_options] +# auto mode: async test fns and async fixtures don't need explicit +# @pytest.mark.asyncio / @pytest_asyncio.fixture decoration. Required +# because the test_api_* fixtures use plain @pytest.fixture on async +# app/client; strict mode errors on those under pytest-asyncio 1.x. +asyncio_mode = "auto" # The fast backend job runs `-m "not integration"`; the dedicated CI # `integration` job (pgvector Postgres + Redis service containers) runs # `-m integration`. Both also runnable locally via docker-compose. diff --git a/tests/conftest.py b/tests/conftest.py index 73b3ef9..619f4b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,8 @@ from sqlalchemy.ext.asyncio import ( ) from sqlalchemy.orm import sessionmaker +from backend.app.models import Base + def _async_database_url() -> str: user = os.environ.get("DB_USER", "fabledcurator") @@ -73,3 +75,27 @@ def db_sync(sync_engine): yield session finally: session.rollback() + + +@pytest.fixture(autouse=True) +def _truncate_after_integration(request): + """Integration tests exercise app/celery code that commits on its own + connection, which the rollback fixtures above can't undo — so data + would leak between tests. Truncate every model table after each + integration-marked test. Scoped to the integration marker so the + DB-less fast unit job never connects here. + """ + yield + if request.node.get_closest_marker("integration") is None: + return + tables = ", ".join(t.name for t in Base.metadata.sorted_tables) + if not tables: + return + eng = create_engine(_sync_database_url(), future=True) + try: + with eng.begin() as conn: + conn.exec_driver_sql( + f"TRUNCATE {tables} RESTART IDENTITY CASCADE" + ) + finally: + eng.dispose() From 7a896605e031c4f51c5059c8cffd9adebb3bb5c6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:35:37 -0400 Subject: [PATCH 098/272] fix(integration): snapshot/restore seeded singletons, dispose app engine per test --- tests/conftest.py | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 619f4b0..e79b80c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,13 +77,32 @@ def db_sync(sync_engine): session.rollback() +@pytest.fixture(scope="session") +def _seed_snapshot(): + """Capture migration-seeded rows (singleton config like import_settings + / ml_settings) once, so the per-test truncation can restore them.""" + eng = create_engine(_sync_database_url(), future=True) + snap: dict[str, list[dict]] = {} + try: + with eng.connect() as conn: + for t in Base.metadata.sorted_tables: + rows = [ + dict(m) for m in conn.execute(t.select()).mappings().all() + ] + if rows: + snap[t.name] = rows + finally: + eng.dispose() + return snap + + @pytest.fixture(autouse=True) -def _truncate_after_integration(request): +def _reset_db_after_integration(request, _seed_snapshot): """Integration tests exercise app/celery code that commits on its own connection, which the rollback fixtures above can't undo — so data - would leak between tests. Truncate every model table after each - integration-marked test. Scoped to the integration marker so the - DB-less fast unit job never connects here. + would leak between tests. After each integration-marked test, truncate + every model table, then restore the migration-seeded rows. Scoped to + the integration marker so the DB-less fast unit job never connects. """ yield if request.node.get_closest_marker("integration") is None: @@ -97,5 +116,24 @@ def _truncate_after_integration(request): conn.exec_driver_sql( f"TRUNCATE {tables} RESTART IDENTITY CASCADE" ) + for t in Base.metadata.sorted_tables: + rows = _seed_snapshot.get(t.name) + if rows: + conn.execute(t.insert(), rows) finally: eng.dispose() + + +@pytest_asyncio.fixture(autouse=True) +async def _dispose_app_engine(request): + """The app's shared async engine (extensions.get_engine) is a process + singleton, but pytest-asyncio gives each test a fresh event loop; + asyncpg connections are loop-bound, so reuse across tests raises + 'attached to a different loop'. Dispose after each integration test so + the next test rebuilds the engine on its own loop. + """ + yield + if request.node.get_closest_marker("integration") is None: + return + from backend.app import extensions + await extensions.dispose_engine() From dfa6a5c895ef03be650879b8ac0b7f59d29d2b8f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:43:46 -0400 Subject: [PATCH 099/272] fix(integration): lazy seed-snapshot (unblock unit job); isolate recover test from eager import --- tests/conftest.py | 56 +++++++++++++++++++++++---------------- tests/test_maintenance.py | 13 ++++++++- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e79b80c..00a8127 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,35 +77,45 @@ def db_sync(sync_engine): session.rollback() -@pytest.fixture(scope="session") -def _seed_snapshot(): - """Capture migration-seeded rows (singleton config like import_settings - / ml_settings) once, so the per-test truncation can restore them.""" - eng = create_engine(_sync_database_url(), future=True) - snap: dict[str, list[dict]] = {} - try: - with eng.connect() as conn: - for t in Base.metadata.sorted_tables: - rows = [ - dict(m) for m in conn.execute(t.select()).mappings().all() - ] - if rows: - snap[t.name] = rows - finally: - eng.dispose() - return snap +# Migration-seeded baseline (singleton config like import_settings / +# ml_settings), captured lazily at the FIRST integration test's setup — +# when the CI integration job has just run `alembic upgrade head` so the DB +# is pristine. Cached module-wide. Never populated in the DB-less unit job +# because only integration-marked tests trigger the connection. +_SEED_SNAPSHOT: dict[str, list[dict]] | None = None @pytest.fixture(autouse=True) -def _reset_db_after_integration(request, _seed_snapshot): +def _reset_db_after_integration(request): """Integration tests exercise app/celery code that commits on its own connection, which the rollback fixtures above can't undo — so data - would leak between tests. After each integration-marked test, truncate - every model table, then restore the migration-seeded rows. Scoped to - the integration marker so the DB-less fast unit job never connects. + would leak between tests. Capture the seeded baseline at the first + integration test, then after each integration-marked test truncate + every model table and restore that baseline. Connects to the DB ONLY + for integration-marked tests, so the no-DB fast unit job is untouched. """ + global _SEED_SNAPSHOT + is_integration = request.node.get_closest_marker("integration") is not None + + if is_integration and _SEED_SNAPSHOT is None: + eng = create_engine(_sync_database_url(), future=True) + snap: dict[str, list[dict]] = {} + try: + with eng.connect() as conn: + for t in Base.metadata.sorted_tables: + rows = [ + dict(m) + for m in conn.execute(t.select()).mappings().all() + ] + if rows: + snap[t.name] = rows + finally: + eng.dispose() + _SEED_SNAPSHOT = snap + yield - if request.node.get_closest_marker("integration") is None: + + if not is_integration: return tables = ", ".join(t.name for t in Base.metadata.sorted_tables) if not tables: @@ -117,7 +127,7 @@ def _reset_db_after_integration(request, _seed_snapshot): f"TRUNCATE {tables} RESTART IDENTITY CASCADE" ) for t in Base.metadata.sorted_tables: - rows = _seed_snapshot.get(t.name) + rows = (_SEED_SNAPSHOT or {}).get(t.name) if rows: conn.execute(t.insert(), rows) finally: diff --git a/tests/test_maintenance.py b/tests/test_maintenance.py index 0e19895..f580916 100644 --- a/tests/test_maintenance.py +++ b/tests/test_maintenance.py @@ -26,7 +26,7 @@ def _make_batch(session) -> int: return batch.id -def test_recover_interrupted_only_old(db_sync): +def test_recover_interrupted_only_old(db_sync, monkeypatch): batch_id = _make_batch(db_sync) now = datetime.now(UTC) @@ -41,6 +41,16 @@ def test_recover_interrupted_only_old(db_sync): db_sync.add_all([fresh, stale]) db_sync.commit() + # Isolate the recover task: under eager Celery, the real + # import_media_file.delay() would run inline against the nonexistent + # /import/b.jpg and flip the just-requeued row 'queued' -> 'skipped'. + from backend.app.tasks import import_file + + dispatched: list[int] = [] + monkeypatch.setattr( + import_file.import_media_file, "delay", dispatched.append + ) + from backend.app.tasks.maintenance import recover_interrupted_tasks recovered = recover_interrupted_tasks.apply().get() assert recovered == 1 @@ -50,6 +60,7 @@ def test_recover_interrupted_only_old(db_sync): assert fresh.status == "processing" assert stale.status == "queued" assert stale.started_at is None + assert dispatched == [stale.id] def test_cleanup_old_deletes_finished_old(db_sync): From 4a29a6d197d9237bf0d41cdecd6e2dc044a1ef0d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 22:53:49 -0400 Subject: [PATCH 100/272] fix(web): pass create_app as a hypercorn factory (backend.app:create_app()) --- entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3d9cdf5..fc9e8b7 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,11 +9,14 @@ case "$ROLE" in echo "[entrypoint] Running alembic upgrade head" alembic upgrade head echo "[entrypoint] Starting hypercorn on :8080" + # create_app is a factory — the `()` tells hypercorn to call it once + # and serve the returned Quart (ASGI) app, rather than treating the + # function itself as the application (which it then mis-invokes as WSGI). exec hypercorn \ --bind 0.0.0.0:8080 \ --workers "${HYPERCORN_WORKERS:-2}" \ --access-logfile - \ - backend.app:create_app + "backend.app:create_app()" ;; worker) From 18ad61265e4bff51851050afc4c9b4b025c7c2a5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 23:13:42 -0400 Subject: [PATCH 101/272] feat(ui): IR-style gradient-fade TopNav (brand, centered links, health) --- frontend/src/components/TopNav.vue | 125 +++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 frontend/src/components/TopNav.vue diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue new file mode 100644 index 0000000..60fd0e8 --- /dev/null +++ b/frontend/src/components/TopNav.vue @@ -0,0 +1,125 @@ + + + + + From 9bb801a5666794f68c71ad97e6bd7d7cd53483bd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 23:14:20 -0400 Subject: [PATCH 102/272] refactor(ui): AppShell = TopNav + content slot (drop left drawer/v-main) --- frontend/src/components/AppShell.vue | 74 +++------------------------- 1 file changed, 6 insertions(+), 68 deletions(-) diff --git a/frontend/src/components/AppShell.vue b/frontend/src/components/AppShell.vue index ce71c61..d4a53dc 100644 --- a/frontend/src/components/AppShell.vue +++ b/frontend/src/components/AppShell.vue @@ -1,78 +1,16 @@ From e92b05028bb61f3c2069635655e126a9a4b55b80 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 23:18:59 -0400 Subject: [PATCH 103/272] =?UTF-8?q?feat(ui):=20nav=20order=20=E2=80=94=20S?= =?UTF-8?q?howcase=20before=20Gallery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/router.js b/frontend/src/router.js index 89b76b7..53a85ac 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -15,8 +15,8 @@ const routes = [ { path: '/', redirect: FRONT_DOOR }, // FC-2: image backbone - { path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase' } }, + { path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, // Artist detail — no meta.title (reached by clicking an artist, not nav). { path: '/artist/:slug', name: 'artist', component: ArtistView }, From 062a87e968e7ba1eac2089049dd476bd7557780b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 15 May 2026 23:27:38 -0400 Subject: [PATCH 104/272] feat(ui): drop active-link underline + redundant page-title headers --- frontend/src/components/TopNav.vue | 6 ++---- frontend/src/views/GalleryView.vue | 7 ------- frontend/src/views/SettingsView.vue | 11 ----------- frontend/src/views/ShowcaseView.vue | 8 +------- frontend/src/views/TagsView.vue | 6 ------ 5 files changed, 3 insertions(+), 35 deletions(-) diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index 60fd0e8..b8c4648 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -96,7 +96,6 @@ const health = computed(() => { text-decoration: none; font-weight: 500; font-size: 1rem; - border-bottom: 2px solid transparent; text-shadow: 0 1px 3px rgba(0, 0, 0, 0.5); transition: background 0.2s ease, transform 0.1s ease; } @@ -104,11 +103,10 @@ const health = computed(() => { background: rgba(232, 228, 216, 0.12); transform: translateY(-1px); } -/* Active route: accent text + thin accent underline (FabledDesignSystem: - accent for nav-active, never on action buttons). */ +/* Active route: accent text only (FabledDesignSystem: accent for + nav-active, never on action buttons). No underline. */ .fc-link.router-link-exact-active { color: rgb(var(--v-theme-accent)); - border-bottom-color: rgb(var(--v-theme-accent)); } .fc-health { diff --git a/frontend/src/views/GalleryView.vue b/frontend/src/views/GalleryView.vue index 32a4488..fe767dc 100644 --- a/frontend/src/views/GalleryView.vue +++ b/frontend/src/views/GalleryView.vue @@ -1,6 +1,5 @@ @@ -107,4 +115,20 @@ function onThumbError() { thumbError.value = true } display: grid; place-items: center; z-index: 11; pointer-events: none; } +.fc-gallery-item__artist { + position: absolute; left: 0; right: 0; bottom: 0; + padding: 14px 8px 6px; + background: linear-gradient( + to top, rgba(20, 23, 26, 0.78), rgba(20, 23, 26, 0) + ); + font-size: 12px; line-height: 1.2; + white-space: nowrap; overflow: hidden; text-overflow: ellipsis; + pointer-events: none; +} +.fc-gallery-item__artist a { + color: rgb(var(--v-theme-parchment, 232 228 216)); + text-decoration: none; + pointer-events: auto; +} +.fc-gallery-item__artist span { color: rgba(232, 228, 216, 0.85); } From 8abdaa4393889db4a372ff2281ee27ba8da81e02 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 18 May 2026 20:17:42 -0400 Subject: [PATCH 162/272] feat(provenance): modal sidebar stacks ProvenancePanel above Tags Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/modal/ImageViewer.vue | 18 ++++++++++++++---- frontend/src/components/modal/TagPanel.vue | 5 ----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue index ef0874a..3f30bff 100644 --- a/frontend/src/components/modal/ImageViewer.vue +++ b/frontend/src/components/modal/ImageViewer.vue @@ -42,7 +42,10 @@ - + @@ -54,6 +57,7 @@ import { useModalStore } from '../../stores/modal.js' import ImageCanvas from './ImageCanvas.vue' import VideoCanvas from './VideoCanvas.vue' import TagPanel from './TagPanel.vue' +import ProvenancePanel from './ProvenancePanel.vue' defineEmits(['close']) @@ -133,13 +137,19 @@ function isTextEntry(el) { flex: 1; display: flex; align-items: center; justify-content: center; min-width: 0; } +.fc-viewer__side { + width: 320px; flex-shrink: 0; + background: rgb(var(--v-theme-surface)); + border-left: 1px solid rgb(var(--v-theme-surface-light)); + overflow-y: auto; +} @media (max-width: 900px) { .fc-viewer__body { flex-direction: column; } - .fc-tag-panel { - width: 100% !important; + .fc-viewer__side { + width: 100%; max-height: 40vh; - border-left: none !important; + border-left: none; border-top: 1px solid rgb(var(--v-theme-surface-light)); } } diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue index 4b3066a..2f1df1c 100644 --- a/frontend/src/components/modal/TagPanel.vue +++ b/frontend/src/components/modal/TagPanel.vue @@ -103,12 +103,7 @@ async function onRenamed() { diff --git a/frontend/src/components/modal/ProvenancePanel.vue b/frontend/src/components/modal/ProvenancePanel.vue index 5b3dcd3..9e428c2 100644 --- a/frontend/src/components/modal/ProvenancePanel.vue +++ b/frontend/src/components/modal/ProvenancePanel.vue @@ -44,6 +44,17 @@ /> + +
+

Attachments

+ ⬇ {{ at.original_filename }} + ({{ at.size_bytes }} B) + +
@@ -77,6 +88,8 @@ const show = computed(() => { return Array.isArray(st.entries) && st.entries.length > 0 }) +const attachments = computed(() => state.value?.attachments || []) + function postDate(e) { return formatPostDate(e.post.date) } function openPost(postId) { @@ -124,4 +137,14 @@ function openPost(postId) { color: rgb(var(--v-theme-on-surface-variant)); } .fc-prov__desc :deep(a) { color: rgb(var(--v-theme-accent)); } +.fc-prov__attach { padding: 0 0 12px; } +.fc-prov__attach-title { + font-size: 13px; color: rgb(var(--v-theme-on-surface-variant)); + margin: 8px 0 4px; +} +.fc-prov__attach-row { + display: block; font-size: 13px; text-decoration: none; + color: rgb(var(--v-theme-accent)); padding: 2px 0; +} +.fc-prov__attach-size { color: rgb(var(--v-theme-on-surface-variant)); } diff --git a/frontend/src/stores/provenance.js b/frontend/src/stores/provenance.js index c87680a..97b28f8 100644 --- a/frontend/src/stores/provenance.js +++ b/frontend/src/stores/provenance.js @@ -16,8 +16,11 @@ export const useProvenanceStore = defineStore('provenance', () => { imageCache.value[id] = { loading: true, error: null, entries: null } try { const body = await api.get(`/api/provenance/image/${id}`) - imageCache.value[id] = - { loading: false, error: null, entries: body.provenance } + imageCache.value[id] = { + loading: false, error: null, + entries: body.provenance, + attachments: body.attachments || [], + } } catch (e) { imageCache.value[id] = { loading: false, error: e.message, entries: null } diff --git a/frontend/test/provenance.spec.js b/frontend/test/provenance.spec.js index 7ade1c7..c5142d6 100644 --- a/frontend/test/provenance.spec.js +++ b/frontend/test/provenance.spec.js @@ -59,4 +59,18 @@ describe('provenance store', () => { expect(globalThis.fetch.mock.calls.length).toBe(1) expect(store.postInfo(3).data.artist.slug).toBe('a') }) + + it('loadForImage keeps attachments from the payload', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ + status: 200, + body: { + image_id: 1, provenance: [], + attachments: [{ id: 9, original_filename: 'p.zip', size_bytes: 5, + ext: '.zip', download_url: '/api/attachments/9/download' }] + } + })) + await store.loadForImage(1) + expect(store.imageProv(1).attachments[0].id).toBe(9) + }) }) From 7dea165fae0b483841c86f175b55241b85eb460d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 11:22:17 -0400 Subject: [PATCH 179/272] fix(test): archive members need structured imgs+threshold0; provenance payload gained attachments key Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_importer_archive.py | 22 ++++++++++++++++++++-- tests/test_provenance_service.py | 4 +++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/test_importer_archive.py b/tests/test_importer_archive.py index 3b72057..713678b 100644 --- a/tests/test_importer_archive.py +++ b/tests/test_importer_archive.py @@ -48,13 +48,31 @@ def _jpeg_bytes(color): return buf.getvalue() +def _split_bytes(orient): + """Structured half/half image — solid colors phash-collapse (distance + 0); orthogonal splits are distinct only at phash_threshold=0 (see + reference-phash-test-images).""" + im = Image.new("L", (256, 256), 0) + px = im.load() + for y in range(256): + for x in range(256): + if (x / 256 if orient == "v" else y / 256) >= 0.5: + px[x, y] = 255 + buf = io.BytesIO() + im.convert("RGB").save(buf, "JPEG") + return buf.getvalue() + + def test_archive_imports_members_and_stores_archive(importer, import_layout): import_root, _ = import_layout + # Distinct structure + threshold 0 so BOTH members import (solid + # colors would phash-collapse; reference-phash-test-images). + importer.settings.phash_threshold = 0 arc = import_root / "Bob" / "set.cbz" arc.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(arc, "w") as zf: - zf.writestr("a.jpg", _jpeg_bytes((200, 10, 10))) - zf.writestr("b.jpg", _jpeg_bytes((10, 200, 10))) + zf.writestr("a.jpg", _split_bytes("v")) + zf.writestr("b.jpg", _split_bytes("h")) zf.writestr("readme.txt", b"hello") arc.with_suffix(".cbz.json").write_text(json.dumps( {"category": "patreon", "id": "777", "title": "Set"})) diff --git a/tests/test_provenance_service.py b/tests/test_provenance_service.py index 5cdbf60..7cc2aed 100644 --- a/tests/test_provenance_service.py +++ b/tests/test_provenance_service.py @@ -58,7 +58,9 @@ async def test_for_image_no_provenance_returns_empty_list(db): rec = await _seed_image(db) svc = ProvenanceService(db) payload = await svc.for_image(rec.id) - assert payload == {"image_id": rec.id, "provenance": []} + assert payload == { + "image_id": rec.id, "provenance": [], "attachments": [] + } @pytest.mark.asyncio From ae67e67299695517adeaeec86aa3ddadb7319804 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 14:59:04 -0400 Subject: [PATCH 180/272] feat(deep-scan): keyset-paginated backfill_phash task Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/tasks/maintenance.py | 46 +++++++++++++++++- tests/test_backfill_phash.py | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 tests/test_backfill_phash.py diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 05dcc4e..1572406 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -1,16 +1,22 @@ """Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks.""" +import logging from datetime import UTC, datetime, timedelta +from PIL import Image from sqlalchemy import create_engine, delete, select, update from sqlalchemy.orm import sessionmaker from ..celery_app import celery from ..config import get_config -from ..models import ImportTask +from ..models import ImageRecord, ImportTask +from ..utils.phash import compute_phash + +log = logging.getLogger(__name__) STUCK_THRESHOLD_MINUTES = 30 OLD_TASK_DAYS = 7 +PHASH_PAGE = 500 def _sync_session_factory(): @@ -71,3 +77,41 @@ def cleanup_old_tasks() -> int: ) session.commit() return result.rowcount or 0 + + +@celery.task(name="backend.app.tasks.maintenance.backfill_phash") +def backfill_phash() -> int: + """Recompute phash for stored images that have none (imported before + FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill, + idempotent. Videos legitimately keep phash NULL. A missing/unreadable + file is logged and left NULL — never fails the task.""" + SessionLocal = _sync_session_factory() + updated = 0 + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord) + .where(ImageRecord.id > last_id) + .where(ImageRecord.phash.is_(None)) + .where(ImageRecord.mime.like("image/%")) + .order_by(ImageRecord.id.asc()) + .limit(PHASH_PAGE) + ).scalars().all() + if not rows: + break + for rec in rows: + try: + with Image.open(rec.path) as im: + ph = compute_phash(im) + except Exception as exc: + log.warning( + "backfill_phash: unreadable %s: %s", rec.path, exc + ) + ph = None + if ph is not None and rec.phash is None: + rec.phash = ph + updated += 1 + session.commit() + last_id = rows[-1].id + return updated diff --git a/tests/test_backfill_phash.py b/tests/test_backfill_phash.py new file mode 100644 index 0000000..b16d1ac --- /dev/null +++ b/tests/test_backfill_phash.py @@ -0,0 +1,81 @@ +"""FC-2d-vi: library-wide phash backfill (keyset, idempotent).""" + +import pytest +from PIL import Image + +from backend.app.models import ImageRecord +from backend.app.tasks.maintenance import backfill_phash + +pytestmark = pytest.mark.integration + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + """sessionmaker-like returning the test's bound session, so the task + runs against the same transaction / sees the seeded rows.""" + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _img(path, color): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (64, 64), color).save(path, "JPEG") + + +def _rec(db_sync, path, *, sha, mime="image/jpeg", phash=None): + rec = ImageRecord( + path=str(path), sha256=sha, phash=phash, size_bytes=1, + mime=mime, width=64, height=64, origin="imported_filesystem", + integrity_status="unknown", + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def test_backfill_fills_null_image_phash(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + p = tmp_path / "a.jpg" + _img(p, (10, 120, 200)) + pre = tmp_path / "preset.jpg" + _img(pre, (5, 5, 5)) + + rec = _rec(db_sync, p, sha="a" + "0" * 63, phash=None) + vid = _rec( + db_sync, tmp_path / "v.mp4", sha="v" + "0" * 63, + mime="video/mp4", phash=None, + ) + missing = _rec( + db_sync, tmp_path / "gone.jpg", sha="g" + "0" * 63, phash=None, + ) + preset = _rec( + db_sync, pre, sha="p" + "0" * 63, phash="deadbeefdeadbeef", + ) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + updated = backfill_phash() + db_sync.expire_all() + assert updated == 1 + assert db_sync.get(ImageRecord, rec.id).phash is not None + assert db_sync.get(ImageRecord, vid.id).phash is None + assert db_sync.get(ImageRecord, missing.id).phash is None + assert db_sync.get(ImageRecord, preset.id).phash == "deadbeefdeadbeef" + + # idempotent: a second run touches nothing + assert backfill_phash() == 0 From 9535533385467cf835d98c7c28bc080db6550d8a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 15:09:19 -0400 Subject: [PATCH 181/272] feat(deep-scan): trigger accepts 'deep'; scan_directory(mode) + enqueue backfill_phash Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/import_admin.py | 7 +++---- backend/app/tasks/scan.py | 15 ++++++++++++--- tests/test_api_import_admin.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py index e4be69a..ee074e3 100644 --- a/backend/app/api/import_admin.py +++ b/backend/app/api/import_admin.py @@ -15,14 +15,13 @@ import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import") async def trigger_scan(): body = await request.get_json(silent=True) or {} mode = body.get("mode", "quick") - if mode != "quick": - # FC-2d will accept 'deep'. For now, reject anything else. - return jsonify({"error": f"mode {mode!r} not supported in FC-2a; use 'quick'"}), 400 + if mode not in ("quick", "deep"): + return jsonify({"error": f"mode {mode!r} not supported; use 'quick' or 'deep'"}), 400 # Enqueue the scan task in Celery. The task creates the batch itself. from ..tasks.scan import scan_directory - async_result = scan_directory.delay(triggered_by="manual") + async_result = scan_directory.delay(triggered_by="manual", mode=mode) return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202 diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py index d162826..54bbe56 100644 --- a/backend/app/tasks/scan.py +++ b/backend/app/tasks/scan.py @@ -34,8 +34,12 @@ def _sync_session_factory(): @celery.task(name="backend.app.tasks.scan.scan_directory", bind=True) -def scan_directory(self, triggered_by: str = "manual") -> int: - """Walks the import root and creates ImportTasks. Returns the batch id.""" +def scan_directory(self, triggered_by: str = "manual", + mode: str = "quick") -> int: + """Walks the import root and creates ImportTasks. `mode` is 'quick' + or 'deep'; deep also enqueues a library-wide phash backfill and (via + Importer.deep) re-derives on already-imported files. Returns the + batch id.""" SessionLocal = _sync_session_factory() with SessionLocal() as session: settings = session.execute( @@ -46,7 +50,7 @@ def scan_directory(self, triggered_by: str = "manual") -> int: batch = ImportBatch( triggered_by=triggered_by, source_path=str(import_root), - scan_mode="quick", # FC-2a is quick only; FC-2d adds 'deep' + scan_mode=mode, status="running", ) session.add(batch) @@ -84,4 +88,9 @@ def scan_directory(self, triggered_by: str = "manual") -> int: import_media_file.delay(task.id) session.commit() + if mode == "deep": + from .maintenance import backfill_phash + + backfill_phash.delay() + return batch_id diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py index c0c82a1..0cdbee5 100644 --- a/tests/test_api_import_admin.py +++ b/tests/test_api_import_admin.py @@ -90,3 +90,33 @@ async def test_clear_completed(client, db): resp = await client.post("/api/import/clear-completed", json={"age_days": 7}) body = await resp.get_json() assert body["deleted"] == 1 + + +@pytest.mark.asyncio +async def test_trigger_accepts_deep(client, monkeypatch): + # Stub the task dispatch — assert the API accepts 'deep' and forwards + # mode, without running the real (eager) scan_directory. + from backend.app.tasks import scan as scan_mod + + seen = {} + + class _Res: + id = "celery-deep" + + def _fake_delay(*, triggered_by, mode): + seen["triggered_by"] = triggered_by + seen["mode"] = mode + return _Res() + + monkeypatch.setattr(scan_mod.scan_directory, "delay", _fake_delay) + resp = await client.post("/api/import/trigger", json={"mode": "deep"}) + assert resp.status_code == 202 + body = await resp.get_json() + assert body["mode"] == "deep" + assert seen == {"triggered_by": "manual", "mode": "deep"} + + +@pytest.mark.asyncio +async def test_trigger_still_rejects_unknown_mode(client): + resp = await client.post("/api/import/trigger", json={"mode": "wat"}) + assert resp.status_code == 400 From 5dfdbf5a60a45d7acafb1706b22661e22928d0dd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 15:32:26 -0400 Subject: [PATCH 182/272] feat(deep-scan): Importer(deep=) re-derive on sha-match; import_file wires batch.scan_mode Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/importer.py | 41 +++++++++++-- backend/app/tasks/import_file.py | 3 + tests/test_importer_deep.py | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 tests/test_importer_deep.py diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index dab757a..88a77e0 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -117,12 +117,14 @@ class Importer: import_root: Path, thumbnailer: Thumbnailer, settings: ImportSettings, + deep: bool = False, ): self.session = session self.images_root = images_root self.import_root = import_root self.thumbnailer = thumbnailer self.settings = settings + self.deep = deep self.attachments = AttachmentStore(images_root) def import_one(self, source: Path) -> ImportResult: @@ -290,10 +292,12 @@ class Importer: existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha) existing = self.session.execute(existing_stmt).scalar_one_or_none() if existing: - return ImportResult( - status="skipped", skip_reason=SkipReason.duplicate_hash, - image_id=existing.id, error="sha256 already present", - ) + if not self.deep: + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="sha256 already present", + ) + return self._deep_rederive(existing, source, attribution_path) # Perceptual near-dup (images only; videos keep phash NULL). phash = None @@ -374,6 +378,35 @@ class Importer: self.session.commit() return ImportResult(status="imported", image_id=record.id) + def _deep_rederive( + self, existing: ImageRecord, source: Path, attribution_path: Path + ) -> ImportResult: + """Deep scan: backfill phash/provenance/artist on an + already-imported record. METADATA ONLY — never re-runs the pHash + near-dup / supersede path. NULL-only, idempotent.""" + if existing.phash is None and not is_video(source): + try: + with Image.open(source) as im: + ph = compute_phash(im) + if ph is not None: + existing.phash = ph + except Exception as exc: + log.warning("deep rephash failed for %s: %s", source, exc) + + artist = None + name = derive_top_level_artist(attribution_path, self.import_root) + if name: + artist = self._upsert_artist(name) + if existing.artist_id is None: + existing.artist_id = artist.id + + self._apply_sidecar(existing, attribution_path, artist) + self.session.commit() + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="deep: re-derived", + ) + def _upsert_artist(self, name: str) -> Artist: slug = slugify(name) artist = self.session.execute( diff --git a/backend/app/tasks/import_file.py b/backend/app/tasks/import_file.py index c1469f0..d53de74 100644 --- a/backend/app/tasks/import_file.py +++ b/backend/app/tasks/import_file.py @@ -55,12 +55,15 @@ def import_media_file(self, import_task_id: int) -> dict: select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() import_root = Path(settings.import_scan_path) + batch = session.get(ImportBatch, task.batch_id) + deep = bool(batch and batch.scan_mode == "deep") importer = Importer( session=session, images_root=IMAGES_ROOT, import_root=import_root, thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), settings=settings, + deep=deep, ) try: diff --git a/tests/test_importer_deep.py b/tests/test_importer_deep.py new file mode 100644 index 0000000..b94f722 --- /dev/null +++ b/tests/test_importer_deep.py @@ -0,0 +1,99 @@ +"""FC-2d-vi: Importer(deep=True) re-derives on an already-imported row.""" + +import json + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, +) +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +def _mk(db_sync, import_layout, *, deep): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, images_root=images_root, import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + deep=deep, + ) + + +def _img(path): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (50, 50), (90, 30, 200)).save(path, "JPEG") + + +def test_deep_rederives_phash_and_provenance(db_sync, import_layout): + import_root, _ = import_layout + src = import_root / "Mae" / "p.jpg" + _img(src) + + # First import (quick). Images get a phash on import; null it + + # primary_post_id to simulate a pre-FC-2d-i+ii/v record, then add a + # sidecar to simulate provenance that didn't exist at first import. + quick = _mk(db_sync, import_layout, deep=False) + r1 = quick.import_one(src) + assert r1.status == "imported" + rec = db_sync.get(ImageRecord, r1.image_id) + rec.phash = None + rec.primary_post_id = None + db_sync.commit() + src.with_suffix(".jpg.json").write_text(json.dumps( + {"category": "patreon", "id": "9", "title": "P"})) + + deep = _mk(db_sync, import_layout, deep=True) + r2 = deep.import_one(src) + assert r2.status == "skipped" + assert "deep" in (r2.error or "") + assert r2.image_id == rec.id + + db_sync.expire_all() + rec2 = db_sync.get(ImageRecord, rec.id) + assert rec2.phash is not None + assert db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 1 # no new record + post = db_sync.execute(select(Post)).scalar_one() + assert db_sync.execute( + select(func.count()).select_from(ImageProvenance) + .where(ImageProvenance.image_record_id == rec.id) + ).scalar_one() == 1 + assert rec2.primary_post_id == post.id + + # Idempotent: a second deep pass adds no duplicate provenance. + _mk(db_sync, import_layout, deep=True).import_one(src) + db_sync.expire_all() + assert db_sync.execute( + select(func.count()).select_from(ImageProvenance) + ).scalar_one() == 1 + + +def test_deep_false_is_plain_skip(db_sync, import_layout): + import_root, _ = import_layout + src = import_root / "Mae" / "q.jpg" + _img(src) + _mk(db_sync, import_layout, deep=False).import_one(src) + r = _mk(db_sync, import_layout, deep=False).import_one(src) + assert r.status == "skipped" + assert r.error == "sha256 already present" From c33741837b690b6c36a514de759f3125ec243bd7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 15:33:16 -0400 Subject: [PATCH 183/272] fix(test): drop obsolete FC-2a deep-rejected assertion (deep now accepted in 2d-vi) Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_api_import_admin.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py index 0cdbee5..9f02861 100644 --- a/tests/test_api_import_admin.py +++ b/tests/test_api_import_admin.py @@ -27,12 +27,6 @@ async def client(app): yield c -@pytest.mark.asyncio -async def test_trigger_rejects_unknown_mode(client): - resp = await client.post("/api/import/trigger", json={"mode": "deep"}) - assert resp.status_code == 400 - - @pytest.mark.asyncio async def test_status_when_idle(client): resp = await client.get("/api/import/status") From e28de547c73bd964775747742e51cb566fc8da45 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 17:51:33 -0400 Subject: [PATCH 184/272] feat(integrity): verify_integrity task (keyset, fail-soft) + weekly beat Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/celery_app.py | 4 + backend/app/tasks/maintenance.py | 81 ++++++++++++++++++ tests/test_verify_integrity.py | 139 +++++++++++++++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 tests/test_verify_integrity.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 1cf8908..a2d3a56 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -66,6 +66,10 @@ def make_celery() -> Celery: "task": "backend.app.tasks.ml.apply_allowlist_tags", "schedule": 86400.0, }, + "integrity-verify-weekly": { + "task": "backend.app.tasks.maintenance.verify_integrity", + "schedule": 604800.0, # weekly + }, }, timezone="UTC", ) diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py index 1572406..be6c932 100644 --- a/backend/app/tasks/maintenance.py +++ b/backend/app/tasks/maintenance.py @@ -1,7 +1,9 @@ """Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks.""" import logging +import subprocess from datetime import UTC, datetime, timedelta +from pathlib import Path from PIL import Image from sqlalchemy import create_engine, delete, select, update @@ -17,6 +19,8 @@ log = logging.getLogger(__name__) STUCK_THRESHOLD_MINUTES = 30 OLD_TASK_DAYS = 7 PHASH_PAGE = 500 +VERIFY_PAGE = 200 +FFPROBE_TIMEOUT_SECONDS = 10 def _sync_session_factory(): @@ -115,3 +119,80 @@ def backfill_phash() -> int: session.commit() last_id = rows[-1].id return updated + + +def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str: + """Compute the integrity verdict for one file. Status precedence: + failed_verification (can't run) > corrupt (sha mismatch / decode + fails) > ok (passes both). Never raises.""" + try: + if not path.is_file(): + return "failed_verification" + try: + actual_sha = sha_fn(path) + except (OSError, PermissionError): + return "failed_verification" + if actual_sha != expected_sha: + return "corrupt" + if mime and mime.startswith("image/"): + try: + with Image.open(path) as im: + im.verify() + except Exception: + return "corrupt" + return "ok" + if mime and mime.startswith("video/"): + try: + proc = subprocess.run( + ["ffprobe", "-v", "error", "-i", str(path)], + capture_output=True, + timeout=FFPROBE_TIMEOUT_SECONDS, + ) + except FileNotFoundError: + # ffprobe binary missing — environment problem, not file. + return "failed_verification" + except subprocess.TimeoutExpired: + return "corrupt" + return "ok" if proc.returncode == 0 else "corrupt" + # Unknown mime — sha matched already; trust that. + return "ok" + except Exception as exc: + log.warning("verify_integrity unexpected error for %s: %s", path, exc) + return "failed_verification" + + +@celery.task(name="backend.app.tasks.maintenance.verify_integrity") +def verify_integrity() -> int: + """Verify every ImageRecord file: sha256 recompute + decode/probe + (PIL for images; ffprobe for videos). Writes integrity_status + (always — the column reflects the most recent verdict). + Keyset-paginated, fail-soft per row, idempotent. Returns the total + count verified.""" + from ..services.importer import _sha256_of # reuse the importer's helper + + SessionLocal = _sync_session_factory() + total = 0 + counts = {"ok": 0, "corrupt": 0, "failed_verification": 0} + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord) + .where(ImageRecord.id > last_id) + .order_by(ImageRecord.id.asc()) + .limit(VERIFY_PAGE) + ).scalars().all() + if not rows: + break + for rec in rows: + rec.integrity_status = _verify_one( + Path(rec.path), rec.sha256, rec.mime, _sha256_of + ) + counts[rec.integrity_status] = ( + counts.get(rec.integrity_status, 0) + 1 + ) + total += 1 + session.commit() + last_id = rows[-1].id + log.info("verify_integrity verdicts: %s (total %d)", counts, total) + return total diff --git a/tests/test_verify_integrity.py b/tests/test_verify_integrity.py new file mode 100644 index 0000000..ee4593a --- /dev/null +++ b/tests/test_verify_integrity.py @@ -0,0 +1,139 @@ +"""FC-2e: verify_integrity — sha + decode/probe per ImageRecord.""" + +import subprocess + +import pytest +from PIL import Image + +from backend.app.models import ImageRecord +from backend.app.tasks.maintenance import verify_integrity + +pytestmark = pytest.mark.integration + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _good_jpeg(path): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (32, 32), (40, 80, 160)).save(path, "JPEG") + + +def _rec(db_sync, path, *, sha, mime="image/jpeg"): + rec = ImageRecord( + path=str(path), sha256=sha, phash=None, size_bytes=1, + mime=mime, width=32, height=32, origin="imported_filesystem", + integrity_status="unknown", + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def _sha_of(path): + import hashlib + + h = hashlib.sha256() + h.update(path.read_bytes()) + return h.hexdigest() + + +def test_ok_corrupt_failed_paths(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + good = tmp_path / "good.jpg" + _good_jpeg(good) + bad = tmp_path / "bad.jpg" + bad.write_bytes(b"not a real jpeg") + truncated = tmp_path / "trunc.jpg" + _good_jpeg(truncated) + truncated.write_bytes(truncated.read_bytes()[:30]) # break the bitstream + + rec_ok = _rec(db_sync, good, sha=_sha_of(good)) + rec_sha_mismatch = _rec(db_sync, good, sha="0" * 64) # bytes intact, sha wrong + rec_decode_fail = _rec(db_sync, truncated, sha=_sha_of(truncated)) + # 'bad' has not-jpeg bytes; sha matches the bad bytes → decode fails + rec_bad_decode = _rec(db_sync, bad, sha=_sha_of(bad)) + rec_missing = _rec( + db_sync, tmp_path / "gone.jpg", sha="g" + "0" * 63, + ) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + total = verify_integrity() + db_sync.expire_all() + assert total == 5 + assert db_sync.get(ImageRecord, rec_ok.id).integrity_status == "ok" + assert db_sync.get(ImageRecord, rec_sha_mismatch.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_decode_fail.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_bad_decode.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_missing.id).integrity_status == "failed_verification" + + # Idempotent — second run re-derives the same verdict. + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec_ok.id).integrity_status == "ok" + + +def test_video_ffprobe_paths(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + vid = tmp_path / "v.mp4" + vid.write_bytes(b"fake but consistent video bytes") + rec = _rec(db_sync, vid, sha=_sha_of(vid), mime="video/mp4") + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + # Case A: ffprobe succeeds → ok + def _run_ok(args, **kw): + return subprocess.CompletedProcess(args, 0, b"", b"") + + monkeypatch.setattr(m.subprocess, "run", _run_ok) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "ok" + + # Case B: ffprobe non-zero → corrupt + def _run_bad(args, **kw): + return subprocess.CompletedProcess(args, 1, b"", b"err") + + monkeypatch.setattr(m.subprocess, "run", _run_bad) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "corrupt" + + # Case C: ffprobe times out → corrupt + def _run_timeout(args, **kw): + raise subprocess.TimeoutExpired(cmd=args, timeout=10) + + monkeypatch.setattr(m.subprocess, "run", _run_timeout) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "corrupt" + + # Case D: ffprobe binary missing → failed_verification + def _run_missing(args, **kw): + raise FileNotFoundError("ffprobe") + + monkeypatch.setattr(m.subprocess, "run", _run_missing) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "failed_verification" From 389d1bb0cfc3191174787642eb725385b713c822 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 17:51:56 -0400 Subject: [PATCH 185/272] =?UTF-8?q?feat(integrity):=20/trigger=20accepts?= =?UTF-8?q?=20mode=3Dverify=20=E2=86=92=20enqueues=20verify=5Fintegrity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/import_admin.py | 13 ++++++++++--- tests/test_api_import_admin.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py index ee074e3..6a16bd3 100644 --- a/backend/app/api/import_admin.py +++ b/backend/app/api/import_admin.py @@ -15,10 +15,17 @@ import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import") async def trigger_scan(): body = await request.get_json(silent=True) or {} mode = body.get("mode", "quick") - if mode not in ("quick", "deep"): - return jsonify({"error": f"mode {mode!r} not supported; use 'quick' or 'deep'"}), 400 + if mode not in ("quick", "deep", "verify"): + return jsonify({"error": f"mode {mode!r} not supported; use 'quick', 'deep', or 'verify'"}), 400 + + # 'verify' is a library task — short-circuit the import_root walk + # (no ImportBatch, no per-file ImportTasks). + if mode == "verify": + from ..tasks.maintenance import verify_integrity + + async_result = verify_integrity.delay() + return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202 - # Enqueue the scan task in Celery. The task creates the batch itself. from ..tasks.scan import scan_directory async_result = scan_directory.delay(triggered_by="manual", mode=mode) diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py index 9f02861..7e15f4b 100644 --- a/tests/test_api_import_admin.py +++ b/tests/test_api_import_admin.py @@ -114,3 +114,26 @@ async def test_trigger_accepts_deep(client, monkeypatch): async def test_trigger_still_rejects_unknown_mode(client): resp = await client.post("/api/import/trigger", json={"mode": "wat"}) assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_trigger_accepts_verify(client, monkeypatch): + # Stub the verify task's dispatch so the API contract is asserted + # without running the real (eager) verify_integrity end-to-end. + from backend.app.tasks import maintenance as m + + seen = {} + + class _Res: + id = "celery-verify" + + def _fake_delay(): + seen["called"] = True + return _Res() + + monkeypatch.setattr(m.verify_integrity, "delay", _fake_delay) + resp = await client.post("/api/import/trigger", json={"mode": "verify"}) + assert resp.status_code == 202 + body = await resp.get_json() + assert body["mode"] == "verify" + assert seen == {"called": True} From c37a3c9b552a5a5fdd2ddf1010551e9251e75154 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 17:53:12 -0400 Subject: [PATCH 186/272] feat(integrity): /api/system/stats exposes integrity counts Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/settings.py | 19 +++++++++++++++++++ tests/test_api_settings.py | 25 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index a91bd60..f5067e4 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -83,6 +83,17 @@ async def system_stats(): ).all() status_counts = {row[0]: row[1] for row in status_rows} + # Integrity counts — FC-2e. + integrity_rows = ( + await session.execute( + select( + ImageRecord.integrity_status, + func.count(ImageRecord.id), + ).group_by(ImageRecord.integrity_status) + ) + ).all() + integrity_counts = {row[0]: row[1] for row in integrity_rows} + # Active batch (most recent running) active_batch_row = ( await session.execute( @@ -116,5 +127,13 @@ async def system_stats(): "skipped": status_counts.get("skipped", 0), "failed": status_counts.get("failed", 0), }, + "integrity": { + "unknown": int(integrity_counts.get("unknown", 0)), + "ok": int(integrity_counts.get("ok", 0)), + "corrupt": int(integrity_counts.get("corrupt", 0)), + "failed_verification": int( + integrity_counts.get("failed_verification", 0) + ), + }, "active_batch": active_batch, }) diff --git a/tests/test_api_settings.py b/tests/test_api_settings.py index a7f30b1..5872a81 100644 --- a/tests/test_api_settings.py +++ b/tests/test_api_settings.py @@ -80,3 +80,28 @@ async def test_system_stats_shape(client): assert key in body for status in ("pending", "queued", "processing", "complete", "skipped", "failed"): assert status in body["tasks"] + + +@pytest.mark.asyncio +async def test_system_stats_integrity_counts(client, db): + from backend.app.models import ImageRecord + + base = "stats_" + "0" * 56 + for i, status in enumerate( + ["ok", "ok", "corrupt", "failed_verification"] + ): + db.add(ImageRecord( + path=f"/images/i/{i}.jpg", sha256=base + f"{i:02d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status=status, + )) + await db.commit() + + resp = await client.get("/api/system/stats") + assert resp.status_code == 200 + body = await resp.get_json() + integ = body["integrity"] + assert integ["ok"] >= 2 + assert integ["corrupt"] >= 1 + assert integ["failed_verification"] >= 1 + assert "unknown" in integ From d4d8976f291fc835e5d89b99f9facbf4478a0674 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 17:53:54 -0400 Subject: [PATCH 187/272] feat(integrity): image-detail payload includes integrity_status Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/gallery_service.py | 1 + tests/test_gallery_service.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 6ef8375..834c6a6 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -245,6 +245,7 @@ class GalleryService: "width": record.width, "height": record.height, "size_bytes": record.size_bytes, + "integrity_status": record.integrity_status, "created_at": record.created_at.isoformat(), "thumbnail_url": thumbnail_url(record.sha256, record.mime), "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py index 7be625f..ecba6bb 100644 --- a/tests/test_gallery_service.py +++ b/tests/test_gallery_service.py @@ -122,3 +122,14 @@ async def test_neighbors(db): async def test_get_image_with_tags_returns_none_for_missing(db): svc = GalleryService(db) assert await svc.get_image_with_tags(99999) is None + + +@pytest.mark.asyncio +async def test_get_image_with_tags_includes_integrity_status(db): + images = await _seed_images(db, 1) + img = images[0] + img.integrity_status = "ok" + await db.flush() + svc = GalleryService(db) + payload = await svc.get_image_with_tags(img.id) + assert payload["integrity_status"] == "ok" From 61b623de48efd55931a122cc3e76bdc15e35581f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 19 May 2026 17:54:25 -0400 Subject: [PATCH 188/272] feat(integrity): ImageViewer badge for corrupt/unverified status Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/modal/ImageViewer.vue | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue index 3f30bff..99ef9e8 100644 --- a/frontend/src/components/modal/ImageViewer.vue +++ b/frontend/src/components/modal/ImageViewer.vue @@ -11,6 +11,13 @@ mdi-close + {{ integrityBadge.label }} + @@ -80,7 +97,7 @@ diff --git a/frontend/src/components/credentials/ExtensionKeyBar.vue b/frontend/src/components/credentials/ExtensionKeyBar.vue new file mode 100644 index 0000000..cf56b3c --- /dev/null +++ b/frontend/src/components/credentials/ExtensionKeyBar.vue @@ -0,0 +1,75 @@ + + + + + diff --git a/frontend/src/components/credentials/PlatformCredentialRow.vue b/frontend/src/components/credentials/PlatformCredentialRow.vue new file mode 100644 index 0000000..07225d8 --- /dev/null +++ b/frontend/src/components/credentials/PlatformCredentialRow.vue @@ -0,0 +1,68 @@ + + + + + diff --git a/frontend/src/router.js b/frontend/src/router.js index 6115b25..31224b4 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -8,6 +8,7 @@ import ArtistView from './views/ArtistView.vue' import SeriesManageView from './views/SeriesManageView.vue' import SeriesReaderView from './views/SeriesReaderView.vue' import SubscriptionsView from './views/SubscriptionsView.vue' +import CredentialsView from './views/CredentialsView.vue' // The application's front door. `/` redirects here. Changing the front door // is a one-line edit (e.g. '/gallery' or '/tags'). @@ -31,7 +32,7 @@ const routes = [ // FC-3: subscription backbone { path: '/subscriptions', name: 'subscriptions', component: SubscriptionsView, meta: { title: 'Subscriptions' } }, - { path: '/credentials', name: 'credentials', component: PlaceholderView, meta: { title: 'Credentials' } }, + { path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } }, { path: '/downloads', name: 'downloads', component: PlaceholderView, meta: { title: 'Downloads' } } ] diff --git a/frontend/src/views/CredentialsView.vue b/frontend/src/views/CredentialsView.vue new file mode 100644 index 0000000..5501d1d --- /dev/null +++ b/frontend/src/views/CredentialsView.vue @@ -0,0 +1,79 @@ + + + From 408971a798c746c3410bff640861d02ebb4b2431 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 20 May 2026 18:39:44 -0400 Subject: [PATCH 212/272] fix(fc3b): SourceFormDialog + SubscriptionsView read platforms store; drop sources.loadPlatforms Co-Authored-By: Claude Opus 4.7 (1M context) --- .../subscriptions/SourceFormDialog.vue | 18 +++++++++++++++--- frontend/src/stores/sources.js | 13 ++----------- frontend/src/views/SubscriptionsView.vue | 4 +++- frontend/test/sources.spec.js | 10 ---------- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/frontend/src/components/subscriptions/SourceFormDialog.vue b/frontend/src/components/subscriptions/SourceFormDialog.vue index 33e1f6c..15b2e63 100644 --- a/frontend/src/components/subscriptions/SourceFormDialog.vue +++ b/frontend/src/components/subscriptions/SourceFormDialog.vue @@ -20,7 +20,13 @@ Adding source for {{ initialArtist.name }} - + @@ -65,6 +71,7 @@ + + diff --git a/frontend/src/components/downloads/DownloadEventRow.vue b/frontend/src/components/downloads/DownloadEventRow.vue new file mode 100644 index 0000000..a4360c0 --- /dev/null +++ b/frontend/src/components/downloads/DownloadEventRow.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/frontend/src/components/downloads/FilterPills.vue b/frontend/src/components/downloads/FilterPills.vue new file mode 100644 index 0000000..20ed1bd --- /dev/null +++ b/frontend/src/components/downloads/FilterPills.vue @@ -0,0 +1,22 @@ + + + diff --git a/frontend/src/router.js b/frontend/src/router.js index 31224b4..ba5ea86 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -1,5 +1,4 @@ import { createRouter, createWebHistory, createMemoryHistory } from 'vue-router' -import PlaceholderView from './views/PlaceholderView.vue' import SettingsView from './views/SettingsView.vue' import GalleryView from './views/GalleryView.vue' import ShowcaseView from './views/ShowcaseView.vue' @@ -9,6 +8,7 @@ import SeriesManageView from './views/SeriesManageView.vue' import SeriesReaderView from './views/SeriesReaderView.vue' import SubscriptionsView from './views/SubscriptionsView.vue' import CredentialsView from './views/CredentialsView.vue' +import DownloadsView from './views/DownloadsView.vue' // The application's front door. `/` redirects here. Changing the front door // is a one-line edit (e.g. '/gallery' or '/tags'). @@ -32,8 +32,8 @@ const routes = [ // FC-3: subscription backbone { path: '/subscriptions', name: 'subscriptions', component: SubscriptionsView, meta: { title: 'Subscriptions' } }, - { path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } }, - { path: '/downloads', name: 'downloads', component: PlaceholderView, meta: { title: 'Downloads' } } + { path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } }, + { path: '/downloads', name: 'downloads', component: DownloadsView, meta: { title: 'Downloads' } } ] // Browser uses HTML5 history; non-browser (Vitest/SSR) falls back to memory diff --git a/frontend/src/views/DownloadsView.vue b/frontend/src/views/DownloadsView.vue new file mode 100644 index 0000000..0979c98 --- /dev/null +++ b/frontend/src/views/DownloadsView.vue @@ -0,0 +1,70 @@ + + + + + From f309a1e79e89ab30809b0af02dcdc658f848bf3d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 20 May 2026 20:47:36 -0400 Subject: [PATCH 226/272] =?UTF-8?q?feat(fc3c):=20/subscriptions=20Check=20?= =?UTF-8?q?Now=20button=20=E2=80=94=20wires=20SourceRow=20=E2=86=92=20stor?= =?UTF-8?q?e.checkNow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../subscriptions/ArtistSection.vue | 5 +++- .../components/subscriptions/SourceRow.vue | 14 ++++++++-- frontend/src/views/SubscriptionsView.vue | 28 ++++++++++++++++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/subscriptions/ArtistSection.vue b/frontend/src/components/subscriptions/ArtistSection.vue index 894f3ab..148d813 100644 --- a/frontend/src/components/subscriptions/ArtistSection.vue +++ b/frontend/src/components/subscriptions/ArtistSection.vue @@ -13,9 +13,11 @@
new Set() }, }) -defineEmits(['toggle', 'edit', 'remove', 'toggle-source', 'add-source']) +defineEmits(['toggle', 'edit', 'remove', 'toggle-source', 'add-source', 'check']) const latestChecked = computed(() => { const dates = props.sources.map(s => s.last_checked_at).filter(Boolean) diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index 66807a3..b98cc88 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -11,14 +11,22 @@ density="compact" hide-details color="accent" @update:model-value="onToggleEnabled" /> +
diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue index b98cc88..cb7a551 100644 --- a/frontend/src/components/subscriptions/SourceRow.vue +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -1,5 +1,6 @@ + + From b1a358c413ff839cc568f9504d70254430b8c352 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 18:56:14 -0400 Subject: [PATCH 251/272] =?UTF-8?q?fc3e:=20PostCard=20=E2=80=94=20header?= =?UTF-8?q?=20+=20title=20+=20truncated-desc=20+=20thumbnail=20strip=20+?= =?UTF-8?q?=20attachment=20chips?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/components/posts/PostCard.vue | 204 +++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 frontend/src/components/posts/PostCard.vue diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue new file mode 100644 index 0000000..68b537e --- /dev/null +++ b/frontend/src/components/posts/PostCard.vue @@ -0,0 +1,204 @@ + + + + + From 6fb9e1ddf8d89e2f0a15df2f8f09b51028ccdb45 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 18:56:40 -0400 Subject: [PATCH 252/272] =?UTF-8?q?fc3e:=20PostsView=20+=20/posts=20route?= =?UTF-8?q?=20=E2=80=94=20IntersectionObserver=20infinite=20scroll,=20URL-?= =?UTF-8?q?driven=20filters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/router.js | 2 + frontend/src/views/PostsView.vue | 106 +++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 frontend/src/views/PostsView.vue diff --git a/frontend/src/router.js b/frontend/src/router.js index ba5ea86..72623bb 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -9,6 +9,7 @@ import SeriesReaderView from './views/SeriesReaderView.vue' import SubscriptionsView from './views/SubscriptionsView.vue' import CredentialsView from './views/CredentialsView.vue' import DownloadsView from './views/DownloadsView.vue' +import PostsView from './views/PostsView.vue' // The application's front door. `/` redirects here. Changing the front door // is a one-line edit (e.g. '/gallery' or '/tags'). @@ -31,6 +32,7 @@ const routes = [ { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, // FC-3: subscription backbone + { path: '/posts', name: 'posts', component: PostsView, meta: { title: 'Posts' } }, { path: '/subscriptions', name: 'subscriptions', component: SubscriptionsView, meta: { title: 'Subscriptions' } }, { path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } }, { path: '/downloads', name: 'downloads', component: DownloadsView, meta: { title: 'Downloads' } } diff --git a/frontend/src/views/PostsView.vue b/frontend/src/views/PostsView.vue new file mode 100644 index 0000000..56a7cef --- /dev/null +++ b/frontend/src/views/PostsView.vue @@ -0,0 +1,106 @@ + + + + + From 9b2a8d7fc69c047b114907e04533b003a93b3183 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 18:57:01 -0400 Subject: [PATCH 253/272] =?UTF-8?q?fc3e:=20ArtistView=20=E2=80=94=20'View?= =?UTF-8?q?=20posts'=20chip=20linking=20to=20/posts=3Fartist=5Fid=3DN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/views/ArtistView.vue | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/views/ArtistView.vue b/frontend/src/views/ArtistView.vue index 7f570e6..7a363f2 100644 --- a/frontend/src/views/ArtistView.vue +++ b/frontend/src/views/ArtistView.vue @@ -34,6 +34,11 @@ :to="`/subscriptions?artist_id=${store.overview.id}`" >{{ store.overview.sources.length }} source{{ store.overview.sources.length === 1 ? '' : 's' }} + View posts + Date: Thu, 21 May 2026 22:14:15 -0400 Subject: [PATCH 254/272] =?UTF-8?q?fc3f:=20ArtistDirectoryService=20?= =?UTF-8?q?=E2=80=94=20cursor-paginated=20artists=20directory=20mirroring?= =?UTF-8?q?=20TagDirectoryService?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../app/services/artist_directory_service.py | 143 ++++++++++ tests/test_artist_directory_service.py | 259 ++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 backend/app/services/artist_directory_service.py create mode 100644 tests/test_artist_directory_service.py diff --git a/backend/app/services/artist_directory_service.py b/backend/app/services/artist_directory_service.py new file mode 100644 index 0000000..a16c1aa --- /dev/null +++ b/backend/app/services/artist_directory_service.py @@ -0,0 +1,143 @@ +"""FC-3f: cursor-paginated artists directory. + +Mirrors TagDirectoryService surface-for-surface, but backed by +Artist + Source + ImageRecord.artist_id (FC-2d-vii-c authoritative +attribution). Pure read-surface; no writes. + +Cursor wire format is identical to tag_directory_service: base64 of +"|". The _encode/_decode helpers are copied (not imported) +to keep the two services decoupled. +""" +from __future__ import annotations + +import base64 +from dataclasses import dataclass + +from sqlalchemy import and_, exists, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, ImageRecord, Source +from .gallery_service import thumbnail_url + +_SEP = "|" +_PREVIEW_COUNT = 3 + + +def _encode(name: str, artist_id: int) -> str: + return base64.urlsafe_b64encode(f"{name}{_SEP}{artist_id}".encode()).decode() + + +def _decode(cursor: str) -> tuple[str, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + name, aid = raw.rsplit(_SEP, 1) + return name, int(aid) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class DirectoryPage: + cards: list[dict] + next_cursor: str | None + + +class ArtistDirectoryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def list_artists( + self, + *, + q: str | None, + platform: str | None, + cursor: str | None, + limit: int = 60, + ) -> DirectoryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + + count_col = func.count(ImageRecord.id).label("image_count") + stmt = ( + select(Artist, count_col) + .outerjoin(ImageRecord, ImageRecord.artist_id == Artist.id) + .group_by(Artist.id) + ) + if q: + stmt = stmt.where(Artist.name.ilike(f"%{q}%")) + if platform is not None: + # Correlated EXISTS — NOT a JOIN. A JOIN to Source duplicates + # the artist row when the artist has multiple sources on this + # platform, breaking the keyset cursor and image_count. + stmt = stmt.where( + exists().where( + and_( + Source.artist_id == Artist.id, + Source.platform == platform, + ) + ) + ) + if cursor: + c_name, c_id = _decode(cursor) + stmt = stmt.where( + or_( + Artist.name > c_name, + and_(Artist.name == c_name, Artist.id > c_id), + ) + ) + stmt = stmt.order_by(Artist.name.asc(), Artist.id.asc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() + + next_cursor: str | None = None + if len(rows) > limit: + last_artist = rows[limit - 1][0] + next_cursor = _encode(last_artist.name, last_artist.id) + rows = rows[:limit] + + artist_ids = [a.id for a, _ in rows] + previews = await self._previews(artist_ids) + + cards = [ + { + "id": artist.id, + "name": artist.name, + "slug": artist.slug, + "is_subscription": bool(artist.is_subscription), + "image_count": int(image_count), + "preview_thumbnails": previews.get(artist.id, []), + } + for artist, image_count in rows + ] + return DirectoryPage(cards=cards, next_cursor=next_cursor) + + async def _previews(self, artist_ids: list[int]) -> dict[int, list[str]]: + """artist_id -> [thumbnail_url, ...up to _PREVIEW_COUNT]. + + Window function ranks ImageRecord rows per artist by id DESC + (newest first) and slices to the top _PREVIEW_COUNT in one query. + """ + if not artist_ids: + return {} + rn = func.row_number().over( + partition_by=ImageRecord.artist_id, + order_by=ImageRecord.id.desc(), + ).label("rn") + sub = ( + select( + ImageRecord.artist_id.label("artist_id"), + ImageRecord.sha256.label("sha256"), + ImageRecord.mime.label("mime"), + rn, + ) + .where(ImageRecord.artist_id.in_(artist_ids)) + .subquery() + ) + stmt = ( + select(sub.c.artist_id, sub.c.sha256, sub.c.mime) + .where(sub.c.rn <= _PREVIEW_COUNT) + .order_by(sub.c.artist_id, sub.c.rn) + ) + out: dict[int, list[str]] = {} + for aid, sha, mime in (await self.session.execute(stmt)).all(): + out.setdefault(aid, []).append(thumbnail_url(sha, mime)) + return out diff --git a/tests/test_artist_directory_service.py b/tests/test_artist_directory_service.py new file mode 100644 index 0000000..fbfbf2f --- /dev/null +++ b/tests/test_artist_directory_service.py @@ -0,0 +1,259 @@ +"""FC-3f: ArtistDirectoryService unit tests. + +Mirrors TagDirectoryService's test shape — covers cursor encode/decode, +ordering by (name ASC, id ASC), substring q filter, platform EXISTS +filter (with the row-duplication trap), pagination, image_count via +LEFT JOIN, and the window-function preview helper. +""" +import pytest + +from backend.app.models import Artist, ImageRecord, Source +from backend.app.services.artist_directory_service import ( + ArtistDirectoryService, + _decode, + _encode, +) + +pytestmark = pytest.mark.integration + + +# --- Cursor round-trip ---------------------------------------------------- + + +def test_cursor_round_trip(): + c = _encode("alice", 5) + assert _decode(c) == ("alice", 5) + + +def test_decode_cursor_rejects_garbage(): + with pytest.raises(ValueError): + _decode("not-base64!!!") + + +def test_cursor_round_trip_handles_pipes_in_name(): + # Names can theoretically contain '|'; rsplit on the LAST '|' must + # still recover (name, id) correctly. + c = _encode("a|b|c", 7) + assert _decode(c) == ("a|b|c", 7) + + +# --- Fixture builders ----------------------------------------------------- + + +async def _seed_artist(db, name: str, *, is_subscription: bool = False): + a = Artist( + name=name, slug=name.lower().replace(" ", "-"), + is_subscription=is_subscription, + ) + db.add(a) + await db.flush() + return a + + +async def _seed_source(db, artist_id: int, platform: str, url: str): + s = Source(artist_id=artist_id, platform=platform, url=url, enabled=True) + db.add(s) + await db.flush() + return s + + +async def _seed_image(db, artist_id: int, suffix: str): + db.add(ImageRecord( + path=f"/images/{suffix}.jpg", + sha256=("0" * 60) + f"{abs(hash(suffix)) % 10000:04d}", + size_bytes=10, mime="image/jpeg", width=10, height=10, + origin="downloaded", artist_id=artist_id, + )) + await db.flush() + + +# --- Ordering + pagination ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_orders_by_name_asc(db): + await _seed_artist(db, "zara-order") + await _seed_artist(db, "alice-order") + await _seed_artist(db, "miles-order") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-order", platform=None, cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-order", "miles-order", "zara-order"] + + +@pytest.mark.asyncio +async def test_list_pagination_across_two_pages(db): + artists = [] + for letter in "abcde": + a = await _seed_artist(db, f"{letter}-page") + artists.append(a) + await db.commit() + + page1 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=None, limit=2, + ) + assert [c["name"] for c in page1.cards] == ["a-page", "b-page"] + assert page1.next_cursor is not None + + page2 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=page1.next_cursor, limit=2, + ) + assert [c["name"] for c in page2.cards] == ["c-page", "d-page"] + assert page2.next_cursor is not None + + page3 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=page2.next_cursor, limit=2, + ) + assert [c["name"] for c in page3.cards] == ["e-page"] + assert page3.next_cursor is None + + +# --- q filter ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_list_q_filter_case_insensitive_substring(db): + await _seed_artist(db, "Alice-Q") + await _seed_artist(db, "Bob-Q") + await _seed_artist(db, "Charlie-Q") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="ALICE", platform=None, cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["Alice-Q"] + + +# --- platform filter (correlated EXISTS, not JOIN) ----------------------- + + +@pytest.mark.asyncio +async def test_list_platform_filter_excludes_no_source(db): + a = await _seed_artist(db, "alice-noplat") + await _seed_source(db, a.id, "patreon", "https://p/alice-np") + await _seed_artist(db, "bob-noplat") # no sources + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-noplat", platform="patreon", cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-noplat"] + + +@pytest.mark.asyncio +async def test_list_platform_filter_excludes_wrong_platform(db): + a = await _seed_artist(db, "alice-wplat") + await _seed_source(db, a.id, "deviantart", "https://d/alice-wp") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-wplat", platform="patreon", cursor=None, limit=60, + ) + assert page.cards == [] + + +@pytest.mark.asyncio +async def test_list_platform_filter_does_not_duplicate_artist_with_multiple_sources(db): + """The EXISTS subquery prevents JOIN-style row duplication when an + artist has multiple sources on the requested platform.""" + a = await _seed_artist(db, "alice-multi") + await _seed_source(db, a.id, "patreon", "https://p/alice-m1") + await _seed_source(db, a.id, "patreon", "https://p/alice-m2") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-multi", platform="patreon", cursor=None, limit=60, + ) + assert len(page.cards) == 1 + assert page.cards[0]["name"] == "alice-multi" + + +# --- image_count via LEFT JOIN ------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_image_count_reflects_artist_id_links(db): + a = await _seed_artist(db, "alice-count") + for i in range(4): + await _seed_image(db, a.id, f"count-{i}") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-count", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-count") + assert target["image_count"] == 4 + + +@pytest.mark.asyncio +async def test_list_zero_image_artist_preserved_by_left_join(db): + """LEFT JOIN ImageRecord keeps the artist row even with no images.""" + await _seed_artist(db, "alice-zero") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-zero", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-zero") + assert target["image_count"] == 0 + assert target["preview_thumbnails"] == [] + + +# --- Preview thumbnails (window function) -------------------------------- + + +@pytest.mark.asyncio +async def test_list_previews_returns_top_three_by_image_id_desc(db): + a = await _seed_artist(db, "alice-prev") + for i in range(5): + await _seed_image(db, a.id, f"prev-{i}") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-prev", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-prev") + assert len(target["preview_thumbnails"]) == 3 + for url in target["preview_thumbnails"]: + assert url.startswith("/images/thumbs/") + + +# --- is_subscription field plumbed through ------------------------------- + + +@pytest.mark.asyncio +async def test_list_is_subscription_field_present(db): + await _seed_artist(db, "alice-sub", is_subscription=True) + await _seed_artist(db, "bob-sub", is_subscription=False) + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-sub", platform=None, cursor=None, limit=60, + ) + alice = next(c for c in page.cards if c["name"] == "alice-sub") + bob = next(c for c in page.cards if c["name"] == "bob-sub") + assert alice["is_subscription"] is True + assert bob["is_subscription"] is False + + +# --- Combined q + platform ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_combined_q_and_platform(db): + alice = await _seed_artist(db, "alice-combo-f") + await _seed_source(db, alice.id, "patreon", "https://p/alice-combo") + bob = await _seed_artist(db, "bob-combo-f") + await _seed_source(db, bob.id, "patreon", "https://p/bob-combo") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="alice", platform="patreon", cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-combo-f"] From 51bdaf167b955c1cfd9105010b2cdc2c0d5adc52 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 22:14:42 -0400 Subject: [PATCH 255/272] =?UTF-8?q?fc3f:=20GET=20/api/artists/directory=20?= =?UTF-8?q?=E2=80=94=20cursor=20+=20q=20+=20platform=20filters=20with=20va?= =?UTF-8?q?lidation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/artists.py | 41 ++++++++++- tests/test_api_artists_directory.py | 104 ++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 tests/test_api_artists_directory.py diff --git a/backend/app/api/artists.py b/backend/app/api/artists.py index f4f5e26..fa6da74 100644 --- a/backend/app/api/artists.py +++ b/backend/app/api/artists.py @@ -1,11 +1,14 @@ """FC-3a: plural /api/artists endpoints — POST (find-or-create) + -autocomplete. The slug-routed singular /api/artist/ blueprint +autocomplete. FC-3f: GET /api/artists/directory (cursor-paginated +browse grid). The slug-routed singular /api/artist/ blueprint stays separate and unmodified.""" from quart import Blueprint, jsonify, request from ..extensions import get_session +from ..services.artist_directory_service import ArtistDirectoryService from ..services.artist_service import ArtistService +from ..services.source_service import KNOWN_PLATFORMS artists_bp = Blueprint("artists", __name__, url_prefix="/api/artists") @@ -42,3 +45,39 @@ async def autocomplete(): return jsonify([ {"id": a.id, "name": a.name, "slug": a.slug} for a in rows ]) + + +@artists_bp.route("/directory", methods=["GET"]) +async def directory(): + """FC-3f: cursor-paginated artists directory. + + Mirrors /api/tags/directory shape: { cards: [...], next_cursor }. + """ + args = request.args + + cursor = args.get("cursor") or None + q = args.get("q") or None + platform = args.get("platform") or None + limit_raw = args.get("limit", "60") + + try: + limit = int(limit_raw) + except ValueError: + return jsonify({"error": "invalid_limit"}), 400 + if limit < 1 or limit > 200: + return jsonify({"error": "invalid_limit"}), 400 + + if platform is not None and platform not in KNOWN_PLATFORMS: + return jsonify({"error": "unknown_platform"}), 400 + + async with get_session() as session: + svc = ArtistDirectoryService(session) + try: + page = await svc.list_artists( + q=q, platform=platform, cursor=cursor, limit=limit, + ) + except ValueError: + # Service raises only on bad cursor (limit was validated above). + return jsonify({"error": "invalid_cursor"}), 400 + + return jsonify({"cards": page.cards, "next_cursor": page.next_cursor}) diff --git a/tests/test_api_artists_directory.py b/tests/test_api_artists_directory.py new file mode 100644 index 0000000..37459fc --- /dev/null +++ b/tests/test_api_artists_directory.py @@ -0,0 +1,104 @@ +"""FC-3f: /api/artists/directory API tests.""" + +import pytest + +from backend.app import create_app +from backend.app.models import Artist, ImageRecord, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def seeded(db): + alice = Artist(name="alice-api", slug="alice-api", is_subscription=True) + bob = Artist(name="bob-api", slug="bob-api", is_subscription=False) + db.add(alice) + db.add(bob) + await db.flush() + db.add(Source( + artist_id=alice.id, platform="patreon", + url="https://p/alice-api", enabled=True, + )) + db.add(ImageRecord( + path="/images/api/alice.jpg", + sha256=("a" * 60) + "0001", + size_bytes=10, mime="image/jpeg", width=10, height=10, + origin="downloaded", artist_id=alice.id, + )) + await db.commit() + return alice, bob + + +@pytest.mark.asyncio +async def test_directory_returns_cards_and_cursor_keys(client, seeded): + resp = await client.get("/api/artists/directory") + assert resp.status_code == 200 + body = await resp.get_json() + assert set(body.keys()) == {"cards", "next_cursor"} + assert isinstance(body["cards"], list) + + +@pytest.mark.asyncio +async def test_directory_card_shape(client, seeded): + resp = await client.get("/api/artists/directory?q=alice-api") + body = await resp.get_json() + card = next(c for c in body["cards"] if c["name"] == "alice-api") + assert set(card.keys()) == { + "id", "name", "slug", "is_subscription", "image_count", "preview_thumbnails", + } + assert card["is_subscription"] is True + assert card["image_count"] == 1 + assert len(card["preview_thumbnails"]) == 1 + + +@pytest.mark.asyncio +async def test_directory_rejects_malformed_cursor(client): + resp = await client.get("/api/artists/directory?cursor=garbage!!!") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "invalid_cursor" + + +@pytest.mark.asyncio +async def test_directory_rejects_unknown_platform(client): + resp = await client.get("/api/artists/directory?platform=myspace") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_platform" + + +@pytest.mark.asyncio +async def test_directory_rejects_limit_out_of_range(client): + resp = await client.get("/api/artists/directory?limit=0") + assert resp.status_code == 400 + resp = await client.get("/api/artists/directory?limit=500") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_directory_q_propagates(client, seeded): + resp = await client.get("/api/artists/directory?q=alice-api") + body = await resp.get_json() + names = [c["name"] for c in body["cards"]] + assert "alice-api" in names + assert "bob-api" not in names + + +@pytest.mark.asyncio +async def test_directory_platform_propagates(client, seeded): + resp = await client.get("/api/artists/directory?platform=patreon") + body = await resp.get_json() + names = [c["name"] for c in body["cards"]] + assert "alice-api" in names + assert "bob-api" not in names # no patreon source From f2f74efff1f4719154fc6cffe646cfe4e299651b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 22:14:53 -0400 Subject: [PATCH 256/272] =?UTF-8?q?fc3f:=20artistDirectory=20Pinia=20store?= =?UTF-8?q?=20=E2=80=94=20cursor=20scroll=20+=20q=20+=20platform=20filters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/stores/artistDirectory.js | 57 ++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 frontend/src/stores/artistDirectory.js diff --git a/frontend/src/stores/artistDirectory.js b/frontend/src/stores/artistDirectory.js new file mode 100644 index 0000000..7550c36 --- /dev/null +++ b/frontend/src/stores/artistDirectory.js @@ -0,0 +1,57 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useArtistDirectoryStore = defineStore('artistDirectory', () => { + const api = useApi() + const cards = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const q = ref('') + const platform = ref(null) + let started = false + + async function loadMore() { + if (loading.value) return + if (started && nextCursor.value === null) return + loading.value = true + error.value = null + try { + const params = { limit: PAGE } + if (q.value) params.q = q.value + if (platform.value) params.platform = platform.value + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/artists/directory', { params }) + cards.value.push(...body.cards) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function reset() { + cards.value = [] + nextCursor.value = null + started = false + await loadMore() + } + + function setQuery(text) { q.value = text; reset() } + function setPlatform(p) { platform.value = p; reset() } + + const hasMore = computed(() => !started || nextCursor.value !== null) + const isEmpty = computed( + () => !loading.value && cards.value.length === 0 && error.value === null + ) + + return { + cards, loading, error, q, platform, hasMore, isEmpty, + loadMore, reset, setQuery, setPlatform, + } +}) From ab2847ad3396ce7bde42ed94af9db72e0750d93d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 22:15:06 -0400 Subject: [PATCH 257/272] =?UTF-8?q?fc3f:=20ArtistCard=20=E2=80=94=20previe?= =?UTF-8?q?w=20row=20+=20name=20+=20subscription=20chip=20+=20image=20coun?= =?UTF-8?q?t,=20clicks=20to=20/artist/:slug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/discovery/ArtistCard.vue | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 frontend/src/components/discovery/ArtistCard.vue diff --git a/frontend/src/components/discovery/ArtistCard.vue b/frontend/src/components/discovery/ArtistCard.vue new file mode 100644 index 0000000..55564cb --- /dev/null +++ b/frontend/src/components/discovery/ArtistCard.vue @@ -0,0 +1,59 @@ + + + + + From 0f31e846357c728cd71432624e2310d8355c9acd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 22:15:28 -0400 Subject: [PATCH 258/272] =?UTF-8?q?fc3f:=20ArtistsView=20+=20/artists=20ro?= =?UTF-8?q?ute=20=E2=80=94=20search=20+=20platform=20filter=20+=20infinite?= =?UTF-8?q?=20scroll=20grid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/router.js | 2 + frontend/src/views/ArtistsView.vue | 93 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 frontend/src/views/ArtistsView.vue diff --git a/frontend/src/router.js b/frontend/src/router.js index 72623bb..b51f37f 100644 --- a/frontend/src/router.js +++ b/frontend/src/router.js @@ -10,6 +10,7 @@ import SubscriptionsView from './views/SubscriptionsView.vue' import CredentialsView from './views/CredentialsView.vue' import DownloadsView from './views/DownloadsView.vue' import PostsView from './views/PostsView.vue' +import ArtistsView from './views/ArtistsView.vue' // The application's front door. `/` redirects here. Changing the front door // is a one-line edit (e.g. '/gallery' or '/tags'). @@ -22,6 +23,7 @@ const routes = [ // FC-2: image backbone { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase' } }, { path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, + { path: '/artists', name: 'artists', component: ArtistsView, meta: { title: 'Artists' } }, { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, // Artist detail — no meta.title (reached by clicking an artist, not nav). { path: '/artist/:slug', name: 'artist', component: ArtistView }, diff --git a/frontend/src/views/ArtistsView.vue b/frontend/src/views/ArtistsView.vue new file mode 100644 index 0000000..dc32f82 --- /dev/null +++ b/frontend/src/views/ArtistsView.vue @@ -0,0 +1,93 @@ + + + + + From da9c19b2dc581afd0b7442278fd7f2346cb48dd4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 08:11:42 -0400 Subject: [PATCH 259/272] fc5: migration 0015 + MigrationRun model (kind/status as String(32), API-validated) Co-Authored-By: Claude Opus 4.7 (1M context) --- alembic/versions/0015_fc5_migration_run.py | 51 ++++++++++++++++++++++ backend/app/models/__init__.py | 2 + backend/app/models/migration_run.py | 37 ++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 alembic/versions/0015_fc5_migration_run.py create mode 100644 backend/app/models/migration_run.py diff --git a/alembic/versions/0015_fc5_migration_run.py b/alembic/versions/0015_fc5_migration_run.py new file mode 100644 index 0000000..89d7d89 --- /dev/null +++ b/alembic/versions/0015_fc5_migration_run.py @@ -0,0 +1,51 @@ +"""fc5: migration_run table + +Revision ID: 0015 +Revises: 0014 +Create Date: 2026-05-22 + +Additive only. New table tracks each invocation of the FC-5 migration +tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are +plain String(32) — values validated at the API layer per the spec, not +a Postgres ENUM (so adding kinds later doesn't need a schema migration). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0015" +down_revision: Union[str, None] = "0014" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "migration_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(32), nullable=False, index=True), + sa.Column("status", sa.String(32), nullable=False, index=True), + sa.Column( + "dry_run", sa.Boolean(), nullable=False, server_default=sa.false(), + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "counts", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + + +def downgrade() -> None: + op.drop_table("migration_run") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 609e2ff..0697c07 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -10,6 +10,7 @@ from .image_record import ImageRecord from .import_batch import ImportBatch from .import_settings import ImportSettings from .import_task import ImportTask +from .migration_run import MigrationRun from .ml_settings import MLSettings from .post import Post from .post_attachment import PostAttachment @@ -40,6 +41,7 @@ __all__ = [ "ImportTask", "ImportSettings", "MLSettings", + "MigrationRun", "TagAlias", "TagAllowlist", "TagReferenceEmbedding", diff --git a/backend/app/models/migration_run.py b/backend/app/models/migration_run.py new file mode 100644 index 0000000..bdb5fed --- /dev/null +++ b/backend/app/models/migration_run.py @@ -0,0 +1,37 @@ +"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc). + +kind/status are String(32) not Postgres ENUM so adding kinds later +doesn't need a schema migration. The API layer validates values. +""" + +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy import Boolean, DateTime, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class MigrationRun(Base): + __tablename__ = "migration_run" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + finished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + counts: Mapped[dict] = mapped_column( + JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"), + ) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + metadata_: Mapped[dict] = mapped_column( + "metadata", JSONB, nullable=False, default=dict, + server_default=sa.text("'{}'::jsonb"), + ) From e1b057f494946a5085565c2fd57409edb5fe7425 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 08:12:17 -0400 Subject: [PATCH 260/272] fc5: backup + rollback services (pg_dump + tar.zst, manifest JSON, find-by-tag) Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/migrators/__init__.py | 6 + backend/app/services/migrators/backup.py | 124 +++++++++++++++++++++ backend/app/services/migrators/rollback.py | 21 ++++ tests/test_migration_backup.py | 64 +++++++++++ 4 files changed, 215 insertions(+) create mode 100644 backend/app/services/migrators/__init__.py create mode 100644 backend/app/services/migrators/backup.py create mode 100644 backend/app/services/migrators/rollback.py create mode 100644 tests/test_migration_backup.py diff --git a/backend/app/services/migrators/__init__.py b/backend/app/services/migrators/__init__.py new file mode 100644 index 0000000..0a7154c --- /dev/null +++ b/backend/app/services/migrators/__init__.py @@ -0,0 +1,6 @@ +"""FC-5 migration tooling. + +One module per concern (backup/rollback/gs/ir/overlap/ml_queue/verify). +Each migrator returns a counts dict; the run_migration task wires +that dict into MigrationRun.counts so the UI polling shows progress. +""" diff --git a/backend/app/services/migrators/backup.py b/backend/app/services/migrators/backup.py new file mode 100644 index 0000000..88040fd --- /dev/null +++ b/backend/app/services/migrators/backup.py @@ -0,0 +1,124 @@ +"""pg_dump + tar.zst-based backup, restorable via pair of subprocess calls. + +Backups live under /_backups/. Each backup is two files +(SQL + tarball) plus a manifest JSON. Tagged backups (e.g. tag='pre_migration') +are how rollback.py finds the most recent restorable snapshot. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +_BACKUPS_DIRNAME = "_backups" + + +def _backups_dir(images_root: Path | None = None) -> Path: + # Overridable for tests via monkeypatch. + root = images_root if images_root is not None else Path("/images") + p = root / _BACKUPS_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p + + +def _run_subprocess(cmd: list[str], **kwargs: Any): + # Overridable for tests via monkeypatch. + return subprocess.run( + cmd, + capture_output=True, + check=True, + **{k: v for k, v in kwargs.items() if not k.startswith("_")}, + ) + + +def create_backup( + *, db_url: str, images_root: Path, tag: str = "manual", +) -> dict: + """Create a backup: pg_dump SQL + tar.zst of images. + + Returns a manifest dict. Writes .sql, .tar.zst, .json into + /_backups/. + """ + ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + out_dir = _backups_dir(images_root) + sql_path = out_dir / f"fc_{ts}.sql" + tar_path = out_dir / f"fc_{ts}.tar.zst" + manifest_path = out_dir / f"fc_{ts}.json" + + _run_subprocess( + ["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), db_url], + _test_ts=ts, + ) + _run_subprocess( + [ + "tar", "--zstd", "-cf", str(tar_path), + "-C", str(images_root.parent), images_root.name, + f"--exclude={images_root.name}/_backups", + f"--exclude={images_root.name}/_quarantine", + ], + _test_ts=ts, + ) + + manifest = { + "backup_id": ts, + "tag": tag, + "created_at": datetime.now(UTC).isoformat(), + "sql_path": str(sql_path), + "tar_path": str(tar_path), + } + manifest_path.write_text(json.dumps(manifest, indent=2)) + return manifest + + +def list_backups(images_root: Path) -> list[dict]: + out_dir = _backups_dir(images_root) + items = [] + for mf in sorted(out_dir.glob("fc_*.json"), reverse=True): + try: + items.append(json.loads(mf.read_text())) + except Exception: + continue + return items + + +def find_latest_backup(images_root: Path, *, tag: str) -> dict | None: + for mf in list_backups(images_root): + if mf.get("tag") == tag: + return mf + return None + + +def restore_backup( + *, manifest: dict, db_url: str, images_root: Path, +) -> dict: + """Restore from a backup manifest. + + 1. Replay the .sql via psql. + 2. Wipe /images/ contents (except _backups/, which holds the file we're using). + 3. Untar the .tar.zst into /images/. + """ + sql_path = Path(manifest["sql_path"]) + tar_path = Path(manifest["tar_path"]) + + _run_subprocess( + ["psql", "-d", db_url, "-f", str(sql_path)], + ) + + # Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup + # we're restoring from!). + for entry in images_root.iterdir(): + if entry.name == _BACKUPS_DIRNAME: + continue + if entry.is_dir(): + shutil.rmtree(entry) + else: + entry.unlink() + + _run_subprocess( + ["tar", "--zstd", "-xf", str(tar_path), "-C", str(images_root.parent)], + ) + + return {"restored_from": manifest["backup_id"]} diff --git a/backend/app/services/migrators/rollback.py b/backend/app/services/migrators/rollback.py new file mode 100644 index 0000000..55ae7ef --- /dev/null +++ b/backend/app/services/migrators/rollback.py @@ -0,0 +1,21 @@ +"""Restore from the most recent 'pre_migration'-tagged backup.""" +from __future__ import annotations + +from pathlib import Path + +from . import backup as backup_mod + + +class NoBackupFoundError(Exception): + """Raised when rollback() is called with no pre_migration backup on disk.""" + + +def rollback_to_pre_migration(*, db_url: str, images_root: Path) -> dict: + manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration") + if manifest is None: + raise NoBackupFoundError( + "no pre_migration-tagged backup found under /_backups/" + ) + return backup_mod.restore_backup( + manifest=manifest, db_url=db_url, images_root=images_root, + ) diff --git a/tests/test_migration_backup.py b/tests/test_migration_backup.py new file mode 100644 index 0000000..5ff3cc1 --- /dev/null +++ b/tests/test_migration_backup.py @@ -0,0 +1,64 @@ +"""FC-5: backup + rollback service tests. + +Uses tmp_path to avoid touching real /images/_backups/. Subprocess +calls (pg_dump, tar) are monkeypatched — the real shell-out is exercised +in operator-side smoke testing on the homelab, not in unit tests. +""" +import pytest + +from backend.app.services.migrators import backup as backup_mod + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_create_backup_writes_sql_and_tar(monkeypatch, tmp_path): + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(cmd) + # Touch the expected output file so existence checks pass downstream. + if "pg_dump" in cmd[0]: + outpath = tmp_path / "_backups" / f"fc_{kwargs['_test_ts']}.sql" + outpath.write_text("-- fake pg_dump output") + else: + outpath = tmp_path / "_backups" / f"fc_{kwargs['_test_ts']}.tar.zst" + outpath.write_bytes(b"\x28\xb5\x2f\xfd") # zstd magic + from types import SimpleNamespace + return SimpleNamespace(returncode=0, stdout=b"", stderr=b"") + + monkeypatch.setattr(backup_mod, "_run_subprocess", fake_run) + + result = backup_mod.create_backup( + db_url="postgresql://x", images_root=tmp_path, tag="pre_migration", + ) + assert "sql_path" in result + assert "tar_path" in result + assert result["tag"] == "pre_migration" + assert any("pg_dump" in c[0] for c in calls) + assert any(c[0] == "tar" for c in calls) + + +@pytest.mark.asyncio +async def test_find_latest_backup_by_tag(monkeypatch, tmp_path): + # Create two manifests with different tags. + backups_dir = tmp_path / "_backups" + backups_dir.mkdir() + import json + (backups_dir / "fc_20260101T000000Z.json").write_text(json.dumps({ + "backup_id": "20260101T000000Z", "tag": "manual", + "created_at": "2026-01-01T00:00:00+00:00", + "sql_path": "/x/a.sql", "tar_path": "/x/a.tar.zst", + })) + (backups_dir / "fc_20260202T000000Z.json").write_text(json.dumps({ + "backup_id": "20260202T000000Z", "tag": "pre_migration", + "created_at": "2026-02-02T00:00:00+00:00", + "sql_path": "/x/b.sql", "tar_path": "/x/b.tar.zst", + })) + + found = backup_mod.find_latest_backup(tmp_path, tag="pre_migration") + assert found is not None + assert found["backup_id"] == "20260202T000000Z" + + missing = backup_mod.find_latest_backup(tmp_path, tag="nonexistent") + assert missing is None From a13a54206e89af298362a1160847be16c3dcb727 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 08:59:43 -0400 Subject: [PATCH 261/272] =?UTF-8?q?fc5:=20gs=5Fingest=20=E2=80=94=20parsed?= =?UTF-8?q?-JSON=20ingest,=20Artist/Source/Credential,=20re-encrypts=20wit?= =?UTF-8?q?h=20FC=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/migrators/gs_ingest.py | 126 ++++++++++++++++++++ tests/test_gs_ingest.py | 122 +++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 backend/app/services/migrators/gs_ingest.py create mode 100644 tests/test_gs_ingest.py diff --git a/backend/app/services/migrators/gs_ingest.py b/backend/app/services/migrators/gs_ingest.py new file mode 100644 index 0000000..b7a3d88 --- /dev/null +++ b/backend/app/services/migrators/gs_ingest.py @@ -0,0 +1,126 @@ +"""GallerySubscriber export → FabledCurator ingest. + +Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection +to GS). Creates Artist (from subscriptions) + Source (nested under each +subscription) + Credential (re-encrypted with FC's key). Idempotent on +natural keys: Artist.slug, (artist_id, platform, url), Credential.platform. + +Credentials arrive plaintext in the export — GS's export script +decrypts using GS's Fernet key in GS's own process. FC re-encrypts +with FC's CredentialCrypto. +""" +from __future__ import annotations + +import json + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, Credential, Source +from ...utils.slug import slugify +from ..credential_crypto import CredentialCrypto + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +async def migrate_async( + db: AsyncSession, + *, + data: dict, + fc_crypto: CredentialCrypto | None = None, + dry_run: bool = False, +) -> dict: + """Ingest a parsed gallerysubscriber-export-v1.json dict.""" + if data.get("source_app") != "gallerysubscriber": + raise ValueError("export source_app must be 'gallerysubscriber'") + if data.get("schema_version") != 1: + raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") + + counts = _zero_counts() + + # Phase 1: subscriptions → Artist; nested sources within each. + for sub in data.get("subscriptions", []): + counts["rows_processed"] += 1 + slug = slugify(sub["name"]) + artist = (await db.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one_or_none() + if artist is None: + if dry_run: + counts["rows_inserted"] += 1 + # Continue to nested sources, but they can't link without an artist row. + continue + notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None + artist = Artist( + name=sub["name"], slug=slug, + is_subscription=True, + auto_check=bool(sub.get("enabled", True)), + notes=notes, + ) + db.add(artist) + await db.flush() + counts["rows_inserted"] += 1 + else: + counts["rows_skipped"] += 1 + + # Nested sources under this subscription. + for src in sub.get("sources", []): + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == src["platform"], + Source.url == src["url"], + ) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Source( + artist_id=artist.id, + platform=src["platform"], + url=src["url"], + enabled=bool(src.get("enabled", True)), + check_interval_override=src.get("check_interval"), + config_overrides=src.get("metadata") or {}, + )) + counts["rows_inserted"] += 1 + + # Phase 2: credentials. + for cred in data.get("credentials", []): + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Credential).where(Credential.platform == cred["platform"]) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + if fc_crypto is None: + # Without a crypto helper we can't encrypt — skip rather than + # store plaintext. + counts["rows_skipped"] += 1 + counts["conflicts"] += 1 + continue + encrypted = fc_crypto.encrypt(cred["plaintext"]) + db.add(Credential( + platform=cred["platform"], + kind=cred.get("credential_type") or "cookies", + encrypted_blob=encrypted, + expires_at=cred.get("expires_at"), + )) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + return counts diff --git a/tests/test_gs_ingest.py b/tests/test_gs_ingest.py new file mode 100644 index 0000000..1574beb --- /dev/null +++ b/tests/test_gs_ingest.py @@ -0,0 +1,122 @@ +"""FC-5: gs_ingest unit tests.""" +from datetime import UTC, datetime + +import pytest +from sqlalchemy import select + +from backend.app.models import Artist, Credential, Source +from backend.app.services.credential_crypto import CredentialCrypto +from backend.app.services.migrators import gs_ingest + +pytestmark = pytest.mark.integration + + +def _gs_export(subscriptions=None, credentials=None) -> dict: + return { + "source_app": "gallerysubscriber", + "schema_version": 1, + "exported_at": datetime.now(UTC).isoformat(), + "subscriptions": subscriptions or [], + "credentials": credentials or [], + } + + +@pytest.mark.asyncio +async def test_subscription_creates_artist_with_slug(db): + data = _gs_export(subscriptions=[ + {"name": "Alice Author", "enabled": True, "metadata": {}, "sources": []}, + ]) + counts = await gs_ingest.migrate_async(db, data=data, dry_run=False) + assert counts["rows_inserted"] >= 1 + + artist = (await db.execute( + select(Artist).where(Artist.name == "Alice Author") + )).scalar_one() + assert artist.slug == "alice-author" + assert artist.is_subscription is True + + +@pytest.mark.asyncio +async def test_nested_source_attached_to_artist(db): + data = _gs_export(subscriptions=[{ + "name": "bob-author", + "enabled": True, + "metadata": {}, + "sources": [{ + "platform": "patreon", + "url": "https://patreon.com/bob", + "enabled": True, + "check_interval": 28800, + "metadata": {"k": "v"}, + }], + }]) + await gs_ingest.migrate_async(db, data=data, dry_run=False) + + src = (await db.execute( + select(Source).where(Source.url == "https://patreon.com/bob") + )).scalar_one() + artist = (await db.execute( + select(Artist).where(Artist.id == src.artist_id) + )).scalar_one() + assert artist.name == "bob-author" + assert src.platform == "patreon" + assert src.check_interval_override == 28800 + assert src.config_overrides == {"k": "v"} + + +@pytest.mark.asyncio +async def test_credential_re_encrypted_with_fc_key(db, tmp_path): + fc_key_path = tmp_path / "credential_key.b64" + fc_crypto = CredentialCrypto(fc_key_path) + + data = _gs_export(credentials=[{ + "platform": "patreon", + "credential_type": "cookies", + "plaintext": "session_cookie=abcdef", + "expires_at": None, + }]) + await gs_ingest.migrate_async(db, data=data, fc_crypto=fc_crypto, dry_run=False) + + cred = (await db.execute( + select(Credential).where(Credential.platform == "patreon") + )).scalar_one() + assert fc_crypto.decrypt(cred.encrypted_blob) == "session_cookie=abcdef" + assert cred.kind == "cookies" + + +@pytest.mark.asyncio +async def test_dry_run_makes_no_changes(db): + data = _gs_export(subscriptions=[ + {"name": "dave", "enabled": True, "metadata": {}, "sources": []}, + ]) + counts = await gs_ingest.migrate_async(db, data=data, dry_run=True) + assert counts["rows_processed"] >= 1 + + after = (await db.execute( + select(Artist).where(Artist.name == "dave") + )).scalar_one_or_none() + assert after is None + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db): + data = _gs_export(subscriptions=[ + {"name": "eve", "enabled": True, "metadata": {}, "sources": []}, + ]) + await gs_ingest.migrate_async(db, data=data, dry_run=False) + counts2 = await gs_ingest.migrate_async(db, data=data, dry_run=False) + assert counts2["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_rejects_wrong_source_app(db): + bad = {"source_app": "wrong", "schema_version": 1, "subscriptions": [], "credentials": []} + with pytest.raises(ValueError): + await gs_ingest.migrate_async(db, data=bad, dry_run=False) + + +@pytest.mark.asyncio +async def test_rejects_unsupported_schema_version(db): + bad = {"source_app": "gallerysubscriber", "schema_version": 99, "subscriptions": [], "credentials": []} + with pytest.raises(ValueError): + await gs_ingest.migrate_async(db, data=bad, dry_run=False) From 77b17f7bb6c5bab2e10fd4ade6c4df464391c9b8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:00:29 -0400 Subject: [PATCH 262/272] =?UTF-8?q?fc5:=20ir=5Fingest=20=E2=80=94=20Tag=20?= =?UTF-8?q?rows=20(skip=20artist/post)=20+=20manifest=20JSON=20for=20sha25?= =?UTF-8?q?6-keyed=20tag=5Fapply?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/migrators/ir_ingest.py | 141 ++++++++++++++++++++ tests/test_ir_ingest.py | 129 ++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 backend/app/services/migrators/ir_ingest.py create mode 100644 tests/test_ir_ingest.py diff --git a/backend/app/services/migrators/ir_ingest.py b/backend/app/services/migrators/ir_ingest.py new file mode 100644 index 0000000..75d12cc --- /dev/null +++ b/backend/app/services/migrators/ir_ingest.py @@ -0,0 +1,141 @@ +"""ImageRepo export → FabledCurator ingest. + +Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR). +Creates Tag rows (skipping artist/post kinds, resolving fandom_name to +FK). Writes the per-image-sha256 artist assignments + tag associations ++ series page assignments to /images/_migration_state/ir_tag_manifest.json +so tag_apply.py can join them to ImageRecord rows AFTER the operator +runs FC's filesystem scan. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Tag, TagKind + +_SKIP_KINDS = frozenset({"artist", "post"}) +_MIGRATION_STATE_DIRNAME = "_migration_state" +_IR_MANIFEST_FILENAME = "ir_tag_manifest.json" + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +def manifest_path(images_root: Path | None = None) -> Path: + root = images_root if images_root is not None else Path("/images") + p = root / _MIGRATION_STATE_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p / _IR_MANIFEST_FILENAME + + +async def _resolve_fandom_id( + db: AsyncSession, fandom_name: str | None, dry_run: bool, +) -> int | None: + """Find-or-create a fandom-kind Tag by name.""" + if not fandom_name: + return None + existing = (await db.execute( + select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + return existing.id + if dry_run: + return None + t = Tag(name=fandom_name, kind=TagKind.fandom) + db.add(t) + await db.flush() + return t.id + + +async def migrate_async( + db: AsyncSession, + *, + data: dict, + images_root: Path | None = None, + dry_run: bool = False, +) -> dict: + """Ingest a parsed imagerepo-export-v1.json dict. + + Creates Tag rows + writes the IR tag manifest file. Tag-to-image + binding happens later in tag_apply.py (after FC's filesystem scan + populates image_record.sha256 → id). + """ + if data.get("source_app") != "imagerepo": + raise ValueError("export source_app must be 'imagerepo'") + if data.get("schema_version") != 1: + raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") + + counts = _zero_counts() + + # Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id). + # First pass: create all fandom-kind tags so they're available for FK resolution. + for tag in data.get("tags", []): + kind = tag.get("kind") or "general" + if kind != "fandom": + continue + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=TagKind.fandom)) + counts["rows_inserted"] += 1 + if not dry_run: + await db.flush() + + # Second pass: every other kind. + for tag in data.get("tags", []): + kind_str = tag.get("kind") or "general" + if kind_str in _SKIP_KINDS: + counts["rows_skipped"] += 1 + continue + if kind_str == "fandom": + continue # handled above + counts["rows_processed"] += 1 + try: + kind = TagKind(kind_str) + except ValueError: + kind = TagKind.general + fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run) + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == kind) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id)) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + + # Phase 2: write the per-image manifest for tag_apply.py to consume later. + manifest = { + "schema_version": 1, + "image_artist_assignments": data.get("image_artist_assignments", []), + "image_tag_associations": data.get("image_tag_associations", []), + "series_pages": data.get("series_pages", []), + } + counts["rows_processed"] += len(manifest["image_artist_assignments"]) + counts["rows_processed"] += len(manifest["image_tag_associations"]) + counts["rows_processed"] += len(manifest["series_pages"]) + if not dry_run: + manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) + + return counts diff --git a/tests/test_ir_ingest.py b/tests/test_ir_ingest.py new file mode 100644 index 0000000..2af8ac8 --- /dev/null +++ b/tests/test_ir_ingest.py @@ -0,0 +1,129 @@ +"""FC-5: ir_ingest unit tests.""" +import json +from datetime import UTC, datetime + +import pytest +from sqlalchemy import select + +from backend.app.models import Tag +from backend.app.services.migrators import ir_ingest + +pytestmark = pytest.mark.integration + + +def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None): + return { + "source_app": "imagerepo", + "schema_version": 1, + "exported_at": datetime.now(UTC).isoformat(), + "tags": tags or [], + "image_artist_assignments": artist_assignments or [], + "image_tag_associations": tag_associations or [], + "series_pages": series_pages or [], + } + + +@pytest.mark.asyncio +async def test_general_tag_created(db, tmp_path): + data = _ir_export(tags=[ + {"name": "blue_eyes", "kind": "general", "fandom_name": None}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + tag = (await db.execute( + select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general") + )).scalar_one() + assert tag.fandom_id is None + + +@pytest.mark.asyncio +async def test_character_tag_resolves_fandom_name(db, tmp_path): + data = _ir_export(tags=[ + {"name": "Wonderland", "kind": "fandom", "fandom_name": None}, + {"name": "Alice", "kind": "character", "fandom_name": "Wonderland"}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + fandom = (await db.execute( + select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom") + )).scalar_one() + alice = (await db.execute( + select(Tag).where(Tag.name == "Alice", Tag.kind == "character") + )).scalar_one() + assert alice.fandom_id == fandom.id + + +@pytest.mark.asyncio +async def test_artist_and_post_kinds_skipped(db, tmp_path): + data = _ir_export(tags=[ + {"name": "BobArtist", "kind": "artist", "fandom_name": None}, + {"name": "PostXYZ", "kind": "post", "fandom_name": None}, + {"name": "general_one", "kind": "general", "fandom_name": None}, + ]) + counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + assert counts["rows_skipped"] >= 2 + + bob = (await db.execute( + select(Tag).where(Tag.name == "BobArtist") + )).scalar_one_or_none() + post = (await db.execute( + select(Tag).where(Tag.name == "PostXYZ") + )).scalar_one_or_none() + assert bob is None + assert post is None + + +@pytest.mark.asyncio +async def test_manifest_written_to_disk(db, tmp_path): + data = _ir_export( + artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}], + tag_associations=[ + {"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None}, + ], + series_pages=[ + {"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1}, + ], + ) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json" + assert manifest_file.exists() + manifest = json.loads(manifest_file.read_text()) + assert manifest["schema_version"] == 1 + assert len(manifest["image_artist_assignments"]) == 1 + assert len(manifest["image_tag_associations"]) == 1 + assert len(manifest["series_pages"]) == 1 + + +@pytest.mark.asyncio +async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path): + data = _ir_export( + tags=[{"name": "dry", "kind": "general", "fandom_name": None}], + artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}], + ) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True) + + tag = (await db.execute( + select(Tag).where(Tag.name == "dry") + )).scalar_one_or_none() + assert tag is None + manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json" + assert not manifest_file.exists() + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db, tmp_path): + data = _ir_export(tags=[ + {"name": "rerun", "kind": "general", "fandom_name": None}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + assert counts2["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_rejects_wrong_source_app(db, tmp_path): + bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [], + "image_artist_assignments": [], "image_tag_associations": [], "series_pages": []} + with pytest.raises(ValueError): + await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False) From 177d6728e17fe2894bda3686b358d1c5a7450861 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:01:17 -0400 Subject: [PATCH 263/272] =?UTF-8?q?fc5:=20tag=5Fapply=20=E2=80=94=20joins?= =?UTF-8?q?=20manifest=20entries=20to=20ImageRecord=20by=20sha256=20after?= =?UTF-8?q?=20filesystem=20scan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/migrators/tag_apply.py | 172 ++++++++++++++++++++ tests/test_tag_apply.py | 156 ++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 backend/app/services/migrators/tag_apply.py create mode 100644 tests/test_tag_apply.py diff --git a/backend/app/services/migrators/tag_apply.py b/backend/app/services/migrators/tag_apply.py new file mode 100644 index 0000000..b02d8ab --- /dev/null +++ b/backend/app/services/migrators/tag_apply.py @@ -0,0 +1,172 @@ +"""Apply the IR tag manifest after FC's filesystem scan. + +Reads /images/_migration_state/ir_tag_manifest.json and joins each entry +to an ImageRecord row by sha256 (which exists after the operator runs +FC's filesystem scan over the mounted IR images dir). + +- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug). +- image_tag_associations → image_tag insert (idempotent). +- series_pages → series_page insert (idempotent on image_id unique). + +Unmatched sha256s are logged into the result's `unmatched` list so the +Celery task can drop them into MigrationRun.metadata for the operator +to inspect. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from ...utils.slug import slugify +from .ir_ingest import manifest_path + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +async def _ensure_artist_id( + db: AsyncSession, artist_name: str, dry_run: bool, +) -> int | None: + if not artist_name or not artist_name.strip(): + return None + slug = slugify(artist_name) + existing = (await db.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one_or_none() + if existing is not None: + return existing.id + if dry_run: + return None + a = Artist(name=artist_name, slug=slug, is_subscription=False) + db.add(a) + await db.flush() + return a.id + + +async def _resolve_tag_id( + db: AsyncSession, tag_name: str, tag_kind: str, +) -> int | None: + try: + kind = TagKind(tag_kind) + except ValueError: + kind = TagKind.general + row = (await db.execute( + select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind) + )).scalar_one_or_none() + return row + + +async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None: + return (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one_or_none() + + +async def apply_async( + db: AsyncSession, + *, + images_root: Path | None = None, + dry_run: bool = False, +) -> dict: + """Apply the manifest. Returns counts + an `unmatched` list of sha256s.""" + mf_path = manifest_path(images_root) + if not mf_path.exists(): + raise FileNotFoundError(f"no IR tag manifest at {mf_path}") + + manifest = json.loads(mf_path.read_text()) + counts = _zero_counts() + unmatched: list[dict] = [] + + # 1. Artist assignments. + for entry in manifest.get("image_artist_assignments", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "artist", **entry}) + counts["rows_skipped"] += 1 + continue + aid = await _ensure_artist_id(db, entry["artist_name"], dry_run) + if aid is None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + img = await db.get(ImageRecord, img_id) + if img is not None and img.artist_id != aid: + img.artist_id = aid + counts["rows_inserted"] += 1 + else: + counts["rows_skipped"] += 1 + + # 2. Tag associations. + for entry in manifest.get("image_tag_associations", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "tag", **entry}) + counts["rows_skipped"] += 1 + continue + tag_id = await _resolve_tag_id( + db, entry["tag_name"], entry.get("tag_kind") or "general", + ) + if tag_id is None: + counts["rows_skipped"] += 1 + continue + # Skip if association already exists. + already = (await db.execute( + select(image_tag.c.image_record_id).where( + image_tag.c.image_record_id == img_id, + image_tag.c.tag_id == tag_id, + ) + )).first() + if already is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + await db.execute(image_tag.insert().values( + image_record_id=img_id, tag_id=tag_id, source="manual", + )) + counts["rows_inserted"] += 1 + + # 3. Series pages. + for entry in manifest.get("series_pages", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "series", **entry}) + counts["rows_skipped"] += 1 + continue + series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series") + if series_tag_id is None: + counts["rows_skipped"] += 1 + continue + existing = (await db.execute( + select(SeriesPage).where(SeriesPage.image_id == img_id) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(SeriesPage( + series_tag_id=series_tag_id, + image_id=img_id, + page_number=entry["page_number"], + )) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + return {"counts": counts, "unmatched": unmatched} diff --git a/tests/test_tag_apply.py b/tests/test_tag_apply.py new file mode 100644 index 0000000..3d9ea6b --- /dev/null +++ b/tests/test_tag_apply.py @@ -0,0 +1,156 @@ +"""FC-5: tag_apply unit tests.""" +import json + +import pytest +from sqlalchemy import select + +from backend.app.models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from backend.app.services.migrators import tag_apply + +pytestmark = pytest.mark.integration + + +async def _seed_image(db, sha: str, *, suffix="x"): + img = ImageRecord( + path=f"/images/imported/aa/{suffix}.jpg", + sha256=sha, size_bytes=10, mime="image/jpeg", + width=10, height=10, origin="imported_filesystem", + ) + db.add(img) + await db.flush() + return img + + +def _write_manifest(tmp_path, *, artist_assignments=None, tag_associations=None, series_pages=None): + d = tmp_path / "_migration_state" + d.mkdir(parents=True, exist_ok=True) + (d / "ir_tag_manifest.json").write_text(json.dumps({ + "schema_version": 1, + "image_artist_assignments": artist_assignments or [], + "image_tag_associations": tag_associations or [], + "series_pages": series_pages or [], + })) + + +@pytest.mark.asyncio +async def test_artist_assignment_sets_image_artist_id(db, tmp_path): + sha = "a" * 64 + await _seed_image(db, sha) + await db.commit() + + _write_manifest(tmp_path, artist_assignments=[ + {"sha256": sha, "artist_name": "BobAuthor"}, + ]) + + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result["counts"]["rows_inserted"] >= 1 + + img = (await db.execute( + select(ImageRecord).where(ImageRecord.sha256 == sha) + )).scalar_one() + artist = (await db.execute( + select(Artist).where(Artist.id == img.artist_id) + )).scalar_one() + assert artist.name == "BobAuthor" + + +@pytest.mark.asyncio +async def test_tag_association_creates_image_tag(db, tmp_path): + sha = "b" * 64 + await _seed_image(db, sha, suffix="b") + # Seed the tag itself (ir_ingest would have created it). + tag = Tag(name="blue_eyes", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + row = (await db.execute( + select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) + )).all() + assert len(row) == 1 + + +@pytest.mark.asyncio +async def test_unmatched_sha_appears_in_unmatched_list(db, tmp_path): + _write_manifest(tmp_path, tag_associations=[ + {"sha256": "z" * 64, "tag_name": "missing_image", "tag_kind": "general", "fandom_name": None}, + ]) + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert any(e["sha256"] == "z" * 64 for e in result["unmatched"]) + + +@pytest.mark.asyncio +async def test_series_page_created(db, tmp_path): + sha = "c" * 64 + await _seed_image(db, sha, suffix="c") + series_tag = Tag(name="MySeries", kind=TagKind.series) + db.add(series_tag) + await db.commit() + + _write_manifest(tmp_path, series_pages=[ + {"sha256": sha, "series_tag_name": "MySeries", "page_number": 1}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + sp = (await db.execute( + select(SeriesPage).where(SeriesPage.image_id == img_id) + )).scalar_one() + assert sp.page_number == 1 + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db, tmp_path): + sha = "d" * 64 + await _seed_image(db, sha, suffix="d") + tag = Tag(name="rerun_tag", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "rerun_tag", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result2["counts"]["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_missing_manifest_raises(db, tmp_path): + with pytest.raises(FileNotFoundError): + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + +@pytest.mark.asyncio +async def test_dry_run_makes_no_changes(db, tmp_path): + sha = "e" * 64 + await _seed_image(db, sha, suffix="e") + tag = Tag(name="dry_tag", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "dry_tag", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + rows = (await db.execute( + select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) + )).all() + assert len(rows) == 0 From b5896427d462ff53871aa56367a1dce95f197e51 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:01:46 -0400 Subject: [PATCH 264/272] fc5: ml_queue (fire tag_and_embed for unprocessed) + verify (row counts + sha256 sample) Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/services/migrators/ml_queue.py | 21 ++++++ backend/app/services/migrators/verify.py | 77 ++++++++++++++++++++++ tests/test_migration_verify.py | 72 ++++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 backend/app/services/migrators/ml_queue.py create mode 100644 backend/app/services/migrators/verify.py create mode 100644 tests/test_migration_verify.py diff --git a/backend/app/services/migrators/ml_queue.py b/backend/app/services/migrators/ml_queue.py new file mode 100644 index 0000000..6d9e92f --- /dev/null +++ b/backend/app/services/migrators/ml_queue.py @@ -0,0 +1,21 @@ +"""Queue every migrated image_record with no embedding for ML re-processing.""" +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import ImageRecord + + +async def queue_all_unprocessed_async(db: AsyncSession) -> int: + """Find every ImageRecord with siglip_embedding IS NULL, fire + tag_and_embed.delay(id) for each. Returns count queued. + """ + from ...tasks.ml import tag_and_embed + + rows = (await db.execute( + select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None)) + )).scalars().all() + for image_id in rows: + tag_and_embed.delay(image_id) + return len(rows) diff --git a/backend/app/services/migrators/verify.py b/backend/app/services/migrators/verify.py new file mode 100644 index 0000000..df8c951 --- /dev/null +++ b/backend/app/services/migrators/verify.py @@ -0,0 +1,77 @@ +"""Post-migration verification: row counts + sha256 sampling.""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, Credential, ImageRecord, Source, Tag + + +async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict: + """Return per-check status dicts. `expected` is optional row-count + assertions; checks default to status='ok' when no expected provided.""" + expected = expected or {} + results: dict[str, dict] = {} + + checks = { + "artist_subscriptions": ( + select(func.count(Artist.id)).where(Artist.is_subscription == True) + ), + "source_count": select(func.count(Source.id)), + "credential_count": select(func.count(Credential.id)), + "tag_count": select(func.count(Tag.id)), + "image_record_imported_or_downloaded": ( + select(func.count(ImageRecord.id)) + .where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"])) + ), + } + for name, stmt in checks.items(): + actual = (await db.execute(stmt)).scalar_one() + exp = expected.get(name) + status = "ok" if exp is None or exp == actual else "mismatch" + results[name] = {"status": status, "actual": int(actual), "expected": exp} + return results + + +async def verify_sha256_sample( + db: AsyncSession, *, sample_size: int = 20, +) -> dict: + """Sample N image_records; verify file exists + sha256 matches.""" + rows = (await db.execute( + select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256) + .order_by(func.random()).limit(sample_size) + )).all() + + matched = 0 + mismatched = 0 + missing = 0 + samples: list[dict] = [] + for img_id, path, expected_sha in rows: + p = Path(path) + if not p.exists(): + missing += 1 + samples.append({"id": img_id, "path": path, "result": "missing"}) + continue + h = hashlib.sha256() + with p.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + if h.hexdigest() == expected_sha: + matched += 1 + samples.append({"id": img_id, "result": "ok"}) + else: + mismatched += 1 + samples.append({ + "id": img_id, "path": path, "result": "mismatch", + "expected_sha": expected_sha, "actual_sha": h.hexdigest(), + }) + return { + "sample_size": len(rows), + "matched": matched, + "mismatched": mismatched, + "missing": missing, + "samples": samples, + } diff --git a/tests/test_migration_verify.py b/tests/test_migration_verify.py new file mode 100644 index 0000000..549cabb --- /dev/null +++ b/tests/test_migration_verify.py @@ -0,0 +1,72 @@ +"""FC-5: verify + ml_queue unit tests.""" +import hashlib + +import pytest + +from backend.app.models import Artist, ImageRecord +from backend.app.services.migrators import ml_queue, verify + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_verify_row_counts_match(db): + db.add(Artist(name="vAlpha", slug="valpha", is_subscription=True)) + db.add(Artist(name="vBeta", slug="vbeta", is_subscription=False)) + await db.commit() + result = await verify.verify_async( + db, expected={"artist_subscriptions": 1}, + ) + assert result["artist_subscriptions"]["status"] == "ok" + + +@pytest.mark.asyncio +async def test_verify_sha256_sample_matches_disk(db, tmp_path): + f = tmp_path / "sample.bin" + f.write_bytes(b"hello world") + sha = hashlib.sha256(b"hello world").hexdigest() + db.add(ImageRecord( + path=str(f), sha256=sha, size_bytes=11, mime="application/octet-stream", + origin="imported_filesystem", + )) + await db.commit() + result = await verify.verify_sha256_sample(db, sample_size=10) + assert result["matched"] >= 1 + assert result["mismatched"] == 0 + + +@pytest.mark.asyncio +async def test_verify_sha256_sample_detects_missing_file(db, tmp_path): + sha = "f" * 64 + db.add(ImageRecord( + path=str(tmp_path / "does-not-exist.bin"), + sha256=sha, size_bytes=1, mime="application/octet-stream", + origin="imported_filesystem", + )) + await db.commit() + result = await verify.verify_sha256_sample(db, sample_size=10) + assert result["missing"] >= 1 + + +@pytest.mark.asyncio +async def test_ml_queue_returns_count_for_unprocessed(db, tmp_path, monkeypatch): + queued: list[int] = [] + + class _FakeTask: + def delay(self, image_id): + queued.append(image_id) + + monkeypatch.setattr( + "backend.app.tasks.ml.tag_and_embed", _FakeTask(), + ) + + f = tmp_path / "q.bin" + f.write_bytes(b"x") + db.add(ImageRecord( + path=str(f), sha256="q" * 64, size_bytes=1, + mime="application/octet-stream", origin="imported_filesystem", + )) + await db.commit() + + count = await ml_queue.queue_all_unprocessed_async(db) + assert count >= 1 From c880bf825912e24e591cd2dfca6fe958df186874 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:02:18 -0400 Subject: [PATCH 265/272] fc5: run_migration Celery task on maintenance queue + dispatch by kind Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/celery_app.py | 2 + backend/app/tasks/migration.py | 186 +++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 backend/app/tasks/migration.py diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 613a78f..e5d91f2 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -28,6 +28,7 @@ def make_celery() -> Celery: "backend.app.tasks.import_file", "backend.app.tasks.thumbnail", "backend.app.tasks.maintenance", + "backend.app.tasks.migration", "backend.app.tasks.ml", "backend.app.tasks.download", ], @@ -41,6 +42,7 @@ def make_celery() -> Celery: "backend.app.tasks.download.*": {"queue": "download"}, "backend.app.tasks.scan.*": {"queue": "scan"}, "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, + "backend.app.tasks.migration.*": {"queue": "maintenance"}, }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, diff --git a/backend/app/tasks/migration.py b/backend/app/tasks/migration.py new file mode 100644 index 0000000..dd200bd --- /dev/null +++ b/backend/app/tasks/migration.py @@ -0,0 +1,186 @@ +"""FC-5 run_migration Celery task. + +Dispatches to the right migrator based on `kind`. Updates MigrationRun +row's status/counts/finished_at as it runs. Failures set status='error' +with the error message preserved. + +kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback +""" +from __future__ import annotations + +import asyncio +import logging +from datetime import UTC, datetime +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from ..celery_app import celery +from ..config import get_config +from ..models import MigrationRun +from ..services.credential_crypto import CredentialCrypto +from ..services.migrators import ( + backup as backup_mod, + gs_ingest, + ir_ingest, + ml_queue, + rollback as rollback_mod, + tag_apply, + verify, +) + +log = logging.getLogger(__name__) + +IMAGES_ROOT = Path("/images") +_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64" + + +def _async_session_factory(): + cfg = get_config() + engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True) + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine + + +async def _update_run( + db: AsyncSession, run_id: int, *, + status: str | None = None, counts: dict | None = None, + error: str | None = None, finished_at: datetime | None = None, + metadata_patch: dict | None = None, +) -> None: + run = (await db.execute( + select(MigrationRun).where(MigrationRun.id == run_id) + )).scalar_one() + if status is not None: + run.status = status + if counts is not None: + run.counts = counts + if error is not None: + run.error = error + if finished_at is not None: + run.finished_at = finished_at + if metadata_patch: + run.metadata_ = {**(run.metadata_ or {}), **metadata_patch} + await db.commit() + + +async def _run_async(run_id: int, kind: str, params: dict) -> dict: + factory, engine = _async_session_factory() + try: + async with factory() as db: + await _update_run(db, run_id, status="running") + try: + if kind == "backup": + manifest = backup_mod.create_backup( + db_url=get_config().database_url_sync, + images_root=IMAGES_ROOT, + tag=params.get("tag", "manual"), + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": 0, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + metadata_patch={"manifest": manifest}, + ) + return manifest + + elif kind == "gs_ingest": + fc_crypto = CredentialCrypto(_KEY_PATH) + counts = await gs_ingest.migrate_async( + db, data=params["data"], + fc_crypto=fc_crypto, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", counts=counts, + finished_at=datetime.now(UTC), + ) + return counts + + elif kind == "ir_ingest": + counts = await ir_ingest.migrate_async( + db, data=params["data"], + images_root=IMAGES_ROOT, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", counts=counts, + finished_at=datetime.now(UTC), + ) + return counts + + elif kind == "tag_apply": + result = await tag_apply.apply_async( + db, images_root=IMAGES_ROOT, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", + counts=result["counts"], + finished_at=datetime.now(UTC), + metadata_patch={"unmatched": result["unmatched"]}, + ) + return result + + elif kind == "ml_queue": + count = await ml_queue.queue_all_unprocessed_async(db) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": count, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + ) + return {"queued": count} + + elif kind == "verify": + checks = await verify.verify_async(db, expected=params.get("expected")) + sample = await verify.verify_sha256_sample( + db, sample_size=params.get("sample_size", 20), + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": sample["sample_size"], + "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, + "conflicts": sample["mismatched"] + sample["missing"]}, + finished_at=datetime.now(UTC), + metadata_patch={"checks": checks, "sample": sample}, + ) + return {"checks": checks, "sample": sample} + + elif kind == "rollback": + result = rollback_mod.rollback_to_pre_migration( + db_url=get_config().database_url_sync, + images_root=IMAGES_ROOT, + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": 0, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + metadata_patch={"rollback_result": result}, + ) + return result + + else: + raise ValueError(f"unknown kind: {kind}") + + except Exception as exc: + log.exception("migration kind=%s failed", kind) + await _update_run( + db, run_id, status="error", error=str(exc), + finished_at=datetime.now(UTC), + ) + raise + finally: + await engine.dispose() + + +@celery.task(name="backend.app.tasks.migration.run_migration", bind=True, acks_late=True) +def run_migration(self, run_id: int, kind: str, params: dict) -> dict: + """FC-5: dispatch a migration kind. Updates MigrationRun row as it goes.""" + return asyncio.run(_run_async(run_id, kind, params)) From 89224a789523d864d5c4da66714affd935706e03 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:03:24 -0400 Subject: [PATCH 266/272] =?UTF-8?q?fc5:=20/api/migrate=20blueprint=20?= =?UTF-8?q?=E2=80=94=20multipart=20upload=20for=20ingest=20kinds=20+=20JSO?= =?UTF-8?q?N=20for=20the=20rest=20+=20apply-without-backup=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/__init__.py | 2 + backend/app/api/migrate.py | 141 ++++++++++++++++++++++++++++++++++++ tests/test_api_migrate.py | 127 ++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 backend/app/api/migrate.py create mode 100644 tests/test_api_migrate.py diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py index 2c7ac84..2de6a32 100644 --- a/backend/app/api/__init__.py +++ b/backend/app/api/__init__.py @@ -23,6 +23,7 @@ def all_blueprints() -> list[Blueprint]: from .downloads import downloads_bp from .gallery import gallery_bp from .import_admin import import_admin_bp + from .migrate import migrate_bp from .ml_admin import ml_admin_bp from .platforms import platforms_bp from .posts import posts_bp @@ -43,6 +44,7 @@ def all_blueprints() -> list[Blueprint]: showcase_bp, settings_bp, import_admin_bp, + migrate_bp, suggestions_bp, allowlist_bp, aliases_bp, diff --git a/backend/app/api/migrate.py b/backend/app/api/migrate.py new file mode 100644 index 0000000..6c5f86b --- /dev/null +++ b/backend/app/api/migrate.py @@ -0,0 +1,141 @@ +"""FC-5: /api/migrate — trigger and poll migration runs. + +Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an +`export_file` field. All other kinds accept JSON. Apply-without-backup +guard rejects non-dry-run ingests unless a pre_migration-tagged backup +exists in the last 24h (override with body.force=true). +""" + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import MigrationRun +from ..tasks.migration import run_migration + +migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate") + +_VALID_KINDS = frozenset({ + "backup", "gs_ingest", "ir_ingest", "tag_apply", + "ml_queue", "verify", "rollback", +}) +_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"}) +_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback"}) + + +def _bad(error: str, *, status: int = 400, **extra): + body = {"error": error} + body.update(extra) + return jsonify(body), status + + +def _has_recent_pre_migration_backup() -> bool: + from ..services.migrators import backup as backup_mod + images_root = Path("/images") + manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration") + if manifest is None: + return False + created_at_str = manifest.get("created_at") + if not created_at_str: + return False + created_at = datetime.fromisoformat(created_at_str) + return (datetime.now(UTC) - created_at) < timedelta(hours=24) + + +def _run_to_dict(run: MigrationRun) -> dict: + return { + "id": run.id, + "kind": run.kind, + "status": run.status, + "dry_run": run.dry_run, + "started_at": run.started_at.isoformat(), + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + "counts": run.counts or {}, + "error": run.error, + "metadata": run.metadata_ or {}, + } + + +@migrate_bp.route("/", methods=["POST"]) +async def create_run(kind: str): + if kind not in _VALID_KINDS: + return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}") + + # Ingest kinds accept multipart/form-data; everything else takes JSON. + if kind in _INGEST_KINDS: + form = await request.form + files = await request.files + if "export_file" not in files: + return _bad("missing_export_file", detail="multipart export_file required") + export_file = files["export_file"] + try: + raw = export_file.read() + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return _bad("invalid_export_file", detail=str(exc)) + dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes") + force = str(form.get("force", "false")).lower() in ("true", "1", "yes") + params: dict = {"data": data, "dry_run": dry_run} + else: + body = await request.get_json() + if body is None: + body = {} + if not isinstance(body, dict): + return _bad("invalid_body") + dry_run = bool(body.get("dry_run", False)) + force = bool(body.get("force", False)) + params = dict(body) + + is_apply = (kind in _APPLY_KINDS) and not dry_run + if is_apply and not force and not _has_recent_pre_migration_backup(): + return _bad( + "no_backup", + detail="apply action requires a pre_migration-tagged backup " + "in the last 24h (or force=true).", + ) + + if kind == "backup": + params.setdefault("tag", "pre_migration") + + async with get_session() as session: + run = MigrationRun(kind=kind, status="pending", dry_run=dry_run) + session.add(run) + await session.commit() + await session.refresh(run) + run_id = run.id + + run_migration.delay(run_id, kind, params) + return jsonify({"run_id": run_id, "status": "pending"}), 202 + + +@migrate_bp.route("/runs/", methods=["GET"]) +async def get_run(run_id: int): + async with get_session() as session: + run = (await session.execute( + select(MigrationRun).where(MigrationRun.id == run_id) + )).scalar_one_or_none() + if run is None: + return _bad("not_found", status=404) + return jsonify(_run_to_dict(run)) + + +@migrate_bp.route("/runs", methods=["GET"]) +async def list_runs(): + try: + limit = int(request.args.get("limit", "10")) + except ValueError: + return _bad("invalid_limit") + if limit < 1 or limit > 100: + return _bad("invalid_limit") + + async with get_session() as session: + rows = (await session.execute( + select(MigrationRun) + .order_by(MigrationRun.id.desc()) + .limit(limit) + )).scalars().all() + return jsonify([_run_to_dict(r) for r in rows]) diff --git a/tests/test_api_migrate.py b/tests/test_api_migrate.py new file mode 100644 index 0000000..0c7bbf2 --- /dev/null +++ b/tests/test_api_migrate.py @@ -0,0 +1,127 @@ +"""FC-5: /api/migrate API tests.""" +import io +import json + +import pytest + +import backend.app.tasks.migration # noqa: F401 — register celery task + +from backend.app import create_app +from backend.app.celery_app import celery + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_post_backup_returns_202_and_id(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + resp = await client.post("/api/migrate/backup", json={}) + assert resp.status_code == 202 + body = await resp.get_json() + assert "run_id" in body + assert body["status"] == "pending" + + +@pytest.mark.asyncio +async def test_post_rejects_unknown_kind(client): + resp = await client.post("/api/migrate/destroy", json={}) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_kind" + + +@pytest.mark.asyncio +async def test_post_ingest_rejects_missing_file(client): + resp = await client.post("/api/migrate/gs_ingest", data={}) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "missing_export_file" + + +@pytest.mark.asyncio +async def test_post_ingest_accepts_multipart_file(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: True, # pretend backup exists + ) + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + data = { + "export_file": (io.BytesIO(json.dumps(export).encode()), "gs-export.json"), + "dry_run": "false", + } + resp = await client.post("/api/migrate/gs_ingest", files=data) + assert resp.status_code == 202 + + +@pytest.mark.asyncio +async def test_post_apply_without_backup_rejected(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: False, + ) + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + data = { + "export_file": (io.BytesIO(json.dumps(export).encode()), "gs-export.json"), + "dry_run": "false", + } + resp = await client.post("/api/migrate/gs_ingest", files=data) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "no_backup" + + +@pytest.mark.asyncio +async def test_post_dry_run_allowed_without_backup(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: False, + ) + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + data = { + "export_file": (io.BytesIO(json.dumps(export).encode()), "gs-export.json"), + "dry_run": "true", + } + resp = await client.post("/api/migrate/gs_ingest", files=data) + assert resp.status_code == 202 + + +@pytest.mark.asyncio +async def test_get_run_404_unknown(client): + resp = await client.get("/api/migrate/runs/9999999") + assert resp.status_code == 404 + + +def test_run_migration_task_registered(): + assert "backend.app.tasks.migration.run_migration" in celery.tasks From b600cf6e86a68d1eef2e633139681a265103efd7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:03:38 -0400 Subject: [PATCH 267/272] =?UTF-8?q?fc5:=20migration=20Pinia=20store=20?= =?UTF-8?q?=E2=80=94=20trigger=20+=20FormData=20upload=20+=202s=20poll=20+?= =?UTF-8?q?=20history=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/stores/migration.js | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 frontend/src/stores/migration.js diff --git a/frontend/src/stores/migration.js b/frontend/src/stores/migration.js new file mode 100644 index 0000000..5bdebd1 --- /dev/null +++ b/frontend/src/stores/migration.js @@ -0,0 +1,83 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const POLL_MS = 2000 + +export const useMigrationStore = defineStore('migration', () => { + const api = useApi() + + const activeRun = ref(null) + const recentRuns = ref([]) + const error = ref(null) + const loading = ref(false) + + let pollHandle = null + + async function trigger(kind, params, file = null) { + loading.value = true + error.value = null + try { + let body + if (file) { + // Multipart upload for ingest kinds. + const form = new FormData() + form.append('export_file', file) + if (params && params.dry_run) form.append('dry_run', 'true') + if (params && params.force) form.append('force', 'true') + body = await api.post(`/api/migrate/${kind}`, { body: form }) + } else { + body = await api.post(`/api/migrate/${kind}`, { body: params || {} }) + } + await loadRun(body.run_id) + pollActive(body.run_id) + return body + } catch (e) { + error.value = e + throw e + } finally { + loading.value = false + } + } + + async function loadRun(runId) { + activeRun.value = await api.get(`/api/migrate/runs/${runId}`) + } + + async function loadRecent() { + recentRuns.value = await api.get('/api/migrate/runs', { params: { limit: 10 } }) + } + + function pollActive(runId) { + stopPolling() + pollHandle = setInterval(async () => { + try { + const run = await api.get(`/api/migrate/runs/${runId}`) + activeRun.value = run + if (run.status === 'ok' || run.status === 'error') { + stopPolling() + await loadRecent() + } + } catch (e) { + error.value = e + stopPolling() + } + }, POLL_MS) + } + + function stopPolling() { + if (pollHandle) { + clearInterval(pollHandle) + pollHandle = null + } + } + + const isRunning = computed( + () => activeRun.value && activeRun.value.status === 'running', + ) + + return { + activeRun, recentRuns, error, loading, isRunning, + trigger, loadRun, loadRecent, pollActive, stopPolling, + } +}) From a9b354780bb8a95b54792a536cd8953f90192b70 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 22 May 2026 09:04:24 -0400 Subject: [PATCH 268/272] =?UTF-8?q?fc5:=20LegacyMigrationCard=20=E2=80=94?= =?UTF-8?q?=20file=20uploads=20+=20step=20buttons=20+=20active-run=20+=20h?= =?UTF-8?q?istory;=20slotted=20into=20MaintenancePanel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../settings/LegacyMigrationCard.vue | 245 ++++++++++++++++++ .../components/settings/MaintenancePanel.vue | 2 + 2 files changed, 247 insertions(+) create mode 100644 frontend/src/components/settings/LegacyMigrationCard.vue diff --git a/frontend/src/components/settings/LegacyMigrationCard.vue b/frontend/src/components/settings/LegacyMigrationCard.vue new file mode 100644 index 0000000..be2dfd1 --- /dev/null +++ b/frontend/src/components/settings/LegacyMigrationCard.vue @@ -0,0 +1,245 @@ + + + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue index e6e963d..9639edf 100644 --- a/frontend/src/components/settings/MaintenancePanel.vue +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -12,6 +12,7 @@ + @@ -21,6 +22,7 @@ import CentroidRecomputeCard from './CentroidRecomputeCard.vue' import MLThresholdSliders from './MLThresholdSliders.vue' import AllowlistTable from './AllowlistTable.vue' import AliasTable from './AliasTable.vue' +import LegacyMigrationCard from './LegacyMigrationCard.vue'
PlatformURLImages