diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d78fd8f --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Database +DB_USER=fabledcurator +DB_PASSWORD=changeme_use_a_real_password +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=fabledcurator + +# Redis / Celery +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 + +# App +# Generate with: openssl rand -hex 32 +SECRET_KEY=changeme_32_byte_hex_secret + +# Extension API key — used in FC-3, lands later but reserved now +# Generate with: openssl rand -hex 32 +EXTENSION_API_KEY= + +# Logging +LOG_LEVEL=INFO + +# Deployment posture: plain HTTP (no TLS in the app; reverse proxy if needed) +# See docs/superpowers/specs/2026-05-13-fabledcurator-merge-design.md §2.1 diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 0000000..3743cd0 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,74 @@ +name: Build images + +on: + push: + branches: [dev, main] + +# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: +# - write:package, read:package (for docker push to git.fabledsword.com) +# - write:release (for future release-cutting workflows) +# - write:issue (for future issue-management automation) +# The injected GITHUB_TOKEN cannot be used — it lacks write:package. + +jobs: + build-web: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + + - name: Determine tag + id: tag + run: | + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT" + else + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT" + fi + + - name: Login to Forgejo registry + uses: docker/login-action@v3 + with: + registry: git.fabledsword.com + username: ${{ github.actor }} + password: ${{ secrets.RELEASE_TOKEN }} + + - name: Build and push web image + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile + push: true + tags: ${{ steps.tag.outputs.tags }} + + build-ml: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + + - name: Determine tag + id: tag + run: | + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT" + else + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT" + fi + + - name: Login to Forgejo registry + uses: docker/login-action@v3 + with: + registry: git.fabledsword.com + username: ${{ github.actor }} + password: ${{ secrets.RELEASE_TOKEN }} + + - name: Build and push ml image + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.ml + push: true + tags: ${{ steps.tag.outputs.tags }} diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 0000000..829aa4c --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,124 @@ +name: CI + +# CI lanes per FabledRulebook/forgejo.md "CI philosophy": +# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers. +# - frontend-build: vitest unit + vite build. +# - integration: pgvector + redis service containers; alembic + `pytest -m integration`. + +on: + push: + branches: [dev, main] + pull_request: + branches: [main] + +jobs: + backend-lint-and-test: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + # DB_PASSWORD and SECRET_KEY are required by config.py at import time + # even though unit tests don't actually touch the DB or use the secret. + DB_PASSWORD: ci_unit_test_placeholder + SECRET_KEY: ci_unit_test_placeholder + steps: + - uses: actions/checkout@v4 + + - name: Install Python deps + # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ + # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain + # versions live on the runner image, not here. + run: pip install -r requirements.txt pytest pytest-asyncio + + - name: Ruff lint + run: ruff check backend/ tests/ alembic/ + + - name: Pytest (unit only — integration runs in the integration job) + run: pytest tests/ -v -m "not integration" + + frontend-build: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + # No package-lock.json is tracked yet (we don't run npm locally per + # feedback-no-local-runs). Using `npm install` instead of `npm ci`. + # If we want strict lockfile-based reproducibility later, commit a + # package-lock.json and flip this back to `npm ci`. + - run: npm install --no-audit --no-fund + # `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS + # with no .ts files and no JSDoc annotations, so vue-tsc has nothing + # to type-check. Re-enable once we add a tsconfig.json and either + # convert to TS or add JSDoc. + - run: npm run test:unit + - run: npm run build + + integration: + # This act_runner (swarm-runner v0.6.1) puts service containers on the + # default bridge with NO service-name DNS, and publishing fixed host + # ports collides with the operator's running docker-compose dev stack on + # the same shared daemon. Workaround: publish NO host ports, and reach + # each service by its bridge IP — discovered at runtime via the mounted + # docker socket (the ci-python image ships /usr/bin/docker). Default-bridge + # containers can talk by IP (only embedded DNS is missing), so IP + # addressing is reliable here. Everything runs in ONE step so resolved + # values don't depend on cross-step env passing. Pattern documented in + # FabledRulebook/forgejo.md "CI philosophy". + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + + - name: Integration suite (resolve service IPs, migrate, test) + run: | + set -eux + # Scope to THIS job's service containers (act_runner names them + # ...JOB-integration...); the operator's compose stack uses the + # same images but different names, so it won't match. + PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1) + test -n "$PG" && test -n "$RD" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + export DB_HOST="$PG_IP" + export CELERY_BROKER_URL="redis://$RD_IP:6379/0" + export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" + # Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools). + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + pip install -r requirements.txt pytest pytest-asyncio + alembic upgrade head + pytest tests/ -v -m integration diff --git a/.gitignore b/.gitignore index 4a5091a..06d2c93 100644 --- a/.gitignore +++ b/.gitignore @@ -42,13 +42,14 @@ yarn-error.log* extension/*.xpi extension/web-ext-artifacts/ -# Runtime volumes (must never be tracked) -images/ -import/ -downloads/ -models/ -postgres_data/ -redis_data/ +# Runtime volumes (must never be tracked) — anchored to repo root so they +# don't accidentally match Python package dirs like backend/app/models/ +/images/ +/import/ +/downloads/ +/models/ +/postgres_data/ +/redis_data/ # IDE / OS .vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..66bf33f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,47 @@ +# syntax=docker/dockerfile:1.7 + +FROM node:22-alpine AS frontend-builder +WORKDIR /build +COPY frontend/package.json frontend/package-lock.json* ./ +# No package-lock.json is tracked yet (we don't run npm locally per +# feedback-no-local-runs), so `npm install` instead of `npm ci`. Flip to +# `npm ci` once a lockfile is committed. +RUN npm install --no-audit --no-fund +COPY frontend/ ./ +RUN npm run build + +FROM python:3.14-slim AS runtime +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +# System deps: ffmpeg (transcode + thumbnails, used in FC-2), unar (archives, FC-2), +# libpq for psycopg, image libs. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + unar \ + libpq5 \ + libjpeg62-turbo \ + libwebp7 \ + libpng16-16 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install -r requirements.txt + +COPY backend/ ./backend/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./ +COPY entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +COPY --from=frontend-builder /build/dist ./frontend/dist + +EXPOSE 8080 + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["web"] diff --git a/Dockerfile.ml b/Dockerfile.ml new file mode 100644 index 0000000..ee548fb --- /dev/null +++ b/Dockerfile.ml @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:1.7 + +FROM python:3.14-slim +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + HF_HOME=/models/.huggingface \ + TRANSFORMERS_CACHE=/models/.huggingface \ + ML_MODEL_DIR=/models + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + libpq5 \ + libjpeg62-turbo \ + libwebp7 \ + libpng16-16 \ + libgl1 \ + libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements-ml.txt requirements.txt ./ +# CPU-only torch: the default PyPI wheel bundles the CUDA runtime (~5.6GB +# layer); this pipeline never uses a GPU. --index-url (not --extra-index-url) +# guarantees only +cpu wheels are considered, so no nvidia-*-cu12 deps. +RUN pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.12,<3.0" "torchvision>=0.27,<0.28" +RUN pip install -r requirements-ml.txt + +COPY backend/ ./backend/ +COPY alembic/ ./alembic/ +COPY alembic.ini ./ +COPY entrypoint.sh ./ +RUN chmod +x entrypoint.sh + +# Models self-heal into /models on first start (FC-2 implements this) +VOLUME ["/models"] + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["ml-worker"] diff --git a/README.md b/README.md index 990356f..225f318 100644 --- a/README.md +++ b/README.md @@ -10,17 +10,41 @@ Pre-v1. Not yet functional. ## Quick start +For local development and testing, just: + ```bash -cp .env.example .env -# edit .env: set DB_PASSWORD and SECRET_KEY docker compose up -d # UI: http://localhost:8080 ``` +That uses sane dev defaults baked into `docker-compose.yml` and the dev +override (`docker-compose.override.yml`, auto-merged) — local builds, DEBUG +logging, exposed Postgres + Redis ports on the host. No `.env` required. + +For a production-like deployment, override the dev defaults via shell env +or a `.env` file (see `.env.example` for the variable names) and use: + +```bash +docker compose -f docker-compose.yml up -d +# (skips the override so containers pull registry images) +``` + ## Deployment posture FabledCurator is designed to run inside a self-hosted homelab environment over plain HTTP. If you want TLS, terminate it at your reverse proxy. The app does not generate certificates, redirect to HTTPS, or set HSTS. +## CI / Forgejo setup + +The repo's workflows expect: + +- **Runner label `python-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. The runner image (`runner-base:python-ci`) is built from `CI-Runner/CI-python/` in the operator's workspace; `make push` from that directory builds and pushes a new image when toolchain pins change. +- **Repo secret `RELEASE_TOKEN`** — a Forgejo PAT with the following scopes: + - `write:package` + `read:package` — for `docker push` to `git.fabledsword.com` + - `write:release` — for future release-cutting workflows + - `write:issue` — for future issue-management automation + + Generate at https://git.fabledsword.com/user/settings/applications. The injected `GITHUB_TOKEN` cannot be used because it lacks `write:package`. + ## License Personal project; use at your own discretion. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..8b17767 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +file_template = %%(rev)s_%%(slug)s +version_path_separator = os +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..0530946 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,53 @@ +"""Alembic environment — reads DATABASE_URL from app config.""" + +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +from alembic import context +from backend.app.config import get_config +from backend.app.models import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +config.set_main_option("sqlalchemy.url", get_config().database_url_sync) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..17dcba0 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/.gitkeep b/alembic/versions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/alembic/versions/0001_initial_unified_schema.py b/alembic/versions/0001_initial_unified_schema.py new file mode 100644 index 0000000..0580b45 --- /dev/null +++ b/alembic/versions/0001_initial_unified_schema.py @@ -0,0 +1,277 @@ +"""initial unified schema + +Revision ID: 0001 +Revises: +Create Date: 2026-05-13 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + "artist", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("check_interval_seconds", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_artist"), + sa.UniqueConstraint("name", name="uq_artist_name"), + sa.UniqueConstraint("slug", name="uq_artist_slug"), + ) + + op.create_table( + "source", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("artist_id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("config_overrides", sa.JSON(), nullable=True), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("check_interval_override", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_source"), + ) + op.create_index("ix_source_artist_id", "source", ["artist_id"]) + + op.create_table( + "credential", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False, server_default="active"), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id", name="pk_credential"), + sa.UniqueConstraint("platform", name="uq_credential_platform"), + ) + + op.create_table( + "post", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("external_post_id", sa.String(length=128), nullable=False), + sa.Column("post_url", sa.Text(), nullable=True), + sa.Column("post_title", sa.Text(), nullable=True), + sa.Column("post_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("raw_metadata", sa.JSON(), nullable=True), + sa.Column( + "downloaded_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_post"), + sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), + ) + op.create_index("ix_post_source_id", "post", ["source_id"]) + + op.create_table( + "image_record", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.Column("phash", sa.String(length=32), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("mime", sa.String(length=64), nullable=False), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("thumbnail_path", sa.Text(), nullable=True), + sa.Column( + "origin", + sa.Enum( + "downloaded", + "imported_filesystem", + "uploaded", + name="origin_enum", + ), + nullable=False, + ), + sa.Column("primary_post_id", sa.Integer(), nullable=True), + sa.Column("wd14_predictions", sa.JSON(), nullable=True), + sa.Column("wd14_model_version", sa.String(length=128), nullable=True), + sa.Column("siglip_embedding", Vector(1152), nullable=True), + sa.Column("siglip_model_version", sa.String(length=128), nullable=True), + sa.Column("centroid_scores", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["primary_post_id"], + ["post.id"], + name="fk_image_record_primary_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_record"), + sa.UniqueConstraint("path", name="uq_image_record_path"), + sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), + ) + op.create_index("ix_image_record_sha256", "image_record", ["sha256"]) + op.create_index("ix_image_record_phash", "image_record", ["phash"]) + op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"]) + + op.create_table( + "image_provenance", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("captured_metadata", sa.JSON(), nullable=True), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_provenance_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_image_provenance_post_id_post", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_image_provenance_source_id_source", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_provenance"), + ) + op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"]) + op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"]) + op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"]) + + op.create_table( + "tag", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("namespace", sa.String(length=64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_tag"), + sa.UniqueConstraint("name", name="uq_tag_name"), + ) + op.create_index("ix_tag_name", "tag", ["name"]) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + + op.create_table( + "image_tag", + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_tag_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"), + ) + + op.create_table( + "download_event", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_download_event_source_id_source", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_download_event_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_download_event"), + ) + op.create_index("ix_download_event_source_id", "download_event", ["source_id"]) + op.create_index("ix_download_event_post_id", "download_event", ["post_id"]) + + +def downgrade() -> None: + op.drop_table("download_event") + op.drop_table("image_tag") + op.drop_table("tag") + op.drop_table("image_provenance") + op.drop_table("image_record") + op.execute("DROP TYPE IF EXISTS origin_enum") + op.drop_table("post") + op.drop_table("credential") + op.drop_table("source") + op.drop_table("artist") + op.execute("DROP EXTENSION IF EXISTS vector") diff --git a/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py new file mode 100644 index 0000000..b9dac2c --- /dev/null +++ b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py @@ -0,0 +1,208 @@ +"""fc2a: tag kinds, import_task, import_batch, integrity_status + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-05-14 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0002" +down_revision: Union[str, None] = "0001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +TAG_KINDS = ( + "artist", + "character", + "fandom", + "general", + "series", + "archive", + "post", + "meta", + "rating", +) + + +def upgrade() -> None: + # --- Tag kind enum + fandom_id --- + tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind") + tag_kind.create(op.get_bind(), checkfirst=True) + + op.add_column( + "tag", + sa.Column("kind", tag_kind, nullable=False, server_default="general"), + ) + op.add_column( + "tag", + sa.Column("fandom_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_tag_fandom_id_tag", + "tag", + "tag", + ["fandom_id"], + ["id"], + ondelete="SET NULL", + ) + + # Drop the old global uniqueness on name; add kind+fandom-aware uniqueness. + op.drop_constraint("uq_tag_name", "tag", type_="unique") + op.drop_index("ix_tag_name", table_name="tag") + op.execute( + """ + CREATE UNIQUE INDEX uq_tag_name_kind_fandom + ON tag (name, kind, COALESCE(fandom_id, 0)) + """ + ) + + # CHECK: fandom_id is only allowed for character kind. + op.create_check_constraint( + "ck_tag_fandom_requires_character", + "tag", + "(fandom_id IS NULL) OR (kind = 'character')", + ) + + # Drop the old namespace column — superseded by kind. + op.drop_index("ix_tag_namespace", table_name="tag") + op.drop_column("tag", "namespace") + + # --- ImportBatch --- + op.create_table( + "import_batch", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("triggered_by", sa.String(length=32), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("scan_mode", sa.String(length=16), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"), + sa.Column("imported", sa.Integer(), nullable=False, server_default="0"), + sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"), + sa.Column("failed", sa.Integer(), nullable=False, server_default="0"), + sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), + sa.PrimaryKeyConstraint("id", name="pk_import_batch"), + ) + op.create_index("ix_import_batch_status", "import_batch", ["status"]) + + # --- ImportTask --- + op.create_table( + "import_task", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("batch_id", sa.Integer(), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("task_type", sa.String(length=16), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"), + sa.Column("result_image_id", sa.Integer(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["batch_id"], + ["import_batch.id"], + name="fk_import_task_batch_id_import_batch", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["result_image_id"], + ["image_record.id"], + name="fk_import_task_result_image_id_image_record", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_import_task"), + ) + op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"]) + op.create_index("ix_import_task_status", "import_task", ["status"]) + op.create_index( + "ix_import_task_created_at_desc", + "import_task", + [sa.text("created_at DESC")], + ) + + # --- ImportSettings (single-row table) --- + op.create_table( + "import_settings", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"), + sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"), + sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "transparency_threshold", + sa.Float(), + nullable=False, + server_default="0.9", + ), + sa.Column( + "skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "single_color_threshold", + sa.Float(), + nullable=False, + server_default="0.95", + ), + sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"), + sa.PrimaryKeyConstraint("id", name="pk_import_settings"), + sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"), + ) + # Seed the single row immediately so callers can always SELECT id=1. + op.execute("INSERT INTO import_settings (id) VALUES (1)") + + # --- ImageRecord additions --- + op.add_column( + "image_record", + sa.Column( + "integrity_status", + sa.String(length=24), + nullable=False, + server_default="unknown", + ), + ) + op.create_index( + "ix_image_record_integrity_status", + "image_record", + ["integrity_status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_integrity_status", table_name="image_record") + op.drop_column("image_record", "integrity_status") + + op.drop_table("import_settings") + op.drop_index("ix_import_task_created_at_desc", table_name="import_task") + op.drop_index("ix_import_task_status", table_name="import_task") + op.drop_index("ix_import_task_batch_id", table_name="import_task") + op.drop_table("import_task") + op.drop_index("ix_import_batch_status", table_name="import_batch") + op.drop_table("import_batch") + + op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check") + op.execute("DROP INDEX uq_tag_name_kind_fandom") + op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True)) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + op.create_index("ix_tag_name", "tag", ["name"], unique=False) + op.create_unique_constraint("uq_tag_name", "tag", ["name"]) + op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey") + op.drop_column("tag", "fandom_id") + op.drop_column("tag", "kind") + sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True) diff --git a/alembic/versions/0003_fc2b_ml_pipeline.py b/alembic/versions/0003_fc2b_ml_pipeline.py new file mode 100644 index 0000000..584bffe --- /dev/null +++ b/alembic/versions/0003_fc2b_ml_pipeline.py @@ -0,0 +1,172 @@ +"""fc2b: ML pipeline — allowlist, aliases, centroids, ml_settings + +Revision ID: 0003 +Revises: 0002 +Create Date: 2026-05-15 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0003" +down_revision: Union[str, None] = "0002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 3.1 rename wd14_* -> tagger_* + op.alter_column("image_record", "wd14_predictions", new_column_name="tagger_predictions") + op.alter_column( + "image_record", "wd14_model_version", new_column_name="tagger_model_version" + ) + + # 3.2 tag_allowlist + op.create_table( + "tag_allowlist", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column( + "min_confidence", sa.Float(), nullable=False, server_default="0.95" + ), + sa.Column( + "added_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_tag_allowlist_tag_id_tag", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("tag_id", name="pk_tag_allowlist"), + sa.CheckConstraint( + "min_confidence > 0 AND min_confidence <= 1", + name="ck_tag_allowlist_confidence_range", + ), + ) + + # 3.3 tag_suggestion_rejection + op.create_table( + "tag_suggestion_rejection", + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column( + "rejected_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], ["image_record.id"], + name="fk_tsr_image_record_id_image_record", ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_tsr_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "image_record_id", "tag_id", name="pk_tag_suggestion_rejection" + ), + ) + op.create_index( + "ix_tag_suggestion_rejection_tag", "tag_suggestion_rejection", ["tag_id"] + ) + + # 3.4 tag_alias + op.create_table( + "tag_alias", + sa.Column("alias_string", sa.String(length=255), nullable=False), + sa.Column("alias_category", sa.String(length=32), nullable=False), + sa.Column("canonical_tag_id", sa.Integer(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["canonical_tag_id"], ["tag.id"], + name="fk_tag_alias_canonical_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "alias_string", "alias_category", name="pk_tag_alias" + ), + ) + op.create_index("ix_tag_alias_canonical", "tag_alias", ["canonical_tag_id"]) + + # 3.5 tag_reference_embedding (centroids) + op.create_table( + "tag_reference_embedding", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("embedding", Vector(1152), nullable=False), + sa.Column("reference_count", sa.Integer(), nullable=False), + sa.Column("model_version", sa.String(length=128), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], + name="fk_tag_reference_embedding_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("tag_id", name="pk_tag_reference_embedding"), + ) + + # 3.6 ml_settings singleton + op.create_table( + "ml_settings", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "suggestion_threshold_artist", sa.Float(), nullable=False, + server_default="0.30", + ), + sa.Column( + "suggestion_threshold_character", sa.Float(), nullable=False, + server_default="0.50", + ), + sa.Column( + "suggestion_threshold_copyright", sa.Float(), nullable=False, + server_default="0.50", + ), + sa.Column( + "suggestion_threshold_general", sa.Float(), nullable=False, + server_default="0.95", + ), + sa.Column( + "centroid_similarity_threshold", sa.Float(), nullable=False, + server_default="0.55", + ), + sa.Column( + "min_reference_images", sa.Integer(), nullable=False, + server_default="5", + ), + sa.Column( + "tagger_model_version", sa.String(length=128), nullable=False, + server_default="camie-tagger-v2", + ), + sa.Column( + "embedder_model_version", sa.String(length=128), nullable=False, + server_default="siglip-so400m-patch14-384", + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_ml_settings"), + sa.CheckConstraint("id = 1", name="ck_ml_settings_singleton"), + ) + op.execute("INSERT INTO ml_settings (id) VALUES (1)") + + +def downgrade() -> None: + op.drop_table("ml_settings") + op.drop_table("tag_reference_embedding") + op.drop_index("ix_tag_alias_canonical", table_name="tag_alias") + op.drop_table("tag_alias") + op.drop_index( + "ix_tag_suggestion_rejection_tag", table_name="tag_suggestion_rejection" + ) + op.drop_table("tag_suggestion_rejection") + op.drop_table("tag_allowlist") + op.alter_column( + "image_record", "tagger_model_version", new_column_name="wd14_model_version" + ) + op.alter_column( + "image_record", "tagger_predictions", new_column_name="wd14_predictions" + ) diff --git a/alembic/versions/0004_fc2c_i_tsm_system_rows.py b/alembic/versions/0004_fc2c_i_tsm_system_rows.py new file mode 100644 index 0000000..e8bd920 --- /dev/null +++ b/alembic/versions/0004_fc2c_i_tsm_system_rows.py @@ -0,0 +1,23 @@ +"""fc2c-i: enable tsm_system_rows for scalable random sampling + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-05-15 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0004" +down_revision: Union[str, None] = "0003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") + + +def downgrade() -> None: + op.execute("DROP EXTENSION IF EXISTS tsm_system_rows") diff --git a/alembic/versions/0005_fc2c_iii_a_series_page.py b/alembic/versions/0005_fc2c_iii_a_series_page.py new file mode 100644 index 0000000..ffe397e --- /dev/null +++ b/alembic/versions/0005_fc2c_iii_a_series_page.py @@ -0,0 +1,50 @@ +"""fc2c-iii-a: series_page ordered membership + +Revision ID: 0005 +Revises: 0004 +Create Date: 2026-05-16 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0005" +down_revision: Union[str, None] = "0004" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_page", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("series_tag_id", sa.Integer(), nullable=False), + sa.Column("image_id", sa.Integer(), nullable=False), + sa.Column("page_number", sa.Integer(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["series_tag_id"], ["tag.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["image_id"], ["image_record.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("image_id", name="uq_series_page_image"), + ) + op.create_index( + "ix_series_page_series_tag_id", "series_page", ["series_tag_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_series_page_series_tag_id", table_name="series_page") + op.drop_table("series_page") diff --git a/alembic/versions/0006_fc2d_phash_threshold.py b/alembic/versions/0006_fc2d_phash_threshold.py new file mode 100644 index 0000000..895ed46 --- /dev/null +++ b/alembic/versions/0006_fc2d_phash_threshold.py @@ -0,0 +1,30 @@ +"""fc2d: import_settings.phash_threshold + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-05-17 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0006" +down_revision: Union[str, None] = "0005" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "phash_threshold", sa.Integer(), + nullable=False, server_default="10", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "phash_threshold") diff --git a/alembic/versions/0007_fc2d_post_metadata_fields.py b/alembic/versions/0007_fc2d_post_metadata_fields.py new file mode 100644 index 0000000..24e8ba9 --- /dev/null +++ b/alembic/versions/0007_fc2d_post_metadata_fields.py @@ -0,0 +1,31 @@ +"""fc2d-iv: post.description + post.attachment_count + +Revision ID: 0007 +Revises: 0006 +Create Date: 2026-05-18 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0007" +down_revision: Union[str, None] = "0006" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "post", sa.Column("description", sa.Text(), nullable=True) + ) + op.add_column( + "post", + sa.Column("attachment_count", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("post", "attachment_count") + op.drop_column("post", "description") diff --git a/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py b/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py new file mode 100644 index 0000000..4019e2d --- /dev/null +++ b/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py @@ -0,0 +1,52 @@ +"""fc2d-vii-c: image_record.artist_id + backfill + drop artist tags + +Revision ID: 0008 +Revises: 0007 +Create Date: 2026-05-18 + +Internal forward-correctness migration (the big legacy-import migration +stays deferred). downgrade() does NOT recreate deleted artist tags; +downgrade is dev-only and the data is reconstructable by re-import. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +from backend.app.utils.artist_backfill import ( + BACKFILL_PRIMARY_SQL, + BACKFILL_PROVENANCE_SQL, + BACKFILL_TAG_SQL, + DELETE_ARTIST_TAGS_SQL, +) + +revision: str = "0008" +down_revision: Union[str, None] = "0007" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "image_record", + sa.Column("artist_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_image_record_artist_id", "image_record", "artist", + ["artist_id"], ["id"], ondelete="SET NULL", + ) + op.create_index( + "ix_image_record_artist_id", "image_record", ["artist_id"], + ) + op.execute(BACKFILL_PRIMARY_SQL) + op.execute(BACKFILL_PROVENANCE_SQL) + op.execute(BACKFILL_TAG_SQL) + op.execute(DELETE_ARTIST_TAGS_SQL) + + +def downgrade() -> None: + op.drop_index("ix_image_record_artist_id", table_name="image_record") + op.drop_constraint( + "fk_image_record_artist_id", "image_record", type_="foreignkey" + ) + op.drop_column("image_record", "artist_id") diff --git a/alembic/versions/0009_fc2d_iii_post_attachment.py b/alembic/versions/0009_fc2d_iii_post_attachment.py new file mode 100644 index 0000000..4820fa4 --- /dev/null +++ b/alembic/versions/0009_fc2d_iii_post_attachment.py @@ -0,0 +1,68 @@ +"""fc2d-iii: post_attachment + import_batch.attachments + +Revision ID: 0009 +Revises: 0008 +Create Date: 2026-05-19 + +Internal forward-correctness migration (big legacy-import migration +stays deferred). No backfill — no attachments exist yet. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0009" +down_revision: Union[str, None] = "0008" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "post_attachment", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "post_id", sa.Integer(), + sa.ForeignKey("post.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column( + "artist_id", sa.Integer(), + sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("original_filename", sa.Text(), nullable=False), + sa.Column("ext", sa.String(32), nullable=False), + sa.Column("mime", sa.String(128), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column( + "captured_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False, + ), + ) + op.create_index( + "ix_post_attachment_sha256", "post_attachment", ["sha256"], + unique=True, + ) + op.create_index( + "ix_post_attachment_post_id", "post_attachment", ["post_id"], + ) + op.create_index( + "ix_post_attachment_artist_id", "post_attachment", ["artist_id"], + ) + op.add_column( + "import_batch", + sa.Column( + "attachments", sa.Integer(), nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_batch", "attachments") + op.drop_index("ix_post_attachment_artist_id", table_name="post_attachment") + op.drop_index("ix_post_attachment_post_id", table_name="post_attachment") + op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") + op.drop_table("post_attachment") diff --git a/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py b/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py new file mode 100644 index 0000000..10f502b --- /dev/null +++ b/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py @@ -0,0 +1,32 @@ +"""fc3a: unique(source.artist_id, source.platform, source.url) + +Revision ID: 0010 +Revises: 0009 +Create Date: 2026-05-20 + +Enforces FC-3a's dedup invariant at the DB level. No backfill — no +existing rows are expected to collide; if they do the migration will +fail loudly (intended). +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0010" +down_revision: Union[str, None] = "0009" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_unique_constraint( + "uq_source_artist_platform_url", + "source", + ["artist_id", "platform", "url"], + ) + + +def downgrade() -> None: + op.drop_constraint( + "uq_source_artist_platform_url", "source", type_="unique" + ) diff --git a/alembic/versions/0011_fc3b_credential_schema_alignment.py b/alembic/versions/0011_fc3b_credential_schema_alignment.py new file mode 100644 index 0000000..00a31a5 --- /dev/null +++ b/alembic/versions/0011_fc3b_credential_schema_alignment.py @@ -0,0 +1,41 @@ +"""fc3b: rename credential.kind -> credential_type, drop status, add last_verified + +Revision ID: 0011 +Revises: 0010 +Create Date: 2026-05-20 + +Aligns the credential table with the GallerySubscriber wire-field names +so the existing browser extension can POST to FC unmodified. Greenfield — +no rows exist in production yet, so no data preservation logic is +needed; the rename uses ALTER COLUMN rather than copy-then-drop. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0011" +down_revision: Union[str, None] = "0010" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("credential", "kind", new_column_name="credential_type") + op.drop_column("credential", "status") + op.add_column( + "credential", + sa.Column("last_verified", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("credential", "last_verified") + op.add_column( + "credential", + sa.Column( + "status", sa.String(length=32), nullable=False, + server_default="active", + ), + ) + op.alter_column("credential", "credential_type", new_column_name="kind") diff --git a/alembic/versions/0012_fc3b_app_setting.py b/alembic/versions/0012_fc3b_app_setting.py new file mode 100644 index 0000000..42c06eb --- /dev/null +++ b/alembic/versions/0012_fc3b_app_setting.py @@ -0,0 +1,36 @@ +"""fc3b: app_setting key/value table + +Revision ID: 0012 +Revises: 0011 +Create Date: 2026-05-20 + +A simple key/value table for small app settings that don't fit +ImportSettings. Initially seeds only `extension_api_key` (done in +create_app on first boot — not in the migration, to keep it +deterministic and independent of randomness). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0012" +down_revision: Union[str, None] = "0011" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "app_setting", + sa.Column("key", sa.String(length=64), primary_key=True), + sa.Column("value", sa.Text(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + ) + + +def downgrade() -> None: + op.drop_table("app_setting") diff --git a/alembic/versions/0013_fc3c_download_event_metadata.py b/alembic/versions/0013_fc3c_download_event_metadata.py new file mode 100644 index 0000000..c88b3f9 --- /dev/null +++ b/alembic/versions/0013_fc3c_download_event_metadata.py @@ -0,0 +1,52 @@ +"""fc3c: download_event.metadata + import_settings downloader fields + +Revision ID: 0013 +Revises: 0012 +Create Date: 2026-05-20 + +Additive only. download_event.metadata is the rich JSONB blob FC-3c +populates per run (run_stats, stdout/stderr, quarantined paths, import +summary). import_settings gains two operator-tunable downloader knobs: +download_rate_limit_seconds (gallery-dl extractor.sleep) and +download_validate_files (toggle the magic-byte validator). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0013" +down_revision: Union[str, None] = "0012" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "download_event", + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_rate_limit_seconds", sa.Float(), + nullable=False, server_default="3.0", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_validate_files", sa.Boolean(), + nullable=False, server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "download_validate_files") + op.drop_column("import_settings", "download_rate_limit_seconds") + op.drop_column("download_event", "metadata") diff --git a/alembic/versions/0014_fc3d_scheduling.py b/alembic/versions/0014_fc3d_scheduling.py new file mode 100644 index 0000000..955e956 --- /dev/null +++ b/alembic/versions/0014_fc3d_scheduling.py @@ -0,0 +1,58 @@ +"""fc3d: scheduling + source health columns + +Revision ID: 0014 +Revises: 0013 +Create Date: 2026-05-21 + +Additive only. source.consecutive_failures (default 0, DownloadService +finalize hook owns the writes). import_settings gains the three +scheduling knobs (global default interval, event retention, failure +warning threshold). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0014" +down_revision: Union[str, None] = "0013" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column( + "consecutive_failures", sa.Integer(), + nullable=False, server_default="0", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_schedule_default_seconds", sa.Integer(), + nullable=False, server_default="28800", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_event_retention_days", sa.Integer(), + nullable=False, server_default="90", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_failure_warning_threshold", sa.Integer(), + nullable=False, server_default="5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "download_failure_warning_threshold") + op.drop_column("import_settings", "download_event_retention_days") + op.drop_column("import_settings", "download_schedule_default_seconds") + op.drop_column("source", "consecutive_failures") diff --git a/alembic/versions/0015_fc5_migration_run.py b/alembic/versions/0015_fc5_migration_run.py new file mode 100644 index 0000000..89d7d89 --- /dev/null +++ b/alembic/versions/0015_fc5_migration_run.py @@ -0,0 +1,51 @@ +"""fc5: migration_run table + +Revision ID: 0015 +Revises: 0014 +Create Date: 2026-05-22 + +Additive only. New table tracks each invocation of the FC-5 migration +tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are +plain String(32) — values validated at the API layer per the spec, not +a Postgres ENUM (so adding kinds later doesn't need a schema migration). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0015" +down_revision: Union[str, None] = "0014" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "migration_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(32), nullable=False, index=True), + sa.Column("status", sa.String(32), nullable=False, index=True), + sa.Column( + "dry_run", sa.Boolean(), nullable=False, server_default=sa.false(), + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "counts", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + + +def downgrade() -> None: + op.drop_table("migration_run") diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..a566a62 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1,37 @@ +"""Quart app factory.""" + +import logging +from pathlib import Path + +from quart import Quart + +from .api import all_blueprints +from .config import get_config +from .frontend import frontend_bp +from .services.credential_crypto import CredentialCrypto + +_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64") + + +def create_app() -> Quart: + cfg = get_config() + logging.basicConfig(level=cfg.log_level) + + # Bootstrap the credential encryption key file (mode 0600) before + # any request can land. FC-3b: file-based system-generated key. + CredentialCrypto(_CREDENTIAL_KEY_PATH) + + app = Quart(__name__) + app.secret_key = cfg.secret_key + + for bp in all_blueprints(): + app.register_blueprint(bp) + # Registered last so /api/* routes win over the SPA catch-all. + app.register_blueprint(frontend_bp) + + @app.after_serving + async def _dispose_db_engine() -> None: + from .extensions import dispose_engine + await dispose_engine() + + return app diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..2de6a32 --- /dev/null +++ b/backend/app/api/__init__.py @@ -0,0 +1,57 @@ +"""API blueprint registration. + +This module is imported by the Quart app factory; it just exposes the +top-level `api_bp` for backward compatibility with FC-1's health route, +and ALL_BLUEPRINTS for the factory to register sibling blueprints. +""" + +from quart import Blueprint + +from . import health + +api_bp = Blueprint("api", __name__, url_prefix="/api") +api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"]) + + +def all_blueprints() -> list[Blueprint]: + from .aliases import aliases_bp + from .allowlist import allowlist_bp + from .artist import artist_bp + from .artists import artists_bp + from .attachments import attachments_bp + from .credentials import credentials_bp + from .downloads import downloads_bp + from .gallery import gallery_bp + from .import_admin import import_admin_bp + from .migrate import migrate_bp + from .ml_admin import ml_admin_bp + from .platforms import platforms_bp + from .posts import posts_bp + from .provenance import provenance_bp + from .settings import settings_bp + from .showcase import showcase_bp + from .sources import sources_bp + from .suggestions import suggestions_bp + from .tags import tags_bp + return [ + api_bp, + attachments_bp, + gallery_bp, + provenance_bp, + tags_bp, + artist_bp, + artists_bp, + showcase_bp, + settings_bp, + import_admin_bp, + migrate_bp, + suggestions_bp, + allowlist_bp, + aliases_bp, + ml_admin_bp, + sources_bp, + platforms_bp, + posts_bp, + credentials_bp, + downloads_bp, + ] diff --git a/backend/app/api/aliases.py b/backend/app/api/aliases.py new file mode 100644 index 0000000..3c77510 --- /dev/null +++ b/backend/app/api/aliases.py @@ -0,0 +1,51 @@ +"""Aliases API: list, create, remove.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.ml.aliases import AliasService + +aliases_bp = Blueprint("aliases", __name__, url_prefix="/api") + + +@aliases_bp.route("/aliases", methods=["GET"]) +async def list_aliases(): + async with get_session() as session: + rows = await AliasService(session).list_all() + return jsonify( + [ + { + "alias_string": r.alias_string, + "alias_category": r.alias_category, + "canonical_tag_id": r.canonical_tag_id, + "canonical_tag_name": r.canonical_tag_name, + } + for r in rows + ] + ) + + +@aliases_bp.route("/aliases", methods=["POST"]) +async def create_alias(): + body = await request.get_json() + required = {"alias_string", "alias_category", "canonical_tag_id"} + if not body or not required.issubset(body): + return jsonify({"error": f"required: {sorted(required)}"}), 400 + async with get_session() as session: + await AliasService(session).create( + body["alias_string"], + body["alias_category"], + body["canonical_tag_id"], + ) + await session.commit() + return "", 201 + + +@aliases_bp.route( + "/aliases//", methods=["DELETE"] +) +async def remove_alias(alias_string: str, alias_category: str): + async with get_session() as session: + await AliasService(session).remove(alias_string, alias_category) + await session.commit() + return "", 204 diff --git a/backend/app/api/allowlist.py b/backend/app/api/allowlist.py new file mode 100644 index 0000000..53f38bb --- /dev/null +++ b/backend/app/api/allowlist.py @@ -0,0 +1,59 @@ +"""Allowlist API: list, adjust threshold, remove.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..models import TagAllowlist +from ..services.ml.allowlist import AllowlistService + +allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api") + + +@allowlist_bp.route("/allowlist", methods=["GET"]) +async def list_allowlist(): + async with get_session() as session: + rows = await AllowlistService(session).list_all() + return jsonify( + [ + { + "tag_id": r.tag_id, + "tag_name": r.tag_name, + "tag_kind": r.tag_kind, + "min_confidence": r.min_confidence, + } + for r in rows + ] + ) + + +@allowlist_bp.route("/tags//allowlist", methods=["GET"]) +async def get_one(tag_id: int): + async with get_session() as session: + row = await session.get(TagAllowlist, tag_id) + if row is None: + return jsonify({"error": "not on allowlist"}), 404 + return jsonify( + {"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()} + ) + + +@allowlist_bp.route("/tags//allowlist", methods=["PATCH"]) +async def patch_threshold(tag_id: int): + body = await request.get_json() + if not body or "min_confidence" not in body: + return jsonify({"error": "min_confidence required"}), 400 + mc = float(body["min_confidence"]) + if not (0 < mc <= 1): + return jsonify({"error": "min_confidence must be in (0, 1]"}), 400 + async with get_session() as session: + await AllowlistService(session).update_threshold(tag_id, mc) + await session.commit() + return "", 204 + + +@allowlist_bp.route("/tags//allowlist", methods=["DELETE"]) +async def remove(tag_id: int): + async with get_session() as session: + await AllowlistService(session).remove(tag_id) + await session.commit() + return "", 204 diff --git a/backend/app/api/artist.py b/backend/app/api/artist.py new file mode 100644 index 0000000..4b755e6 --- /dev/null +++ b/backend/app/api/artist.py @@ -0,0 +1,36 @@ +"""Artist API: overview aggregates + paged images.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.artist_service import ArtistService + +artist_bp = Blueprint("artist", __name__, url_prefix="/api/artist") + + +@artist_bp.route("/", methods=["GET"]) +async def overview(slug: str): + async with get_session() as session: + svc = ArtistService(session) + data = await svc.overview(slug) + if data is None: + return jsonify({"error": "artist not found"}), 404 + return jsonify(data) + + +@artist_bp.route("//images", methods=["GET"]) +async def images(slug: str): + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = ArtistService(session) + try: + page = await svc.images(slug, cursor=cursor, limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + if page is None: + return jsonify({"error": "artist not found"}), 404 + return jsonify({"images": page.images, "next_cursor": page.next_cursor}) diff --git a/backend/app/api/artists.py b/backend/app/api/artists.py new file mode 100644 index 0000000..fa6da74 --- /dev/null +++ b/backend/app/api/artists.py @@ -0,0 +1,83 @@ +"""FC-3a: plural /api/artists endpoints — POST (find-or-create) + +autocomplete. FC-3f: GET /api/artists/directory (cursor-paginated +browse grid). The slug-routed singular /api/artist/ blueprint +stays separate and unmodified.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.artist_directory_service import ArtistDirectoryService +from ..services.artist_service import ArtistService +from ..services.source_service import KNOWN_PLATFORMS + +artists_bp = Blueprint("artists", __name__, url_prefix="/api/artists") + + +@artists_bp.route("", methods=["POST"]) +async def create_or_get(): + body = await request.get_json() + if not isinstance(body, dict): + return jsonify({"error": "invalid_body"}), 400 + name = body.get("name", "") + async with get_session() as session: + svc = ArtistService(session) + try: + artist, created = await svc.find_or_create(name) + except ValueError as exc: + return jsonify({"error": "empty_name", "detail": str(exc)}), 400 + return jsonify({ + "id": artist.id, "name": artist.name, "slug": artist.slug, + "created": created, + }), 201 + + +@artists_bp.route("/autocomplete", methods=["GET"]) +async def autocomplete(): + q = request.args.get("q") or "" + try: + limit = int(request.args.get("limit", "20")) + except ValueError: + return jsonify({"error": "invalid_limit"}), 400 + if limit < 1 or limit > 100: + return jsonify({"error": "invalid_limit"}), 400 + async with get_session() as session: + rows = await ArtistService(session).autocomplete(q, limit=limit) + return jsonify([ + {"id": a.id, "name": a.name, "slug": a.slug} for a in rows + ]) + + +@artists_bp.route("/directory", methods=["GET"]) +async def directory(): + """FC-3f: cursor-paginated artists directory. + + Mirrors /api/tags/directory shape: { cards: [...], next_cursor }. + """ + args = request.args + + cursor = args.get("cursor") or None + q = args.get("q") or None + platform = args.get("platform") or None + limit_raw = args.get("limit", "60") + + try: + limit = int(limit_raw) + except ValueError: + return jsonify({"error": "invalid_limit"}), 400 + if limit < 1 or limit > 200: + return jsonify({"error": "invalid_limit"}), 400 + + if platform is not None and platform not in KNOWN_PLATFORMS: + return jsonify({"error": "unknown_platform"}), 400 + + async with get_session() as session: + svc = ArtistDirectoryService(session) + try: + page = await svc.list_artists( + q=q, platform=platform, cursor=cursor, limit=limit, + ) + except ValueError: + # Service raises only on bad cursor (limit was validated above). + return jsonify({"error": "invalid_cursor"}), 400 + + return jsonify({"cards": page.cards, "next_cursor": page.next_cursor}) diff --git a/backend/app/api/attachments.py b/backend/app/api/attachments.py new file mode 100644 index 0000000..1fa8800 --- /dev/null +++ b/backend/app/api/attachments.py @@ -0,0 +1,27 @@ +"""Attachment download: streams a preserved non-art post file.""" + +from quart import Blueprint, jsonify, send_file + +from ..extensions import get_session +from ..models import PostAttachment + +attachments_bp = Blueprint( + "attachments", __name__, url_prefix="/api/attachments" +) + + +@attachments_bp.route("//download", methods=["GET"]) +async def download(attachment_id: int): + async with get_session() as session: + att = await session.get(PostAttachment, attachment_id) + if att is None: + return jsonify({"error": "not found"}), 404 + path = att.path + filename = att.original_filename + mimetype = att.mime or "application/octet-stream" + return await send_file( + path, + mimetype=mimetype, + as_attachment=True, + attachment_filename=filename, + ) diff --git a/backend/app/api/credentials.py b/backend/app/api/credentials.py new file mode 100644 index 0000000..182792a --- /dev/null +++ b/backend/app/api/credentials.py @@ -0,0 +1,126 @@ +"""FC-3b: /api/credentials — CRUD with X-Extension-Key auth. + +Browser requests (no header) are accepted per the homelab posture — +there is no user-session model in FC. The X-Extension-Key check +exists to give the extension a revocable path: a request that +supplies the header must match `app_setting.extension_api_key`. +""" + +from pathlib import Path + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import AppSetting, Credential +from ..services.credential_crypto import CredentialCrypto +from ..services.credential_service import ( + CredentialService, + EmptyDataError, + UnknownPlatformError, + WrongAuthTypeError, +) + +credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials") + + +# Production key path; tests pass `cookies_dir=tmp_path` via the +# service constructor (the API uses the default). The Fernet key +# file is also at `/images/secrets/credential_key.b64`. +_KEY_PATH = Path("/images/secrets/credential_key.b64") +_crypto: CredentialCrypto | None = None + + +def _get_crypto() -> CredentialCrypto: + global _crypto + if _crypto is None: + _crypto = CredentialCrypto(_KEY_PATH) + return _crypto + + +def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra): + body = {"error": error} + if detail is not None: + body["detail"] = detail + body.update(extra) + return jsonify(body), status + + +async def _ext_key_ok(session) -> bool: + """If X-Extension-Key is supplied, it must match the stored value. + Missing header → True (browser path; accepted per homelab posture). + """ + supplied = request.headers.get("X-Extension-Key") + if supplied is None: + return True + stored = (await session.execute( + select(AppSetting.value).where(AppSetting.key == "extension_api_key") + )).scalar_one_or_none() + return stored is not None and supplied == stored + + +@credentials_bp.route("", methods=["GET"]) +async def list_credentials(): + async with get_session() as session: + if not await _ext_key_ok(session): + return _bad("unauthorized", status=401) + records = await CredentialService(session, _get_crypto()).list() + return jsonify([r.to_dict() for r in records]) + + +@credentials_bp.route("/", methods=["GET"]) +async def get_credential(platform: str): + async with get_session() as session: + if not await _ext_key_ok(session): + return _bad("unauthorized", status=401) + record = await CredentialService(session, _get_crypto()).get(platform) + if record is None: + return _bad("not_found", status=404) + return jsonify(record.to_dict()) + + +@credentials_bp.route("", methods=["POST"]) +async def upsert_credential(): + body = await request.get_json() + if not isinstance(body, dict): + return _bad("invalid_body", detail="body must be a JSON object") + try: + platform = body["platform"] + credential_type = body["credential_type"] + data = body["data"] + except KeyError: + return _bad("invalid_body", detail="platform, credential_type, data are required") + if not isinstance(data, str): + return _bad("invalid_body", detail="data must be a string") + + async with get_session() as session: + if not await _ext_key_ok(session): + return _bad("unauthorized", status=401) + existed = (await session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() is not None + svc = CredentialService(session, _get_crypto()) + try: + record = await svc.upsert( + platform=platform, credential_type=credential_type, data=data, + ) + except UnknownPlatformError as exc: + return _bad("unknown_platform", detail=str(exc)) + except WrongAuthTypeError as exc: + return _bad("wrong_auth_type", detail=str(exc), expected=exc.expected) + except EmptyDataError as exc: + return _bad("empty_data", detail=str(exc)) + return jsonify(record.to_dict()), (200 if existed else 201) + + +@credentials_bp.route("/", methods=["DELETE"]) +async def delete_credential(platform: str): + async with get_session() as session: + if not await _ext_key_ok(session): + return _bad("unauthorized", status=401) + svc = CredentialService(session, _get_crypto()) + try: + await svc.delete(platform) + except LookupError: + return _bad("not_found", status=404) + return "", 204 diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py new file mode 100644 index 0000000..cac9015 --- /dev/null +++ b/backend/app/api/downloads.py @@ -0,0 +1,110 @@ +"""FC-3c: GET /api/downloads (list) + GET /api/downloads/ (detail). + +List view: keyset-paginated, newest first, optional filtering by +status/source/artist. Returns slim records. +Detail view: full DownloadEvent including the metadata JSONB. +""" + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import Artist, DownloadEvent, Source + +downloads_bp = Blueprint("downloads", __name__, url_prefix="/api/downloads") + +_ALLOWED_STATUSES = frozenset({"pending", "running", "ok", "error", "skipped"}) + + +def _summary_from_metadata(metadata: dict | None) -> dict: + metadata = metadata or {} + run_stats = metadata.get("run_stats") or {} + return { + "downloaded": int(run_stats.get("downloaded_count", 0)), + "skipped": int(run_stats.get("skipped_count", 0)), + "quarantined": int(run_stats.get("quarantined_count", 0)), + "duration_seconds": metadata.get("duration_seconds"), + "error_type": metadata.get("error_type"), + } + + +def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | None) -> dict: + return { + "id": event.id, + "source_id": event.source_id, + "artist_name": artist.name if artist else None, + "artist_slug": artist.slug if artist else None, + "platform": source.platform if source else None, + "status": event.status, + "started_at": event.started_at.isoformat() if event.started_at else None, + "finished_at": event.finished_at.isoformat() if event.finished_at else None, + "files_count": event.files_count, + "bytes_downloaded": event.bytes_downloaded, + "error": event.error, + "summary": _summary_from_metadata(event.metadata_), + } + + +def _detail_record(event: DownloadEvent, source: Source | None, artist: Artist | None) -> dict: + base = _list_record(event, source, artist) + base["metadata"] = event.metadata_ or {} + return base + + +@downloads_bp.route("", methods=["GET"]) +async def list_downloads(): + status = request.args.get("status") + if status and status not in _ALLOWED_STATUSES: + return jsonify({"error": "invalid_status"}), 400 + try: + limit = int(request.args.get("limit", "50")) + except ValueError: + return jsonify({"error": "invalid_limit"}), 400 + limit = max(1, min(200, limit)) + before_raw = request.args.get("before") + before = None + if before_raw is not None: + try: + before = int(before_raw) + except ValueError: + return jsonify({"error": "invalid_before"}), 400 + try: + source_id = int(request.args["source_id"]) if "source_id" in request.args else None + artist_id = int(request.args["artist_id"]) if "artist_id" in request.args else None + except (ValueError, KeyError): + return jsonify({"error": "invalid_filter"}), 400 + + async with get_session() as session: + stmt = ( + select(DownloadEvent, Source, Artist) + .outerjoin(Source, Source.id == DownloadEvent.source_id) + .outerjoin(Artist, Artist.id == Source.artist_id) + .order_by(DownloadEvent.id.desc()) + .limit(limit) + ) + if status: + stmt = stmt.where(DownloadEvent.status == status) + if source_id is not None: + stmt = stmt.where(DownloadEvent.source_id == source_id) + if artist_id is not None: + stmt = stmt.where(Source.artist_id == artist_id) + if before is not None: + stmt = stmt.where(DownloadEvent.id < before) + rows = (await session.execute(stmt)).all() + + return jsonify([_list_record(e, s, a) for e, s, a in rows]) + + +@downloads_bp.route("/", methods=["GET"]) +async def get_download(event_id: int): + async with get_session() as session: + row = (await session.execute( + select(DownloadEvent, Source, Artist) + .outerjoin(Source, Source.id == DownloadEvent.source_id) + .outerjoin(Artist, Artist.id == Source.artist_id) + .where(DownloadEvent.id == event_id) + )).first() + if row is None: + return jsonify({"error": "not_found"}), 404 + event, source, artist = row + return jsonify(_detail_record(event, source, artist)) diff --git a/backend/app/api/gallery.py b/backend/app/api/gallery.py new file mode 100644 index 0000000..b14d414 --- /dev/null +++ b/backend/app/api/gallery.py @@ -0,0 +1,111 @@ +"""Gallery API: cursor scroll, timeline, jump, image detail.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.gallery_service import GalleryService + +gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery") + + +@gallery_bp.route("/scroll", methods=["GET"]) +async def scroll(): + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "50")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + post_id_raw = request.args.get("post_id") + post_id = int(post_id_raw) if post_id_raw else None + artist_id_raw = request.args.get("artist_id") + artist_id = int(artist_id_raw) if artist_id_raw else None + + async with get_session() as session: + svc = GalleryService(session) + try: + page = await svc.scroll( + cursor=cursor, limit=limit, tag_id=tag_id, + post_id=post_id, artist_id=artist_id, + ) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + + return jsonify( + { + "images": [ + { + "id": i.id, + "sha256": i.sha256, + "mime": i.mime, + "width": i.width, + "height": i.height, + "created_at": i.created_at.isoformat(), + "thumbnail_url": i.thumbnail_url, + "artist": i.artist, + } + for i in page.images + ], + "next_cursor": page.next_cursor, + "date_groups": [ + {"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups + ], + } + ) + + +@gallery_bp.route("/timeline", methods=["GET"]) +async def timeline(): + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + post_id_raw = request.args.get("post_id") + post_id = int(post_id_raw) if post_id_raw else None + artist_id_raw = request.args.get("artist_id") + artist_id = int(artist_id_raw) if artist_id_raw else None + async with get_session() as session: + svc = GalleryService(session) + try: + buckets = await svc.timeline( + tag_id=tag_id, post_id=post_id, artist_id=artist_id + ) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify( + [{"year": b.year, "month": b.month, "count": b.count} for b in buckets] + ) + + +@gallery_bp.route("/jump", methods=["GET"]) +async def jump(): + try: + year = int(request.args["year"]) + month = int(request.args["month"]) + except (KeyError, ValueError): + return jsonify({"error": "year and month query params required"}), 400 + tag_id_raw = request.args.get("tag_id") + tag_id = int(tag_id_raw) if tag_id_raw else None + post_id_raw = request.args.get("post_id") + post_id = int(post_id_raw) if post_id_raw else None + artist_id_raw = request.args.get("artist_id") + artist_id = int(artist_id_raw) if artist_id_raw else None + async with get_session() as session: + svc = GalleryService(session) + try: + cursor = await svc.jump_cursor( + year=year, month=month, tag_id=tag_id, + post_id=post_id, artist_id=artist_id, + ) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify({"cursor": cursor}) + + +@gallery_bp.route("/image/", methods=["GET"]) +async def image_detail(image_id: int): + async with get_session() as session: + svc = GalleryService(session) + payload = await svc.get_image_with_tags(image_id) + if payload is None: + return jsonify({"error": "not found"}), 404 + return jsonify(payload) diff --git a/backend/app/api/health.py b/backend/app/api/health.py new file mode 100644 index 0000000..ab4918e --- /dev/null +++ b/backend/app/api/health.py @@ -0,0 +1,5 @@ +"""Health endpoint — no DB or Redis touch; just liveness.""" + + +async def get_health(): + return {"status": "ok"}, 200 diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py new file mode 100644 index 0000000..6a16bd3 --- /dev/null +++ b/backend/app/api/import_admin.py @@ -0,0 +1,142 @@ +"""Import admin API: trigger scan, list tasks, retry, clear.""" + +from datetime import UTC, datetime, timedelta + +from quart import Blueprint, jsonify, request +from sqlalchemy import delete, select, update + +from ..extensions import get_session +from ..models import ImportBatch, ImportTask + +import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import") + + +@import_admin_bp.route("/trigger", methods=["POST"]) +async def trigger_scan(): + body = await request.get_json(silent=True) or {} + mode = body.get("mode", "quick") + if mode not in ("quick", "deep", "verify"): + return jsonify({"error": f"mode {mode!r} not supported; use 'quick', 'deep', or 'verify'"}), 400 + + # 'verify' is a library task — short-circuit the import_root walk + # (no ImportBatch, no per-file ImportTasks). + if mode == "verify": + from ..tasks.maintenance import verify_integrity + + async_result = verify_integrity.delay() + return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202 + + from ..tasks.scan import scan_directory + + async_result = scan_directory.delay(triggered_by="manual", mode=mode) + return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202 + + +@import_admin_bp.route("/status", methods=["GET"]) +async def status(): + async with get_session() as session: + active = ( + await session.execute( + select(ImportBatch) + .where(ImportBatch.status == "running") + .order_by(ImportBatch.started_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + payload = {"active_batch": None} + if active: + payload["active_batch"] = { + "id": active.id, + "total_files": active.total_files, + "imported": active.imported, + "skipped": active.skipped, + "failed": active.failed, + "started_at": active.started_at.isoformat(), + } + return jsonify(payload) + + +@import_admin_bp.route("/tasks", methods=["GET"]) +async def list_tasks(): + status_filter = request.args.get("status") + try: + limit = min(int(request.args.get("limit", "50")), 200) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + cursor_raw = request.args.get("cursor") + cursor_id = int(cursor_raw) if cursor_raw else None + + async with get_session() as session: + stmt = select(ImportTask).order_by(ImportTask.created_at.desc(), ImportTask.id.desc()) + if status_filter: + stmt = stmt.where(ImportTask.status == status_filter) + if cursor_id is not None: + stmt = stmt.where(ImportTask.id < cursor_id) + stmt = stmt.limit(limit + 1) + rows = (await session.execute(stmt)).scalars().all() + + has_more = len(rows) > limit + rows = rows[:limit] + return jsonify({ + "tasks": [ + { + "id": t.id, + "batch_id": t.batch_id, + "source_path": t.source_path, + "task_type": t.task_type, + "status": t.status, + "result_image_id": t.result_image_id, + "error": t.error, + "size_bytes": t.size_bytes, + "created_at": t.created_at.isoformat(), + "started_at": t.started_at.isoformat() if t.started_at else None, + "finished_at": t.finished_at.isoformat() if t.finished_at else None, + } + for t in rows + ], + "next_cursor": rows[-1].id if has_more and rows else None, + }) + + +@import_admin_bp.route("/retry-failed", methods=["POST"]) +async def retry_failed(): + async with get_session() as session: + failed_ids = ( + await session.execute(select(ImportTask.id).where(ImportTask.status == "failed")) + ).scalars().all() + if not failed_ids: + return jsonify({"retried": 0}) + await session.execute( + update(ImportTask) + .where(ImportTask.id.in_(failed_ids)) + .values(status="queued", error=None, started_at=None, finished_at=None) + ) + await session.commit() + + from ..tasks.import_file import import_media_file + for tid in failed_ids: + import_media_file.delay(tid) + + return jsonify({"retried": len(failed_ids)}) + + +@import_admin_bp.route("/clear-completed", methods=["POST"]) +async def clear_completed(): + body = await request.get_json(silent=True) or {} + age_days = body.get("age_days", 0) + status_filter = body.get("status", ["complete", "skipped"]) + if not isinstance(status_filter, list): + return jsonify({"error": "status must be a list"}), 400 + + cutoff = ( + datetime.now(UTC) - timedelta(days=int(age_days)) if age_days else None + ) + + async with get_session() as session: + stmt = delete(ImportTask).where(ImportTask.status.in_(status_filter)) + if cutoff is not None: + stmt = stmt.where(ImportTask.finished_at < cutoff) + result = await session.execute(stmt) + await session.commit() + + return jsonify({"deleted": result.rowcount or 0}) diff --git a/backend/app/api/migrate.py b/backend/app/api/migrate.py new file mode 100644 index 0000000..6c5f86b --- /dev/null +++ b/backend/app/api/migrate.py @@ -0,0 +1,141 @@ +"""FC-5: /api/migrate — trigger and poll migration runs. + +Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an +`export_file` field. All other kinds accept JSON. Apply-without-backup +guard rejects non-dry-run ingests unless a pre_migration-tagged backup +exists in the last 24h (override with body.force=true). +""" + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import MigrationRun +from ..tasks.migration import run_migration + +migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate") + +_VALID_KINDS = frozenset({ + "backup", "gs_ingest", "ir_ingest", "tag_apply", + "ml_queue", "verify", "rollback", +}) +_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"}) +_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback"}) + + +def _bad(error: str, *, status: int = 400, **extra): + body = {"error": error} + body.update(extra) + return jsonify(body), status + + +def _has_recent_pre_migration_backup() -> bool: + from ..services.migrators import backup as backup_mod + images_root = Path("/images") + manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration") + if manifest is None: + return False + created_at_str = manifest.get("created_at") + if not created_at_str: + return False + created_at = datetime.fromisoformat(created_at_str) + return (datetime.now(UTC) - created_at) < timedelta(hours=24) + + +def _run_to_dict(run: MigrationRun) -> dict: + return { + "id": run.id, + "kind": run.kind, + "status": run.status, + "dry_run": run.dry_run, + "started_at": run.started_at.isoformat(), + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + "counts": run.counts or {}, + "error": run.error, + "metadata": run.metadata_ or {}, + } + + +@migrate_bp.route("/", methods=["POST"]) +async def create_run(kind: str): + if kind not in _VALID_KINDS: + return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}") + + # Ingest kinds accept multipart/form-data; everything else takes JSON. + if kind in _INGEST_KINDS: + form = await request.form + files = await request.files + if "export_file" not in files: + return _bad("missing_export_file", detail="multipart export_file required") + export_file = files["export_file"] + try: + raw = export_file.read() + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return _bad("invalid_export_file", detail=str(exc)) + dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes") + force = str(form.get("force", "false")).lower() in ("true", "1", "yes") + params: dict = {"data": data, "dry_run": dry_run} + else: + body = await request.get_json() + if body is None: + body = {} + if not isinstance(body, dict): + return _bad("invalid_body") + dry_run = bool(body.get("dry_run", False)) + force = bool(body.get("force", False)) + params = dict(body) + + is_apply = (kind in _APPLY_KINDS) and not dry_run + if is_apply and not force and not _has_recent_pre_migration_backup(): + return _bad( + "no_backup", + detail="apply action requires a pre_migration-tagged backup " + "in the last 24h (or force=true).", + ) + + if kind == "backup": + params.setdefault("tag", "pre_migration") + + async with get_session() as session: + run = MigrationRun(kind=kind, status="pending", dry_run=dry_run) + session.add(run) + await session.commit() + await session.refresh(run) + run_id = run.id + + run_migration.delay(run_id, kind, params) + return jsonify({"run_id": run_id, "status": "pending"}), 202 + + +@migrate_bp.route("/runs/", methods=["GET"]) +async def get_run(run_id: int): + async with get_session() as session: + run = (await session.execute( + select(MigrationRun).where(MigrationRun.id == run_id) + )).scalar_one_or_none() + if run is None: + return _bad("not_found", status=404) + return jsonify(_run_to_dict(run)) + + +@migrate_bp.route("/runs", methods=["GET"]) +async def list_runs(): + try: + limit = int(request.args.get("limit", "10")) + except ValueError: + return _bad("invalid_limit") + if limit < 1 or limit > 100: + return _bad("invalid_limit") + + async with get_session() as session: + rows = (await session.execute( + select(MigrationRun) + .order_by(MigrationRun.id.desc()) + .limit(limit) + )).scalars().all() + return jsonify([_run_to_dict(r) for r in rows]) diff --git a/backend/app/api/ml_admin.py b/backend/app/api/ml_admin.py new file mode 100644 index 0000000..a346175 --- /dev/null +++ b/backend/app/api/ml_admin.py @@ -0,0 +1,74 @@ +"""ML admin API: settings, backfill trigger, centroid recompute trigger.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..models import MLSettings + +ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml") + + +_EDITABLE = ( + "suggestion_threshold_artist", + "suggestion_threshold_character", + "suggestion_threshold_copyright", + "suggestion_threshold_general", + "centroid_similarity_threshold", + "min_reference_images", +) + + +@ml_admin_bp.route("/settings", methods=["GET"]) +async def get_settings(): + from sqlalchemy import select + + async with get_session() as session: + s = ( + await session.execute(select(MLSettings).where(MLSettings.id == 1)) + ).scalar_one() + return jsonify( + { + "suggestion_threshold_artist": s.suggestion_threshold_artist, + "suggestion_threshold_character": s.suggestion_threshold_character, + "suggestion_threshold_copyright": s.suggestion_threshold_copyright, + "suggestion_threshold_general": s.suggestion_threshold_general, + "centroid_similarity_threshold": s.centroid_similarity_threshold, + "min_reference_images": s.min_reference_images, + "tagger_model_version": s.tagger_model_version, + "embedder_model_version": s.embedder_model_version, + } + ) + + +@ml_admin_bp.route("/settings", methods=["PATCH"]) +async def patch_settings(): + from sqlalchemy import select + + body = await request.get_json() + if not isinstance(body, dict): + return jsonify({"error": "body must be an object"}), 400 + async with get_session() as session: + s = ( + await session.execute(select(MLSettings).where(MLSettings.id == 1)) + ).scalar_one() + for field in _EDITABLE: + if field in body: + setattr(s, field, body[field]) + await session.commit() + return await get_settings() + + +@ml_admin_bp.route("/backfill", methods=["POST"]) +async def trigger_backfill(): + from ..tasks.ml import backfill + + r = backfill.delay() + return jsonify({"celery_task_id": r.id}), 202 + + +@ml_admin_bp.route("/recompute-centroids", methods=["POST"]) +async def trigger_recompute(): + from ..tasks.ml import recompute_centroids + + r = recompute_centroids.delay() + return jsonify({"celery_task_id": r.id}), 202 diff --git a/backend/app/api/platforms.py b/backend/app/api/platforms.py new file mode 100644 index 0000000..d22834e --- /dev/null +++ b/backend/app/api/platforms.py @@ -0,0 +1,12 @@ +"""FC-3b: /api/platforms — informational, no auth, matches GS shape.""" + +from quart import Blueprint, jsonify + +from ..services.platforms import PLATFORMS, to_dict + +platforms_bp = Blueprint("platforms", __name__, url_prefix="/api/platforms") + + +@platforms_bp.route("", methods=["GET"]) +async def list_platforms(): + return jsonify({"platforms": {k: to_dict(v) for k, v in PLATFORMS.items()}}) diff --git a/backend/app/api/posts.py b/backend/app/api/posts.py new file mode 100644 index 0000000..1f9dd3e --- /dev/null +++ b/backend/app/api/posts.py @@ -0,0 +1,69 @@ +"""FC-3e: /api/posts — cursor-paginated unified posts feed.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.post_feed_service import PostFeedService +from ..services.source_service import KNOWN_PLATFORMS + +posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts") + + +def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra): + body = {"error": error} + if detail is not None: + body["detail"] = detail + body.update(extra) + return jsonify(body), status + + +@posts_bp.route("", methods=["GET"]) +async def list_posts(): + args = request.args + + cursor = args.get("cursor") or None + artist_id_raw = args.get("artist_id") + platform = args.get("platform") or None + limit_raw = args.get("limit", "24") + + try: + limit = int(limit_raw) + except ValueError: + return _bad("invalid_limit", detail="limit must be an integer") + if limit < 1 or limit > 100: + return _bad("invalid_limit", detail="limit must be between 1 and 100") + + artist_id = None + if artist_id_raw is not None: + try: + artist_id = int(artist_id_raw) + except ValueError: + return _bad("invalid_artist_id", detail="artist_id must be an integer") + + if platform is not None and platform not in KNOWN_PLATFORMS: + return _bad( + "unknown_platform", + detail=f"platform must be one of {sorted(KNOWN_PLATFORMS)}", + ) + + async with get_session() as session: + try: + page = await PostFeedService(session).scroll( + cursor=cursor, artist_id=artist_id, + platform=platform, limit=limit, + ) + except ValueError as exc: + # Service raises ValueError for malformed cursors only; + # limit bounds are validated above. + return _bad("invalid_cursor", detail=str(exc)) + + return jsonify(page) + + +@posts_bp.route("/", methods=["GET"]) +async def get_post(post_id: int): + async with get_session() as session: + item = await PostFeedService(session).get_post(post_id) + if item is None: + return _bad("not_found", status=404, detail=f"post id={post_id}") + return jsonify(item) diff --git a/backend/app/api/provenance.py b/backend/app/api/provenance.py new file mode 100644 index 0000000..cb7799f --- /dev/null +++ b/backend/app/api/provenance.py @@ -0,0 +1,33 @@ +"""Provenance API: image -> posts, post -> header payload. + +Separate from the gallery/tag APIs by design (provenance is its own +system). Read-only. +""" + +from quart import Blueprint, jsonify + +from ..extensions import get_session +from ..services.provenance_service import ProvenanceService + +provenance_bp = Blueprint("provenance", __name__, + url_prefix="/api/provenance") + + +@provenance_bp.route("/image/", methods=["GET"]) +async def image_provenance(image_id: int): + async with get_session() as session: + svc = ProvenanceService(session) + payload = await svc.for_image(image_id) + if payload is None: + return jsonify({"error": "not found"}), 404 + return jsonify(payload) + + +@provenance_bp.route("/post/", methods=["GET"]) +async def post_provenance(post_id: int): + async with get_session() as session: + svc = ProvenanceService(session) + payload = await svc.for_post(post_id) + if payload is None: + return jsonify({"error": "not found"}), 404 + return jsonify(payload) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py new file mode 100644 index 0000000..6ad49b8 --- /dev/null +++ b/backend/app/api/settings.py @@ -0,0 +1,221 @@ +"""Settings API: import filters, system stats.""" + +import secrets + +from quart import Blueprint, jsonify, request +from sqlalchemy import func, select + +from ..extensions import get_session +from ..models import AppSetting, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag + +settings_bp = Blueprint("settings", __name__, url_prefix="/api") + + +_EDITABLE_FIELDS = ( + "min_width", + "min_height", + "skip_transparent", + "transparency_threshold", + "skip_single_color", + "single_color_threshold", + "single_color_tolerance", + "phash_threshold", + "download_rate_limit_seconds", + "download_validate_files", + "download_schedule_default_seconds", + "download_event_retention_days", + "download_failure_warning_threshold", +) + + +@settings_bp.route("/settings/import", methods=["GET"]) +async def get_import_settings(): + async with get_session() as session: + row = ( + await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) + ).scalar_one() + return jsonify({ + "min_width": row.min_width, + "min_height": row.min_height, + "skip_transparent": row.skip_transparent, + "transparency_threshold": row.transparency_threshold, + "skip_single_color": row.skip_single_color, + "single_color_threshold": row.single_color_threshold, + "single_color_tolerance": row.single_color_tolerance, + "phash_threshold": row.phash_threshold, + "download_rate_limit_seconds": row.download_rate_limit_seconds, + "download_validate_files": row.download_validate_files, + "download_schedule_default_seconds": row.download_schedule_default_seconds, + "download_event_retention_days": row.download_event_retention_days, + "download_failure_warning_threshold": row.download_failure_warning_threshold, + }) + + +@settings_bp.route("/settings/import", methods=["PATCH"]) +async def update_import_settings(): + body = await request.get_json() + if not isinstance(body, dict): + return jsonify({"error": "body must be a JSON object"}), 400 + + if "phash_threshold" in body and ( + not isinstance(body["phash_threshold"], int) + or isinstance(body["phash_threshold"], bool) + or body["phash_threshold"] < 0 + ): + return jsonify( + {"error": "phash_threshold must be a non-negative integer"} + ), 400 + + if "download_rate_limit_seconds" in body: + val = body["download_rate_limit_seconds"] + if not isinstance(val, (int, float)) or isinstance(val, bool) or val < 0: + return jsonify( + {"error": "download_rate_limit_seconds must be a non-negative number"} + ), 400 + if "download_validate_files" in body and not isinstance( + body["download_validate_files"], bool + ): + return jsonify( + {"error": "download_validate_files must be a boolean"} + ), 400 + + # FC-3d scheduling knobs — bounds validation. + def _bad_int(name: str, lo: int, hi: int): + return jsonify( + {"error": f"{name} must be an integer in [{lo}, {hi}]"} + ), 400 + + if "download_schedule_default_seconds" in body: + v = body["download_schedule_default_seconds"] + if not isinstance(v, int) or isinstance(v, bool) or v < 60 or v > 86400: + return _bad_int("download_schedule_default_seconds", 60, 86400) + if "download_event_retention_days" in body: + v = body["download_event_retention_days"] + if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 3650: + return _bad_int("download_event_retention_days", 1, 3650) + if "download_failure_warning_threshold" in body: + v = body["download_failure_warning_threshold"] + if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100: + return _bad_int("download_failure_warning_threshold", 1, 100) + + async with get_session() as session: + row = ( + await session.execute(select(ImportSettings).where(ImportSettings.id == 1)) + ).scalar_one() + for field in _EDITABLE_FIELDS: + if field in body: + setattr(row, field, body[field]) + await session.commit() + + return await get_import_settings() + + +@settings_bp.route("/system/stats", methods=["GET"]) +async def system_stats(): + async with get_session() as session: + total_images = (await session.execute(select(func.count(ImageRecord.id)))).scalar_one() + total_tags = (await session.execute(select(func.count(Tag.id)))).scalar_one() + storage_bytes = ( + (await session.execute(select(func.coalesce(func.sum(ImageRecord.size_bytes), 0)))).scalar_one() + ) + + # Task counts grouped by status + status_rows = ( + await session.execute( + select(ImportTask.status, func.count(ImportTask.id)).group_by(ImportTask.status) + ) + ).all() + status_counts = {row[0]: row[1] for row in status_rows} + + # Integrity counts — FC-2e. + integrity_rows = ( + await session.execute( + select( + ImageRecord.integrity_status, + func.count(ImageRecord.id), + ).group_by(ImageRecord.integrity_status) + ) + ).all() + integrity_counts = {row[0]: row[1] for row in integrity_rows} + + # Active batch (most recent running) + active_batch_row = ( + await session.execute( + select(ImportBatch) + .where(ImportBatch.status == "running") + .order_by(ImportBatch.started_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + active_batch = None + if active_batch_row: + active_batch = { + "id": active_batch_row.id, + "source_path": active_batch_row.source_path, + "started_at": active_batch_row.started_at.isoformat(), + "total_files": active_batch_row.total_files, + "imported": active_batch_row.imported, + "skipped": active_batch_row.skipped, + "failed": active_batch_row.failed, + } + + return jsonify({ + "total_images": total_images, + "total_tags": total_tags, + "storage_bytes": storage_bytes, + "tasks": { + "pending": status_counts.get("pending", 0), + "queued": status_counts.get("queued", 0), + "processing": status_counts.get("processing", 0), + "complete": status_counts.get("complete", 0), + "skipped": status_counts.get("skipped", 0), + "failed": status_counts.get("failed", 0), + }, + "integrity": { + "unknown": int(integrity_counts.get("unknown", 0)), + "ok": int(integrity_counts.get("ok", 0)), + "corrupt": int(integrity_counts.get("corrupt", 0)), + "failed_verification": int( + integrity_counts.get("failed_verification", 0) + ), + }, + "active_batch": active_batch, + }) + + +# --- FC-3b: extension API key (lives in app_setting) ----------------------- + + +async def _get_or_seed_extension_api_key(session) -> str: + row = (await session.execute( + select(AppSetting).where(AppSetting.key == "extension_api_key") + )).scalar_one_or_none() + if row is None: + row = AppSetting(key="extension_api_key", value=secrets.token_urlsafe(32)) + session.add(row) + await session.commit() + await session.refresh(row) + return row.value + + +@settings_bp.route("/settings/extension_api_key", methods=["GET"]) +async def get_extension_api_key(): + async with get_session() as session: + key = await _get_or_seed_extension_api_key(session) + return jsonify({"key": key}) + + +@settings_bp.route("/settings/extension_api_key/rotate", methods=["POST"]) +async def rotate_extension_api_key(): + async with get_session() as session: + row = (await session.execute( + select(AppSetting).where(AppSetting.key == "extension_api_key") + )).scalar_one_or_none() + new_value = secrets.token_urlsafe(32) + if row is None: + row = AppSetting(key="extension_api_key", value=new_value) + session.add(row) + else: + row.value = new_value + await session.commit() + return jsonify({"key": new_value}) diff --git a/backend/app/api/showcase.py b/backend/app/api/showcase.py new file mode 100644 index 0000000..232cc86 --- /dev/null +++ b/backend/app/api/showcase.py @@ -0,0 +1,23 @@ +"""Showcase API: scalable random sample of images.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.showcase_service import ShowcaseService + +showcase_bp = Blueprint("showcase", __name__, url_prefix="/api/showcase") + + +@showcase_bp.route("", methods=["GET"]) +async def random_showcase(): + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = ShowcaseService(session) + try: + images = await svc.random_sample(limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify({"images": images}) diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py new file mode 100644 index 0000000..fd2b109 --- /dev/null +++ b/backend/app/api/sources.py @@ -0,0 +1,156 @@ +"""FC-3a: CRUD over Source rows. FC-3c adds POST //check.""" + +from quart import Blueprint, jsonify, request +from sqlalchemy import select + +from ..extensions import get_session +from ..models import DownloadEvent, Source +from ..services.source_service import ( + KNOWN_PLATFORMS, + ArtistNotFoundError, + DuplicateSourceError, + EmptyUrlError, + InvalidConfigError, + SourceService, + UnknownPlatformError, +) + +sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources") + + +def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra): + body = {"error": error} + if detail is not None: + body["detail"] = detail + body.update(extra) + return jsonify(body), status + + +@sources_bp.route("", methods=["GET"]) +async def list_sources(): + artist_id_raw = request.args.get("artist_id") + artist_id = None + if artist_id_raw is not None: + try: + artist_id = int(artist_id_raw) + except ValueError: + return _bad("invalid_artist_id", detail="artist_id must be an integer") + async with get_session() as session: + records = await SourceService(session).list(artist_id=artist_id) + return jsonify([r.to_dict() for r in records]) + + +@sources_bp.route("/", methods=["GET"]) +async def get_source(source_id: int): + async with get_session() as session: + record = await SourceService(session).get(source_id) + if record is None: + return _bad("not_found", status=404, detail=f"source id={source_id}") + return jsonify(record.to_dict()) + + +@sources_bp.route("", methods=["POST"]) +async def create_source(): + body = await request.get_json() + if not isinstance(body, dict): + return _bad("invalid_body", detail="body must be a JSON object") + try: + artist_id = int(body["artist_id"]) + platform = body["platform"] + url = body["url"] + except (KeyError, TypeError, ValueError): + return _bad("invalid_body", detail="artist_id, platform, url are required") + + optional = { + k: body[k] + for k in ("enabled", "config_overrides", "check_interval_override") + if k in body + } + + async with get_session() as session: + svc = SourceService(session) + try: + record = await svc.create( + artist_id=artist_id, platform=platform, url=url, **optional, + ) + except ArtistNotFoundError: + return _bad("artist_not_found", status=404, detail=f"artist id={artist_id}") + except UnknownPlatformError as exc: + return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS)) + except InvalidConfigError as exc: + return _bad("invalid_config", detail=str(exc)) + except EmptyUrlError as exc: + return _bad("empty_url", detail=str(exc)) + except DuplicateSourceError as exc: + return _bad("duplicate", status=409, existing_id=exc.existing_id) + return jsonify(record.to_dict()), 201 + + +@sources_bp.route("/", methods=["PATCH"]) +async def patch_source(source_id: int): + body = await request.get_json() + if not isinstance(body, dict): + return _bad("invalid_body", detail="body must be a JSON object") + async with get_session() as session: + svc = SourceService(session) + try: + record = await svc.update(source_id, **body) + except LookupError: + return _bad("not_found", status=404) + except UnknownPlatformError as exc: + return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS)) + except InvalidConfigError as exc: + return _bad("invalid_config", detail=str(exc)) + except EmptyUrlError as exc: + return _bad("empty_url", detail=str(exc)) + except DuplicateSourceError as exc: + return _bad("duplicate", status=409, existing_id=exc.existing_id) + return jsonify(record.to_dict()) + + +@sources_bp.route("/", methods=["DELETE"]) +async def delete_source(source_id: int): + async with get_session() as session: + try: + await SourceService(session).delete(source_id) + except LookupError: + return _bad("not_found", status=404) + return "", 204 + + +@sources_bp.route("//check", methods=["POST"]) +async def check_source(source_id: int): + """FC-3c: enqueue a download for this source. + + Returns 202 with the new DownloadEvent id. If a pending/running + event already exists for this source, returns 409 with that id.""" + async with get_session() as session: + source = (await session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + return _bad("not_found", status=404, detail=f"source id={source_id}") + if not source.enabled: + return _bad("source_disabled", detail="enable the source first") + + in_flight = (await session.execute( + select(DownloadEvent.id).where( + DownloadEvent.source_id == source_id, + DownloadEvent.status.in_(["pending", "running"]), + ).order_by(DownloadEvent.id.desc()).limit(1) + )).scalar_one_or_none() + if in_flight is not None: + return jsonify( + {"download_event_id": in_flight, "status": "already_running"} + ), 409 + + event = DownloadEvent(source_id=source_id, status="pending") + session.add(event) + await session.commit() + await session.refresh(event) + event_id = event.id + + from ..tasks.download import download_source + download_source.delay(source_id) + + return jsonify({"download_event_id": event_id, "status": "pending"}), 202 diff --git a/backend/app/api/suggestions.py b/backend/app/api/suggestions.py new file mode 100644 index 0000000..25bd2fe --- /dev/null +++ b/backend/app/api/suggestions.py @@ -0,0 +1,119 @@ +"""Suggestions API: per-image ranked suggestions + accept/alias/dismiss.""" + +from quart import Blueprint, jsonify, request + +from ..extensions import get_session +from ..services.ml.allowlist import AllowlistService +from ..services.ml.suggestions import SuggestionService + +suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api") + + +@suggestions_bp.route("/images//suggestions", methods=["GET"]) +async def get_suggestions(image_id: int): + async with get_session() as session: + sl = await SuggestionService(session).for_image(image_id) + return jsonify( + { + "by_category": { + cat: [ + { + "canonical_tag_id": s.canonical_tag_id, + "display_name": s.display_name, + "category": s.category, + "score": round(s.score, 4), + "source": s.source, + "creates_new_tag": s.creates_new_tag, + } + for s in items + ] + for cat, items in sl.by_category.items() + } + } + ) + + +@suggestions_bp.route( + "/images//suggestions/accept", methods=["POST"] +) +async def accept_suggestion(image_id: int): + body = await request.get_json() + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + tag_id = body["tag_id"] + async with get_session() as session: + newly_added = await AllowlistService(session).accept(image_id, tag_id) + await session.commit() + if newly_added: + from ..tasks.ml import apply_allowlist_tags + + apply_allowlist_tags.delay(tag_id=tag_id) + return "", 204 + + +@suggestions_bp.route( + "/images//suggestions/alias", methods=["POST"] +) +async def alias_suggestion(image_id: int): + body = await request.get_json() + required = {"alias_string", "alias_category", "canonical_tag_id"} + if not body or not required.issubset(body): + return jsonify({"error": f"required: {sorted(required)}"}), 400 + async with get_session() as session: + newly_added = await AllowlistService(session).add_alias_and_accept( + image_id, + body["alias_string"], + body["alias_category"], + body["canonical_tag_id"], + ) + await session.commit() + if newly_added: + from ..tasks.ml import apply_allowlist_tags + + apply_allowlist_tags.delay(tag_id=body["canonical_tag_id"]) + return "", 204 + + +@suggestions_bp.route( + "/images//suggestions/dismiss", methods=["POST"] +) +async def dismiss_suggestion(image_id: int): + body = await request.get_json() + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + async with get_session() as session: + await AllowlistService(session).dismiss(image_id, body["tag_id"]) + await session.commit() + return "", 204 + + +@suggestions_bp.route("/suggestions/bulk", methods=["POST"]) +async def bulk_suggestions(): + body = await request.get_json() + if not body or "image_ids" not in body: + return jsonify({"error": "image_ids required"}), 400 + raw = body["image_ids"] + if not isinstance(raw, list) or not raw: + return jsonify({"error": "image_ids must be a non-empty list"}), 400 + try: + ids = [int(x) for x in raw] + except (TypeError, ValueError): + return jsonify({"error": "image_ids must be integers"}), 400 + if len(ids) > 200: + return jsonify({"error": "selection too large (max 200)"}), 400 + try: + threshold = float(body.get("threshold", 0.8)) + except (TypeError, ValueError): + threshold = 0.8 + threshold = min(1.0, max(0.0, threshold)) + async with get_session() as session: + suggestions = await SuggestionService(session).for_selection( + ids, threshold=threshold + ) + return jsonify( + { + "suggestions": suggestions, + "evaluated": len(ids), + "threshold": threshold, + } + ) diff --git a/backend/app/api/tags.py b/backend/app/api/tags.py new file mode 100644 index 0000000..64f16bf --- /dev/null +++ b/backend/app/api/tags.py @@ -0,0 +1,366 @@ +"""Tags API: autocomplete, create, list/add/remove for an image.""" + +from quart import Blueprint, jsonify, request +from sqlalchemy import exists, select +from sqlalchemy.exc import IntegrityError + +from ..extensions import get_session +from ..models import Tag, TagKind +from ..models.tag_allowlist import TagAllowlist +from ..services.bulk_tag_service import BulkTagService +from ..services.series_service import SeriesError, SeriesService +from ..services.tag_directory_service import TagDirectoryService +from ..services.tag_service import ( + TagMergeConflict, + TagService, + TagValidationError, +) + +tags_bp = Blueprint("tags", __name__, url_prefix="/api") + + +def _coerce_kind(raw: str | None) -> TagKind | None: + if raw is None: + return None + try: + return TagKind(raw) + except ValueError: + return None + + +def _parse_bulk_ids( + body, *, max_ids: int = 200 +) -> tuple[list[int] | None, tuple | None]: + """Returns (image_ids, error). error is (json, status) or None.""" + if not body or "image_ids" not in body: + return None, (jsonify({"error": "image_ids required"}), 400) + raw = body["image_ids"] + if not isinstance(raw, list) or not raw: + return None, ( + jsonify({"error": "image_ids must be a non-empty list"}), + 400, + ) + try: + ids = [int(x) for x in raw] + except (TypeError, ValueError): + return None, ( + jsonify({"error": "image_ids must be integers"}), + 400, + ) + if len(ids) > max_ids: + return None, ( + jsonify({"error": f"selection too large (max {max_ids})"}), + 400, + ) + return ids, None + + +@tags_bp.route("/tags/autocomplete", methods=["GET"]) +async def autocomplete(): + q = request.args.get("q", "") + kind = _coerce_kind(request.args.get("kind")) + try: + limit = int(request.args.get("limit", "20")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + + async with get_session() as session: + svc = TagService(session) + hits = await svc.autocomplete(q, kind=kind, limit=limit) + + return jsonify( + [ + { + "id": h.id, + "name": h.name, + "kind": h.kind, + "fandom_id": h.fandom_id, + "fandom_name": h.fandom_name, + "image_count": h.image_count, + } + for h in hits + ] + ) + + +@tags_bp.route("/tags/directory", methods=["GET"]) +async def directory(): + kind = request.args.get("kind") or None + if kind is not None and _coerce_kind(kind) is None: + return jsonify({"error": f"invalid kind {kind!r}"}), 400 + q = request.args.get("q") or None + cursor = request.args.get("cursor") or None + try: + limit = int(request.args.get("limit", "60")) + except ValueError: + return jsonify({"error": "limit must be an integer"}), 400 + async with get_session() as session: + svc = TagDirectoryService(session) + try: + page = await svc.list_tags(kind=kind, q=q, cursor=cursor, limit=limit) + except ValueError as exc: + return jsonify({"error": str(exc)}), 400 + return jsonify({"cards": page.cards, "next_cursor": page.next_cursor}) + + +@tags_bp.route("/tags", methods=["POST"]) +async def create_tag(): + body = await request.get_json() + if not body or "name" not in body or "kind" not in body: + return jsonify({"error": "name and kind required"}), 400 + name = body["name"] + kind = _coerce_kind(body["kind"]) + if kind is None: + return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400 + fandom_id = body.get("fandom_id") + + async with get_session() as session: + svc = TagService(session) + try: + tag = await svc.find_or_create(name, kind, fandom_id=fandom_id) + except TagValidationError as exc: + return jsonify({"error": str(exc)}), 400 + await session.commit() + return jsonify( + {"id": tag.id, "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id} + ), 201 + + +@tags_bp.route("/images//tags", methods=["GET"]) +async def list_tags_for_image(image_id: int): + async with get_session() as session: + svc = TagService(session) + tags = await svc.list_for_image(image_id) + return jsonify( + [ + { + "id": t.id, + "name": t.name, + "kind": t.kind.value, + "fandom_id": t.fandom_id, + } + for t in tags + ] + ) + + +@tags_bp.route("/images//tags", methods=["POST"]) +async def add_tag_to_image(image_id: int): + body = await request.get_json() + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + source = body.get("source", "manual") + + async with get_session() as session: + svc = TagService(session) + await svc.add_to_image(image_id, body["tag_id"], source=source) + await session.commit() + return "", 204 + + +@tags_bp.route("/images//tags/", methods=["DELETE"]) +async def remove_tag_from_image(image_id: int, tag_id: int): + async with get_session() as session: + svc = TagService(session) + await svc.remove_from_image(image_id, tag_id) + await session.commit() + return "", 204 + + +@tags_bp.route("/tags/", methods=["PATCH"]) +async def rename_tag(tag_id: int): + body = await request.get_json() + if not body or "name" not in body: + return jsonify({"error": "name required"}), 400 + async with get_session() as session: + svc = TagService(session) + try: + tag = await svc.rename(tag_id, body["name"]) + except TagMergeConflict as exc: + return jsonify( + { + "error": str(exc), + "target": { + "id": exc.target_id, + "name": exc.target_name, + }, + "source_image_count": exc.source_image_count, + "will_alias": exc.will_alias, + } + ), 409 + except TagValidationError as exc: + return jsonify({"error": str(exc)}), 400 + await session.commit() + return jsonify( + {"id": tag.id, "name": tag.name, "kind": tag.kind.value} + ) + + +@tags_bp.route("/tags//merge", methods=["POST"]) +async def merge_tag(source_id: int): + body = await request.get_json() + if not body or "target_id" not in body: + return jsonify({"error": "target_id required"}), 400 + target_id = body["target_id"] + async with get_session() as session: + svc = TagService(session) + try: + result = await svc.merge(source_id, target_id) + except TagValidationError as exc: + msg = str(exc) + status = 404 if "not found" in msg else 400 + return jsonify({"error": msg}), status + await session.commit() + target_allowlisted = await session.scalar( + select(exists().where(TagAllowlist.tag_id == result.target_id)) + ) + if target_allowlisted: + from ..tasks.ml import apply_allowlist_tags + + apply_allowlist_tags.delay(tag_id=result.target_id) + return jsonify( + { + "target": { + "id": result.target_id, + "name": result.target_name, + "kind": result.target_kind, + }, + "merged_count": result.merged_count, + "alias_created": result.alias_created, + "source_deleted": result.source_deleted, + } + ) + + +@tags_bp.route("/images/common-tags", methods=["POST"]) +async def common_tags(): + body = await request.get_json() + ids, err = _parse_bulk_ids(body) + if err: + return err + async with get_session() as session: + tags = await BulkTagService(session).common_tags(ids) + return jsonify({"tags": tags}) + + +@tags_bp.route("/images/bulk/tags", methods=["POST"]) +async def bulk_add_tag(): + body = await request.get_json() + ids, err = _parse_bulk_ids(body) + if err: + return err + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + source = body.get("source", "manual") + if source not in ("manual", "ml_accepted"): + return jsonify({"error": f"invalid source {source!r}"}), 400 + async with get_session() as session: + if await session.get(Tag, body["tag_id"]) is None: + return jsonify({"error": "tag not found"}), 404 + try: + added = await BulkTagService(session).bulk_add( + ids, body["tag_id"], source=source + ) + except IntegrityError: + return jsonify( + {"error": "one or more image_ids do not exist"} + ), 400 + await session.commit() + return jsonify({"added_count": added, "total": len(ids)}) + + +@tags_bp.route("/images/bulk/tags/remove", methods=["POST"]) +async def bulk_remove_tag(): + body = await request.get_json() + ids, err = _parse_bulk_ids(body) + if err: + return err + if not body or "tag_id" not in body: + return jsonify({"error": "tag_id required"}), 400 + async with get_session() as session: + if await session.get(Tag, body["tag_id"]) is None: + return jsonify({"error": "tag not found"}), 404 + removed = await BulkTagService(session).bulk_remove( + ids, body["tag_id"] + ) + await session.commit() + return jsonify({"removed_count": removed}) + + +def _series_err(exc: SeriesError): + msg = str(exc) + status = 404 if ("not found" in msg or "not a series" in msg) else 400 + return jsonify({"error": msg}), status + + +@tags_bp.route("/series//pages", methods=["GET"]) +async def series_pages(tag_id: int): + async with get_session() as session: + try: + data = await SeriesService(session).list_pages(tag_id) + except SeriesError as exc: + return _series_err(exc) + return jsonify(data) + + +@tags_bp.route("/series//pages", methods=["POST"]) +async def series_add(tag_id: int): + body = await request.get_json() + ids, err = _parse_bulk_ids(body, max_ids=500) + if err: + return err + async with get_session() as session: + try: + n = await SeriesService(session).add_images(tag_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"added_count": n}) + + +@tags_bp.route("/series//pages/remove", methods=["POST"]) +async def series_remove(tag_id: int): + body = await request.get_json() + ids, err = _parse_bulk_ids(body, max_ids=500) + if err: + return err + async with get_session() as session: + try: + n = await SeriesService(session).remove_images(tag_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"removed_count": n}) + + +@tags_bp.route("/series//reorder", methods=["POST"]) +async def series_reorder(tag_id: int): + body = await request.get_json() + ids, err = _parse_bulk_ids(body, max_ids=500) + if err: + return err + async with get_session() as session: + try: + await SeriesService(session).reorder(tag_id, ids) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) + + +@tags_bp.route("/series//cover", methods=["POST"]) +async def series_cover(tag_id: int): + body = await request.get_json() + if not body or "image_id" not in body: + return jsonify({"error": "image_id required"}), 400 + try: + image_id = int(body["image_id"]) + except (TypeError, ValueError): + return jsonify({"error": "image_id must be an integer"}), 400 + async with get_session() as session: + try: + await SeriesService(session).set_cover(tag_id, image_id) + except SeriesError as exc: + return _series_err(exc) + await session.commit() + return jsonify({"ok": True}) diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py new file mode 100644 index 0000000..e5d91f2 --- /dev/null +++ b/backend/app/celery_app.py @@ -0,0 +1,90 @@ +"""Celery configuration with separate queue lanes. + +Queues: + import — filesystem import path (FC-2) + ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image) + thumbnail — image and video thumbnail generation (FC-2) + download — gallery-dl tasks (FC-3) + scan — periodic source checks (FC-3) — kept separate so long imports + don't starve the scheduler + maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3) + default — anything not explicitly routed +""" + +from celery import Celery + +from .config import get_config + + +def make_celery() -> Celery: + cfg = get_config() + app = Celery( + "fabledcurator", + broker=cfg.celery_broker_url, + backend=cfg.celery_result_backend, + include=[ + "backend.app.tasks.smoke", + "backend.app.tasks.scan", + "backend.app.tasks.import_file", + "backend.app.tasks.thumbnail", + "backend.app.tasks.maintenance", + "backend.app.tasks.migration", + "backend.app.tasks.ml", + "backend.app.tasks.download", + ], + ) + app.conf.update( + task_default_queue="default", + task_routes={ + "backend.app.tasks.import_file.*": {"queue": "import"}, + "backend.app.tasks.ml.*": {"queue": "ml"}, + "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"}, + "backend.app.tasks.download.*": {"queue": "download"}, + "backend.app.tasks.scan.*": {"queue": "scan"}, + "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, + "backend.app.tasks.migration.*": {"queue": "maintenance"}, + }, + # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. + task_acks_late=True, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + beat_schedule={ + "recover-interrupted-tasks": { + "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", + "schedule": 300.0, # every 5 minutes + }, + "cleanup-old-tasks": { + "task": "backend.app.tasks.maintenance.cleanup_old_tasks", + "schedule": 86400.0, # daily + }, + "ml-backfill-daily": { + "task": "backend.app.tasks.ml.backfill", + "schedule": 86400.0, + }, + "recompute-centroids-daily": { + "task": "backend.app.tasks.ml.recompute_centroids", + "schedule": 86400.0, + }, + "apply-allowlist-sweep-daily": { + "task": "backend.app.tasks.ml.apply_allowlist_tags", + "schedule": 86400.0, + }, + "integrity-verify-weekly": { + "task": "backend.app.tasks.maintenance.verify_integrity", + "schedule": 604800.0, # weekly + }, + "fc3d-tick-due-sources": { + "task": "backend.app.tasks.scan.tick_due_sources", + "schedule": 60.0, # every minute + }, + "fc3d-cleanup-download-events": { + "task": "backend.app.tasks.maintenance.cleanup_old_download_events", + "schedule": 86400.0, # daily + }, + }, + timezone="UTC", + ) + return app + + +celery = make_celery() diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..2338f25 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,52 @@ +"""Application configuration loaded from environment variables.""" + +import os +from dataclasses import dataclass +from functools import lru_cache + + +@dataclass(frozen=True) +class Config: + db_user: str + db_password: str + db_host: str + db_port: int + db_name: str + + celery_broker_url: str + celery_result_backend: str + + secret_key: str + extension_api_key: str # used by the Firefox extension; lands in FC-3 but read here + log_level: str + + @property + def database_url(self) -> str: + return ( + f"postgresql+asyncpg://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + @property + def database_url_sync(self) -> str: + # Alembic uses sync driver + return ( + f"postgresql+psycopg://{self.db_user}:{self.db_password}" + f"@{self.db_host}:{self.db_port}/{self.db_name}" + ) + + +@lru_cache(maxsize=1) +def get_config() -> Config: + return Config( + db_user=os.environ.get("DB_USER", "fabledcurator"), + db_password=os.environ["DB_PASSWORD"], + db_host=os.environ.get("DB_HOST", "postgres"), + db_port=int(os.environ.get("DB_PORT", "5432")), + db_name=os.environ.get("DB_NAME", "fabledcurator"), + celery_broker_url=os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"), + celery_result_backend=os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"), + secret_key=os.environ["SECRET_KEY"], + extension_api_key=os.environ.get("EXTENSION_API_KEY", ""), + log_level=os.environ.get("LOG_LEVEL", "INFO"), + ) diff --git a/backend/app/extensions.py b/backend/app/extensions.py new file mode 100644 index 0000000..d5af12f --- /dev/null +++ b/backend/app/extensions.py @@ -0,0 +1,52 @@ +"""Process-wide singleton async engine + session helper. + +Previously every API blueprint lazily built its own engine via make_engine(), +so the process ran N connection pools. This module owns ONE engine for the +whole app; create_app() disposes it on shutdown via after_serving. +""" + +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + async_sessionmaker, + create_async_engine, +) + +from .config import get_config + +_engine: AsyncEngine | None = None +_sessionmaker: async_sessionmaker | None = None + + +def get_engine() -> AsyncEngine: + global _engine, _sessionmaker + if _engine is None: + _engine = create_async_engine( + get_config().database_url, pool_pre_ping=True, future=True + ) + _sessionmaker = async_sessionmaker(_engine, expire_on_commit=False) + return _engine + + +def _get_sessionmaker() -> async_sessionmaker: + if _sessionmaker is None: + get_engine() + assert _sessionmaker is not None + return _sessionmaker + + +@asynccontextmanager +async def get_session(): + """`async with get_session() as session:` — one shared pool.""" + Session = _get_sessionmaker() + async with Session() as session: + yield session + + +async def dispose_engine() -> None: + global _engine, _sessionmaker + if _engine is not None: + await _engine.dispose() + _engine = None + _sessionmaker = None diff --git a/backend/app/frontend.py b/backend/app/frontend.py new file mode 100644 index 0000000..879a6c9 --- /dev/null +++ b/backend/app/frontend.py @@ -0,0 +1,17 @@ +"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback.""" + +from pathlib import Path + +from quart import Blueprint, send_from_directory + +FRONTEND_DIST = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist" + +frontend_bp = Blueprint("frontend", __name__) + + +@frontend_bp.route("/") +@frontend_bp.route("/") +async def serve_spa(subpath: str = ""): + if subpath and (FRONTEND_DIST / subpath).is_file(): + return await send_from_directory(FRONTEND_DIST, subpath) + return await send_from_directory(FRONTEND_DIST, "index.html") diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..0697c07 --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,49 @@ +"""All ORM models. Import this module to make every model visible to Alembic.""" + +from .app_setting import AppSetting +from .artist import Artist +from .base import Base +from .credential import Credential +from .download_event import DownloadEvent +from .image_provenance import ImageProvenance +from .image_record import ImageRecord +from .import_batch import ImportBatch +from .import_settings import ImportSettings +from .import_task import ImportTask +from .migration_run import MigrationRun +from .ml_settings import MLSettings +from .post import Post +from .post_attachment import PostAttachment +from .series_page import SeriesPage +from .source import Source +from .tag import Tag, TagKind, image_tag +from .tag_alias import TagAlias +from .tag_allowlist import TagAllowlist +from .tag_reference_embedding import TagReferenceEmbedding +from .tag_suggestion_rejection import TagSuggestionRejection + +__all__ = [ + "Base", + "AppSetting", + "Artist", + "Source", + "Credential", + "Post", + "PostAttachment", + "SeriesPage", + "ImageRecord", + "ImageProvenance", + "Tag", + "TagKind", + "image_tag", + "DownloadEvent", + "ImportBatch", + "ImportTask", + "ImportSettings", + "MLSettings", + "MigrationRun", + "TagAlias", + "TagAllowlist", + "TagReferenceEmbedding", + "TagSuggestionRejection", +] diff --git a/backend/app/models/app_setting.py b/backend/app/models/app_setting.py new file mode 100644 index 0000000..771843d --- /dev/null +++ b/backend/app/models/app_setting.py @@ -0,0 +1,24 @@ +"""AppSetting — small key/value app config (extension API key, etc.). + +For settings that need DB persistence + runtime mutation. Use +`ImportSettings` for the structured import-tuning settings; this +table is for one-off scalar values keyed by string. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class AppSetting(Base): + __tablename__ = "app_setting" + + key: Mapped[str] = mapped_column(String(64), primary_key=True) + value: Mapped[str] = mapped_column(Text, nullable=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, + server_default=func.now(), onupdate=func.now(), + ) diff --git a/backend/app/models/artist.py b/backend/app/models/artist.py new file mode 100644 index 0000000..d6847f4 --- /dev/null +++ b/backend/app/models/artist.py @@ -0,0 +1,33 @@ +"""Artist — unified entity that is both the gallery's ``artist:`` tag concept +and GallerySubscriber's Subscription. ``is_subscription`` is True if any +Sources are attached. +""" + +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class Artist(Base): + __tablename__ = "artist" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + notes: Mapped[str | None] = mapped_column(Text, nullable=True) + + # True once a Source is attached; flips false if all sources removed. + is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + + # Per-artist scheduling overrides; null means "use global default". + auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + sources = relationship("Source", back_populates="artist", cascade="all, delete-orphan") diff --git a/backend/app/models/base.py b/backend/app/models/base.py new file mode 100644 index 0000000..9fd456b --- /dev/null +++ b/backend/app/models/base.py @@ -0,0 +1,17 @@ +"""SQLAlchemy declarative base with standardized constraint naming.""" + +from sqlalchemy import MetaData +from sqlalchemy.orm import DeclarativeBase + +# Stable constraint names so Alembic autogeneration produces clean diffs. +NAMING_CONVENTION = { + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + + +class Base(DeclarativeBase): + metadata = MetaData(naming_convention=NAMING_CONVENTION) diff --git a/backend/app/models/credential.py b/backend/app/models/credential.py new file mode 100644 index 0000000..2f07732 --- /dev/null +++ b/backend/app/models/credential.py @@ -0,0 +1,29 @@ +"""Credential — encrypted blob (cookies/token) scoped per-platform, +not per-source. One Patreon credential serves every Patreon source. + +Schema aligned with GallerySubscriber's wire format (FC-3b) so the +existing browser extension hits FC unmodified. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, LargeBinary, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class Credential(Base): + __tablename__ = "credential" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + platform: Mapped[str] = mapped_column(String(64), nullable=False, unique=True) + credential_type: Mapped[str] = mapped_column(String(32), nullable=False) # cookies|token + encrypted_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + captured_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_verified: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) diff --git a/backend/app/models/download_event.py b/backend/app/models/download_event.py new file mode 100644 index 0000000..7fe5b37 --- /dev/null +++ b/backend/app/models/download_event.py @@ -0,0 +1,34 @@ +"""DownloadEvent — log of every gallery-dl run, populated by FC-3.""" + +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class DownloadEvent(Base): + __tablename__ = "download_event" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error|skipped + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + metadata_: Mapped[dict] = mapped_column( + "metadata", JSONB, nullable=False, default=dict, + server_default=sa.text("'{}'::jsonb"), + ) diff --git a/backend/app/models/image_provenance.py b/backend/app/models/image_provenance.py new file mode 100644 index 0000000..cd50887 --- /dev/null +++ b/backend/app/models/image_provenance.py @@ -0,0 +1,33 @@ +"""ImageProvenance — links an ImageRecord to a Post. + +Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate +rule (spec §3): when a downloaded image is a pHash dupe of an existing +record, we append a new provenance row to the existing record rather than +dropping the metadata. +""" + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class ImageProvenance(Base): + __tablename__ = "image_provenance" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + image_record_id: Mapped[int] = mapped_column( + ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True + ) + post_id: Mapped[int] = mapped_column( + ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + captured_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py new file mode 100644 index 0000000..f43fd60 --- /dev/null +++ b/backend/app/models/image_record.py @@ -0,0 +1,82 @@ +"""ImageRecord — the gallery's primary entity, ported from ImageRepo. + +ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them +without a schema migration. The SigLIP embedding column uses pgvector's Vector +type — pgvector extension is enabled in the initial migration. +""" + +from datetime import datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import ( + JSON, + BigInteger, + DateTime, + Enum, + ForeignKey, + Integer, + String, + Text, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + +ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded") + + +class ImageRecord(Base): + __tablename__ = "image_record" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + + # On-disk identity + path: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + mime: Mapped[str] = mapped_column(String(64), nullable=False) + width: Mapped[int | None] = mapped_column(Integer, nullable=True) + height: Mapped[int | None] = mapped_column(Integer, nullable=True) + + # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'. + # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'. + integrity_status: Mapped[str] = mapped_column( + String(24), nullable=False, default="unknown", index=True + ) + + # Thumbnail (populated by FC-2) + thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True) + + # Origin / provenance pointers + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False) + primary_post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + # FC-2d-vii-c: canonical per-image artist (the single source of truth + # for attribution; provenance posts remain lineage detail). + artist_id: Mapped[int | None] = mapped_column( + ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # ML fields (populated by FC-2's ml-worker) + tagger_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True) + tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + # 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require + # a column-width migration. + siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True) + siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True) + + # Centroid score cache (populated post-tagging) + centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/models/import_batch.py b/backend/app/models/import_batch.py new file mode 100644 index 0000000..d26db4c --- /dev/null +++ b/backend/app/models/import_batch.py @@ -0,0 +1,33 @@ +"""ImportBatch — one scan run, aggregating many ImportTasks.""" + +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class ImportBatch(Base): + __tablename__ = "import_batch" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + triggered_by: Mapped[str] = mapped_column(String(32), nullable=False) + source_path: Mapped[str] = mapped_column(Text, nullable=False) + scan_mode: Mapped[str] = mapped_column(String(16), nullable=False) # quick|deep + + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True) + # running | complete | cancelled + + tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan") diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py new file mode 100644 index 0000000..fca7f88 --- /dev/null +++ b/backend/app/models/import_settings.py @@ -0,0 +1,51 @@ +"""ImportSettings — single-row table holding the importer's tunable knobs. + +Enforced as a single row via a CHECK (id = 1) constraint. The application +always SELECTs id=1 and never inserts/deletes after the initial migration. +""" + +from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class ImportSettings(Base): + __tablename__ = "import_settings" + # Bare constraint name — Base.metadata's naming convention applies the + # ck__ prefix, producing the final ck_import_settings_singleton. + __table_args__ = (CheckConstraint("id = 1", name="singleton"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import") + + min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9) + + skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) + single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30) + + phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + + # FC-3c downloader knobs + download_rate_limit_seconds: Mapped[float] = mapped_column( + Float, nullable=False, default=3.0 + ) + download_validate_files: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True + ) + + # FC-3d scheduling knobs + download_schedule_default_seconds: Mapped[int] = mapped_column( + Integer, nullable=False, default=28800 + ) + download_event_retention_days: Mapped[int] = mapped_column( + Integer, nullable=False, default=90 + ) + download_failure_warning_threshold: Mapped[int] = mapped_column( + Integer, nullable=False, default=5 + ) diff --git a/backend/app/models/import_task.py b/backend/app/models/import_task.py new file mode 100644 index 0000000..da3d4ec --- /dev/null +++ b/backend/app/models/import_task.py @@ -0,0 +1,41 @@ +"""ImportTask — a single source file to import as part of a batch. + +State machine: pending -> queued -> processing -> complete | skipped | failed. +Workers crashing mid-task leave rows in 'processing'; the recovery sweep +(tasks/maintenance.py::recover_interrupted_tasks) re-queues rows that have +been processing longer than the stuck-task threshold. +""" + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class ImportTask(Base): + __tablename__ = "import_task" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + batch_id: Mapped[int] = mapped_column( + ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True + ) + + source_path: Mapped[str] = mapped_column(Text, nullable=False) + task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive + status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True) + + result_image_id: Mapped[int | None] = mapped_column( + ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True + ) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + batch = relationship("ImportBatch", back_populates="tasks") diff --git a/backend/app/models/migration_run.py b/backend/app/models/migration_run.py new file mode 100644 index 0000000..bdb5fed --- /dev/null +++ b/backend/app/models/migration_run.py @@ -0,0 +1,37 @@ +"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc). + +kind/status are String(32) not Postgres ENUM so adding kinds later +doesn't need a schema migration. The API layer validates values. +""" + +from datetime import datetime + +import sqlalchemy as sa +from sqlalchemy import Boolean, DateTime, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class MigrationRun(Base): + __tablename__ = "migration_run" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(32), nullable=False, index=True) + dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + started_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + finished_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True, + ) + counts: Mapped[dict] = mapped_column( + JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"), + ) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + metadata_: Mapped[dict] = mapped_column( + "metadata", JSONB, nullable=False, default=dict, + server_default=sa.text("'{}'::jsonb"), + ) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py new file mode 100644 index 0000000..5ad33ad --- /dev/null +++ b/backend/app/models/ml_settings.py @@ -0,0 +1,44 @@ +"""MLSettings — single-row table holding ML pipeline tunables.""" + +from datetime import datetime + +from sqlalchemy import CheckConstraint, DateTime, Float, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class MLSettings(Base): + __tablename__ = "ml_settings" + # Bare name — Base.metadata's naming convention prepends ck_
_, + # producing the final ck_ml_settings_singleton (matches migration 0003). + __table_args__ = (CheckConstraint("id = 1", name="singleton"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + suggestion_threshold_artist: Mapped[float] = mapped_column( + Float, nullable=False, default=0.30 + ) + suggestion_threshold_character: Mapped[float] = mapped_column( + Float, nullable=False, default=0.50 + ) + suggestion_threshold_copyright: Mapped[float] = mapped_column( + Float, nullable=False, default=0.50 + ) + suggestion_threshold_general: Mapped[float] = mapped_column( + Float, nullable=False, default=0.95 + ) + centroid_similarity_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.55 + ) + min_reference_images: Mapped[int] = mapped_column( + Integer, nullable=False, default=5 + ) + tagger_model_version: Mapped[str] = mapped_column( + String(128), nullable=False, default="camie-tagger-v2" + ) + embedder_model_version: Mapped[str] = mapped_column( + String(128), nullable=False, default="siglip-so400m-patch14-384" + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/post.py b/backend/app/models/post.py new file mode 100644 index 0000000..d5263c9 --- /dev/null +++ b/backend/app/models/post.py @@ -0,0 +1,36 @@ +"""Post — provenance anchor for content downloaded from a Source. + +A Post is one creator post; it may contain many images/videos. +""" + +from datetime import datetime + +from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class Post(Base): + __tablename__ = "post" + __table_args__ = ( + UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True + ) + external_post_id: Mapped[str] = mapped_column(String(128), nullable=False) + post_url: Mapped[str | None] = mapped_column(Text, nullable=True) + post_title: Mapped[str | None] = mapped_column(Text, nullable=True) + post_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + raw_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + description: Mapped[str | None] = mapped_column(Text, nullable=True) + attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True) + + downloaded_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/post_attachment.py b/backend/app/models/post_attachment.py new file mode 100644 index 0000000..4fea125 --- /dev/null +++ b/backend/app/models/post_attachment.py @@ -0,0 +1,47 @@ +"""PostAttachment — a non-art file preserved from a post. + +Art images become ImageRecords; everything else a post contained +(archives, .exe, .pdf, ...) is captured here so nothing is lost. +post_id is nullable (set only when an adjacent sidecar yields a Post); +artist_id mirrors the canonical attribution model (FC-2d-vii-c). Both +FKs are SET NULL so deleting a Post/Artist never deletes the preserved +binary row. +""" + +from datetime import datetime + +from sqlalchemy import ( + BigInteger, + DateTime, + ForeignKey, + Integer, + String, + Text, + func, +) +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class PostAttachment(Base): + __tablename__ = "post_attachment" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + post_id: Mapped[int | None] = mapped_column( + ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True + ) + artist_id: Mapped[int | None] = mapped_column( + ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True + ) + sha256: Mapped[str] = mapped_column( + String(64), nullable=False, unique=True, index=True + ) + path: Mapped[str] = mapped_column(Text, nullable=False) + original_filename: Mapped[str] = mapped_column(Text, nullable=False) + ext: Mapped[str] = mapped_column(String(32), nullable=False) + mime: Mapped[str | None] = mapped_column(String(128), nullable=True) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + captured_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/series_page.py b/backend/app/models/series_page.py new file mode 100644 index 0000000..1ebcd7d --- /dev/null +++ b/backend/app/models/series_page.py @@ -0,0 +1,38 @@ +"""SeriesPage — ordered image membership for a series-kind Tag. + +A series IS a Tag with kind='series'; series_page gives it ordered pages. +An image belongs to at most one series (UNIQUE image_id). Cover = the +lowest page_number. page_number is an ordering key only (not unique) — +reorder rewrites 1..N wholesale. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Integer, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class SeriesPage(Base): + __tablename__ = "series_page" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + series_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + image_id: Mapped[int] = mapped_column( + ForeignKey("image_record.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + page_number: Mapped[int] = mapped_column(Integer, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + onupdate=func.now(), + ) diff --git a/backend/app/models/source.py b/backend/app/models/source.py new file mode 100644 index 0000000..9c90e81 --- /dev/null +++ b/backend/app/models/source.py @@ -0,0 +1,32 @@ +"""Source — a platform-specific URL owned by an Artist (e.g., a Patreon URL). + +Multiple sources per artist support creators with cross-platform presence. +""" + +from datetime import datetime + +from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class Source(Base): + __tablename__ = "source" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + artist_id: Mapped[int] = mapped_column( + ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True + ) + platform: Mapped[str] = mapped_column(String(64), nullable=False) + url: Mapped[str] = mapped_column(Text, nullable=False) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + + config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) + + last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) + consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + artist = relationship("Artist", back_populates="sources") diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py new file mode 100644 index 0000000..c99fc06 --- /dev/null +++ b/backend/app/models/tag.py @@ -0,0 +1,81 @@ +"""Tag + ImageTag association. + +Tags carry a ``kind`` enum and (for kind='character') an optional +``fandom_id`` FK to another tag of kind='fandom'. The uniqueness key is +(name, kind, COALESCE(fandom_id, 0)) so the same name can exist across +different kinds and the same character name can exist in different fandoms. +""" + +from datetime import datetime +from enum import StrEnum + +from sqlalchemy import ( + CheckConstraint, + Column, + DateTime, + ForeignKey, + Integer, + String, + Table, + func, +) +from sqlalchemy import ( + Enum as SQLEnum, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from .base import Base + + +class TagKind(StrEnum): + artist = "artist" + character = "character" + fandom = "fandom" + general = "general" + series = "series" + archive = "archive" + post = "post" + meta = "meta" + rating = "rating" + + +image_tag = Table( + "image_tag", + Base.metadata, + Column( + "image_record_id", + ForeignKey("image_record.id", ondelete="CASCADE"), + primary_key=True, + ), + Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), + Column("source", String(32), nullable=False, default="manual"), + Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), +) + + +class Tag(Base): + __tablename__ = "tag" + __table_args__ = ( + CheckConstraint( + "(fandom_id IS NULL) OR (kind = 'character')", + name="ck_tag_fandom_requires_character", + ), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + kind: Mapped[TagKind] = mapped_column( + SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]), + nullable=False, + default=TagKind.general, + ) + fandom_id: Mapped[int | None] = mapped_column( + ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True + ) + + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + images = relationship("ImageRecord", secondary=image_tag, backref="tags") + fandom = relationship("Tag", remote_side="Tag.id", foreign_keys=[fandom_id]) diff --git a/backend/app/models/tag_alias.py b/backend/app/models/tag_alias.py new file mode 100644 index 0000000..3a21375 --- /dev/null +++ b/backend/app/models/tag_alias.py @@ -0,0 +1,24 @@ +"""TagAlias — maps a model's (name, category) prediction to the operator's +canonical tag. Resolved at suggestion-read time so raw predictions stay +unmolested in image_record.tagger_predictions. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class TagAlias(Base): + __tablename__ = "tag_alias" + + alias_string: Mapped[str] = mapped_column(String(255), primary_key=True) + alias_category: Mapped[str] = mapped_column(String(32), primary_key=True) + canonical_tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/tag_allowlist.py b/backend/app/models/tag_allowlist.py new file mode 100644 index 0000000..75bd0f2 --- /dev/null +++ b/backend/app/models/tag_allowlist.py @@ -0,0 +1,28 @@ +"""TagAllowlist — tags the operator opted-in to auto-apply via maintenance.""" + +from datetime import datetime + +from sqlalchemy import CheckConstraint, DateTime, Float, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class TagAllowlist(Base): + __tablename__ = "tag_allowlist" + # Bare name — Base.metadata's naming convention prepends ck_
_, + # producing the final ck_tag_allowlist_confidence_range (matches migration 0003). + __table_args__ = ( + CheckConstraint( + "min_confidence > 0 AND min_confidence <= 1", + name="confidence_range", + ), + ) + + tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True + ) + min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) + added_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/tag_reference_embedding.py b/backend/app/models/tag_reference_embedding.py new file mode 100644 index 0000000..2dc841b --- /dev/null +++ b/backend/app/models/tag_reference_embedding.py @@ -0,0 +1,23 @@ +"""TagReferenceEmbedding — per-tag centroid (mean SigLIP embedding of members).""" + +from datetime import datetime + +from pgvector.sqlalchemy import Vector +from sqlalchemy import DateTime, ForeignKey, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class TagReferenceEmbedding(Base): + __tablename__ = "tag_reference_embedding" + + tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True + ) + embedding: Mapped[list[float]] = mapped_column(Vector(1152), nullable=False) + reference_count: Mapped[int] = mapped_column(Integer, nullable=False) + model_version: Mapped[str] = mapped_column(String(128), nullable=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/models/tag_suggestion_rejection.py b/backend/app/models/tag_suggestion_rejection.py new file mode 100644 index 0000000..8a834fe --- /dev/null +++ b/backend/app/models/tag_suggestion_rejection.py @@ -0,0 +1,25 @@ +"""TagSuggestionRejection — per-image dismissed suggestions. + +Prevents re-suggestion AND prevents allowlist auto-apply on that image. +""" + +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, func +from sqlalchemy.orm import Mapped, mapped_column + +from .base import Base + + +class TagSuggestionRejection(Base): + __tablename__ = "tag_suggestion_rejection" + + image_record_id: Mapped[int] = mapped_column( + ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True + ) + tag_id: Mapped[int] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True + ) + rejected_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) diff --git a/backend/app/scripts/__init__.py b/backend/app/scripts/__init__.py new file mode 100644 index 0000000..0d3c54b --- /dev/null +++ b/backend/app/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational scripts run from the container entrypoint or CLI.""" diff --git a/backend/app/scripts/download_models.py b/backend/app/scripts/download_models.py new file mode 100644 index 0000000..39131a1 --- /dev/null +++ b/backend/app/scripts/download_models.py @@ -0,0 +1,53 @@ +"""Self-heal model weights into /models. Idempotent — only missing files +are fetched. Called by the ml-worker entrypoint before Celery starts. +""" + +import os +import sys +from pathlib import Path + +MODEL_ROOT = Path(os.environ.get("ML_MODEL_DIR", "/models")) +CAMIE_REPO = os.environ.get("CAMIE_HF_REPO", "Camais03/camie-tagger-v2") +SIGLIP_REPO = os.environ.get( + "SIGLIP_HF_REPO", "google/siglip-so400m-patch14-384" +) + + +def _snapshot(repo_id: str, dest: Path, allow_patterns: list[str] | None) -> None: + from huggingface_hub import snapshot_download + + dest.mkdir(parents=True, exist_ok=True) + snapshot_download( + repo_id=repo_id, + local_dir=str(dest), + allow_patterns=allow_patterns, + ) + + +def ensure_camie() -> None: + dest = MODEL_ROOT / "camie" + if (dest / "model.onnx").is_file() and (dest / "selected_tags.csv").is_file(): + print(f"[download_models] Camie present at {dest}") + return + print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}") + _snapshot(CAMIE_REPO, dest, ["model.onnx", "selected_tags.csv", "*.json"]) + + +def ensure_siglip() -> None: + dest = MODEL_ROOT / "siglip" + if (dest / "config.json").is_file() and any(dest.glob("*.safetensors")): + print(f"[download_models] SigLIP present at {dest}") + return + print(f"[download_models] Fetching {SIGLIP_REPO} -> {dest}") + _snapshot(SIGLIP_REPO, dest, None) + + +def main() -> int: + ensure_camie() + ensure_siglip() + print("[download_models] Done.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..535d24d --- /dev/null +++ b/backend/app/services/__init__.py @@ -0,0 +1 @@ +"""Business-logic services consumed by API blueprints and Celery tasks.""" diff --git a/backend/app/services/archive_extractor.py b/backend/app/services/archive_extractor.py new file mode 100644 index 0000000..45babef --- /dev/null +++ b/backend/app/services/archive_extractor.py @@ -0,0 +1,57 @@ +"""Best-effort archive member extraction (filesystem-import aid). + +zip/cbz via stdlib zipfile; rar via rarfile (needs an unrar/bsdtar +binary); 7z via py7zr (pure-Python). NEVER raises: on any failure the +context yields an empty list, and the caller still preserves the archive +file itself as a PostAttachment, so nothing is lost. +""" + +import logging +import zipfile +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory + +log = logging.getLogger(__name__) + +ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"} + + +def is_archive(path: Path) -> bool: + return Path(path).suffix.lower() in ARCHIVE_EXTS + + +@contextmanager +def extract_archive(path: Path): + """Context manager yielding a list of (member_name, extracted_path) + for every regular-file member. Temp dir is removed on exit. Members + are only ever under the temp dir (libs' safe extractall + rglob from + base). Fail-soft: any error → yields [].""" + tmp = TemporaryDirectory(prefix="fc_arch_") + base = Path(tmp.name) + members: list[tuple[str, Path]] = [] + try: + try: + ext = Path(path).suffix.lower() + if ext in (".zip", ".cbz"): + with zipfile.ZipFile(path) as zf: + zf.extractall(base) + elif ext == ".rar": + import rarfile + + with rarfile.RarFile(path) as rf: + rf.extractall(base) + elif ext == ".7z": + import py7zr + + with py7zr.SevenZipFile(path, "r") as zf: + zf.extractall(base) + for p in sorted(base.rglob("*")): + if p.is_file(): + members.append((str(p.relative_to(base)), p)) + except Exception as exc: + log.warning("archive extract failed for %s: %s", path, exc) + members = [] + yield members + finally: + tmp.cleanup() diff --git a/backend/app/services/artist_directory_service.py b/backend/app/services/artist_directory_service.py new file mode 100644 index 0000000..a16c1aa --- /dev/null +++ b/backend/app/services/artist_directory_service.py @@ -0,0 +1,143 @@ +"""FC-3f: cursor-paginated artists directory. + +Mirrors TagDirectoryService surface-for-surface, but backed by +Artist + Source + ImageRecord.artist_id (FC-2d-vii-c authoritative +attribution). Pure read-surface; no writes. + +Cursor wire format is identical to tag_directory_service: base64 of +"|". The _encode/_decode helpers are copied (not imported) +to keep the two services decoupled. +""" +from __future__ import annotations + +import base64 +from dataclasses import dataclass + +from sqlalchemy import and_, exists, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, ImageRecord, Source +from .gallery_service import thumbnail_url + +_SEP = "|" +_PREVIEW_COUNT = 3 + + +def _encode(name: str, artist_id: int) -> str: + return base64.urlsafe_b64encode(f"{name}{_SEP}{artist_id}".encode()).decode() + + +def _decode(cursor: str) -> tuple[str, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + name, aid = raw.rsplit(_SEP, 1) + return name, int(aid) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class DirectoryPage: + cards: list[dict] + next_cursor: str | None + + +class ArtistDirectoryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def list_artists( + self, + *, + q: str | None, + platform: str | None, + cursor: str | None, + limit: int = 60, + ) -> DirectoryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + + count_col = func.count(ImageRecord.id).label("image_count") + stmt = ( + select(Artist, count_col) + .outerjoin(ImageRecord, ImageRecord.artist_id == Artist.id) + .group_by(Artist.id) + ) + if q: + stmt = stmt.where(Artist.name.ilike(f"%{q}%")) + if platform is not None: + # Correlated EXISTS — NOT a JOIN. A JOIN to Source duplicates + # the artist row when the artist has multiple sources on this + # platform, breaking the keyset cursor and image_count. + stmt = stmt.where( + exists().where( + and_( + Source.artist_id == Artist.id, + Source.platform == platform, + ) + ) + ) + if cursor: + c_name, c_id = _decode(cursor) + stmt = stmt.where( + or_( + Artist.name > c_name, + and_(Artist.name == c_name, Artist.id > c_id), + ) + ) + stmt = stmt.order_by(Artist.name.asc(), Artist.id.asc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() + + next_cursor: str | None = None + if len(rows) > limit: + last_artist = rows[limit - 1][0] + next_cursor = _encode(last_artist.name, last_artist.id) + rows = rows[:limit] + + artist_ids = [a.id for a, _ in rows] + previews = await self._previews(artist_ids) + + cards = [ + { + "id": artist.id, + "name": artist.name, + "slug": artist.slug, + "is_subscription": bool(artist.is_subscription), + "image_count": int(image_count), + "preview_thumbnails": previews.get(artist.id, []), + } + for artist, image_count in rows + ] + return DirectoryPage(cards=cards, next_cursor=next_cursor) + + async def _previews(self, artist_ids: list[int]) -> dict[int, list[str]]: + """artist_id -> [thumbnail_url, ...up to _PREVIEW_COUNT]. + + Window function ranks ImageRecord rows per artist by id DESC + (newest first) and slices to the top _PREVIEW_COUNT in one query. + """ + if not artist_ids: + return {} + rn = func.row_number().over( + partition_by=ImageRecord.artist_id, + order_by=ImageRecord.id.desc(), + ).label("rn") + sub = ( + select( + ImageRecord.artist_id.label("artist_id"), + ImageRecord.sha256.label("sha256"), + ImageRecord.mime.label("mime"), + rn, + ) + .where(ImageRecord.artist_id.in_(artist_ids)) + .subquery() + ) + stmt = ( + select(sub.c.artist_id, sub.c.sha256, sub.c.mime) + .where(sub.c.rn <= _PREVIEW_COUNT) + .order_by(sub.c.artist_id, sub.c.rn) + ) + out: dict[int, list[str]] = {} + for aid, sha, mime in (await self.session.execute(stmt)).all(): + out.setdefault(aid, []).append(thumbnail_url(sha, mime)) + return out diff --git a/backend/app/services/artist_service.py b/backend/app/services/artist_service.py new file mode 100644 index 0000000..271778e --- /dev/null +++ b/backend/app/services/artist_service.py @@ -0,0 +1,242 @@ +"""Artist overview + paged images. + +Images are linked to an artist through the provenance chain +Source(artist_id) -> ImageProvenance(source_id) -> ImageRecord. DISTINCT +guards against an image having several provenance rows for one artist. +Dates come from Post.post_date via ImageProvenance.post_id. +""" + +from dataclasses import dataclass + +from sqlalchemy import and_, case, func, or_, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, + Tag, +) +from ..models.tag import image_tag +from ..utils.slug import slugify +from .gallery_service import decode_cursor, encode_cursor, thumbnail_url + + +@dataclass(frozen=True) +class ArtistImagesPage: + images: list[dict] + next_cursor: str | None + + +class ArtistService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _artist_by_slug(self, slug: str) -> Artist | None: + return ( + await self.session.execute( + select(Artist).where(Artist.slug == slug) + ) + ).scalar_one_or_none() + + async def overview(self, slug: str) -> dict | None: + artist = await self._artist_by_slug(slug) + if artist is None: + return None + aid = artist.id + + img_join = ( + select(ImageRecord.id).where(ImageRecord.artist_id == aid) + ) + image_count = ( + await self.session.execute( + select(func.count(ImageRecord.id)) + .where(ImageRecord.artist_id == aid) + ) + ).scalar_one() + + date_row = ( + await self.session.execute( + select(func.min(Post.post_date), func.max(Post.post_date)) + .select_from(Post) + .join(ImageProvenance, ImageProvenance.post_id == Post.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(Source.artist_id == aid) + ) + ).first() + dmin, dmax = date_row if date_row else (None, None) + + cooccurring = ( + await self.session.execute( + select( + Tag.id, Tag.name, Tag.kind, + func.count(image_tag.c.image_record_id).label("cnt"), + ) + .select_from(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id.in_(img_join)) + .group_by(Tag.id, Tag.name, Tag.kind) + .order_by(func.count(image_tag.c.image_record_id).desc()) + .limit(20) + ) + ).all() + + sources = ( + await self.session.execute( + select( + Source.id, Source.platform, Source.url, + func.count(func.distinct(ImageProvenance.image_record_id)).label("cnt"), + ) + .select_from(Source) + .outerjoin(ImageProvenance, ImageProvenance.source_id == Source.id) + .where(Source.artist_id == aid) + .group_by(Source.id, Source.platform, Source.url) + .order_by(Source.id) + ) + ).all() + + month = func.date_trunc("month", Post.post_date).label("m") + activity = ( + await self.session.execute( + select(month, func.count(func.distinct(ImageProvenance.image_record_id))) + .select_from(Post) + .join(ImageProvenance, ImageProvenance.post_id == Post.id) + .join(Source, Source.id == ImageProvenance.source_id) + .where(and_(Source.artist_id == aid, Post.post_date.isnot(None))) + .group_by(month) + .order_by(month) + ) + ).all() + + return { + "id": artist.id, + "name": artist.name, + "slug": artist.slug, + "is_subscription": bool(artist.is_subscription), + "image_count": int(image_count), + "date_range": { + "min": dmin.isoformat() if dmin else None, + "max": dmax.isoformat() if dmax else None, + }, + "cooccurring_tags": [ + { + "id": tid, + "name": name, + "kind": kind.value if hasattr(kind, "value") else kind, + "count": int(cnt), + } + for tid, name, kind, cnt in cooccurring + ], + "sources": [ + { + "id": sid, + "platform": platform, + "url": url, + "image_count": int(cnt), + } + for sid, platform, url, cnt in sources + ], + "activity": [ + {"month": m.isoformat(), "count": int(cnt)} + for m, cnt in activity + ], + } + + async def images( + self, slug: str, cursor: str | None, limit: int = 60 + ) -> ArtistImagesPage | None: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + artist = await self._artist_by_slug(slug) + if artist is None: + return None + + # Dedupe via IN-subquery rather than JOIN + DISTINCT: an image can + # have several provenance rows for one artist, and SELECT DISTINCT + # over ImageRecord fails in Postgres because its json columns have + # no equality operator. + stmt = select(ImageRecord).where( + ImageRecord.artist_id == artist.id + ) + if cursor: + cur_ts, cur_id = decode_cursor(cursor) + stmt = stmt.where( + or_( + ImageRecord.created_at < cur_ts, + and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id), + ) + ) + stmt = stmt.order_by( + ImageRecord.created_at.desc(), ImageRecord.id.desc() + ).limit(limit + 1) + rows = (await self.session.execute(stmt)).scalars().all() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = encode_cursor(last.created_at, last.id) + rows = rows[:limit] + + return ArtistImagesPage( + images=[ + { + "id": r.id, + "sha256": r.sha256, + "mime": r.mime, + "width": r.width, + "height": r.height, + "thumbnail_url": thumbnail_url(r.sha256, r.mime), + } + for r in rows + ], + next_cursor=next_cursor, + ) + + async def find_or_create(self, name: str) -> tuple[Artist, bool]: + """Return (artist, created). Slug-keyed; idempotent under races.""" + cleaned = (name or "").strip() + if not cleaned: + raise ValueError("artist name must not be empty") + slug = slugify(cleaned) + + existing = (await self.session.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one_or_none() + if existing is not None: + return existing, False + + artist = Artist(name=cleaned, slug=slug) + self.session.add(artist) + try: + await self.session.flush() + except IntegrityError: + await self.session.rollback() + existing = (await self.session.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one() + return existing, False + await self.session.commit() + return artist, True + + async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]: + cleaned = (prefix or "").strip() + if not cleaned: + return [] + like = f"%{cleaned.lower()}%" + prefix_like = f"{cleaned.lower()}%" + # Rank: exact (0) < prefix (1) < substring (2). + rank = case( + (func.lower(Artist.name) == cleaned.lower(), 0), + (func.lower(Artist.name).like(prefix_like), 1), + else_=2, + ).label("rank") + rows = (await self.session.execute( + select(Artist, rank) + .where(func.lower(Artist.name).like(like)) + .order_by(rank, Artist.name.asc()) + .limit(limit) + )).all() + return [a for a, _ in rows] diff --git a/backend/app/services/attachment_store.py b/backend/app/services/attachment_store.py new file mode 100644 index 0000000..6b29c24 --- /dev/null +++ b/backend/app/services/attachment_store.py @@ -0,0 +1,27 @@ +"""sha-addressed store for preserved non-art post files. + +Arbitrary binaries — never opened or validated as images. Parallels the +thumbnail store layout (/images/attachments//). +""" + +import shutil +from pathlib import Path + + +class AttachmentStore: + def __init__(self, images_root: Path): + self.root = Path(images_root) / "attachments" + + def store(self, src: Path, sha256: str) -> str: + """Copy src into the sha-addressed store; idempotent on sha. + Returns the stored absolute path as a string.""" + ext = Path(src).suffix.lower() + dest_dir = self.root / sha256[:3] + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / f"{sha256}{ext}" + if dest.exists(): + return str(dest) + partial = dest.with_suffix(dest.suffix + ".partial") + shutil.copy2(src, partial) + partial.rename(dest) + return str(dest) diff --git a/backend/app/services/bulk_tag_service.py b/backend/app/services/bulk_tag_service.py new file mode 100644 index 0000000..b573ef9 --- /dev/null +++ b/backend/app/services/bulk_tag_service.py @@ -0,0 +1,93 @@ +"""Bulk tag operations over a set of images (Core, set-based, atomic).""" + +from sqlalchemy import and_, func, select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Tag +from ..models.tag import image_tag +from ..models.tag_suggestion_rejection import TagSuggestionRejection + + +class BulkTagService: + def __init__(self, session: AsyncSession): + self.session = session + + async def common_tags(self, image_ids: list[int]) -> list[dict]: + """Tags present on EVERY image in image_ids.""" + if not image_ids: + return [] + n = len(set(image_ids)) + stmt = ( + select(Tag.id, Tag.name, Tag.kind) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id.in_(image_ids)) + .group_by(Tag.id, Tag.name, Tag.kind) + .having( + func.count(func.distinct(image_tag.c.image_record_id)) == n + ) + .order_by(Tag.name.asc()) + ) + rows = (await self.session.execute(stmt)).all() + return [ + { + "id": r.id, + "name": r.name, + "kind": r.kind.value if hasattr(r.kind, "value") else r.kind, + } + for r in rows + ] + + async def bulk_add( + self, image_ids: list[int], tag_id: int, source: str = "manual" + ) -> int: + """Apply tag_id to every image. Idempotent per (image, tag). + Returns the number of links actually created.""" + if not image_ids: + return 0 + stmt = ( + pg_insert(image_tag) + .values( + [ + { + "image_record_id": iid, + "tag_id": tag_id, + "source": source, + } + for iid in set(image_ids) + ] + ) + .on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + ) + res = await self.session.execute(stmt) + return res.rowcount or 0 + + async def bulk_remove(self, image_ids: list[int], tag_id: int) -> int: + """Remove tag_id from every image, and record a per-image rejection + so the proactive allowlist sweep won't immediately re-add it + (mirrors single-image removal semantics). Returns links removed.""" + if not image_ids: + return 0 + res = await self.session.execute( + image_tag.delete().where( + and_( + image_tag.c.tag_id == tag_id, + image_tag.c.image_record_id.in_(image_ids), + ) + ) + ) + await self.session.execute( + pg_insert(TagSuggestionRejection) + .values( + [ + {"image_record_id": iid, "tag_id": tag_id} + for iid in set(image_ids) + ] + ) + .on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + ) + return res.rowcount or 0 diff --git a/backend/app/services/credential_crypto.py b/backend/app/services/credential_crypto.py new file mode 100644 index 0000000..86a45ce --- /dev/null +++ b/backend/app/services/credential_crypto.py @@ -0,0 +1,55 @@ +"""Fernet-based encryption for credential blobs. + +The key is a single 32-byte value (urlsafe-base64-encoded; what +Fernet.generate_key produces) stored at a fixed path inside the +images/data root. Created on first boot if absent; mode 0600. No KDF +needed — the file contents are already maximum-entropy random bytes. + +Operator backup procedure must include this file alongside the rest +of /images/ — losing it makes existing encrypted_blob rows +undecryptable (recovery = delete the rows and re-upload). +""" + +import os +from pathlib import Path + +from cryptography.fernet import Fernet, InvalidToken + + +class InvalidCredentialBlob(Exception): + """Raised when decryption fails (wrong key, tampered blob, …).""" + + +class CredentialCrypto: + """Fernet encrypt/decrypt with an on-disk key file. + + Instantiate with a path; the file is created on first access and + reused thereafter. Tests pass a tmp_path; production calls with + `IMAGES_ROOT / "secrets" / "credential_key.b64"`. + """ + + def __init__(self, key_path: Path): + self._key_path = Path(key_path) + self._fernet = Fernet(self._load_or_create_key()) + + def _load_or_create_key(self) -> bytes: + if self._key_path.exists(): + return self._key_path.read_bytes() + parent = self._key_path.parent + parent.mkdir(parents=True, exist_ok=True) + os.chmod(parent, 0o700) + key = Fernet.generate_key() + self._key_path.write_bytes(key) + os.chmod(self._key_path, 0o600) + return key + + def encrypt(self, plaintext: str) -> bytes: + return self._fernet.encrypt(plaintext.encode("utf-8")) + + def decrypt(self, ciphertext: bytes) -> str: + try: + return self._fernet.decrypt(ciphertext).decode("utf-8") + except InvalidToken as exc: + raise InvalidCredentialBlob( + "credential blob is corrupt or encrypted with a different key" + ) from exc diff --git a/backend/app/services/credential_service.py b/backend/app/services/credential_service.py new file mode 100644 index 0000000..a2cd324 --- /dev/null +++ b/backend/app/services/credential_service.py @@ -0,0 +1,199 @@ +"""FC-3b: CRUD over Credential rows + on-demand cookies-file +materialisation for gallery-dl (consumed by FC-3c). +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Credential +from .credential_crypto import CredentialCrypto +from .platforms import PLATFORMS, auth_type_for + + +class CredentialServiceError(Exception): + """Base.""" + + +class UnknownPlatformError(CredentialServiceError): + pass + + +class WrongAuthTypeError(CredentialServiceError): + def __init__(self, expected: str): + super().__init__(f"wrong credential_type for platform; expected {expected!r}") + self.expected = expected + + +class EmptyDataError(CredentialServiceError): + pass + + +@dataclass(frozen=True) +class CredentialRecord: + platform: str + credential_type: str + captured_at: str + expires_at: str | None + last_verified: str | None + + def to_dict(self) -> dict: + return { + "platform": self.platform, + "credential_type": self.credential_type, + "captured_at": self.captured_at, + "expires_at": self.expires_at, + "last_verified": self.last_verified, + } + + +def _to_record(row: Credential) -> CredentialRecord: + return CredentialRecord( + platform=row.platform, + credential_type=row.credential_type, + captured_at=row.captured_at.isoformat() if row.captured_at else "", + expires_at=row.expires_at.isoformat() if row.expires_at else None, + last_verified=row.last_verified.isoformat() if row.last_verified else None, + ) + + +# Default cookies dir under the image root; tests override. +_DEFAULT_COOKIES_DIR = Path("/images/cookies") + + +class CredentialService: + def __init__( + self, + session: AsyncSession, + crypto: CredentialCrypto, + cookies_dir: Path | None = None, + ): + self.session = session + self.crypto = crypto + self.cookies_dir = Path(cookies_dir) if cookies_dir else _DEFAULT_COOKIES_DIR + + async def list(self) -> list[CredentialRecord]: + rows = (await self.session.execute( + select(Credential).order_by(Credential.platform.asc()) + )).scalars().all() + return [_to_record(r) for r in rows] + + async def get(self, platform: str) -> CredentialRecord | None: + row = (await self.session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() + return _to_record(row) if row else None + + async def upsert( + self, + *, + platform: str, + credential_type: str, + data: str, + expires_at: datetime | None = None, + ) -> CredentialRecord: + if platform not in PLATFORMS: + raise UnknownPlatformError(f"unknown platform: {platform!r}") + expected = auth_type_for(platform) + if credential_type != expected: + raise WrongAuthTypeError(expected=expected or "") + cleaned = (data or "").strip() + if not cleaned: + raise EmptyDataError("data must be a non-empty string") + blob = self.crypto.encrypt(cleaned) + + row = (await self.session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() + if row is None: + row = Credential( + platform=platform, + credential_type=credential_type, + encrypted_blob=blob, + expires_at=expires_at, + ) + self.session.add(row) + else: + row.credential_type = credential_type + row.encrypted_blob = blob + row.expires_at = expires_at + row.last_verified = None # invalidate prior verification + await self.session.commit() + await self.session.refresh(row) + return _to_record(row) + + async def delete(self, platform: str) -> None: + row = (await self.session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() + if row is None: + raise LookupError(f"no credential for platform {platform!r}") + await self.session.delete(row) + await self.session.commit() + + async def get_cookies_path(self, platform: str) -> Path | None: + """Decrypt the credential and write a Netscape cookies.txt for + gallery-dl. Returns None if no credential or wrong kind.""" + row = (await self.session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() + if row is None or row.credential_type != "cookies": + return None + plaintext = self.crypto.decrypt(row.encrypted_blob) + netscape = _to_netscape(plaintext) + self.cookies_dir.mkdir(parents=True, exist_ok=True) + out = self.cookies_dir / f"{platform}_cookies.txt" + out.write_text(netscape) + os.chmod(out, 0o600) + return out + + async def get_token(self, platform: str) -> str | None: + row = (await self.session.execute( + select(Credential).where(Credential.platform == platform) + )).scalar_one_or_none() + if row is None or row.credential_type != "token": + return None + return self.crypto.decrypt(row.encrypted_blob) + + +def _to_netscape(plaintext: str) -> str: + """Accept either Netscape-format text (the extension's output) or a + JSON array of cookie dicts (a manual-paste edge case); produce + Netscape-format text suitable for gallery-dl --cookies.""" + stripped = plaintext.strip() + if not stripped: + return "" + if stripped.startswith("#") or "\t" in stripped: + return plaintext # already Netscape + try: + loaded = json.loads(stripped) + except json.JSONDecodeError: + return plaintext # write as-is and hope; gallery-dl will complain if invalid + if isinstance(loaded, dict): + loaded = [loaded] + if not isinstance(loaded, list): + return plaintext + lines = ["# Netscape HTTP Cookie File"] + for c in loaded: + domain = str(c.get("domain", "")) + if domain and not domain.startswith("."): + domain = "." + domain + flag = "TRUE" + path = str(c.get("path", "/")) + secure = "TRUE" if c.get("secure", False) else "FALSE" + expiration = c.get("expiration") or c.get("expirationDate") or 0 + try: + expiration_str = str(int(float(expiration))) + except (TypeError, ValueError): + expiration_str = "0" + name = str(c.get("name", "")) + value = str(c.get("value", "")) + lines.append("\t".join([domain, flag, path, secure, expiration_str, name, value])) + return "\n".join(lines) + "\n" diff --git a/backend/app/services/download_service.py b/backend/app/services/download_service.py new file mode 100644 index 0000000..347590b --- /dev/null +++ b/backend/app/services/download_service.py @@ -0,0 +1,323 @@ +"""FC-3c download orchestrator. + +Three-phase pipeline: + Phase 1 — DB setup (brief session): load source, mark event running, + fetch credentials + settings. + Phase 2 — Execute (no DB connection): call GalleryDLService; on + Patreon campaign-ID failure, resolve vanity and retry once. + Phase 3 — Import + persist (fresh session): attach_in_place each + written file; clean up duplicates; persist rich metadata. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import Session as SyncSession +from sqlalchemy.orm import joinedload + +from ..models import Artist, DownloadEvent, Source +from .credential_service import CredentialService +from .gallery_dl import GalleryDLService, SourceConfig +from .importer import Importer +from .patreon_resolver import resolve_campaign_id + +log = logging.getLogger(__name__) + + +_PATREON_VANITY_RE = re.compile( + r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)", + re.IGNORECASE, +) +_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id" + + +def _extract_patreon_vanity(url: str) -> str | None: + m = _PATREON_VANITY_RE.match(url) + return m.group(1) if m else None + + +def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool: + return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower() + + +def _effective_url(platform: str, source_url: str, overrides: dict) -> str: + if platform == "patreon" and overrides.get("patreon_campaign_id"): + return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}" + return source_url + + +class DownloadService: + """Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`. + + The Importer is sync (existing FC convention); calls to attach_in_place + happen via run_in_executor. + """ + + def __init__( + self, + async_session: AsyncSession, + sync_session: SyncSession, + gdl: GalleryDLService, + importer: Importer, + cred_service: CredentialService, + ): + self.async_session = async_session + self.sync_session = sync_session + self.gdl = gdl + self.importer = importer + self.cred_service = cred_service + + async def download_source(self, source_id: int) -> int: + """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is.""" + setup = await self._phase1_setup(source_id) + if setup["status"] in ("skipped", "in_flight"): + return setup["event_id"] + ctx = setup + + source_config = SourceConfig.from_dict(ctx["config_overrides"] or {}) + effective_url = _effective_url( + ctx["platform"], ctx["url"], ctx["config_overrides"] or {} + ) + + dl_result = await self.gdl.download( + url=effective_url, + artist_slug=ctx["artist_slug"], + platform=ctx["platform"], + source_config=source_config, + cookies_path=ctx["cookies_path"], + auth_token=ctx["auth_token"], + ) + + resolved_campaign_id: str | None = None + if ( + ctx["platform"] == "patreon" + and not dl_result.success + and "patreon_campaign_id" not in (ctx["config_overrides"] or {}) + and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr) + ): + vanity = _extract_patreon_vanity(ctx["url"]) + if vanity: + log.info( + "Attempting campaign-ID resolution for %s (%s)", + ctx["artist_slug"], vanity, + ) + resolved_campaign_id = await resolve_campaign_id( + vanity, ctx["cookies_path"] + ) + if resolved_campaign_id: + dl_result = await self.gdl.download( + url=f"https://www.patreon.com/id:{resolved_campaign_id}", + artist_slug=ctx["artist_slug"], + platform=ctx["platform"], + source_config=source_config, + cookies_path=ctx["cookies_path"], + auth_token=ctx["auth_token"], + ) + + return await self._phase3_persist( + ctx["event_id"], ctx, dl_result, resolved_campaign_id, + ) + + async def _phase1_setup(self, source_id: int) -> dict[str, Any]: + source = (await self.async_session.execute( + select(Source).options(joinedload(Source.artist)).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source {source_id} not found") + + if not source.enabled: + ev = DownloadEvent(source_id=source_id, status="pending") + self.async_session.add(ev) + await self.async_session.commit() + await self.async_session.refresh(ev) + await self._finalize_event_and_source( + event_id=ev.id, source_id=source_id, status="skipped", + error_message="source disabled", + ) + return {"status": "skipped", "event_id": ev.id} + + existing = (await self.async_session.execute( + select(DownloadEvent).where( + DownloadEvent.source_id == source_id, + DownloadEvent.status.in_(["pending", "running"]), + ).order_by(DownloadEvent.id.desc()).limit(1) + )).scalar_one_or_none() + + if existing and existing.status == "running": + return {"status": "in_flight", "event_id": existing.id} + if existing and existing.status == "pending": + existing.status = "running" + await self.async_session.commit() + event_id = existing.id + else: + ev = DownloadEvent(source_id=source_id, status="running") + self.async_session.add(ev) + await self.async_session.commit() + await self.async_session.refresh(ev) + event_id = ev.id + + artist = source.artist + if source.platform in ("discord", "pixiv"): + cookies_path = None + auth_token = await self.cred_service.get_token(source.platform) + else: + cookies_path_obj = await self.cred_service.get_cookies_path(source.platform) + cookies_path = str(cookies_path_obj) if cookies_path_obj else None + auth_token = None + + return { + "status": "ok", + "event_id": event_id, + "source_id": source_id, + "url": source.url, + "platform": source.platform, + "artist_slug": artist.slug if artist else "_unknown", + "artist_id": artist.id if artist else None, + "config_overrides": dict(source.config_overrides or {}), + "cookies_path": cookies_path, + "auth_token": auth_token, + } + + async def _phase3_persist( + self, + event_id: int, + ctx: dict, + dl_result, + resolved_campaign_id: str | None, + ) -> int: + # Cache Patreon campaign id on the source. + if resolved_campaign_id: + src = self.sync_session.get(Source, ctx["source_id"]) + if src is not None: + src.config_overrides = { + **(src.config_overrides or {}), + "patreon_campaign_id": resolved_campaign_id, + } + self.sync_session.commit() + + artist = None + source_row = None + if ctx.get("artist_id"): + artist = self.sync_session.get(Artist, ctx["artist_id"]) + source_row = self.sync_session.get(Source, ctx["source_id"]) + + import_summary = {"attached": 0, "skipped": 0, "errors": 0} + bytes_downloaded = 0 + + loop = asyncio.get_running_loop() + + for path_str in dl_result.written_paths or []: + if path_str in (dl_result.quarantined_paths or []): + continue + path = Path(path_str) + # Sync stdlib filesystem ops are intentional here: this orchestrator + # runs under `asyncio.run` inside a Celery task — no other awaitables + # compete for the loop. ASYNC240 suppressed below. + if not path.exists(): # noqa: ASYNC240 + continue + + def _attach(p=path): + return self.importer.attach_in_place( + p, artist=artist, source=source_row, + ) + + result = await loop.run_in_executor(None, _attach) + + if result.status in ("imported", "superseded"): + import_summary["attached"] += 1 + try: + bytes_downloaded += path.stat().st_size # noqa: ASYNC240 + except OSError: + pass + elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in ( + "duplicate_hash", "duplicate_phash", + ): + import_summary["skipped"] += 1 + try: + path.unlink(missing_ok=True) # noqa: ASYNC240 + except OSError: + pass + else: + import_summary["errors"] += 1 + + ev = (await self.async_session.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + + run_stats = self.gdl._compute_run_stats( + dl_result.return_code, dl_result.stdout, dl_result.stderr + ) + run_stats["quarantined_count"] = dl_result.files_quarantined + stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr) + + status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error" + ev.status = status + ev.finished_at = datetime.now(UTC) + ev.files_count = import_summary["attached"] + ev.bytes_downloaded = bytes_downloaded + ev.error = dl_result.error_message if not dl_result.success else None + ev.metadata_ = { + "run_stats": run_stats, + "error_type": dl_result.error_type.value if dl_result.error_type else None, + "stdout": self.gdl._truncate_log(dl_result.stdout) or None, + "stderr": self.gdl._truncate_log(dl_result.stderr) or None, + "stderr_errors_warnings": stderr_summary or None, + "duration_seconds": dl_result.duration_seconds, + "quarantined_paths": dl_result.quarantined_paths or None, + "import_summary": import_summary, + } + await self._update_source_health( + source_id=ctx["source_id"], status=status, error_message=ev.error, + ) + await self.async_session.commit() + return event_id + + async def _update_source_health( + self, *, source_id: int, status: str, error_message: str | None, + ) -> None: + """FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}. + + ok -> failures = 0, error = None, checked_at = now + error -> failures += 1, error = error_message, checked_at = now + skipped -> failures unchanged, error = None, checked_at = now + """ + source = (await self.async_session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one() + now = datetime.now(UTC) + if status == "ok": + source.consecutive_failures = 0 + source.last_error = None + elif status == "error": + source.consecutive_failures = (source.consecutive_failures or 0) + 1 + source.last_error = error_message + elif status == "skipped": + source.last_error = None + source.last_checked_at = now + + async def _finalize_event_and_source( + self, *, event_id: int, source_id: int, status: str, + error_message: str | None, + ) -> None: + """Finalize the event AND the source-health columns in one + transactional step. Used by the skipped early-out path and by + unit tests that exercise the source-health writes directly. + """ + ev = (await self.async_session.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + ev.status = status + ev.finished_at = datetime.now(UTC) + ev.error = error_message + await self._update_source_health( + source_id=source_id, status=status, error_message=error_message, + ) + await self.async_session.commit() diff --git a/backend/app/services/file_validator.py b/backend/app/services/file_validator.py new file mode 100644 index 0000000..ca6e40f --- /dev/null +++ b/backend/app/services/file_validator.py @@ -0,0 +1,110 @@ +"""Magic-byte head/tail validator for fresh gallery-dl writes. + +Catches truncated downloads (servers lying about Content-Length, TCP +RST mid-transfer, etc.) before they enter the library. O(1) per file +— reads only head and tail bytes; never decodes pixels. + +Fail-open: unknown formats and unreadable files pass through. Format +detection is by extension; we only validate what the downloader is +expected to write. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +JPEG_HEAD = b"\xff\xd8\xff" +JPEG_TAIL = b"\xff\xd9" + +PNG_HEAD = b"\x89PNG\r\n\x1a\n" +PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82" + +GIF_HEAD_87A = b"GIF87a" +GIF_HEAD_89A = b"GIF89a" +GIF_TAIL = b"\x3b" + +WEBP_RIFF = b"RIFF" +WEBP_FORMAT = b"WEBP" + + +@dataclass(frozen=True) +class ValidationResult: + ok: bool + format: str | None = None + reason: str | None = None + size: int = 0 + + +VALIDATED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp"} + + +def is_validatable(path: Path) -> bool: + return path.suffix.lower() in VALIDATED_EXTENSIONS + + +def validate_file(path: Path) -> ValidationResult: + """Returns ok=False if the file is structurally incomplete. + + Never raises. Unknown formats return ok=True with format=None. + Unreadable files surface as ok=False so callers can quarantine. + """ + ext = path.suffix.lower() + if ext not in VALIDATED_EXTENSIONS: + return ValidationResult(ok=True, format=None) + + try: + size = path.stat().st_size + except FileNotFoundError: + return ValidationResult(ok=False, reason="file not found") + except OSError as exc: + return ValidationResult(ok=False, reason=f"stat failed: {exc}") + + if size < 16: + return ValidationResult(ok=False, format=ext.lstrip("."), reason="too small", size=size) + + try: + with path.open("rb") as fh: + head = fh.read(32) + fh.seek(max(0, size - 32)) + tail = fh.read(32) + except OSError as exc: + return ValidationResult(ok=False, reason=f"read failed: {exc}", size=size) + + if ext in (".jpg", ".jpeg"): + if not head.startswith(JPEG_HEAD): + return ValidationResult(ok=False, format="jpeg", reason="missing JPEG SOI marker", size=size) + if not tail.endswith(JPEG_TAIL): + return ValidationResult(ok=False, format="jpeg", reason="missing JPEG EOI marker", size=size) + return ValidationResult(ok=True, format="jpeg", size=size) + + if ext == ".png": + if not head.startswith(PNG_HEAD): + return ValidationResult(ok=False, format="png", reason="missing PNG signature", size=size) + if not tail.endswith(PNG_TAIL): + return ValidationResult(ok=False, format="png", reason="missing PNG IEND chunk", size=size) + return ValidationResult(ok=True, format="png", size=size) + + if ext == ".gif": + if not (head.startswith(GIF_HEAD_87A) or head.startswith(GIF_HEAD_89A)): + return ValidationResult(ok=False, format="gif", reason="missing GIF signature", size=size) + if not tail.endswith(GIF_TAIL): + return ValidationResult(ok=False, format="gif", reason="missing GIF trailer", size=size) + return ValidationResult(ok=True, format="gif", size=size) + + if ext == ".webp": + if not (head.startswith(WEBP_RIFF) and WEBP_FORMAT in head[:16]): + return ValidationResult(ok=False, format="webp", reason="missing WebP RIFF/WEBP marker", size=size) + try: + chunk_size = int.from_bytes(head[4:8], "little") + except Exception: + return ValidationResult(ok=False, format="webp", reason="malformed WebP chunk size", size=size) + if chunk_size + 8 != size: + return ValidationResult( + ok=False, format="webp", + reason=f"WebP RIFF size mismatch: header says {chunk_size + 8}, file is {size}", + size=size, + ) + return ValidationResult(ok=True, format="webp", size=size) + + return ValidationResult(ok=True, format=None, size=size) diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py new file mode 100644 index 0000000..5e9b17f --- /dev/null +++ b/backend/app/services/gallery_dl.py @@ -0,0 +1,660 @@ +"""Gallery-dl subprocess wrapper. + +Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/ +backend/app/services/gallery_dl.py). FC adaptations: + - `artist_slug` replaces GS's `subscription_name` throughout + - Validation uses backend/app/services/file_validator.py + - Config dir under `/.gallery-dl/`, not a separate config_path + - PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path + +from .file_validator import is_validatable, validate_file + +log = logging.getLogger(__name__) + + +class ErrorType(StrEnum): + AUTH_ERROR = "auth_error" + RATE_LIMITED = "rate_limited" + NOT_FOUND = "not_found" + ACCESS_DENIED = "access_denied" + NETWORK_ERROR = "network_error" + NO_NEW_CONTENT = "no_new_content" + TIER_LIMITED = "tier_limited" + TIMEOUT = "timeout" + HTTP_ERROR = "http_error" + UNSUPPORTED_URL = "unsupported_url" + VALIDATION_FAILED = "validation_failed" + UNKNOWN_ERROR = "unknown_error" + + +@dataclass +class SourceConfig: + content_types: list[str] = field(default_factory=lambda: ["all"]) + sleep: float | None = None + sleep_request: float | None = None + directory_pattern: str | None = None + filename_pattern: str | None = None + skip_existing: bool = True + save_metadata: bool = True + timeout: int = 3600 + + @classmethod + def from_dict(cls, data: dict) -> SourceConfig: + return cls( + content_types=data.get("content_types", ["all"]), + sleep=data.get("sleep"), + sleep_request=data.get("sleep_request"), + directory_pattern=data.get("directory_pattern"), + filename_pattern=data.get("filename_pattern"), + skip_existing=data.get("skip_existing", True), + save_metadata=data.get("save_metadata", True), + timeout=data.get("timeout", 3600), + ) + + +@dataclass +class DownloadResult: + success: bool + url: str + artist_slug: str + platform: str + files_downloaded: int = 0 + files_quarantined: int = 0 + quarantined_paths: list[str] = field(default_factory=list) + written_paths: list[str] = field(default_factory=list) + stdout: str = "" + stderr: str = "" + return_code: int = 0 + error_type: ErrorType | None = None + error_message: str | None = None + duration_seconds: float = 0.0 + started_at: str | None = None + completed_at: str | None = None + + +def _summarize_validation_failures(failures: list[dict]) -> str: + if not failures: + return "Validation failed" + reasons: dict[str, int] = {} + for f in failures: + key = f.get("reason") or "unknown" + reasons[key] = reasons.get(key, 0) + 1 + top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1]) + n = len(failures) + if top_count == n: + return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}" + return f"{n} files quarantined ({top_count}× {top_reason}, mixed)" + + +class GalleryDLService: + """Service for executing gallery-dl downloads.""" + + AUTH_ERROR_PATTERNS = [ + "unauthorized", "login required", "please login", "must be logged in", + "authentication required", "authenticationerror", "refresh-token", + "session expired", "invalid cookie", "cookies are expired", + "cookies have expired", "not logged in", + ] + RATE_LIMIT_PATTERNS = [ + '" 429 ', "rate limit", "too many requests", "ratelimit", "throttl", + ] + NOT_FOUND_PATTERNS = [ + '" 404 ', "error 404", "status: 404", "status code: 404", + "404 not found", "page not found", "user not found", + "creator not found", "artist not found", + "content no longer available", "has been deleted", + "account deleted", "profile does not exist", + ] + GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"] + NETWORK_ERROR_PATTERNS = [ + "timed out", "read timed out", "connect timed out", + "connection timed out", "connection refused", "connection reset", + "network unreachable", "name resolution failed", + "name or service not known", "nodename nor servname provided", + "ssl: certificate_verify_failed", "ssl: wrong_version_number", + "certificate verify failed", "[errno", + ] + ACCESS_DENIED_PATTERNS = [ + '" 403 ', "error 403", "forbidden", "access denied", + "permission denied", "tier required", "pledge required", + ] + + # Per-platform defaults. Lifted from GS — same six platforms FC supports. + PLATFORM_DEFAULTS = { + "patreon": { + "content_types": ["images", "image_large", "attachments", "postfile", "content"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + "videos": True, + "embeds": True, + "cursor": True, + }, + "subscribestar": { + "content_types": ["all"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + }, + "hentaifoundry": { + "content_types": ["all"], + "directory": [], + "filename": "{category}_{index:>03}_{title[:50]}.{extension}", + "include": "all", + }, + "discord": { + "content_types": ["all"], + "directory": ["{channel[name]}"], + "filename": "{date:%Y%m%d}_{id}_{filename}.{extension}", + "embeds": "all", + "stickers": True, + "reactions": False, + "threads": True, + }, + "pixiv": { + "content_types": ["all"], + "directory": ["{category}"], + "filename": "{id}_{title[:50]}_{num:>02}.{extension}", + "ugoira": True, + }, + "deviantart": { + "content_types": ["all"], + "directory": [], + "filename": "{index:>03}_{title[:50]}.{extension}", + "flat": True, + "original": True, + "mature": True, + "metadata": True, + }, + } + + def __init__( + self, + images_root: Path, + rate_limit: float = 3.0, + validate_files: bool = True, + ): + self.images_root = Path(images_root) + self._rate_limit = rate_limit + self._validate_files = validate_files + self._config_dir = self.images_root / ".gallery-dl" + self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + (self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700) + + def _get_default_config(self) -> dict: + config = { + "extractor": { + "base-directory": str(self.images_root), + "archive": str(self._config_dir / "archive.sqlite3"), + "skip": True, + "sleep": self._rate_limit, + "sleep-request": max(0.5, self._rate_limit / 4), + "retries": 3, + "timeout": 30.0, + "verify": True, + "postprocessors": [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ], + }, + "downloader": { + "part": True, + "part-directory": str(self._config_dir / "temp"), + "retries": 3, + "timeout": 120.0, + }, + "output": {"progress": True}, + } + for platform_name, defaults in self.PLATFORM_DEFAULTS.items(): + config["extractor"][platform_name] = { + key: value for key, value in defaults.items() if key != "content_types" + } + return config + + def _build_config_for_source( + self, + platform: str, + source_config: SourceConfig, + artist_slug: str, + ) -> dict: + config = json.loads(json.dumps(self._get_default_config())) # deep copy + + destination = str(self.images_root / artist_slug / platform) + config["extractor"]["base-directory"] = destination + + if source_config.sleep is not None: + config["extractor"]["sleep"] = source_config.sleep + if source_config.sleep_request is not None: + config["extractor"]["sleep-request"] = source_config.sleep_request + config["extractor"]["skip"] = source_config.skip_existing + + if source_config.save_metadata: + config["extractor"]["postprocessors"] = [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ] + else: + config["extractor"].pop("postprocessors", None) + + platform_section = config["extractor"].setdefault(platform, {}) + + if platform == "patreon": + if "all" in source_config.content_types: + platform_section["files"] = [ + "images", "image_large", "attachments", "postfile", "content", + ] + else: + platform_section["files"] = source_config.content_types + elif platform == "hentaifoundry": + if "pictures" in source_config.content_types or "all" in source_config.content_types: + platform_section["include"] = "all" + + if source_config.directory_pattern: + platform_section["directory"] = source_config.directory_pattern + if source_config.filename_pattern: + platform_section["filename"] = source_config.filename_pattern + + platform_section["metadata"] = source_config.save_metadata + + return config + + def _categorize_error( + self, return_code: int, stdout: str, stderr: str, + ) -> tuple[ErrorType, str]: + combined = f"{stdout} {stderr}".lower() + + skip_line_count = len([ + line for line in stdout.split("\n") if line.strip().startswith("#") + ]) + skip_text_indicators = ["skipping", "already exists", "archive"] + has_skip_text = any(ind in combined for ind in skip_text_indicators) + + actual_error_indicators = [ + "][error]", "exception", "traceback", + "download failed", "extraction failed", + ] + per_item_patterns = [ + "not allowed to view", "unable to get post", + "failed to extract campaign id", "failed to download", + "cannot import yt-dlp", "cannot import youtube-dl", + ] + all_lines = combined.split("\n") + error_lines = [ + line for line in all_lines + if any(ind in line for ind in actual_error_indicators) + ] + per_item_error_lines = [ + line for line in error_lines + if any(p in line for p in per_item_patterns) + ] + source_level_error_lines = [ + line for line in error_lines + if line not in set(per_item_error_lines) + ] + has_actual_error = bool(source_level_error_lines) + + if per_item_error_lines: + per_item_set = set(per_item_error_lines) + combined = "\n".join(line for line in all_lines if line not in per_item_set) + + if (skip_line_count > 0 or has_skip_text) and not has_actual_error: + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + if return_code == 0 and not stdout.strip(): + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + has_401_error = ( + '" 401 ' in combined + or "401 unauthorized" in combined + or "error 401" in combined + or "status: 401" in combined + or "status code: 401" in combined + ) + if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS): + return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid" + + if any(p in combined for p in self.RATE_LIMIT_PATTERNS): + return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time" + + has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS) + has_generic_with_error = any( + p in combined and "][error]" in combined + for p in self.GENERIC_NOT_FOUND_PATTERNS + ) + if has_specific_404 or has_generic_with_error: + return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content" + + if any(p in combined for p in self.NETWORK_ERROR_PATTERNS): + return ErrorType.NETWORK_ERROR, "Network error — check internet connection" + + if any(p in combined for p in self.ACCESS_DENIED_PATTERNS): + return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier" + + if "http error" in combined or "httperror" in combined: + return ErrorType.HTTP_ERROR, "HTTP error occurred" + + if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined: + return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" + + if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error: + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + if return_code in (1, 4) and not has_actual_error: + tier_gated_lines = [ + line for line in combined.split("\n") + if "][warning]" in line and "not allowed to view post" in line + ] + if tier_gated_lines: + count = len(tier_gated_lines) + return ( + ErrorType.TIER_LIMITED, + f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}", + ) + + return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" + + def _count_downloaded_files(self, stdout: str) -> int: + if not stdout: + return 0 + return sum(1 for line in stdout.splitlines() if line.strip().startswith("/")) + + def _written_paths(self, stdout: str) -> list[Path]: + if not stdout: + return [] + return [ + Path(stripped) + for line in stdout.splitlines() + if (stripped := line.strip()).startswith("/") + ] + + def _validate_and_quarantine( + self, + written_paths: list[Path], + artist_slug: str, + platform: str, + url: str, + ) -> tuple[list[str], list[dict]]: + quarantined_relpaths: list[str] = [] + failures: list[dict] = [] + if not written_paths: + return quarantined_relpaths, failures + + quarantine_root = ( + self.images_root / "_quarantine" / artist_slug / platform + ) + + for path in written_paths: + if not is_validatable(path): + continue + try: + result = validate_file(path) + except Exception as exc: + log.warning("Validator raised on %s: %s", path, exc) + continue + if result.ok: + continue + try: + quarantine_root.mkdir(parents=True, exist_ok=True) + dest = quarantine_root / path.name + counter = 1 + while dest.exists(): + dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}" + counter += 1 + path.rename(dest) + sidecar = dest.with_suffix(dest.suffix + ".quarantine.json") + sidecar.write_text( + json.dumps( + { + "original_path": str(path), + "source_url": url, + "artist_slug": artist_slug, + "platform": platform, + "format": result.format, + "reason": result.reason, + "size": result.size, + "quarantined_at": datetime.now(UTC).isoformat(), + }, + indent=2, + ) + ) + except OSError as exc: + log.error("Failed to quarantine %s: %s. File left in place.", path, exc) + continue + + log.warning( + "Quarantined corrupt file: %s → %s (%s: %s)", + path, dest, result.format, result.reason, + ) + quarantined_relpaths.append(str(dest)) + failures.append( + { + "path": str(path), + "format": result.format, + "reason": result.reason, + "size": result.size, + } + ) + return quarantined_relpaths, failures + + def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict: + stdout = stdout or "" + stderr = stderr or "" + + skipped_stdout = sum( + 1 for line in stdout.splitlines() if line.strip().startswith("#") + ) + skipped_stderr = sum( + 1 for line in stderr.splitlines() if "] skipping " in line.lower() + ) + per_item_failures = sum( + 1 for line in stderr.splitlines() + if "[download][error]" in line.lower() and "failed to download" in line.lower() + ) + warning_count = sum( + 1 for line in stderr.splitlines() if "][warning]" in line.lower() + ) + tier_gated_count = sum( + 1 for line in stderr.splitlines() + if "][warning]" in line.lower() and "not allowed to view post" in line.lower() + ) + + return { + "exit_code": return_code, + "downloaded_count": self._count_downloaded_files(stdout), + "skipped_count": skipped_stdout + skipped_stderr, + "per_item_failures": per_item_failures, + "warning_count": warning_count, + "tier_gated_count": tier_gated_count, + } + + @staticmethod + def _extract_errors_warnings(stderr: str) -> str: + if not stderr: + return "" + kept = [ + line for line in stderr.splitlines() + if "][error]" in line.lower() or "][warning]" in line.lower() + ] + return "\n".join(kept) + + @staticmethod + def _truncate_log(text: str, max_bytes: int = 500_000) -> str: + if not text: + return text + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + half = max_bytes // 2 + head = encoded[:half].decode("utf-8", errors="ignore") + tail = encoded[-half:].decode("utf-8", errors="ignore") + head_lines = head.count("\n") + tail_lines = tail.count("\n") + total_lines = text.count("\n") + elided = max(0, total_lines - head_lines - tail_lines) + marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n" + return head + marker + tail + + async def download( + self, + url: str, + artist_slug: str, + platform: str, + source_config: SourceConfig | None = None, + cookies_path: str | None = None, + auth_token: str | None = None, + ) -> DownloadResult: + start_time = time.time() + started_at = datetime.now(UTC).isoformat() + + if source_config is None: + source_config = SourceConfig() + + config = self._build_config_for_source(platform, source_config, artist_slug) + + if cookies_path: + config["extractor"]["cookies"] = cookies_path + if auth_token and platform == "discord": + config["extractor"].setdefault("discord", {}) + config["extractor"]["discord"]["token"] = auth_token + if auth_token and platform == "pixiv": + config["extractor"].setdefault("pixiv", {}) + config["extractor"]["pixiv"]["refresh-token"] = auth_token + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, dir=str(self._config_dir), + ) as fh: + json.dump(config, fh, indent=2) + temp_config_path = fh.name + + try: + cmd = [ + sys.executable, "-m", "gallery_dl", + "--config", temp_config_path, "--verbose", url, + ] + log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url) + + loop = asyncio.get_running_loop() + proc = await loop.run_in_executor( + None, + lambda: subprocess.run( + cmd, capture_output=True, text=True, + timeout=source_config.timeout, + ), + ) + + duration = time.time() - start_time + completed_at = datetime.now(UTC).isoformat() + + quarantined_paths: list[str] = [] + quarantine_failures: list[dict] = [] + if self._validate_files: + quarantined_paths, quarantine_failures = self._validate_and_quarantine( + written_paths=self._written_paths(proc.stdout), + artist_slug=artist_slug, + platform=platform, + url=url, + ) + + written_paths = [ + str(p) for p in self._written_paths(proc.stdout) + if str(p) not in set(quarantined_paths) + ] + files_written = self._count_downloaded_files(proc.stdout) + files_quarantined = len(quarantined_paths) + files_kept = max(0, files_written - files_quarantined) + + if proc.returncode == 0: + if files_quarantined > 0: + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept, + files_quarantined=files_quarantined, + quarantined_paths=quarantined_paths, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + error_type=ErrorType.VALIDATION_FAILED, + error_message=_summarize_validation_failures(quarantine_failures), + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + return DownloadResult( + success=True, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + + error_type, error_message = self._categorize_error( + proc.returncode, proc.stdout, proc.stderr + ) + success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED) + + if files_quarantined > 0: + error_type = ErrorType.VALIDATION_FAILED + error_message = _summarize_validation_failures(quarantine_failures) + success = False + + return DownloadResult( + success=success, url=url, artist_slug=artist_slug, platform=platform, + files_downloaded=files_kept if success else 0, + files_quarantined=files_quarantined, + quarantined_paths=quarantined_paths, + written_paths=written_paths, + stdout=proc.stdout, stderr=proc.stderr, + return_code=proc.returncode, + error_type=error_type, error_message=error_message, + duration_seconds=duration, + started_at=started_at, completed_at=completed_at, + ) + + except subprocess.TimeoutExpired: + duration = time.time() - start_time + log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration) + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + error_type=ErrorType.TIMEOUT, + error_message=f"Download timed out after {source_config.timeout} seconds", + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.now(UTC).isoformat(), + ) + except Exception as exc: + duration = time.time() - start_time + log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc) + return DownloadResult( + success=False, url=url, artist_slug=artist_slug, platform=platform, + error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc), + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.now(UTC).isoformat(), + ) + finally: + try: + Path(temp_config_path).unlink() # noqa: ASYNC240 + except Exception: + pass diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py new file mode 100644 index 0000000..834c6a6 --- /dev/null +++ b/backend/app/services/gallery_service.py @@ -0,0 +1,308 @@ +"""Cursor-paginated gallery queries. + +Cursor format: opaque base64-encoded ":". +Pagination key is (created_at DESC, id DESC) so we don't drift when new +imports arrive between page loads. Decoding rejects malformed cursors with +a ValueError; the API layer translates that to HTTP 400. +""" + +import base64 +from dataclasses import dataclass +from datetime import datetime + +from sqlalchemy import and_, exists, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, ImageProvenance, ImageRecord, Source, Tag +from ..models.tag import image_tag + +CURSOR_SEPARATOR = "|" + + +def encode_cursor(created_at: datetime, image_id: int) -> str: + raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}" + return base64.urlsafe_b64encode(raw.encode()).decode() + + +def decode_cursor(cursor: str) -> tuple[datetime, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1) + return datetime.fromisoformat(ts_part), int(id_part) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class GalleryImage: + id: int + path: str + sha256: str + mime: str + width: int | None + height: int | None + created_at: datetime + thumbnail_url: str + artist: dict | None = None + + +@dataclass(frozen=True) +class GalleryPage: + images: list[GalleryImage] + next_cursor: str | None + date_groups: list[tuple[int, int, list[int]]] # (year, month, [image_id...]) + + +@dataclass(frozen=True) +class TimelineBucket: + year: int + month: int + count: int + + +def thumbnail_url(sha256_hex: str, mime: str) -> str: + # Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go + # under /images/thumbs/. The MIME determines the extension. + ext = ".png" if mime in ("image/png", "image/gif") else ".jpg" + bucket = sha256_hex[:3] + return f"/images/thumbs/{bucket}/{sha256_hex}{ext}" + + +def _require_single_filter(tag_id, post_id, artist_id) -> None: + if sum(x is not None for x in (tag_id, post_id, artist_id)) > 1: + raise ValueError( + "tag_id, post_id, artist_id are mutually exclusive" + ) + + +def _provenance_clause(post_id, artist_id): + """Correlated EXISTS clause (NOT a join) so an image with multiple + matching provenance rows is returned exactly once and the + (created_at DESC, id DESC) cursor ordering is unaffected.""" + if post_id is not None: + return exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.post_id == post_id, + ) + if artist_id is not None: + return exists().where( + ImageProvenance.image_record_id == ImageRecord.id, + ImageProvenance.source_id == Source.id, + Source.artist_id == artist_id, + ) + return None + + +async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]: + """Map image_id -> {"name","slug"} via the canonical + image_record.artist_id (FC-2d-vii-c). Bounded by page size.""" + if not image_ids: + return {} + stmt = ( + select(ImageRecord.id, Artist.name, Artist.slug) + .join(Artist, Artist.id == ImageRecord.artist_id) + .where(ImageRecord.id.in_(image_ids)) + ) + return { + img_id: {"name": name, "slug": slug} + for img_id, name, slug in (await session.execute(stmt)).all() + } + + +class GalleryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def scroll( + self, + cursor: str | None, + limit: int = 50, + tag_id: int | None = None, + post_id: int | None = None, + artist_id: int | None = None, + ) -> GalleryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + _require_single_filter(tag_id, post_id, artist_id) + + stmt = select(ImageRecord) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + prov = _provenance_clause(post_id, artist_id) + if prov is not None: + stmt = stmt.where(prov) + + if cursor: + cur_ts, cur_id = decode_cursor(cursor) + stmt = stmt.where( + or_( + ImageRecord.created_at < cur_ts, + and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id), + ) + ) + + stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).scalars().all() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = encode_cursor(last.created_at, last.id) + rows = rows[:limit] + + artists = await _artists_for(self.session, [r.id for r in rows]) + images = [ + GalleryImage( + id=r.id, + path=r.path, + sha256=r.sha256, + mime=r.mime, + width=r.width, + height=r.height, + created_at=r.created_at, + thumbnail_url=thumbnail_url(r.sha256, r.mime), + artist=artists.get(r.id), + ) + for r in rows + ] + return GalleryPage( + images=images, + next_cursor=next_cursor, + date_groups=_group_by_year_month(images), + ) + + async def timeline( + self, + tag_id: int | None = None, + post_id: int | None = None, + artist_id: int | None = None, + ) -> list[TimelineBucket]: + year_col = func.date_part("year", ImageRecord.created_at).label("yr") + month_col = func.date_part("month", ImageRecord.created_at).label("mo") + stmt = select( + year_col, month_col, func.count(ImageRecord.id).label("cnt") + ) + _require_single_filter(tag_id, post_id, artist_id) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + prov = _provenance_clause(post_id, artist_id) + if prov is not None: + stmt = stmt.where(prov) + stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc()) + rows = (await self.session.execute(stmt)).all() + return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows] + + async def jump_cursor( + self, year: int, month: int, tag_id: int | None = None, + post_id: int | None = None, artist_id: int | None = None, + ) -> str | None: + """Returns a cursor that, when passed to scroll(), positions at the + first image of the given year-month. None if the bucket is empty. + """ + from sqlalchemy import extract + + stmt = select(ImageRecord).where( + extract("year", ImageRecord.created_at) == year, + extract("month", ImageRecord.created_at) == month, + ) + _require_single_filter(tag_id, post_id, artist_id) + if tag_id is not None: + stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where( + image_tag.c.tag_id == tag_id + ) + prov = _provenance_clause(post_id, artist_id) + if prov is not None: + stmt = stmt.where(prov) + stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1) + first = (await self.session.execute(stmt)).scalar_one_or_none() + if first is None: + return None + # Cursor is exclusive; we encode a cursor with id+1 so the row itself + # is the first result in the next scroll(). + return encode_cursor(first.created_at, first.id + 1) + + async def get_image_with_tags(self, image_id: int) -> dict | None: + record = await self.session.get(ImageRecord, image_id) + if record is None: + return None + tag_stmt = ( + select(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id == image_id) + .order_by(Tag.kind.asc(), Tag.name.asc()) + ) + tags = (await self.session.execute(tag_stmt)).scalars().all() + neighbors = await self._neighbors(record) + return { + "id": record.id, + "path": record.path, + "sha256": record.sha256, + "mime": record.mime, + "width": record.width, + "height": record.height, + "size_bytes": record.size_bytes, + "integrity_status": record.integrity_status, + "created_at": record.created_at.isoformat(), + "thumbnail_url": thumbnail_url(record.sha256, record.mime), + "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", + "tags": [ + { + "id": t.id, + "name": t.name, + "kind": t.kind.value if hasattr(t.kind, "value") else t.kind, + "fandom_id": t.fandom_id, + } + for t in tags + ], + "neighbors": neighbors, + } + + async def _neighbors(self, record: ImageRecord) -> dict: + prev_stmt = ( + select(ImageRecord.id) + .where( + or_( + ImageRecord.created_at > record.created_at, + and_( + ImageRecord.created_at == record.created_at, + ImageRecord.id > record.id, + ), + ) + ) + .order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc()) + .limit(1) + ) + next_stmt = ( + select(ImageRecord.id) + .where( + or_( + ImageRecord.created_at < record.created_at, + and_( + ImageRecord.created_at == record.created_at, + ImageRecord.id < record.id, + ), + ) + ) + .order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()) + .limit(1) + ) + prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none() + next_id = (await self.session.execute(next_stmt)).scalar_one_or_none() + return {"prev_id": prev_id, "next_id": next_id} + + +def _group_by_year_month( + images: list[GalleryImage], +) -> list[tuple[int, int, list[int]]]: + groups: list[tuple[int, int, list[int]]] = [] + for img in images: + y, m = img.created_at.year, img.created_at.month + if groups and groups[-1][0] == y and groups[-1][1] == m: + groups[-1][2].append(img.id) + else: + groups.append((y, m, [img.id])) + return groups diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py new file mode 100644 index 0000000..51550c2 --- /dev/null +++ b/backend/app/services/importer.py @@ -0,0 +1,761 @@ +"""Single-file import pipeline. + +The Importer is the synchronous core of FC-2a's import path. A Celery task +(tasks/import_file.py) instantiates one per job and calls import_one(). + +The service is intentionally not async — Celery tasks are run in a +synchronous worker process, and the DB session is a sync session passed in +by the task. The Quart side uses an async session via the same models. +""" + +import hashlib +import json +import logging +import shutil +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +from PIL import Image +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..models import ( + Artist, + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + PostAttachment, + Source, +) +from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name +from ..utils.phash import compute_phash, find_similar +from ..utils.sidecar import find_sidecar, parse_sidecar +from ..utils.slug import slugify +from .archive_extractor import extract_archive, is_archive +from .attachment_store import AttachmentStore +from .thumbnailer import Thumbnailer + +log = logging.getLogger(__name__) + + +class SkipReason(StrEnum): + too_small = "too_small" + too_transparent = "too_transparent" + single_color = "single_color" + duplicate_hash = "duplicate_hash" + duplicate_phash = "duplicate_phash" + invalid_image = "invalid_image" + + +@dataclass(frozen=True) +class ImportResult: + status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached' + image_id: int | None = None + skip_reason: SkipReason | None = None + error: str | None = None + member_image_ids: list[int] = field(default_factory=list) + + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"} +VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"} +ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS + + +def is_supported(path: Path) -> bool: + return path.suffix.lower() in ALL_EXTS + + +def is_video(path: Path) -> bool: + return path.suffix.lower() in VIDEO_EXTS + + +def _mime_for(path: Path) -> str: + suffix = path.suffix.lower() + image_mimes = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", + ".tif": "image/tiff", + ".tiff": "image/tiff", + } + video_mimes = { + ".mp4": "video/mp4", + ".mov": "video/quicktime", + ".avi": "video/x-msvideo", + ".mkv": "video/x-matroska", + ".webm": "video/webm", + ".m4v": "video/x-m4v", + ".wmv": "video/x-ms-wmv", + ".flv": "video/x-flv", + } + return image_mimes.get(suffix) or video_mimes.get(suffix) or "application/octet-stream" + + +def _sha256_of(path: Path, chunk_size: int = 1 << 20) -> str: + h = hashlib.sha256() + with path.open("rb") as fp: + while True: + block = fp.read(chunk_size) + if not block: + break + h.update(block) + return h.hexdigest() + + +class Importer: + """Imports one file at a time. Constructed per-task by the Celery task.""" + + def __init__( + self, + session: Session, + images_root: Path, + import_root: Path, + thumbnailer: Thumbnailer, + settings: ImportSettings, + deep: bool = False, + ): + self.session = session + self.images_root = images_root + self.import_root = import_root + self.thumbnailer = thumbnailer + self.settings = settings + self.deep = deep + self.attachments = AttachmentStore(images_root) + + def import_one(self, source: Path) -> ImportResult: + """Dispatch by kind. Media → normal pipeline. Archive → extract + media members (one Post via the archive-adjacent sidecar) and + preserve the archive itself. Other non-media → PostAttachment. + Nothing is skipped for being non-media (FC-2d-iii).""" + if source.suffix.lower() == ".json": + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error="sidecar json is metadata, not content", + ) + if is_archive(source): + return self._import_archive(source) + if not is_supported(source): + return self._capture_attachment(source) + return self._import_media(source, source) + + def _resolve_artist(self, attribution_path: Path) -> Artist | None: + name = derive_top_level_artist(attribution_path, self.import_root) + return self._upsert_artist(name) if name else None + + def _post_for_sidecar( + self, source: Path, artist: Artist | None + ) -> Post | None: + """If a sidecar sits next to `source`, ensure its Source+Post + exist (idempotent) and return the Post — so attachments can link + to the same Post the per-member _apply_sidecar will reuse.""" + sc = find_sidecar(source) + if sc is None or artist is None: + return None + try: + data = json.loads(sc.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("sidecar parse failed for %s: %s", sc, exc) + return None + sd = parse_sidecar(data) + platform = sd.platform or "unknown" + url = sd.post_url or f"sidecar:{platform}" + src = self.session.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == platform, + Source.url == url, + ) + ).scalar_one_or_none() + if src is None: + src = Source(artist_id=artist.id, platform=platform, url=url) + self.session.add(src) + self.session.flush() + epid = sd.external_post_id or sc.stem + post = self.session.execute( + select(Post).where( + Post.source_id == src.id, + Post.external_post_id == epid, + ) + ).scalar_one_or_none() + if post is None: + post = Post(source_id=src.id, external_post_id=epid) + self.session.add(post) + self.session.flush() + return post + + def _capture_attachment( + self, source: Path, *, post: Post | None = None, + artist: Artist | None = None, resolved: bool = False, + ) -> ImportResult: + if not resolved: + artist = self._resolve_artist(source) + post = self._post_for_sidecar(source, artist) + sha = _sha256_of(source) + existing = self.session.execute( + select(PostAttachment).where(PostAttachment.sha256 == sha) + ).scalar_one_or_none() + if existing is None: + stored = self.attachments.store(source, sha) + self.session.add(PostAttachment( + post_id=post.id if post else None, + artist_id=artist.id if artist else None, + sha256=sha, + path=stored, + original_filename=source.name, + ext=source.suffix.lower(), + mime=_mime_for(source), + size_bytes=source.stat().st_size, + )) + self.session.flush() + self.session.commit() + return ImportResult(status="attached") + + def _import_archive(self, source: Path) -> ImportResult: + artist = self._resolve_artist(source) + post = self._post_for_sidecar(source, artist) + member_ids: list[int] = [] + with extract_archive(source) as members: + for _name, member_path in members: + if not is_supported(member_path): + continue # non-media preserved via the stored archive + res = self._import_media(member_path, source) + if res.status in ("imported", "superseded") and res.image_id: + member_ids.append(res.image_id) + # Preserve the archive itself (links to the same Post/Artist). + self._capture_attachment( + source, post=post, artist=artist, resolved=True + ) + if member_ids: + return ImportResult( + status="imported", image_id=member_ids[0], + member_image_ids=member_ids, + ) + return ImportResult(status="attached") + + def _import_media( + self, source: Path, attribution_path: Path + ) -> ImportResult: + """The media import pipeline (filters, dedup, copy, provenance). + + `attribution_path` anchors artist/subdir/sidecar derivation: for + a normal file it equals `source`; for an archive member it is the + archive's real path (the member lives in a tempdir). + """ + if not is_supported(source): + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error=f"unsupported extension {source.suffix}", + ) + + # Compute file dimensions (images only) and apply filters. + width = height = None + has_alpha = False + if not is_video(source): + try: + with Image.open(source) as im: + im.verify() + with Image.open(source) as im: + width, height = im.size + has_alpha = im.mode in ("RGBA", "LA") or ( + im.mode == "P" and "transparency" in im.info + ) + except Exception as exc: + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc), + ) + + if (self.settings.min_width and width < self.settings.min_width) or ( + self.settings.min_height and height < self.settings.min_height + ): + return ImportResult( + status="skipped", skip_reason=SkipReason.too_small, + error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}", + ) + + if self.settings.skip_transparent and has_alpha: + pct = self._transparency_pct(source) + if pct >= self.settings.transparency_threshold: + return ImportResult( + status="skipped", skip_reason=SkipReason.too_transparent, + error=f"{pct:.2%} transparent", + ) + + # Hash dedup (exact). + sha = _sha256_of(source) + existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha) + existing = self.session.execute(existing_stmt).scalar_one_or_none() + if existing: + if not self.deep: + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="sha256 already present", + ) + return self._deep_rederive(existing, source, attribution_path) + + # Perceptual near-dup (images only; videos keep phash NULL). + phash = None + if not is_video(source): + with Image.open(source) as im: + phash = compute_phash(im) + if phash is not None: + cand_rows = self.session.execute( + select( + ImageRecord.phash, + ImageRecord.width, + ImageRecord.height, + ImageRecord.id, + ).where(ImageRecord.phash.is_not(None)) + ).all() + candidates = [ + (c.phash, c.width or 0, c.height or 0, c.id) + for c in cand_rows + ] + rel, match_id = find_similar( + phash, width or 0, height or 0, + candidates, self.settings.phash_threshold, + ) + if rel == "larger_exists": + return ImportResult( + status="skipped", + skip_reason=SkipReason.duplicate_phash, + image_id=match_id, + error="perceptual near-duplicate of larger existing image", + ) + if rel == "smaller_exists": + target = self.session.get(ImageRecord, match_id) + self._supersede(target, source, sha, phash, width, height) + return ImportResult( + status="superseded", image_id=match_id + ) + + dest = self._copy_to_library(source, sha, attribution_path) + + record = ImageRecord( + path=str(dest), + sha256=sha, + phash=phash, + size_bytes=dest.stat().st_size, + mime=_mime_for(source), + width=width, + height=height, + origin="imported_filesystem", + integrity_status="unknown", + ) + self.session.add(record) + self.session.flush() + + # Folder→artist (anchored to attribution_path). + artist = None + artist_name = derive_top_level_artist( + attribution_path, self.import_root + ) + if artist_name: + artist = self._attach_artist(record, artist_name) + + # Sidecar provenance (best-effort; never fails the import). + self._apply_sidecar(record, attribution_path, artist) + + # Thumbnail is queued separately by the calling task; the importer + # does not generate thumbnails inline so the import queue stays moving. + + self.session.commit() + return ImportResult(status="imported", image_id=record.id) + + def _deep_rederive( + self, existing: ImageRecord, source: Path, attribution_path: Path + ) -> ImportResult: + """Deep scan: backfill phash/provenance/artist on an + already-imported record. METADATA ONLY — never re-runs the pHash + near-dup / supersede path. NULL-only, idempotent.""" + if existing.phash is None and not is_video(source): + try: + with Image.open(source) as im: + ph = compute_phash(im) + if ph is not None: + existing.phash = ph + except Exception as exc: + log.warning("deep rephash failed for %s: %s", source, exc) + + artist = None + name = derive_top_level_artist(attribution_path, self.import_root) + if name: + artist = self._upsert_artist(name) + if existing.artist_id is None: + existing.artist_id = artist.id + + self._apply_sidecar(existing, attribution_path, artist) + self.session.commit() + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="deep: re-derived", + ) + + def attach_in_place( + self, + path: Path, + *, + sidecar_path: Path | None = None, + artist: Artist | None = None, + source: Source | None = None, + ) -> ImportResult: + """Attach a file ALREADY at its final library location (FC-3c). + + Mirrors _import_media but skips the copy step — the file is + where it'll permanently live. Caller (DownloadService) already + has artist/source context from the subscription Source row; pass + them through. The sidecar JSON gallery-dl emits next to each + downloaded file is read by `_apply_sidecar` via `find_sidecar`. + + Caller's responsibilities after this returns: + - duplicate_hash / duplicate_phash skip → delete the on-disk file + - superseded → file stays where it is (now canonical) + - imported → file stays where it is + - failed → file untouched; caller decides + """ + if not is_supported(path): + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error=f"unsupported extension {path.suffix}", + ) + + # Format / dimension / transparency filters (mirror _import_media). + width = height = None + has_alpha = False + if not is_video(path): + try: + with Image.open(path) as im: + im.verify() + with Image.open(path) as im: + width, height = im.size + has_alpha = im.mode in ("RGBA", "LA") or ( + im.mode == "P" and "transparency" in im.info + ) + except Exception as exc: + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error=str(exc), + ) + + if (self.settings.min_width and width < self.settings.min_width) or ( + self.settings.min_height and height < self.settings.min_height + ): + return ImportResult( + status="skipped", skip_reason=SkipReason.too_small, + error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}", + ) + + if self.settings.skip_transparent and has_alpha: + pct = self._transparency_pct(path) + if pct >= self.settings.transparency_threshold: + return ImportResult( + status="skipped", skip_reason=SkipReason.too_transparent, + error=f"{pct:.2%} transparent", + ) + + # Hash dedup + sha = _sha256_of(path) + existing = self.session.execute( + select(ImageRecord).where(ImageRecord.sha256 == sha) + ).scalar_one_or_none() + if existing: + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="sha256 already present", + ) + + # phash dedup + supersede + phash = None + if not is_video(path): + try: + with Image.open(path) as im: + phash = compute_phash(im) + except Exception: + phash = None + if phash is not None: + cand_rows = self.session.execute( + select( + ImageRecord.phash, + ImageRecord.width, + ImageRecord.height, + ImageRecord.id, + ).where(ImageRecord.phash.is_not(None)) + ).all() + candidates = [ + (c.phash, c.width or 0, c.height or 0, c.id) + for c in cand_rows + ] + rel, match_id = find_similar( + phash, width or 0, height or 0, + candidates, self.settings.phash_threshold, + ) + if rel == "larger_exists": + return ImportResult( + status="skipped", + skip_reason=SkipReason.duplicate_phash, + image_id=match_id, + error="perceptual near-duplicate of larger existing image", + ) + if rel == "smaller_exists": + target = self.session.get(ImageRecord, match_id) + self._supersede( + target, path, sha, phash, width, height, new_path=path + ) + return ImportResult(status="superseded", image_id=match_id) + + # Create record — the path IS where the file lives. + record = ImageRecord( + path=str(path), + sha256=sha, + phash=phash, + size_bytes=path.stat().st_size, + mime=_mime_for(path), + width=width, + height=height, + origin="downloaded", + integrity_status="unknown", + ) + if artist is not None: + record.artist_id = artist.id + self.session.add(record) + self.session.flush() + + # Sidecar provenance (best-effort). When `source` is passed, link + # the post to that subscription Source instead of creating a new + # per-post Source row. + self._apply_sidecar(record, path, artist, explicit_source=source) + + self.session.commit() + return ImportResult(status="imported", image_id=record.id) + + def _upsert_artist(self, name: str) -> Artist: + slug = slugify(name) + artist = self.session.execute( + select(Artist).where(Artist.slug == slug) + ).scalar_one_or_none() + if artist is None: + artist = Artist(name=name, slug=slug, is_subscription=False) + self.session.add(artist) + self.session.flush() + return artist + + def _attach_artist(self, record: ImageRecord, artist_name: str) -> Artist: + """Upsert the Artist and set it as the image's canonical artist. + Artist-kind tags were retired in FC-2d-vii-c — the Artist row is + the single source of truth. Returns the Artist (sidecar + provenance attaches its Source to it).""" + artist = self._upsert_artist(artist_name) + if record.artist_id is None: + record.artist_id = artist.id + self.session.flush() + return artist + + @staticmethod + def _sidecar_artist_name(data: dict) -> str | None: + for k in ("artist", "author_name"): + v = data.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + for k in ("user", "creator"): + v = data.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + if isinstance(v, dict): + for sub in ("name", "full_name", "vanity", "account"): + sv = v.get(sub) + if isinstance(sv, str) and sv.strip(): + return sv.strip() + return None + + def _apply_sidecar( + self, + record: ImageRecord, + source: Path, + artist: Artist | None, + *, + explicit_source: Source | None = None, + ) -> None: + """Read the sidecar adjacent to `source` and wire up provenance. + + If `explicit_source` is passed (FC-3c attach_in_place case), the + sidecar's post is linked to that Source instead of looking up / + creating one keyed by the sidecar's post_url. Without it, the + filesystem importer's per-post-Source creation is preserved. + """ + sc = find_sidecar(source) + if sc is None: + return + try: + data = json.loads(sc.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("sidecar parse failed for %s: %s", sc, exc) + return + + sd = parse_sidecar(data) + + if artist is None: + name = self._sidecar_artist_name(data) + if name: + artist = self._upsert_artist(name) + if artist is None: + log.warning( + "sidecar present for %s but no artist; skipping provenance", + source, + ) + return + + if record.artist_id is None: + record.artist_id = artist.id + + if explicit_source is not None: + src = explicit_source + else: + platform = sd.platform or "unknown" + url = sd.post_url or f"sidecar:{platform}" + src = self.session.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == platform, + Source.url == url, + ) + ).scalar_one_or_none() + if src is None: + src = Source(artist_id=artist.id, platform=platform, url=url) + self.session.add(src) + self.session.flush() + + epid = sd.external_post_id or sc.stem + post = self.session.execute( + select(Post).where( + Post.source_id == src.id, + Post.external_post_id == epid, + ) + ).scalar_one_or_none() + if post is None: + post = Post(source_id=src.id, external_post_id=epid) + self.session.add(post) + self.session.flush() + if sd.post_url is not None: + post.post_url = sd.post_url + if sd.post_title is not None: + post.post_title = sd.post_title + if sd.post_date is not None: + post.post_date = sd.post_date + if sd.description is not None: + post.description = sd.description + if sd.attachment_count is not None: + post.attachment_count = sd.attachment_count + post.raw_metadata = sd.raw + + exists = self.session.execute( + select(ImageProvenance.id).where( + ImageProvenance.image_record_id == record.id, + ImageProvenance.post_id == post.id, + ) + ).scalar_one_or_none() + if exists is None: + self.session.add( + ImageProvenance( + image_record_id=record.id, + post_id=post.id, + source_id=src.id, + captured_metadata=sd.raw, + ) + ) + if record.primary_post_id is None: + record.primary_post_id = post.id + self.session.flush() + + def _copy_to_library( + self, source: Path, sha: str, attribution_path: Path + ) -> Path: + """Copy `source` to its final library path. Returns the destination. + + Atomic write: copies to .partial then renames. Shared by + _import_media (filesystem scan) and _supersede (when new_path is + not passed). FC-3c's attach_in_place skips this helper entirely + — the file is already at its final home. + """ + subdir = derive_subdir(attribution_path, self.import_root) + dest_dir = self.images_root / subdir if subdir else self.images_root + dest_dir.mkdir(parents=True, exist_ok=True) + dest_name = hash_suffixed_name(source.stem, sha, source.suffix) + dest = dest_dir / dest_name + partial = dest.with_suffix(dest.suffix + ".partial") + shutil.copy2(source, partial) + partial.rename(dest) + return dest + + def _supersede( + self, existing: ImageRecord, source: Path, sha: str, + phash: str, width: int | None, height: int | None, + *, new_path: Path | None = None, + ) -> None: + """Replace `existing`'s file with the larger `source`, keeping the + row id (so tags/series/curation stay attached). ML is cleared so + the import task re-derives it on the new pixels. + + If `new_path` is provided, `source` is assumed to ALREADY be at + that path (FC-3c attach_in_place case) — skip the copy step. + Otherwise the file is copied via _copy_to_library.""" + if new_path is None: + dest = self._copy_to_library(source, sha, source) + else: + dest = new_path + + old_path = existing.path + old_thumb = existing.thumbnail_path + + existing.path = str(dest) + existing.sha256 = sha + existing.phash = phash + existing.size_bytes = dest.stat().st_size + existing.mime = _mime_for(source) + existing.width = width + existing.height = height + existing.thumbnail_path = None + existing.integrity_status = "unknown" + existing.tagger_predictions = None + existing.tagger_model_version = None + existing.siglip_embedding = None + existing.siglip_model_version = None + existing.centroid_scores = None + # created_at intentionally preserved; updated_at auto-bumps. + self.session.flush() + self.session.commit() + + for stale in (old_path, old_thumb): + if not stale or stale == str(dest): + # If the supersede kept the file in place (new_path == old + # location), don't delete it. + continue + try: + p = Path(stale) + if p.exists(): + p.unlink() + except Exception: + # Benign orphan; the DB swap already committed. Don't undo it. + pass + + def _transparency_pct(self, source: Path) -> float: + """Fraction of fully-transparent pixels in the image. 0.0 if no alpha.""" + with Image.open(source) as im: + if im.mode not in ("RGBA", "LA") and not ( + im.mode == "P" and "transparency" in im.info + ): + return 0.0 + if im.mode != "RGBA": + im = im.convert("RGBA") + alpha = im.getchannel("A") + histogram = alpha.histogram() + transparent = histogram[0] + total = sum(histogram) + return transparent / total if total else 0.0 diff --git a/backend/app/services/migrators/__init__.py b/backend/app/services/migrators/__init__.py new file mode 100644 index 0000000..0a7154c --- /dev/null +++ b/backend/app/services/migrators/__init__.py @@ -0,0 +1,6 @@ +"""FC-5 migration tooling. + +One module per concern (backup/rollback/gs/ir/overlap/ml_queue/verify). +Each migrator returns a counts dict; the run_migration task wires +that dict into MigrationRun.counts so the UI polling shows progress. +""" diff --git a/backend/app/services/migrators/backup.py b/backend/app/services/migrators/backup.py new file mode 100644 index 0000000..88040fd --- /dev/null +++ b/backend/app/services/migrators/backup.py @@ -0,0 +1,124 @@ +"""pg_dump + tar.zst-based backup, restorable via pair of subprocess calls. + +Backups live under /_backups/. Each backup is two files +(SQL + tarball) plus a manifest JSON. Tagged backups (e.g. tag='pre_migration') +are how rollback.py finds the most recent restorable snapshot. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +_BACKUPS_DIRNAME = "_backups" + + +def _backups_dir(images_root: Path | None = None) -> Path: + # Overridable for tests via monkeypatch. + root = images_root if images_root is not None else Path("/images") + p = root / _BACKUPS_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p + + +def _run_subprocess(cmd: list[str], **kwargs: Any): + # Overridable for tests via monkeypatch. + return subprocess.run( + cmd, + capture_output=True, + check=True, + **{k: v for k, v in kwargs.items() if not k.startswith("_")}, + ) + + +def create_backup( + *, db_url: str, images_root: Path, tag: str = "manual", +) -> dict: + """Create a backup: pg_dump SQL + tar.zst of images. + + Returns a manifest dict. Writes .sql, .tar.zst, .json into + /_backups/. + """ + ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + out_dir = _backups_dir(images_root) + sql_path = out_dir / f"fc_{ts}.sql" + tar_path = out_dir / f"fc_{ts}.tar.zst" + manifest_path = out_dir / f"fc_{ts}.json" + + _run_subprocess( + ["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), db_url], + _test_ts=ts, + ) + _run_subprocess( + [ + "tar", "--zstd", "-cf", str(tar_path), + "-C", str(images_root.parent), images_root.name, + f"--exclude={images_root.name}/_backups", + f"--exclude={images_root.name}/_quarantine", + ], + _test_ts=ts, + ) + + manifest = { + "backup_id": ts, + "tag": tag, + "created_at": datetime.now(UTC).isoformat(), + "sql_path": str(sql_path), + "tar_path": str(tar_path), + } + manifest_path.write_text(json.dumps(manifest, indent=2)) + return manifest + + +def list_backups(images_root: Path) -> list[dict]: + out_dir = _backups_dir(images_root) + items = [] + for mf in sorted(out_dir.glob("fc_*.json"), reverse=True): + try: + items.append(json.loads(mf.read_text())) + except Exception: + continue + return items + + +def find_latest_backup(images_root: Path, *, tag: str) -> dict | None: + for mf in list_backups(images_root): + if mf.get("tag") == tag: + return mf + return None + + +def restore_backup( + *, manifest: dict, db_url: str, images_root: Path, +) -> dict: + """Restore from a backup manifest. + + 1. Replay the .sql via psql. + 2. Wipe /images/ contents (except _backups/, which holds the file we're using). + 3. Untar the .tar.zst into /images/. + """ + sql_path = Path(manifest["sql_path"]) + tar_path = Path(manifest["tar_path"]) + + _run_subprocess( + ["psql", "-d", db_url, "-f", str(sql_path)], + ) + + # Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup + # we're restoring from!). + for entry in images_root.iterdir(): + if entry.name == _BACKUPS_DIRNAME: + continue + if entry.is_dir(): + shutil.rmtree(entry) + else: + entry.unlink() + + _run_subprocess( + ["tar", "--zstd", "-xf", str(tar_path), "-C", str(images_root.parent)], + ) + + return {"restored_from": manifest["backup_id"]} diff --git a/backend/app/services/migrators/gs_ingest.py b/backend/app/services/migrators/gs_ingest.py new file mode 100644 index 0000000..18b832a --- /dev/null +++ b/backend/app/services/migrators/gs_ingest.py @@ -0,0 +1,126 @@ +"""GallerySubscriber export → FabledCurator ingest. + +Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection +to GS). Creates Artist (from subscriptions) + Source (nested under each +subscription) + Credential (re-encrypted with FC's key). Idempotent on +natural keys: Artist.slug, (artist_id, platform, url), Credential.platform. + +Credentials arrive plaintext in the export — GS's export script +decrypts using GS's Fernet key in GS's own process. FC re-encrypts +with FC's CredentialCrypto. +""" +from __future__ import annotations + +import json + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, Credential, Source +from ...utils.slug import slugify +from ..credential_crypto import CredentialCrypto + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +async def migrate_async( + db: AsyncSession, + *, + data: dict, + fc_crypto: CredentialCrypto | None = None, + dry_run: bool = False, +) -> dict: + """Ingest a parsed gallerysubscriber-export-v1.json dict.""" + if data.get("source_app") != "gallerysubscriber": + raise ValueError("export source_app must be 'gallerysubscriber'") + if data.get("schema_version") != 1: + raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") + + counts = _zero_counts() + + # Phase 1: subscriptions → Artist; nested sources within each. + for sub in data.get("subscriptions", []): + counts["rows_processed"] += 1 + slug = slugify(sub["name"]) + artist = (await db.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one_or_none() + if artist is None: + if dry_run: + counts["rows_inserted"] += 1 + # Continue to nested sources, but they can't link without an artist row. + continue + notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None + artist = Artist( + name=sub["name"], slug=slug, + is_subscription=True, + auto_check=bool(sub.get("enabled", True)), + notes=notes, + ) + db.add(artist) + await db.flush() + counts["rows_inserted"] += 1 + else: + counts["rows_skipped"] += 1 + + # Nested sources under this subscription. + for src in sub.get("sources", []): + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == src["platform"], + Source.url == src["url"], + ) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Source( + artist_id=artist.id, + platform=src["platform"], + url=src["url"], + enabled=bool(src.get("enabled", True)), + check_interval_override=src.get("check_interval"), + config_overrides=src.get("metadata") or {}, + )) + counts["rows_inserted"] += 1 + + # Phase 2: credentials. + for cred in data.get("credentials", []): + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Credential).where(Credential.platform == cred["platform"]) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + if fc_crypto is None: + # Without a crypto helper we can't encrypt — skip rather than + # store plaintext. + counts["rows_skipped"] += 1 + counts["conflicts"] += 1 + continue + encrypted = fc_crypto.encrypt(cred["plaintext"]) + db.add(Credential( + platform=cred["platform"], + credential_type=cred.get("credential_type") or "cookies", + encrypted_blob=encrypted, + expires_at=cred.get("expires_at"), + )) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + return counts diff --git a/backend/app/services/migrators/ir_ingest.py b/backend/app/services/migrators/ir_ingest.py new file mode 100644 index 0000000..75d12cc --- /dev/null +++ b/backend/app/services/migrators/ir_ingest.py @@ -0,0 +1,141 @@ +"""ImageRepo export → FabledCurator ingest. + +Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR). +Creates Tag rows (skipping artist/post kinds, resolving fandom_name to +FK). Writes the per-image-sha256 artist assignments + tag associations ++ series page assignments to /images/_migration_state/ir_tag_manifest.json +so tag_apply.py can join them to ImageRecord rows AFTER the operator +runs FC's filesystem scan. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Tag, TagKind + +_SKIP_KINDS = frozenset({"artist", "post"}) +_MIGRATION_STATE_DIRNAME = "_migration_state" +_IR_MANIFEST_FILENAME = "ir_tag_manifest.json" + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +def manifest_path(images_root: Path | None = None) -> Path: + root = images_root if images_root is not None else Path("/images") + p = root / _MIGRATION_STATE_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p / _IR_MANIFEST_FILENAME + + +async def _resolve_fandom_id( + db: AsyncSession, fandom_name: str | None, dry_run: bool, +) -> int | None: + """Find-or-create a fandom-kind Tag by name.""" + if not fandom_name: + return None + existing = (await db.execute( + select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + return existing.id + if dry_run: + return None + t = Tag(name=fandom_name, kind=TagKind.fandom) + db.add(t) + await db.flush() + return t.id + + +async def migrate_async( + db: AsyncSession, + *, + data: dict, + images_root: Path | None = None, + dry_run: bool = False, +) -> dict: + """Ingest a parsed imagerepo-export-v1.json dict. + + Creates Tag rows + writes the IR tag manifest file. Tag-to-image + binding happens later in tag_apply.py (after FC's filesystem scan + populates image_record.sha256 → id). + """ + if data.get("source_app") != "imagerepo": + raise ValueError("export source_app must be 'imagerepo'") + if data.get("schema_version") != 1: + raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") + + counts = _zero_counts() + + # Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id). + # First pass: create all fandom-kind tags so they're available for FK resolution. + for tag in data.get("tags", []): + kind = tag.get("kind") or "general" + if kind != "fandom": + continue + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=TagKind.fandom)) + counts["rows_inserted"] += 1 + if not dry_run: + await db.flush() + + # Second pass: every other kind. + for tag in data.get("tags", []): + kind_str = tag.get("kind") or "general" + if kind_str in _SKIP_KINDS: + counts["rows_skipped"] += 1 + continue + if kind_str == "fandom": + continue # handled above + counts["rows_processed"] += 1 + try: + kind = TagKind(kind_str) + except ValueError: + kind = TagKind.general + fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run) + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == kind) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id)) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + + # Phase 2: write the per-image manifest for tag_apply.py to consume later. + manifest = { + "schema_version": 1, + "image_artist_assignments": data.get("image_artist_assignments", []), + "image_tag_associations": data.get("image_tag_associations", []), + "series_pages": data.get("series_pages", []), + } + counts["rows_processed"] += len(manifest["image_artist_assignments"]) + counts["rows_processed"] += len(manifest["image_tag_associations"]) + counts["rows_processed"] += len(manifest["series_pages"]) + if not dry_run: + manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) + + return counts diff --git a/backend/app/services/migrators/ml_queue.py b/backend/app/services/migrators/ml_queue.py new file mode 100644 index 0000000..6d9e92f --- /dev/null +++ b/backend/app/services/migrators/ml_queue.py @@ -0,0 +1,21 @@ +"""Queue every migrated image_record with no embedding for ML re-processing.""" +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import ImageRecord + + +async def queue_all_unprocessed_async(db: AsyncSession) -> int: + """Find every ImageRecord with siglip_embedding IS NULL, fire + tag_and_embed.delay(id) for each. Returns count queued. + """ + from ...tasks.ml import tag_and_embed + + rows = (await db.execute( + select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None)) + )).scalars().all() + for image_id in rows: + tag_and_embed.delay(image_id) + return len(rows) diff --git a/backend/app/services/migrators/rollback.py b/backend/app/services/migrators/rollback.py new file mode 100644 index 0000000..55ae7ef --- /dev/null +++ b/backend/app/services/migrators/rollback.py @@ -0,0 +1,21 @@ +"""Restore from the most recent 'pre_migration'-tagged backup.""" +from __future__ import annotations + +from pathlib import Path + +from . import backup as backup_mod + + +class NoBackupFoundError(Exception): + """Raised when rollback() is called with no pre_migration backup on disk.""" + + +def rollback_to_pre_migration(*, db_url: str, images_root: Path) -> dict: + manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration") + if manifest is None: + raise NoBackupFoundError( + "no pre_migration-tagged backup found under /_backups/" + ) + return backup_mod.restore_backup( + manifest=manifest, db_url=db_url, images_root=images_root, + ) diff --git a/backend/app/services/migrators/tag_apply.py b/backend/app/services/migrators/tag_apply.py new file mode 100644 index 0000000..b02d8ab --- /dev/null +++ b/backend/app/services/migrators/tag_apply.py @@ -0,0 +1,172 @@ +"""Apply the IR tag manifest after FC's filesystem scan. + +Reads /images/_migration_state/ir_tag_manifest.json and joins each entry +to an ImageRecord row by sha256 (which exists after the operator runs +FC's filesystem scan over the mounted IR images dir). + +- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug). +- image_tag_associations → image_tag insert (idempotent). +- series_pages → series_page insert (idempotent on image_id unique). + +Unmatched sha256s are logged into the result's `unmatched` list so the +Celery task can drop them into MigrationRun.metadata for the operator +to inspect. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from ...utils.slug import slugify +from .ir_ingest import manifest_path + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +async def _ensure_artist_id( + db: AsyncSession, artist_name: str, dry_run: bool, +) -> int | None: + if not artist_name or not artist_name.strip(): + return None + slug = slugify(artist_name) + existing = (await db.execute( + select(Artist).where(Artist.slug == slug) + )).scalar_one_or_none() + if existing is not None: + return existing.id + if dry_run: + return None + a = Artist(name=artist_name, slug=slug, is_subscription=False) + db.add(a) + await db.flush() + return a.id + + +async def _resolve_tag_id( + db: AsyncSession, tag_name: str, tag_kind: str, +) -> int | None: + try: + kind = TagKind(tag_kind) + except ValueError: + kind = TagKind.general + row = (await db.execute( + select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind) + )).scalar_one_or_none() + return row + + +async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None: + return (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one_or_none() + + +async def apply_async( + db: AsyncSession, + *, + images_root: Path | None = None, + dry_run: bool = False, +) -> dict: + """Apply the manifest. Returns counts + an `unmatched` list of sha256s.""" + mf_path = manifest_path(images_root) + if not mf_path.exists(): + raise FileNotFoundError(f"no IR tag manifest at {mf_path}") + + manifest = json.loads(mf_path.read_text()) + counts = _zero_counts() + unmatched: list[dict] = [] + + # 1. Artist assignments. + for entry in manifest.get("image_artist_assignments", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "artist", **entry}) + counts["rows_skipped"] += 1 + continue + aid = await _ensure_artist_id(db, entry["artist_name"], dry_run) + if aid is None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + img = await db.get(ImageRecord, img_id) + if img is not None and img.artist_id != aid: + img.artist_id = aid + counts["rows_inserted"] += 1 + else: + counts["rows_skipped"] += 1 + + # 2. Tag associations. + for entry in manifest.get("image_tag_associations", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "tag", **entry}) + counts["rows_skipped"] += 1 + continue + tag_id = await _resolve_tag_id( + db, entry["tag_name"], entry.get("tag_kind") or "general", + ) + if tag_id is None: + counts["rows_skipped"] += 1 + continue + # Skip if association already exists. + already = (await db.execute( + select(image_tag.c.image_record_id).where( + image_tag.c.image_record_id == img_id, + image_tag.c.tag_id == tag_id, + ) + )).first() + if already is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + await db.execute(image_tag.insert().values( + image_record_id=img_id, tag_id=tag_id, source="manual", + )) + counts["rows_inserted"] += 1 + + # 3. Series pages. + for entry in manifest.get("series_pages", []): + counts["rows_processed"] += 1 + img_id = await _sha_to_image_id(db, entry["sha256"]) + if img_id is None: + unmatched.append({"kind": "series", **entry}) + counts["rows_skipped"] += 1 + continue + series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series") + if series_tag_id is None: + counts["rows_skipped"] += 1 + continue + existing = (await db.execute( + select(SeriesPage).where(SeriesPage.image_id == img_id) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(SeriesPage( + series_tag_id=series_tag_id, + image_id=img_id, + page_number=entry["page_number"], + )) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + return {"counts": counts, "unmatched": unmatched} diff --git a/backend/app/services/migrators/verify.py b/backend/app/services/migrators/verify.py new file mode 100644 index 0000000..52bf37c --- /dev/null +++ b/backend/app/services/migrators/verify.py @@ -0,0 +1,80 @@ +"""Post-migration verification: row counts + sha256 sampling.""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Artist, Credential, ImageRecord, Source, Tag + + +async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict: + """Return per-check status dicts. `expected` is optional row-count + assertions; checks default to status='ok' when no expected provided.""" + expected = expected or {} + results: dict[str, dict] = {} + + checks = { + "artist_subscriptions": ( + select(func.count(Artist.id)).where(Artist.is_subscription.is_(True)) + ), + "source_count": select(func.count(Source.id)), + "credential_count": select(func.count(Credential.id)), + "tag_count": select(func.count(Tag.id)), + "image_record_imported_or_downloaded": ( + select(func.count(ImageRecord.id)) + .where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"])) + ), + } + for name, stmt in checks.items(): + actual = (await db.execute(stmt)).scalar_one() + exp = expected.get(name) + status = "ok" if exp is None or exp == actual else "mismatch" + results[name] = {"status": status, "actual": int(actual), "expected": exp} + return results + + +async def verify_sha256_sample( + db: AsyncSession, *, sample_size: int = 20, +) -> dict: + """Sample N image_records; verify file exists + sha256 matches.""" + rows = (await db.execute( + select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256) + .order_by(func.random()).limit(sample_size) + )).all() + + matched = 0 + mismatched = 0 + missing = 0 + samples: list[dict] = [] + for img_id, path, expected_sha in rows: + p = Path(path) + # Sync stdlib filesystem ops are intentional: this verify pass runs + # inside a Celery task under asyncio.run; no other awaitables compete + # for the loop. Same pattern as download_service.py. + if not p.exists(): # noqa: ASYNC240 + missing += 1 + samples.append({"id": img_id, "path": path, "result": "missing"}) + continue + h = hashlib.sha256() + with p.open("rb") as f: # noqa: ASYNC230 + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + if h.hexdigest() == expected_sha: + matched += 1 + samples.append({"id": img_id, "result": "ok"}) + else: + mismatched += 1 + samples.append({ + "id": img_id, "path": path, "result": "mismatch", + "expected_sha": expected_sha, "actual_sha": h.hexdigest(), + }) + return { + "sample_size": len(rows), + "matched": matched, + "mismatched": mismatched, + "missing": missing, + "samples": samples, + } diff --git a/backend/app/services/ml/__init__.py b/backend/app/services/ml/__init__.py new file mode 100644 index 0000000..82ab712 --- /dev/null +++ b/backend/app/services/ml/__init__.py @@ -0,0 +1 @@ +"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases.""" diff --git a/backend/app/services/ml/aliases.py b/backend/app/services/ml/aliases.py new file mode 100644 index 0000000..4d65ecd --- /dev/null +++ b/backend/app/services/ml/aliases.py @@ -0,0 +1,104 @@ +"""Alias resolution + CRUD. + +A tag_alias maps (model_name, model_category) -> canonical Tag. Resolution +happens at suggestion-read time so raw tagger_predictions stay unmolested. +""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from sqlalchemy import delete, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Tag, TagAlias + + +@dataclass(frozen=True) +class AliasRow: + alias_string: str + alias_category: str + canonical_tag_id: int + canonical_tag_name: str + + +class AliasService: + def __init__(self, session: AsyncSession): + self.session = session + + async def resolve(self, name: str, category: str) -> Tag | None: + """Return the canonical Tag for (name, category), or None if no alias.""" + stmt = ( + select(Tag) + .join(TagAlias, TagAlias.canonical_tag_id == Tag.id) + .where(TagAlias.alias_string == name) + .where(TagAlias.alias_category == category) + ) + return (await self.session.execute(stmt)).scalar_one_or_none() + + async def resolve_many( + self, pairs: list[tuple[str, str]] + ) -> dict[tuple[str, str], Tag]: + """Batch-resolve. Returns only the pairs that have an alias. + + Used by SuggestionService so it does one query instead of N. + """ + if not pairs: + return {} + strings = {p[0] for p in pairs} + stmt = ( + select(TagAlias, Tag) + .join(Tag, Tag.id == TagAlias.canonical_tag_id) + .where(TagAlias.alias_string.in_(strings)) + ) + rows = (await self.session.execute(stmt)).all() + wanted = set(pairs) + out: dict[tuple[str, str], Tag] = {} + for alias, tag in rows: + key = (alias.alias_string, alias.alias_category) + if key in wanted: + out[key] = tag + return out + + async def create( + self, alias_string: str, alias_category: str, canonical_tag_id: int + ) -> None: + """Idempotent create (ON CONFLICT DO NOTHING).""" + stmt = insert(TagAlias).values( + alias_string=alias_string, + alias_category=alias_category, + canonical_tag_id=canonical_tag_id, + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["alias_string", "alias_category"] + ) + await self.session.execute(stmt) + + async def remove(self, alias_string: str, alias_category: str) -> None: + await self.session.execute( + delete(TagAlias) + .where(TagAlias.alias_string == alias_string) + .where(TagAlias.alias_category == alias_category) + ) + + async def list_all(self) -> Sequence[AliasRow]: + stmt = ( + select( + TagAlias.alias_string, + TagAlias.alias_category, + TagAlias.canonical_tag_id, + Tag.name, + ) + .join(Tag, Tag.id == TagAlias.canonical_tag_id) + .order_by(TagAlias.alias_string.asc()) + ) + rows = (await self.session.execute(stmt)).all() + return [ + AliasRow( + alias_string=r[0], + alias_category=r[1], + canonical_tag_id=r[2], + canonical_tag_name=r[3], + ) + for r in rows + ] diff --git a/backend/app/services/ml/allowlist.py b/backend/app/services/ml/allowlist.py new file mode 100644 index 0000000..dc8aeab --- /dev/null +++ b/backend/app/services/ml/allowlist.py @@ -0,0 +1,126 @@ +"""Allowlist semantics: accepting a suggestion adds the canonical tag to +image_tag AND to tag_allowlist; per-image removal/dismiss writes a rejection. +""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from sqlalchemy import delete, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Tag, TagAllowlist, TagSuggestionRejection +from ...models.tag import image_tag +from .aliases import AliasService + + +@dataclass(frozen=True) +class AllowlistRow: + tag_id: int + tag_name: str + tag_kind: str + min_confidence: float + + +class AllowlistService: + def __init__(self, session: AsyncSession): + self.session = session + self.aliases = AliasService(session) + + async def _apply_image_tag(self, image_id: int, tag_id: int, source: str): + stmt = insert(image_tag).values( + image_record_id=image_id, tag_id=tag_id, source=source + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + await self.session.execute(stmt) + + async def _add_to_allowlist(self, tag_id: int) -> bool: + """Returns True if newly added (caller should kick off retro-apply).""" + exists = await self.session.get(TagAllowlist, tag_id) + if exists is not None: + return False + self.session.add(TagAllowlist(tag_id=tag_id)) + await self.session.flush() + return True + + async def _clear_rejection(self, image_id: int, tag_id: int): + await self.session.execute( + delete(TagSuggestionRejection) + .where(TagSuggestionRejection.image_record_id == image_id) + .where(TagSuggestionRejection.tag_id == tag_id) + ) + + async def accept(self, image_id: int, tag_id: int) -> bool: + """Accept a suggestion. Returns True if the tag was newly added to + the allowlist (the API layer enqueues apply_allowlist_tags then).""" + await self._apply_image_tag(image_id, tag_id, source="ml_accepted") + await self._clear_rejection(image_id, tag_id) + return await self._add_to_allowlist(tag_id) + + async def add_alias_and_accept( + self, + image_id: int, + alias_string: str, + alias_category: str, + canonical_tag_id: int, + ) -> bool: + await self.aliases.create( + alias_string, alias_category, canonical_tag_id + ) + return await self.accept(image_id, canonical_tag_id) + + async def dismiss(self, image_id: int, tag_id: int) -> None: + stmt = insert(TagSuggestionRejection).values( + image_record_id=image_id, tag_id=tag_id + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + await self.session.execute(stmt) + + async def reject_applied_tag(self, image_id: int, tag_id: int) -> None: + """Operator removed an applied tag from an image. Remove the + image_tag row AND record a rejection so the allowlist won't + re-apply it on the next maintenance sweep.""" + await self.session.execute( + image_tag.delete() + .where(image_tag.c.image_record_id == image_id) + .where(image_tag.c.tag_id == tag_id) + ) + await self.dismiss(image_id, tag_id) + + async def update_threshold( + self, tag_id: int, min_confidence: float + ) -> None: + row = await self.session.get(TagAllowlist, tag_id) + if row is not None: + row.min_confidence = min_confidence + + async def remove(self, tag_id: int) -> None: + await self.session.execute( + delete(TagAllowlist).where(TagAllowlist.tag_id == tag_id) + ) + + async def list_all(self) -> Sequence[AllowlistRow]: + stmt = ( + select( + TagAllowlist.tag_id, + Tag.name, + Tag.kind, + TagAllowlist.min_confidence, + ) + .join(Tag, Tag.id == TagAllowlist.tag_id) + .order_by(Tag.name.asc()) + ) + rows = (await self.session.execute(stmt)).all() + return [ + AllowlistRow( + tag_id=r[0], + tag_name=r[1], + tag_kind=r[2].value if hasattr(r[2], "value") else str(r[2]), + min_confidence=r[3], + ) + for r in rows + ] diff --git a/backend/app/services/ml/centroids.py b/backend/app/services/ml/centroids.py new file mode 100644 index 0000000..6a21d77 --- /dev/null +++ b/backend/app/services/ml/centroids.py @@ -0,0 +1,147 @@ +"""Tag centroids: the mean SigLIP embedding of a tag's member images. + +Powers centroid-augmented suggestions (a tag whose centroid is close to an +image's embedding becomes a suggestion even if Camie didn't predict it). +""" + +from dataclasses import dataclass + +import numpy as np +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import ( + ImageRecord, + MLSettings, + Tag, + TagKind, + TagReferenceEmbedding, +) +from ...models.tag import image_tag +from .embedder import MODEL_VERSION as SIGLIP_VERSION + +ELIGIBLE_KINDS = { + TagKind.character, + TagKind.fandom, + TagKind.general, + TagKind.series, +} + + +@dataclass(frozen=True) +class CentroidHit: + tag_id: int + similarity: float + + +class CentroidService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _min_reference_images(self) -> int: + return ( + await self.session.execute( + select(MLSettings.min_reference_images).where(MLSettings.id == 1) + ) + ).scalar_one() + + async def recompute_for_tag(self, tag_id: int) -> bool: + """Recompute one tag's centroid. Returns True if a centroid was + written, False if skipped (ineligible kind or too few members).""" + tag = await self.session.get(Tag, tag_id) + if tag is None or tag.kind not in ELIGIBLE_KINDS: + return False + + min_refs = await self._min_reference_images() + + stmt = ( + select(ImageRecord.siglip_embedding) + .join(image_tag, image_tag.c.image_record_id == ImageRecord.id) + .where(image_tag.c.tag_id == tag_id) + .where(ImageRecord.siglip_embedding.is_not(None)) + ) + embeddings = [ + np.array(e, dtype=np.float32) + for e in (await self.session.execute(stmt)).scalars().all() + ] + if len(embeddings) < min_refs: + return False + + centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32) + + stmt = insert(TagReferenceEmbedding).values( + tag_id=tag_id, + embedding=centroid.tolist(), + reference_count=len(embeddings), + model_version=SIGLIP_VERSION, + ) + stmt = stmt.on_conflict_do_update( + index_elements=["tag_id"], + set_={ + "embedding": centroid.tolist(), + "reference_count": len(embeddings), + "model_version": SIGLIP_VERSION, + "updated_at": func.now(), + }, + ) + await self.session.execute(stmt) + return True + + async def list_drifted(self) -> list[int]: + """Tag ids whose centroid is stale: member count != reference_count, + OR no centroid row, OR centroid built on a different SigLIP version. + Only considers eligible-kind tags with embeddings present.""" + member_counts = ( + select( + image_tag.c.tag_id.label("tag_id"), + func.count(image_tag.c.image_record_id).label("members"), + ) + .join(ImageRecord, ImageRecord.id == image_tag.c.image_record_id) + .where(ImageRecord.siglip_embedding.is_not(None)) + .group_by(image_tag.c.tag_id) + .subquery() + ) + stmt = ( + select(Tag.id) + .join(member_counts, member_counts.c.tag_id == Tag.id) + .outerjoin( + TagReferenceEmbedding, + TagReferenceEmbedding.tag_id == Tag.id, + ) + .where(Tag.kind.in_(ELIGIBLE_KINDS)) + .where( + (TagReferenceEmbedding.tag_id.is_(None)) + | ( + TagReferenceEmbedding.reference_count + != member_counts.c.members + ) + | (TagReferenceEmbedding.model_version != SIGLIP_VERSION) + ) + ) + return list((await self.session.execute(stmt)).scalars().all()) + + async def find_similar_tags( + self, image_id: int, limit: int = 20 + ) -> list[CentroidHit]: + """Cosine similarity between an image's embedding and stored + centroids. Returns top-`limit` by similarity DESC. pgvector's + cosine_distance gives 1 - cosine_similarity.""" + img = await self.session.get(ImageRecord, image_id) + if img is None or img.siglip_embedding is None: + return [] + emb = img.siglip_embedding + distance = TagReferenceEmbedding.embedding.cosine_distance(emb) + stmt = ( + select( + TagReferenceEmbedding.tag_id, + (1 - distance).label("similarity"), + ) + .order_by(distance.asc()) + .limit(limit) + ) + rows = (await self.session.execute(stmt)).all() + return [ + CentroidHit(tag_id=r.tag_id, similarity=float(r.similarity)) + for r in rows + ] diff --git a/backend/app/services/ml/embedder.py b/backend/app/services/ml/embedder.py new file mode 100644 index 0000000..49c40f4 --- /dev/null +++ b/backend/app/services/ml/embedder.py @@ -0,0 +1,62 @@ +"""SigLIP SO400M image-embedding wrapper (PyTorch CPU). + +Direct port of ImageRepo's siglip.py. torch/transformers are imported +lazily inside load() so this module can be imported in the web container +(which never runs inference) without paying the torch import cost. +""" + +import os +from pathlib import Path + +import numpy as np +from PIL import Image, ImageFile + +ImageFile.LOAD_TRUNCATED_IMAGES = True + +MODEL_NAME = os.environ.get( + "SIGLIP_MODEL_NAME", "google/siglip-so400m-patch14-384" +) +MODEL_VERSION = os.environ.get( + "SIGLIP_MODEL_VERSION", "siglip-so400m-patch14-384" +) +EMBED_DIM = 1152 +_LOCAL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "siglip" + + +class Embedder: + def __init__(self, model_dir: Path | None = None): + self._model_dir = model_dir or _LOCAL_DIR + self._model = None + self._processor = None + self._torch = None + + def load(self) -> None: + if self._model is not None: + return + import torch + from transformers import AutoModel, AutoProcessor + + self._torch = torch + self._processor = AutoProcessor.from_pretrained(str(self._model_dir)) + self._model = AutoModel.from_pretrained(str(self._model_dir)) + self._model.eval() + + def infer(self, image_path: Path) -> np.ndarray: + """Return a 1152-dim float32 embedding (SigLIP MAP-pooled output).""" + self.load() + img = Image.open(image_path).convert("RGB") + with self._torch.no_grad(): + inputs = self._processor(images=img, return_tensors="pt") + out = self._model.get_image_features(**inputs) + pooled = out.pooler_output if hasattr(out, "pooler_output") else out + return pooled[0].numpy().astype(np.float32) + + +_default_embedder: Embedder | None = None + + +def get_embedder() -> Embedder: + global _default_embedder + if _default_embedder is None: + _default_embedder = Embedder() + return _default_embedder diff --git a/backend/app/services/ml/suggestions.py b/backend/app/services/ml/suggestions.py new file mode 100644 index 0000000..0b8d805 --- /dev/null +++ b/backend/app/services/ml/suggestions.py @@ -0,0 +1,274 @@ +"""The suggestion read-path: raw predictions + centroids -> alias-resolved, +threshold-filtered, category-grouped, ranked suggestions for one image. +""" + +from dataclasses import dataclass, field + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import ( + ImageRecord, + MLSettings, + Tag, + TagSuggestionRejection, +) +from ...models.tag import image_tag +from .aliases import AliasService +from .centroids import CentroidService +from .tagger import SURFACED_CATEGORIES + + +@dataclass(frozen=True) +class Suggestion: + # canonical_tag_id is None when this is a raw Camie tag with no alias and + # no existing Tag row — accepting it will create the tag. + canonical_tag_id: int | None + display_name: str + category: str + score: float + source: str # 'tagger' | 'centroid' | 'both' + creates_new_tag: bool + + +@dataclass +class SuggestionList: + by_category: dict[str, list[Suggestion]] = field(default_factory=dict) + + +class SuggestionService: + def __init__(self, session: AsyncSession): + self.session = session + self.aliases = AliasService(session) + self.centroids = CentroidService(session) + + async def _settings(self) -> MLSettings: + return ( + await self.session.execute(select(MLSettings).where(MLSettings.id == 1)) + ).scalar_one() + + def _threshold_for(self, s: MLSettings, category: str) -> float: + # 'artist' intentionally absent (FC-2d-vii-c) — falls through to + # the 1.01 "never surfaces" default like any unsurfaced category. + return { + "character": s.suggestion_threshold_character, + "copyright": s.suggestion_threshold_copyright, + "general": s.suggestion_threshold_general, + }.get(category, 1.01) + + async def for_image(self, image_id: int) -> SuggestionList: + img = await self.session.get(ImageRecord, image_id) + if img is None: + return SuggestionList() + + settings = await self._settings() + predictions: dict = img.tagger_predictions or {} + + applied = set( + ( + await self.session.execute( + select(image_tag.c.tag_id).where( + image_tag.c.image_record_id == image_id + ) + ) + ).scalars().all() + ) + rejected = set( + ( + await self.session.execute( + select(TagSuggestionRejection.tag_id).where( + TagSuggestionRejection.image_record_id == image_id + ) + ) + ).scalars().all() + ) + + # --- Camie predictions --- + candidates: list[tuple[str, str, float]] = [] + for name, p in predictions.items(): + category = p.get("category", "general") + if category not in SURFACED_CATEGORIES: + continue + conf = float(p.get("confidence", 0.0)) + if conf < self._threshold_for(settings, category): + continue + candidates.append((name, category, conf)) + + alias_map = await self.aliases.resolve_many( + [(n, c) for n, c, _ in candidates] + ) + + merged: dict[object, Suggestion] = {} + + def _merge(key, sug: Suggestion): + existing = merged.get(key) + if existing is None: + merged[key] = sug + elif sug.score > existing.score: + merged[key] = Suggestion( + canonical_tag_id=existing.canonical_tag_id, + display_name=existing.display_name, + category=existing.category, + score=sug.score, + source="both" + if existing.source != sug.source + else existing.source, + creates_new_tag=existing.creates_new_tag, + ) + + for name, category, conf in candidates: + canonical = alias_map.get((name, category)) + if canonical is not None: + if canonical.id in applied or canonical.id in rejected: + continue + _merge( + canonical.id, + Suggestion( + canonical_tag_id=canonical.id, + display_name=canonical.name, + category=category, + score=conf, + source="tagger", + creates_new_tag=False, + ), + ) + else: + existing_tag = ( + await self.session.execute( + select(Tag).where(Tag.name == name) + ) + ).scalars().first() + if existing_tag is not None: + if ( + existing_tag.id in applied + or existing_tag.id in rejected + ): + continue + _merge( + existing_tag.id, + Suggestion( + canonical_tag_id=existing_tag.id, + display_name=existing_tag.name, + category=category, + score=conf, + source="tagger", + creates_new_tag=False, + ), + ) + else: + _merge( + f"raw:{name}:{category}", + Suggestion( + canonical_tag_id=None, + display_name=name, + category=category, + score=conf, + source="tagger", + creates_new_tag=True, + ), + ) + + # --- Centroid augmentation --- + hits = await self.centroids.find_similar_tags(image_id, limit=30) + for hit in hits: + if hit.similarity < settings.centroid_similarity_threshold: + continue + if hit.tag_id in applied or hit.tag_id in rejected: + continue + tag = await self.session.get(Tag, hit.tag_id) + if tag is None: + continue + cat = tag.kind.value if hasattr(tag.kind, "value") else str(tag.kind) + display_cat = cat if cat in SURFACED_CATEGORIES else "general" + _merge( + tag.id, + Suggestion( + canonical_tag_id=tag.id, + display_name=tag.name, + category=display_cat, + score=hit.similarity, + source="centroid", + creates_new_tag=False, + ), + ) + + result = SuggestionList() + for sug in merged.values(): + result.by_category.setdefault(sug.category, []).append(sug) + for cat in result.by_category: + result.by_category[cat].sort(key=lambda s: s.score, reverse=True) + return result + + async def for_selection( + self, + image_ids: list[int], + threshold: float = 0.8, + top_k: int = 10, + ) -> dict[str, list[dict]]: + """Consensus suggestions across image_ids. A tag is included iff it + was suggested for (or already applied to) >= threshold fraction of + the selection AND was acceptable on >= 1 image. Confidence is the + mean over images where it was suggested. Aggregated by + canonical_tag_id; creates-new (no canonical id) suggestions are + skipped (bulk Accept applies by tag id).""" + if not image_ids: + return {} + threshold = min(1.0, max(0.0, threshold)) + total = len(image_ids) + + stats: dict[int, dict] = {} + for image_id in image_ids: + sl = await self.for_image(image_id) + for category, items in sl.by_category.items(): + for s in items: + if s.canonical_tag_id is None or s.creates_new_tag: + continue + st = stats.get(s.canonical_tag_id) + if st is None: + st = { + "tag_id": s.canonical_tag_id, + "name": s.display_name, + "category": category, + "source": s.source, + "suggested_count": 0, + "sum_score": 0.0, + } + stats[s.canonical_tag_id] = st + st["suggested_count"] += 1 + st["sum_score"] += s.score + + rows = ( + await self.session.execute( + select( + image_tag.c.image_record_id, image_tag.c.tag_id + ).where(image_tag.c.image_record_id.in_(image_ids)) + ) + ).all() + applied_by_tag: dict[int, set[int]] = {} + for iid, tid in rows: + applied_by_tag.setdefault(tid, set()).add(iid) + + result: dict[str, list[dict]] = {} + for st in stats.values(): + existing_count = len(applied_by_tag.get(st["tag_id"], set())) + covered = st["suggested_count"] + existing_count + coverage = covered / total + if coverage < threshold or st["suggested_count"] < 1: + continue + result.setdefault(st["category"], []).append( + { + "canonical_tag_id": st["tag_id"], + "name": st["name"], + "category": st["category"], + "confidence": round( + st["sum_score"] / st["suggested_count"], 4 + ), + "coverage": round(coverage, 4), + "covered_count": covered, + "source": st["source"], + } + ) + for cat in result: + result[cat].sort(key=lambda x: x["confidence"], reverse=True) + result[cat] = result[cat][:top_k] + return result diff --git a/backend/app/services/ml/tagger.py b/backend/app/services/ml/tagger.py new file mode 100644 index 0000000..92de3a3 --- /dev/null +++ b/backend/app/services/ml/tagger.py @@ -0,0 +1,149 @@ +"""Camie-tagger-v2 ONNX wrapper. + +CPU-only, single-image at a time. Loaded lazily inside the ml-worker +process; NOT thread-safe — the ml queue worker must run --concurrency=1 +(set by the FC-1 entrypoint). + +Camie's selected_tags.csv columns: tag_id,name,category,count +where category is a string: general|character|copyright|artist|meta|rating|year +(unlike WD14's integer Danbooru category ids). +""" + +import csv +import os +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +from PIL import Image, ImageFile + +# onnxruntime lives in requirements-ml.txt only — it is NOT installed in the +# lean web image or in CI. Imported lazily inside Tagger.load() so this module +# imports fine without it (the suggestion service imports SURFACED_CATEGORIES +# from here in the web container, and CI collects the pure-logic tests). + +# Tolerate minutely-truncated source images (same rationale as IR's wd14.py: +# a few missing bytes at the JPEG EOI shouldn't block tagging the whole image). +ImageFile.LOAD_TRUNCATED_IMAGES = True + +MODEL_NAME = os.environ.get("CAMIE_MODEL_NAME", "camie-tagger-v2") +_MODEL_DIR = Path(os.environ.get("ML_MODEL_DIR", "/models")) / "camie" + +# Below this confidence, predictions aren't stored (keeps the JSON compact). +STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05")) + +# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are +# still stored but the suggestion service filters them out. +# FC-2d-vii-c: 'artist' retired — artist identity is acquisition-derived +# (image_record.artist_id), never ML-inferred. Raw predictions are still +# stored at STORE_FLOOR but artist never surfaces. +SURFACED_CATEGORIES = {"character", "copyright", "general"} + + +@dataclass(frozen=True) +class TagPrediction: + name: str + category: str + confidence: float + + +class Tagger: + def __init__(self, model_dir: Path | None = None): + self._model_dir = model_dir or _MODEL_DIR + self._session = None # onnxruntime.InferenceSession once load()ed + self._tag_meta: list[dict] | None = None + self._input_name: str | None = None + self._output_name: str | None = None + self._input_size: int = 448 + + def load(self) -> None: + if self._session is not None: + return + model_path = self._model_dir / "model.onnx" + tags_path = self._model_dir / "selected_tags.csv" + if not model_path.is_file(): + raise RuntimeError( + f"Camie model.onnx missing at {model_path}. " + f"Populate /models via the ml-worker downloader." + ) + if not tags_path.is_file(): + raise RuntimeError( + f"Camie selected_tags.csv missing at {tags_path}. " + f"Populate /models via the ml-worker downloader." + ) + + tag_meta: list[dict] = [] + with open(tags_path, newline="") as f: + reader = csv.DictReader(f) + for row in reader: + tag_meta.append( + {"name": row["name"], "category": row["category"]} + ) + + # Lazy import — kept after the file-existence checks so the + # missing-model RuntimeError still fires first in environments + # without onnxruntime (CI / lean web image). + import onnxruntime as ort + + session = ort.InferenceSession( + str(model_path), providers=["CPUExecutionProvider"] + ) + self._input_name = session.get_inputs()[0].name + self._output_name = session.get_outputs()[0].name + input_shape = session.get_inputs()[0].shape + for dim in input_shape: + if isinstance(dim, int) and dim > 1: + self._input_size = dim + break + # Assign sentinels last so a partial load isn't observable. + self._tag_meta = tag_meta + self._session = session + + def _preprocess(self, image_path: Path) -> np.ndarray: + img = Image.open(image_path) + # Camie handles RGBA natively but we still composite onto white so + # transparency doesn't bias the model (same as IR's WD14 path). + if img.mode != "RGBA": + img = img.convert("RGBA") + bg = Image.new("RGBA", img.size, (255, 255, 255, 255)) + bg.paste(img, mask=img.split()[3]) + img = bg.convert("RGB") + + w, h = img.size + side = max(w, h) + square = Image.new("RGB", (side, side), (255, 255, 255)) + square.paste(img, ((side - w) // 2, (side - h) // 2)) + square = square.resize( + (self._input_size, self._input_size), Image.BICUBIC + ) + arr = np.array(square, dtype=np.float32) + return arr[np.newaxis, :, :, :] # NHWC + + def infer(self, image_path: Path) -> dict[str, TagPrediction]: + """Run Camie on one image. Returns {name: TagPrediction}, only + entries with confidence >= STORE_FLOOR (across all categories — + the suggestion service does category filtering later).""" + self.load() + x = self._preprocess(image_path) + out = self._session.run([self._output_name], {self._input_name: x})[0][0] + results: dict[str, TagPrediction] = {} + for idx, score in enumerate(out): + conf = float(score) + if conf < STORE_FLOOR: + continue + meta = self._tag_meta[idx] + results[meta["name"]] = TagPrediction( + name=meta["name"], category=meta["category"], confidence=conf + ) + return results + + +_default_tagger: Tagger | None = None + + +def get_tagger() -> Tagger: + """Process-level singleton so the ONNX session loads once per worker.""" + global _default_tagger + if _default_tagger is None: + _default_tagger = Tagger() + return _default_tagger diff --git a/backend/app/services/patreon_resolver.py b/backend/app/services/patreon_resolver.py new file mode 100644 index 0000000..64797c9 --- /dev/null +++ b/backend/app/services/patreon_resolver.py @@ -0,0 +1,102 @@ +"""Patreon vanity → campaign ID resolver. + +When gallery-dl fails with "Failed to extract campaign ID" on a Patreon +source URL, call resolve_campaign_id(vanity, cookies_path) to look up +the campaign ID via Patreon's public campaigns API. Caller then retries +with `patreon.com/id:` to bypass the broken vanity path. + +Uses the stdlib `requests` (already a transitive dep via gallery-dl) so +we don't add `aiohttp`. The sync call is wrapped in run_in_executor so +the calling async code stays non-blocking. + +Never raises. Returns None on any error (network, auth, parse, missing +match). +""" + +from __future__ import annotations + +import asyncio +import http.cookiejar +import logging +import os + +import requests + +log = logging.getLogger(__name__) + +_CAMPAIGNS_URL = "https://www.patreon.com/api/campaigns" +_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" +) +_TIMEOUT_SECONDS = 10.0 + + +def _load_cookie_jar(cookies_path: str | None) -> http.cookiejar.MozillaCookieJar | None: + if not cookies_path or not os.path.isfile(cookies_path): + return None + try: + jar = http.cookiejar.MozillaCookieJar(cookies_path) + jar.load(ignore_discard=True, ignore_expires=True) + return jar + except (OSError, http.cookiejar.LoadError) as exc: + log.debug("Could not load cookies from %s: %s", cookies_path, exc) + return None + + +def _sync_lookup(vanity: str, cookies_path: str | None) -> str | None: + jar = _load_cookie_jar(cookies_path) + headers = { + "User-Agent": _USER_AGENT, + "Accept": "application/vnd.api+json", + } + params = { + "filter[vanity]": vanity, + "fields[campaign]": "name", + } + try: + resp = requests.get( + _CAMPAIGNS_URL, + params=params, + headers=headers, + cookies=jar, + timeout=_TIMEOUT_SECONDS, + ) + except requests.RequestException as exc: + log.warning("Patreon campaigns API request failed for vanity=%s: %s", vanity, exc) + return None + + if resp.status_code != 200: + log.warning( + "Patreon campaigns API returned HTTP %d for vanity=%s", + resp.status_code, vanity, + ) + return None + + try: + payload = resp.json() + except ValueError as exc: + log.warning("Patreon campaigns API returned non-JSON for vanity=%s: %s", vanity, exc) + return None + + if not isinstance(payload, dict): + return None + data = payload.get("data") + if not isinstance(data, list) or not data: + return None + first = data[0] if isinstance(data[0], dict) else None + campaign_id = first.get("id") if first else None + if not isinstance(campaign_id, str) or not campaign_id: + return None + log.info("Resolved Patreon vanity=%s → campaign_id=%s", vanity, campaign_id) + return campaign_id + + +async def resolve_campaign_id( + vanity: str, + cookies_path: str | None, +) -> str | None: + """Async wrapper. Returns the campaign id string or None on any failure. + Never raises.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, _sync_lookup, vanity, cookies_path) diff --git a/backend/app/services/platforms.py b/backend/app/services/platforms.py new file mode 100644 index 0000000..bd3723a --- /dev/null +++ b/backend/app/services/platforms.py @@ -0,0 +1,140 @@ +"""FC-3b platforms registry — the single source of truth for what +FabledCurator supports. + +Lifted from GallerySubscriber's +~/Nextcloud/Projects/GallerySubscriber/backend/app/api/platforms.py +and ~/.../extension/lib/platforms.js. Six platforms; auth_type and +URL patterns match GS exactly so the existing browser extension +hits FC unmodified. +""" + +from dataclasses import dataclass +from typing import Literal + + +@dataclass(frozen=True) +class PlatformInfo: + key: str + name: str + description: str + auth_type: Literal["cookies", "token"] + requires_auth: bool + url_pattern: str + url_examples: list[str] + default_config: dict + notes: str | None = None + + +# Common defaults used across most platforms; embedded per-platform +# below so per-platform overrides remain explicit. +_DEFAULTS = { + "sleep": 3.0, + "sleep_request": 1.5, + "skip_existing": True, + "save_metadata": True, + "timeout": 3600, +} + + +PLATFORMS: dict[str, PlatformInfo] = { + "patreon": PlatformInfo( + key="patreon", + name="Patreon", + description="Download posts from Patreon creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?patreon\.com/", + url_examples=[ + "https://www.patreon.com/example_artist", + "https://www.patreon.com/user?u=12345678", + ], + default_config={**_DEFAULTS, "content_types": ["images", "attachments"]}, + ), + "subscribestar": PlatformInfo( + key="subscribestar", + name="SubscribeStar", + description="Download posts from SubscribeStar creators", + auth_type="cookies", + requires_auth=True, + url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/", + url_examples=[ + "https://subscribestar.adult/example_artist", + "https://www.subscribestar.com/example_artist", + ], + default_config={**_DEFAULTS, "content_types": ["all"]}, + ), + "hentaifoundry": PlatformInfo( + key="hentaifoundry", + name="Hentai Foundry", + description="Download artwork from Hentai Foundry artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?hentai-foundry\.com/", + url_examples=[ + "https://www.hentai-foundry.com/user/example_artist", + "https://www.hentai-foundry.com/pictures/user/example_artist", + ], + default_config={**_DEFAULTS, "content_types": ["pictures"]}, + ), + "discord": PlatformInfo( + key="discord", + name="Discord", + description="Download attachments from Discord channels", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?discord\.com/channels/", + url_examples=["https://discord.com/channels/123456789/987654321"], + default_config={**_DEFAULTS, "content_types": ["all"]}, + notes="Requires Discord user token (not bot token).", + ), + "pixiv": PlatformInfo( + key="pixiv", + name="Pixiv", + description="Download artwork from Pixiv artists", + auth_type="token", + requires_auth=True, + url_pattern=r"^https?://(www\.)?pixiv\.net/", + url_examples=[ + "https://www.pixiv.net/users/12345678", + "https://www.pixiv.net/en/users/12345678", + ], + default_config={**_DEFAULTS, "content_types": ["all"]}, + notes="Requires OAuth refresh token. Run `gallery-dl oauth:pixiv` to obtain one.", + ), + "deviantart": PlatformInfo( + key="deviantart", + name="DeviantArt", + description="Download artwork from DeviantArt artists", + auth_type="cookies", + requires_auth=False, + url_pattern=r"^https?://(www\.)?deviantart\.com/", + url_examples=[ + "https://www.deviantart.com/example-artist", + "https://www.deviantart.com/example-artist/gallery", + ], + default_config={**_DEFAULTS, "content_types": ["gallery"]}, + ), +} + + +def known_platform_keys() -> frozenset[str]: + return frozenset(PLATFORMS.keys()) + + +def auth_type_for(platform: str) -> str | None: + info = PLATFORMS.get(platform) + return info.auth_type if info else None + + +def to_dict(info: PlatformInfo) -> dict: + return { + "key": info.key, + "name": info.name, + "description": info.description, + "auth_type": info.auth_type, + "requires_auth": info.requires_auth, + "url_pattern": info.url_pattern, + "url_examples": info.url_examples, + "default_config": info.default_config, + "notes": info.notes, + } diff --git a/backend/app/services/post_feed_service.py b/backend/app/services/post_feed_service.py new file mode 100644 index 0000000..d137188 --- /dev/null +++ b/backend/app/services/post_feed_service.py @@ -0,0 +1,209 @@ +"""FC-3e: cursor-paginated read service for the Posts stream. + +Mirrors GalleryService.scroll's cursor encoding so the frontend pattern +is identical: base64 of "|". Sort key is +COALESCE(Post.post_date, Post.downloaded_at) so posts without a +publish date sort by when we captured them. + +Pure read-surface; no writes. The service composes the post dict +(including thumbnails from ImageRecord.primary_post_id and non-media +attachments from PostAttachment) so the API layer can jsonify directly. +""" +from __future__ import annotations + +import base64 +from datetime import datetime + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, ImageRecord, Post, PostAttachment, Source +from ..utils.text import html_to_plain, truncate_at_word +from .gallery_service import thumbnail_url + +CURSOR_SEPARATOR = "|" +DESCRIPTION_LIMIT = 280 +THUMBNAIL_LIMIT = 6 + + +def encode_cursor(sort_key: datetime, post_id: int) -> str: + raw = f"{sort_key.isoformat()}{CURSOR_SEPARATOR}{post_id}" + return base64.urlsafe_b64encode(raw.encode()).decode() + + +def decode_cursor(cursor: str) -> tuple[datetime, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1) + return datetime.fromisoformat(ts_part), int(id_part) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +def _sort_key(): + """Postgres COALESCE expression used in ORDER BY and WHERE clauses.""" + return func.coalesce(Post.post_date, Post.downloaded_at) + + +class PostFeedService: + def __init__(self, session: AsyncSession): + self.session = session + + async def scroll( + self, + *, + cursor: str | None = None, + artist_id: int | None = None, + platform: str | None = None, + limit: int = 24, + ) -> dict: + if limit < 1 or limit > 100: + raise ValueError("limit must be between 1 and 100") + + sort_key = _sort_key() + stmt = ( + select(Post, Artist, Source) + .join(Source, Post.source_id == Source.id) + .join(Artist, Source.artist_id == Artist.id) + ) + if artist_id is not None: + stmt = stmt.where(Source.artist_id == artist_id) + if platform is not None: + stmt = stmt.where(Source.platform == platform) + if cursor: + cur_ts, cur_id = decode_cursor(cursor) + stmt = stmt.where( + or_( + sort_key < cur_ts, + and_(sort_key == cur_ts, Post.id < cur_id), + ) + ) + + stmt = stmt.order_by(sort_key.desc(), Post.id.desc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() + + next_cursor: str | None = None + if len(rows) > limit: + last_post, _, _ = rows[limit - 1] + last_key = last_post.post_date or last_post.downloaded_at + next_cursor = encode_cursor(last_key, last_post.id) + rows = rows[:limit] + + post_ids = [p.id for p, _, _ in rows] + thumbs_map = await self._thumbnails_for(post_ids) + atts_map = await self._attachments_for(post_ids) + + items = [ + self._to_dict(post, artist, source, thumbs_map, atts_map) + for post, artist, source in rows + ] + return {"items": items, "next_cursor": next_cursor} + + async def get_post(self, post_id: int) -> dict | None: + row = (await self.session.execute( + select(Post, Artist, Source) + .join(Source, Post.source_id == Source.id) + .join(Artist, Source.artist_id == Artist.id) + .where(Post.id == post_id) + )).one_or_none() + if row is None: + return None + post, artist, source = row + thumbs_map = await self._thumbnails_for([post.id]) + atts_map = await self._attachments_for([post.id]) + item = self._to_dict(post, artist, source, thumbs_map, atts_map) + item["description_full"] = html_to_plain(post.description) + return item + + # --- composition helpers --------------------------------------------- + + async def _thumbnails_for(self, post_ids: list[int]) -> dict[int, dict]: + """post_id -> {"thumbs": [...up to 6], "more": int}. + + Selects THUMBNAIL_LIMIT+1 images per post via window function so we + can detect overflow in a single query. + """ + if not post_ids: + return {} + # Rank images within each post and fetch only the top THUMBNAIL_LIMIT+1. + ranked = ( + select( + ImageRecord.id, + ImageRecord.primary_post_id, + ImageRecord.sha256, + ImageRecord.mime, + func.row_number().over( + partition_by=ImageRecord.primary_post_id, + order_by=ImageRecord.id.asc(), + ).label("rn"), + func.count(ImageRecord.id).over( + partition_by=ImageRecord.primary_post_id, + ).label("total"), + ) + .where(ImageRecord.primary_post_id.in_(post_ids)) + .subquery() + ) + rows = (await self.session.execute( + select( + ranked.c.id, ranked.c.primary_post_id, + ranked.c.sha256, ranked.c.mime, ranked.c.total, + ).where(ranked.c.rn <= THUMBNAIL_LIMIT) + )).all() + + out: dict[int, dict] = {pid: {"thumbs": [], "more": 0} for pid in post_ids} + for img_id, pid, sha, mime, total in rows: + entry = out.setdefault(pid, {"thumbs": [], "more": 0}) + entry["thumbs"].append({ + "image_id": img_id, + "thumbnail_url": thumbnail_url(sha, mime), + "mime": mime, + }) + # `total` is constant per partition; overflow = total - THUMBNAIL_LIMIT. + entry["more"] = max(0, total - THUMBNAIL_LIMIT) + return out + + async def _attachments_for(self, post_ids: list[int]) -> dict[int, list[dict]]: + if not post_ids: + return {} + rows = (await self.session.execute( + select(PostAttachment) + .where(PostAttachment.post_id.in_(post_ids)) + .order_by(PostAttachment.id.asc()) + )).scalars().all() + out: dict[int, list[dict]] = {pid: [] for pid in post_ids} + for att in rows: + out.setdefault(att.post_id, []).append({ + "id": att.id, + "original_filename": att.original_filename, + "ext": att.ext, + "mime": att.mime, + "size_bytes": att.size_bytes, + "download_url": f"/api/attachments/{att.id}/download", + }) + return out + + def _to_dict( + self, post: Post, artist: Artist, source: Source, + thumbs_map: dict, atts_map: dict, + ) -> dict: + plain_full = html_to_plain(post.description) if post.description else None + if plain_full is None: + description_plain, truncated = None, False + else: + description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT) + thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0}) + return { + "id": post.id, + "external_post_id": post.external_post_id, + "post_url": post.post_url, + "post_title": post.post_title, + "post_date": post.post_date.isoformat() if post.post_date else None, + "downloaded_at": post.downloaded_at.isoformat(), + "description_plain": description_plain, + "description_truncated": truncated, + "artist": {"id": artist.id, "name": artist.name, "slug": artist.slug}, + "source": {"id": source.id, "platform": source.platform}, + "thumbnails": thumbs_entry["thumbs"], + "thumbnails_more": thumbs_entry["more"], + "attachments": atts_map.get(post.id, []), + } diff --git a/backend/app/services/provenance_service.py b/backend/app/services/provenance_service.py new file mode 100644 index 0000000..091feef --- /dev/null +++ b/backend/app/services/provenance_service.py @@ -0,0 +1,117 @@ +"""Read-only provenance queries. + +Provenance is its own system, intentionally separate from the tag/ML +system (see project_provenance_separation). This service joins +ImageProvenance -> Post/Source/Artist and returns plain dicts. It never +mutates and never imports tag/ML modules. +""" + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + PostAttachment, + Source, +) +from ..utils.html_sanitize import sanitize_post_html + + +def _post_dict(p: Post) -> dict: + return { + "id": p.id, + "external_post_id": p.external_post_id, + "url": p.post_url, + "title": p.post_title, + "date": p.post_date.isoformat() if p.post_date else None, + "description_html": sanitize_post_html(p.description), + "attachment_count": p.attachment_count, + } + + +def _source_dict(s: Source) -> dict: + return {"id": s.id, "platform": s.platform, "url": s.url} + + +def _artist_dict(a: Artist) -> dict: + return {"id": a.id, "name": a.name, "slug": a.slug} + + +def _attachment_dict(a: PostAttachment) -> dict: + return { + "id": a.id, + "original_filename": a.original_filename, + "size_bytes": a.size_bytes, + "ext": a.ext, + "download_url": f"/api/attachments/{a.id}/download", + } + + +class ProvenanceService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _attachments_for_posts(self, post_ids: list[int]) -> list[dict]: + if not post_ids: + return [] + rows = ( + await self.session.execute( + select(PostAttachment) + .where(PostAttachment.post_id.in_(post_ids)) + .order_by(PostAttachment.id.asc()) + ) + ).scalars().all() + return [_attachment_dict(a) for a in rows] + + async def for_image(self, image_id: int) -> dict | None: + rec = await self.session.get(ImageRecord, image_id) + if rec is None: + return None + stmt = ( + select(ImageProvenance, Post, Source, Artist) + .join(Post, Post.id == ImageProvenance.post_id) + .join(Source, Source.id == ImageProvenance.source_id) + .join(Artist, Artist.id == Source.artist_id) + .where(ImageProvenance.image_record_id == image_id) + .order_by(ImageProvenance.captured_at.asc(), + ImageProvenance.id.asc()) + ) + rows = (await self.session.execute(stmt)).all() + post_ids = [ip.post_id for ip, _p, _s, _a in rows] + attachments = await self._attachments_for_posts(post_ids) + return { + "image_id": image_id, + "provenance": [ + { + "provenance_id": ip.id, + "captured_at": ip.captured_at.isoformat() + if ip.captured_at else None, + "post": _post_dict(post), + "source": _source_dict(src), + "artist": _artist_dict(art), + } + for ip, post, src, art in rows + ], + "attachments": attachments, + } + + async def for_post(self, post_id: int) -> dict | None: + stmt = ( + select(Post, Source, Artist) + .join(Source, Source.id == Post.source_id) + .join(Artist, Artist.id == Source.artist_id) + .where(Post.id == post_id) + ) + row = (await self.session.execute(stmt)).first() + if row is None: + return None + post, src, art = row + return { + "post": _post_dict(post), + "source": _source_dict(src), + "artist": _artist_dict(art), + "attachments": await self._attachments_for_posts([post.id]), + } diff --git a/backend/app/services/scheduler_service.py b/backend/app/services/scheduler_service.py new file mode 100644 index 0000000..58071d0 --- /dev/null +++ b/backend/app/services/scheduler_service.py @@ -0,0 +1,80 @@ +"""FC-3d: pure logic for deciding which sources are due for a check. + +The Celery tick task wraps `select_due_sources` and fires +`download_source.delay()` per result; this module knows nothing about +Celery, gallery-dl, or any side effect. +""" +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from ..models import Artist, ImportSettings, Source + +MIN_INTERVAL_SECONDS = 60 +MAX_INTERVAL_SECONDS = 86400 +MAX_BACKOFF_EXPONENT = 6 + + +def compute_effective_interval( + source: Source, artist: Artist, settings: ImportSettings, +) -> int: + """Return the seconds between scheduled checks for one source. + + Precedence: source.check_interval_override > artist.check_interval_seconds + > settings.download_schedule_default_seconds. Multiplied by + 2 ** min(consecutive_failures, MAX_BACKOFF_EXPONENT) and clamped to + [MIN_INTERVAL_SECONDS, MAX_INTERVAL_SECONDS]. + """ + base = ( + source.check_interval_override + or artist.check_interval_seconds + or settings.download_schedule_default_seconds + ) + exponent = min(max(0, source.consecutive_failures or 0), MAX_BACKOFF_EXPONENT) + factor = 2 ** exponent + raw = base * factor + return max(MIN_INTERVAL_SECONDS, min(MAX_INTERVAL_SECONDS, raw)) + + +async def select_due_sources(session: AsyncSession) -> list[Source]: + """Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval. + + Never-checked sources (last_checked_at IS NULL) are always due. + """ + rows = (await session.execute( + select(Source) + .options(selectinload(Source.artist)) + .join(Artist, Source.artist_id == Artist.id) + .where(Source.enabled.is_(True)) + .where(Artist.auto_check.is_(True)) + )).scalars().all() + + settings = (await session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + )).scalar_one() + + now = datetime.now(UTC) + due: list[Source] = [] + for s in rows: + interval = compute_effective_interval(s, s.artist, settings) + if s.last_checked_at is None: + due.append(s) + continue + elapsed = (now - s.last_checked_at).total_seconds() + if elapsed >= interval: + due.append(s) + return due + + +def compute_next_check_at( + source: Source, artist: Artist, settings: ImportSettings, +) -> datetime | None: + """Return the projected datetime of the next check, or None if never checked.""" + if source.last_checked_at is None: + return None + interval = compute_effective_interval(source, artist, settings) + return source.last_checked_at + timedelta(seconds=interval) diff --git a/backend/app/services/series_service.py b/backend/app/services/series_service.py new file mode 100644 index 0000000..6ed3df8 --- /dev/null +++ b/backend/app/services/series_service.py @@ -0,0 +1,171 @@ +"""Series = a series-kind Tag + ordered series_page membership. + +All mutations are Core/set-based and run in the request transaction. +""" + +from sqlalchemy import and_, func, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord, Tag, TagKind +from ..models.series_page import SeriesPage +from .gallery_service import thumbnail_url + + +class SeriesError(ValueError): + """Series validation failure. The API maps 'not a series'/'not found' + to 404 and everything else to 400.""" + + +class SeriesService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _require_series(self, series_tag_id: int) -> Tag: + tag = await self.session.get(Tag, series_tag_id) + if tag is None: + raise SeriesError(f"Tag {series_tag_id} not found") + if tag.kind != TagKind.series: + raise SeriesError(f"Tag {series_tag_id} is not a series") + return tag + + @staticmethod + def _clean_ids(ids: list[int]) -> list[int]: + if not ids: + raise SeriesError("image_ids must be a non-empty list") + if len(ids) > 500: + raise SeriesError("selection too large (max 500)") + seen: set[int] = set() + out: list[int] = [] + for x in ids: + if x not in seen: + seen.add(x) + out.append(int(x)) + return out + + async def _member_order(self, series_tag_id: int) -> list[int]: + rows = ( + await self.session.execute( + select(SeriesPage.image_id) + .where(SeriesPage.series_tag_id == series_tag_id) + .order_by(SeriesPage.page_number.asc()) + ) + ).scalars().all() + return list(rows) + + async def list_pages(self, series_tag_id: int) -> dict: + tag = await self._require_series(series_tag_id) + rows = ( + await self.session.execute( + select( + SeriesPage.image_id, + SeriesPage.page_number, + ImageRecord.sha256, + ImageRecord.mime, + ImageRecord.path, + ) + .join(ImageRecord, ImageRecord.id == SeriesPage.image_id) + .where(SeriesPage.series_tag_id == series_tag_id) + .order_by(SeriesPage.page_number.asc()) + ) + ).all() + return { + "series": {"id": tag.id, "name": tag.name}, + "pages": [ + { + "image_id": r.image_id, + "page_number": r.page_number, + "thumbnail_url": thumbnail_url(r.sha256, r.mime), + "image_url": f"/images/{r.path.split('/images/', 1)[-1]}", + } + for r in rows + ], + } + + async def add_images( + self, series_tag_id: int, image_ids: list[int] + ) -> int: + await self._require_series(series_tag_id) + ids = self._clean_ids(image_ids) + existing = dict( + ( + await self.session.execute( + select(SeriesPage.image_id, SeriesPage.series_tag_id) + .where(SeriesPage.image_id.in_(ids)) + ) + ).all() + ) + # Already in THIS series → leave untouched (no churn). + to_add = [i for i in ids if existing.get(i) != series_tag_id] + if not to_add: + return 0 + # Remove any prior membership for the to_add images (the "move"). + await self.session.execute( + SeriesPage.__table__.delete().where( + SeriesPage.image_id.in_(to_add) + ) + ) + max_pn = ( + await self.session.scalar( + select(func.coalesce(func.max(SeriesPage.page_number), 0)) + .where(SeriesPage.series_tag_id == series_tag_id) + ) + ) or 0 + await self.session.execute( + pg_insert(SeriesPage).values( + [ + { + "series_tag_id": series_tag_id, + "image_id": iid, + "page_number": max_pn + offset, + } + for offset, iid in enumerate(to_add, start=1) + ] + ) + ) + return len(to_add) + + async def remove_images( + self, series_tag_id: int, image_ids: list[int] + ) -> int: + await self._require_series(series_tag_id) + ids = self._clean_ids(image_ids) + res = await self.session.execute( + SeriesPage.__table__.delete().where( + and_( + SeriesPage.series_tag_id == series_tag_id, + SeriesPage.image_id.in_(ids), + ) + ) + ) + return res.rowcount or 0 + + async def reorder( + self, series_tag_id: int, ordered_image_ids: list[int] + ) -> None: + await self._require_series(series_tag_id) + ordered = self._clean_ids(ordered_image_ids) + current = set(await self._member_order(series_tag_id)) + if set(ordered) != current or len(ordered) != len(current): + raise SeriesError( + "ordered image_ids must exactly match series membership" + ) + for idx, iid in enumerate(ordered, start=1): + await self.session.execute( + update(SeriesPage) + .where( + and_( + SeriesPage.series_tag_id == series_tag_id, + SeriesPage.image_id == iid, + ) + ) + .values(page_number=idx) + ) + + async def set_cover(self, series_tag_id: int, image_id: int) -> None: + await self._require_series(series_tag_id) + order = await self._member_order(series_tag_id) + if image_id not in order: + raise SeriesError(f"image {image_id} is not in this series") + new_order = [image_id] + [i for i in order if i != image_id] + await self.reorder(series_tag_id, new_order) diff --git a/backend/app/services/showcase_service.py b/backend/app/services/showcase_service.py new file mode 100644 index 0000000..b45e0b5 --- /dev/null +++ b/backend/app/services/showcase_service.py @@ -0,0 +1,40 @@ +"""Random-sample query for the showcase. + +Uses the tsm_system_rows TABLESAMPLE method (migration 0004) instead of +ORDER BY random(): sampling cost scales with the sample size, not the table, +so it stays fast as the collection grows. SYSTEM_ROWS(n) returns up to n +rows; an empty table yields none. +""" + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import ImageRecord +from .gallery_service import thumbnail_url + + +class ShowcaseService: + def __init__(self, session: AsyncSession): + self.session = session + + async def random_sample(self, limit: int = 60) -> list[dict]: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + stmt = select(ImageRecord).from_statement( + text( + "SELECT * FROM image_record " + "TABLESAMPLE SYSTEM_ROWS(:n)" + ).bindparams(n=limit) + ) + rows = (await self.session.execute(stmt)).scalars().all() + return [ + { + "id": r.id, + "sha256": r.sha256, + "mime": r.mime, + "width": r.width, + "height": r.height, + "thumbnail_url": thumbnail_url(r.sha256, r.mime), + } + for r in rows + ] diff --git a/backend/app/services/source_service.py b/backend/app/services/source_service.py new file mode 100644 index 0000000..df75e3e --- /dev/null +++ b/backend/app/services/source_service.py @@ -0,0 +1,275 @@ +"""FC-3a: CRUD over Source rows, with platform/url/config validation and +is_subscription auto-flip on first add / last delete. +""" + +from dataclasses import dataclass + +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Artist, ImportSettings, Source +from .platforms import known_platform_keys +from .scheduler_service import compute_next_check_at + +# Re-exported for back-compat with FC-3a; the registry in +# services/platforms.py is the single source of truth. +KNOWN_PLATFORMS = known_platform_keys() + + +# --- Errors ----------------------------------------------------------------- + +class SourceServiceError(Exception): + """Base.""" + + +class ArtistNotFoundError(SourceServiceError): + pass + + +class UnknownPlatformError(SourceServiceError): + def __init__(self, platform: str): + super().__init__(f"unknown platform: {platform!r}") + self.platform = platform + + +class InvalidConfigError(SourceServiceError): + pass + + +class EmptyUrlError(SourceServiceError): + pass + + +class DuplicateSourceError(SourceServiceError): + def __init__(self, existing_id: int): + super().__init__(f"duplicate source (existing id={existing_id})") + self.existing_id = existing_id + + +@dataclass(frozen=True) +class SourceRecord: + id: int + artist_id: int + artist_name: str + artist_slug: str + platform: str + url: str + enabled: bool + config_overrides: dict | None + last_checked_at: str | None + last_error: str | None + check_interval_override: int | None + consecutive_failures: int + next_check_at: str | None + + def to_dict(self) -> dict: + return { + "id": self.id, + "artist_id": self.artist_id, + "artist_name": self.artist_name, + "artist_slug": self.artist_slug, + "platform": self.platform, + "url": self.url, + "enabled": self.enabled, + "config_overrides": self.config_overrides, + "last_checked_at": self.last_checked_at, + "last_error": self.last_error, + "check_interval_override": self.check_interval_override, + "consecutive_failures": self.consecutive_failures, + "next_check_at": self.next_check_at, + } + + +# --- Service ---------------------------------------------------------------- + +_EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"} + + +class SourceService: + def __init__(self, session: AsyncSession): + self.session = session + + async def _artist_or_raise(self, artist_id: int) -> Artist: + row = (await self.session.execute( + select(Artist).where(Artist.id == artist_id) + )).scalar_one_or_none() + if row is None: + raise ArtistNotFoundError(f"artist id={artist_id} not found") + return row + + @staticmethod + def _validate_platform(platform: str) -> str: + if platform not in KNOWN_PLATFORMS: + raise UnknownPlatformError(platform) + return platform + + @staticmethod + def _validate_url(url: str) -> str: + cleaned = (url or "").strip() + if not cleaned: + raise EmptyUrlError("url must not be empty") + return cleaned + + @staticmethod + def _validate_config(config: object) -> dict | None: + if config is None: + return None + if not isinstance(config, dict): + raise InvalidConfigError("config_overrides must be a JSON object") + return config + + async def _load_settings(self) -> ImportSettings: + return (await self.session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + )).scalar_one() + + def _build_record( + self, source: Source, artist: Artist, settings: ImportSettings, + ) -> SourceRecord: + nxt = compute_next_check_at(source, artist, settings) + return SourceRecord( + id=source.id, + artist_id=source.artist_id, + artist_name=artist.name, + artist_slug=artist.slug, + platform=source.platform, + url=source.url, + enabled=source.enabled, + config_overrides=source.config_overrides, + last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None, + last_error=source.last_error, + check_interval_override=source.check_interval_override, + consecutive_failures=source.consecutive_failures or 0, + next_check_at=nxt.isoformat() if nxt else None, + ) + + async def _row_to_record(self, source: Source) -> SourceRecord: + artist = (await self.session.execute( + select(Artist).where(Artist.id == source.artist_id) + )).scalar_one() + settings = await self._load_settings() + return self._build_record(source, artist, settings) + + async def list(self, artist_id: int | None = None) -> list[SourceRecord]: + stmt = ( + select(Source, Artist) + .join(Artist, Artist.id == Source.artist_id) + .order_by(Artist.name.asc(), Source.id.asc()) + ) + if artist_id is not None: + stmt = stmt.where(Source.artist_id == artist_id) + rows = (await self.session.execute(stmt)).all() + settings = await self._load_settings() + return [self._build_record(s, a, settings) for s, a in rows] + + async def get(self, source_id: int) -> SourceRecord | None: + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + return None + return await self._row_to_record(source) + + async def create( + self, + *, + artist_id: int, + platform: str, + url: str, + enabled: bool = True, + config_overrides: dict | None = None, + check_interval_override: int | None = None, + ) -> SourceRecord: + await self._artist_or_raise(artist_id) + platform = self._validate_platform(platform) + url = self._validate_url(url) + config_overrides = self._validate_config(config_overrides) + + prior_count = (await self.session.execute( + select(func.count(Source.id)).where(Source.artist_id == artist_id) + )).scalar_one() + + source = Source( + artist_id=artist_id, platform=platform, url=url, + enabled=enabled, config_overrides=config_overrides, + check_interval_override=check_interval_override, + ) + self.session.add(source) + try: + await self.session.flush() + except IntegrityError as exc: + await self.session.rollback() + existing = (await self.session.execute( + select(Source.id).where( + Source.artist_id == artist_id, + Source.platform == platform, + Source.url == url, + ) + )).scalar_one() + raise DuplicateSourceError(existing_id=existing) from exc + + if prior_count == 0: + await self.session.execute( + Artist.__table__.update() + .where(Artist.id == artist_id) + .values(is_subscription=True) + ) + await self.session.commit() + return await self._row_to_record(source) + + async def update(self, source_id: int, **fields) -> SourceRecord: + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + + for key in list(fields.keys()): + if key not in _EDITABLE: + fields.pop(key) + if "platform" in fields: + fields["platform"] = self._validate_platform(fields["platform"]) + if "url" in fields: + fields["url"] = self._validate_url(fields["url"]) + if "config_overrides" in fields: + fields["config_overrides"] = self._validate_config(fields["config_overrides"]) + + for key, value in fields.items(): + setattr(source, key, value) + try: + await self.session.flush() + except IntegrityError as exc: + await self.session.rollback() + existing = (await self.session.execute( + select(Source.id).where( + Source.artist_id == source.artist_id, + Source.platform == source.platform, + Source.url == source.url, + Source.id != source.id, + ) + )).scalar_one() + raise DuplicateSourceError(existing_id=existing) from exc + await self.session.commit() + return await self._row_to_record(source) + + async def delete(self, source_id: int) -> None: + source = (await self.session.execute( + select(Source).where(Source.id == source_id) + )).scalar_one_or_none() + if source is None: + raise LookupError(f"source id={source_id} not found") + artist_id = source.artist_id + await self.session.delete(source) + await self.session.flush() + + remaining = (await self.session.execute( + select(func.count(Source.id)).where(Source.artist_id == artist_id) + )).scalar_one() + if remaining == 0: + await self.session.execute( + Artist.__table__.update() + .where(Artist.id == artist_id) + .values(is_subscription=False) + ) + await self.session.commit() diff --git a/backend/app/services/tag_directory_service.py b/backend/app/services/tag_directory_service.py new file mode 100644 index 0000000..a0ffaaa --- /dev/null +++ b/backend/app/services/tag_directory_service.py @@ -0,0 +1,126 @@ +"""Paged tag-directory cards with window-function preview thumbnails. + +Cursor orders by (Tag.name ASC, Tag.id ASC). Preview thumbnails for the +page's tags are fetched in ONE windowed query (ROW_NUMBER <= 3) to avoid +an N+1. fandom_name is resolved via a self-join on Tag (fandom_id -> Tag). +""" + +import base64 +from dataclasses import dataclass + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased + +from ..models import ImageRecord, Tag +from ..models.tag import image_tag +from .gallery_service import thumbnail_url + +_SEP = "|" + + +def _encode(name: str, tag_id: int) -> str: + return base64.urlsafe_b64encode(f"{name}{_SEP}{tag_id}".encode()).decode() + + +def _decode(cursor: str) -> tuple[str, int]: + try: + raw = base64.urlsafe_b64decode(cursor.encode()).decode() + name, tid = raw.rsplit(_SEP, 1) + return name, int(tid) + except Exception as exc: + raise ValueError(f"invalid cursor: {cursor!r}") from exc + + +@dataclass(frozen=True) +class DirectoryPage: + cards: list[dict] + next_cursor: str | None + + +class TagDirectoryService: + def __init__(self, session: AsyncSession): + self.session = session + + async def list_tags( + self, + kind: str | None, + q: str | None, + cursor: str | None, + limit: int = 60, + ) -> DirectoryPage: + if limit < 1 or limit > 200: + raise ValueError("limit must be between 1 and 200") + + fandom = aliased(Tag) + count_col = func.count(image_tag.c.image_record_id).label("image_count") + stmt = ( + select(Tag, fandom.name.label("fandom_name"), count_col) + .outerjoin(fandom, Tag.fandom_id == fandom.id) + .outerjoin(image_tag, image_tag.c.tag_id == Tag.id) + .group_by(Tag.id, fandom.name) + ) + if kind is not None: + stmt = stmt.where(Tag.kind == kind) + if q: + stmt = stmt.where(Tag.name.ilike(f"%{q}%")) + if cursor: + c_name, c_id = _decode(cursor) + stmt = stmt.where( + or_( + Tag.name > c_name, + and_(Tag.name == c_name, Tag.id > c_id), + ) + ) + stmt = stmt.order_by(Tag.name.asc(), Tag.id.asc()).limit(limit + 1) + rows = (await self.session.execute(stmt)).all() + + next_cursor = None + if len(rows) > limit: + last_tag = rows[limit - 1][0] + next_cursor = _encode(last_tag.name, last_tag.id) + rows = rows[:limit] + + tag_ids = [r[0].id for r in rows] + previews = await self._previews(tag_ids) + + cards = [ + { + "id": tag.id, + "name": tag.name, + "kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind, + "fandom_id": tag.fandom_id, + "fandom_name": fandom_name, + "image_count": int(image_count), + "preview_thumbnails": previews.get(tag.id, []), + } + for tag, fandom_name, image_count in rows + ] + return DirectoryPage(cards=cards, next_cursor=next_cursor) + + async def _previews(self, tag_ids: list[int]) -> dict[int, list[str]]: + if not tag_ids: + return {} + rn = func.row_number().over( + partition_by=image_tag.c.tag_id, + order_by=image_tag.c.image_record_id.desc(), + ).label("rn") + sub = ( + select( + image_tag.c.tag_id.label("tag_id"), + image_tag.c.image_record_id.label("image_record_id"), + rn, + ) + .where(image_tag.c.tag_id.in_(tag_ids)) + .subquery() + ) + stmt = ( + select(sub.c.tag_id, ImageRecord.sha256, ImageRecord.mime) + .join(ImageRecord, ImageRecord.id == sub.c.image_record_id) + .where(sub.c.rn <= 3) + .order_by(sub.c.tag_id, sub.c.rn) + ) + out: dict[int, list[str]] = {} + for tag_id, sha, mime in (await self.session.execute(stmt)).all(): + out.setdefault(tag_id, []).append(thumbnail_url(sha, mime)) + return out diff --git a/backend/app/services/tag_service.py b/backend/app/services/tag_service.py new file mode 100644 index 0000000..da9e499 --- /dev/null +++ b/backend/app/services/tag_service.py @@ -0,0 +1,470 @@ +"""Tag CRUD + autocomplete + image-tag association.""" + +from collections.abc import Sequence +from dataclasses import dataclass + +from sqlalchemy import and_, case, exists, func, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from ..models import Tag, TagKind, image_tag +from ..models.tag_allowlist import TagAllowlist +from ..models.tag_reference_embedding import TagReferenceEmbedding + + +class TagValidationError(ValueError): + """Raised when tag construction breaks the kind/fandom rules.""" + + +class TagMergeConflict(TagValidationError): + """Rename hit a same-(kind, fandom) name collision. Carries the + colliding target and the merge preview the 409 needs.""" + + def __init__( + self, + message: str, + *, + target_id: int, + target_name: str, + source_image_count: int, + will_alias: bool, + ): + super().__init__(message) + self.target_id = target_id + self.target_name = target_name + self.source_image_count = source_image_count + self.will_alias = will_alias + + +@dataclass(frozen=True) +class TagAutocompleteHit: + id: int + name: str + kind: str + fandom_id: int | None + fandom_name: str | None + image_count: int + + +@dataclass(frozen=True) +class MergeResult: + target_id: int + target_name: str + target_kind: str + merged_count: int + alias_created: bool + source_deleted: bool + + +class TagService: + def __init__(self, session: AsyncSession): + self.session = session + + async def find_or_create( + self, name: str, kind: TagKind, fandom_id: int | None = None + ) -> Tag: + """Upserts (name, kind, fandom_id) into the tag table. + + Validates kind/fandom rules before touching the DB: + - fandom_id may only be set when kind == TagKind.character + - if fandom_id is set, the referenced tag must exist and have + kind == TagKind.fandom + """ + name = name.strip() + if not name: + raise TagValidationError("Tag name cannot be empty") + + if fandom_id is not None and kind != TagKind.character: + raise TagValidationError( + "fandom_id is only valid for character tags" + ) + + if fandom_id is not None: + fandom = await self.session.get(Tag, fandom_id) + if fandom is None or fandom.kind != TagKind.fandom: + raise TagValidationError( + f"fandom_id {fandom_id} does not reference a fandom tag" + ) + + # Upsert via INSERT ... ON CONFLICT DO NOTHING. We can't use the + # uniqueness index name directly (it's a partial coalesce-based + # expression), so we re-select after insert. + stmt = ( + select(Tag) + .where(Tag.name == name) + .where(Tag.kind == kind) + .where( + Tag.fandom_id.is_(None) if fandom_id is None else Tag.fandom_id == fandom_id + ) + ) + existing = (await self.session.execute(stmt)).scalar_one_or_none() + if existing: + return existing + + new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id) + self.session.add(new_tag) + await self.session.flush() + return new_tag + + async def autocomplete( + self, + query: str, + kind: TagKind | None = None, + limit: int = 20, + ) -> list[TagAutocompleteHit]: + """ILIKE search ranked exact > prefix > substring, then by image count. + + Returns at most `limit` hits. + """ + q = (query or "").strip() + if not q: + return [] + + match_rank = case( + (func.lower(Tag.name) == q.lower(), 0), + (func.lower(Tag.name).startswith(q.lower()), 1), + else_=2, + ) + + fandom_alias = Tag.__table__.alias("fandom_lookup") + image_count = ( + select(func.count(image_tag.c.image_record_id)) + .where(image_tag.c.tag_id == Tag.id) + .correlate(Tag.__table__) + .scalar_subquery() + ) + + stmt = ( + select( + Tag.id, + Tag.name, + Tag.kind, + Tag.fandom_id, + fandom_alias.c.name.label("fandom_name"), + image_count.label("image_count"), + ) + .select_from(Tag.__table__.outerjoin( + fandom_alias, Tag.fandom_id == fandom_alias.c.id + )) + .where(func.lower(Tag.name).like(f"%{q.lower()}%")) + .order_by(match_rank.asc(), image_count.desc(), Tag.name.asc()) + .limit(limit) + ) + if kind is not None: + stmt = stmt.where(Tag.kind == kind) + + rows = (await self.session.execute(stmt)).all() + return [ + TagAutocompleteHit( + id=r.id, + name=r.name, + kind=r.kind.value if hasattr(r.kind, "value") else r.kind, + fandom_id=r.fandom_id, + fandom_name=r.fandom_name, + image_count=r.image_count or 0, + ) + for r in rows + ] + + async def add_to_image(self, image_id: int, tag_id: int, source: str = "manual") -> None: + """Idempotent: re-adding an existing tag does nothing.""" + stmt = pg_insert(image_tag).values( + image_record_id=image_id, tag_id=tag_id, source=source + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + await self.session.execute(stmt) + + async def remove_from_image(self, image_id: int, tag_id: int) -> None: + await self.session.execute( + image_tag.delete().where( + and_( + image_tag.c.image_record_id == image_id, + image_tag.c.tag_id == tag_id, + ) + ) + ) + + async def list_for_image(self, image_id: int) -> Sequence[Tag]: + stmt = ( + select(Tag) + .join(image_tag, image_tag.c.tag_id == Tag.id) + .where(image_tag.c.image_record_id == image_id) + .order_by(Tag.kind.asc(), Tag.name.asc()) + ) + return (await self.session.execute(stmt)).scalars().all() + + async def _keep_as_alias(self, tag_id: int) -> bool: + """A merged-away tag's old name must survive as an alias iff the ML + pipeline has ever applied it OR could re-emit it (allowlisted / has + a centroid) — otherwise the proactive apply_allowlist_tags worker + would silently regenerate it. Purely-manual, ML-unknown tags are + deleted outright (no DB bloat).""" + is_machine = await self.session.scalar( + select( + exists().where( + and_( + image_tag.c.tag_id == tag_id, + image_tag.c.source.in_( + ("ml_auto", "ml_accepted", "auto") + ), + ) + ) + ) + ) + if is_machine: + return True + allowlisted = await self.session.scalar( + select(exists().where(TagAllowlist.tag_id == tag_id)) + ) + if allowlisted: + return True + has_centroid = await self.session.scalar( + select( + exists().where(TagReferenceEmbedding.tag_id == tag_id) + ) + ) + return bool(has_centroid) + + async def rename(self, tag_id: int, new_name: str) -> Tag: + """Rename a tag. Raises TagMergeConflict if the new name collides + with an existing tag of the same (kind, fandom_id) — the API turns + that into a 409 merge hint. + """ + new_name = new_name.strip() + if not new_name: + raise TagValidationError("Tag name cannot be empty") + tag = await self.session.get(Tag, tag_id) + if tag is None: + raise TagValidationError(f"Tag {tag_id} not found") + + clash_stmt = ( + select(Tag) + .where(Tag.name == new_name) + .where(Tag.kind == tag.kind) + .where( + Tag.fandom_id.is_(None) + if tag.fandom_id is None + else Tag.fandom_id == tag.fandom_id + ) + .where(Tag.id != tag_id) + ) + clash = (await self.session.execute(clash_stmt)).scalar_one_or_none() + if clash is not None: + source_image_count = await self.session.scalar( + select(func.count()) + .select_from(image_tag) + .where(image_tag.c.tag_id == tag_id) + ) + will_alias = await self._keep_as_alias(tag_id) + raise TagMergeConflict( + f"A {tag.kind} tag named {new_name!r} already exists", + target_id=clash.id, + target_name=clash.name, + source_image_count=int(source_image_count or 0), + will_alias=will_alias, + ) + + tag.name = new_name + await self.session.flush() + return tag + + async def merge(self, source_id: int, target_id: int) -> MergeResult: + """Transactionally repoint every FK from source→target, optionally + keep source's name as a tagger alias, delete source. Atomic: any + exception rolls back via the caller's session.""" + if source_id == target_id: + raise TagValidationError("Cannot merge a tag into itself") + source = await self.session.get(Tag, source_id) + if source is None: + raise TagValidationError(f"Tag {source_id} not found") + target = await self.session.get(Tag, target_id) + if target is None: + raise TagValidationError(f"Tag {target_id} not found") + if source.kind != target.kind or (source.fandom_id or 0) != ( + target.fandom_id or 0 + ): + raise TagValidationError( + "Tags must be the same kind and fandom to merge" + ) + + keep_as_alias = await self._keep_as_alias(source_id) + source_name = source.name + source_kind = source.kind + target_name = target.name + target_kind = ( + target.kind.value + if hasattr(target.kind, "value") + else str(target.kind) + ) + + merged_count = await self._repoint_image_tags(source_id, target_id) + await self._repoint_rejections(source_id, target_id) + await self._repoint_allowlist(source_id, target_id) + await self._repoint_embedding(source_id) + await self._repoint_aliases(source_id, target_id) + await self._repoint_fandom_children( + source_id, target_id, source_kind + ) + await self._repoint_series_pages(source_id, target_id) + + alias_created = False + if keep_as_alias: + alias_created = await self._create_protective_aliases( + source_name, source_kind, target_id + ) + + await self.session.delete(source) + await self.session.flush() + + return MergeResult( + target_id=target_id, + target_name=target_name, + target_kind=target_kind, + merged_count=merged_count, + alias_created=alias_created, + source_deleted=True, + ) + + async def _repoint_image_tags(self, src: int, tgt: int) -> int: + """Drop source links whose image already links target, then move + the rest. Returns the number actually moved.""" + await self.session.execute( + image_tag.delete().where( + and_( + image_tag.c.tag_id == src, + image_tag.c.image_record_id.in_( + select(image_tag.c.image_record_id).where( + image_tag.c.tag_id == tgt + ) + ), + ) + ) + ) + res = await self.session.execute( + image_tag.update() + .where(image_tag.c.tag_id == src) + .values(tag_id=tgt) + ) + return res.rowcount or 0 + + async def _repoint_rejections(self, src: int, tgt: int) -> None: + from ..models.tag_suggestion_rejection import TagSuggestionRejection + + await self.session.execute( + text( + "DELETE FROM tag_suggestion_rejection r " + "WHERE r.tag_id = :src AND EXISTS (" + " SELECT 1 FROM tag_suggestion_rejection r2 " + " WHERE r2.tag_id = :tgt " + " AND r2.image_record_id = r.image_record_id)" + ), + {"src": src, "tgt": tgt}, + ) + await self.session.execute( + update(TagSuggestionRejection) + .where(TagSuggestionRejection.tag_id == src) + .values(tag_id=tgt) + ) + + async def _repoint_allowlist(self, src: int, tgt: int) -> None: + tgt_has = await self.session.scalar( + select(exists().where(TagAllowlist.tag_id == tgt)) + ) + if tgt_has: + await self.session.execute( + text("DELETE FROM tag_allowlist WHERE tag_id = :src"), + {"src": src}, + ) + else: + await self.session.execute( + update(TagAllowlist) + .where(TagAllowlist.tag_id == src) + .values(tag_id=tgt) + ) + + async def _repoint_embedding(self, src: int) -> None: + await self.session.execute( + text( + "DELETE FROM tag_reference_embedding WHERE tag_id = :src" + ), + {"src": src}, + ) + + async def _repoint_aliases(self, src: int, tgt: int) -> None: + from ..models.tag_alias import TagAlias + + await self.session.execute( + update(TagAlias) + .where(TagAlias.canonical_tag_id == src) + .values(canonical_tag_id=tgt) + ) + + async def _repoint_series_pages(self, src: int, tgt: int) -> None: + from ..models.series_page import SeriesPage + + # image_id is UNIQUE across series_page and src != tgt, so an + # image in src's series cannot already be in tgt's — no collision. + await self.session.execute( + update(SeriesPage) + .where(SeriesPage.series_tag_id == src) + .values(series_tag_id=tgt) + ) + + async def _repoint_fandom_children( + self, src: int, tgt: int, src_kind: TagKind + ) -> None: + if src_kind != TagKind.fandom: + return + await self.session.execute( + update(Tag).where(Tag.fandom_id == src).values(fandom_id=tgt) + ) + + async def _create_protective_aliases( + self, src_name: str, src_kind: TagKind, tgt: int + ) -> bool: + """One alias per category the tagger has actually emitted for + src_name (so future predictions resolve to target); fall back to + the tag's kind when the tagger never predicted this exact name. + Idempotent — never clobbers a pre-existing operator alias.""" + from ..models.tag_alias import TagAlias + + rows = ( + await self.session.execute( + text( + "SELECT DISTINCT " + " (tagger_predictions::jsonb -> :n ->> 'category') " + " AS cat " + "FROM image_record " + "WHERE tagger_predictions IS NOT NULL " + " AND (tagger_predictions::jsonb) ? :n" + ), + {"n": src_name}, + ) + ).all() + categories = {r.cat for r in rows if r.cat} + if not categories: + kind_val = ( + src_kind.value + if hasattr(src_kind, "value") + else str(src_kind) + ) + categories = {kind_val} + + created = False + for cat in categories: + res = await self.session.execute( + pg_insert(TagAlias) + .values( + alias_string=src_name, + alias_category=cat, + canonical_tag_id=tgt, + ) + .on_conflict_do_nothing( + index_elements=["alias_string", "alias_category"] + ) + ) + if res.rowcount: + created = True + return created diff --git a/backend/app/services/thumbnailer.py b/backend/app/services/thumbnailer.py new file mode 100644 index 0000000..eea4bed --- /dev/null +++ b/backend/app/services/thumbnailer.py @@ -0,0 +1,102 @@ +"""Image + video thumbnail generation. + +Images go through Pillow; videos go through ffmpeg's first-frame snapshot +at ~5% into the runtime (avoids fade-ins/title cards while still being a +representative frame). + +Thumbnails are stored at /images/thumbs/{sha[:3]}/{sha}.jpg or .png; the +3-char prefix bucket avoids hot single-directory inode pressure. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from PIL import Image + +THUMB_MAX_PX = 400 +JPEG_QUALITY = 85 +FFMPEG_TIMEOUT_SECONDS = 60 + + +@dataclass(frozen=True) +class ThumbnailResult: + path: Path + width: int + height: int + mime: str + + +def thumbnail_path_for(sha256_hex: str, *, alpha: bool, root: Path) -> Path: + ext = ".png" if alpha else ".jpg" + bucket = sha256_hex[:3] + return root / "thumbs" / bucket / f"{sha256_hex}{ext}" + + +class Thumbnailer: + def __init__(self, images_root: Path): + self.images_root = images_root + + def generate_image_thumbnail( + self, source: Path, sha256_hex: str + ) -> ThumbnailResult: + """Reads source, fits inside THUMB_MAX_PX × THUMB_MAX_PX preserving + aspect ratio. Writes PNG if source has alpha, else JPEG. + """ + with Image.open(source) as im: + im.load() + has_alpha = im.mode in ("RGBA", "LA") or ( + im.mode == "P" and "transparency" in im.info + ) + if has_alpha: + im = im.convert("RGBA") + else: + im = im.convert("RGB") + im.thumbnail((THUMB_MAX_PX, THUMB_MAX_PX), Image.Resampling.LANCZOS) + + dest = thumbnail_path_for(sha256_hex, alpha=has_alpha, root=self.images_root) + dest.parent.mkdir(parents=True, exist_ok=True) + if has_alpha: + im.save(dest, "PNG", optimize=True) + mime = "image/png" + else: + im.save(dest, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=True) + mime = "image/jpeg" + return ThumbnailResult(path=dest, width=im.width, height=im.height, mime=mime) + + def generate_video_thumbnail( + self, source: Path, sha256_hex: str, duration_seconds: float | None = None + ) -> ThumbnailResult: + """Snapshots a frame ~5% into the video via ffmpeg, then runs that + frame through the image-thumbnail pipeline. + + duration_seconds is optional; if absent, we snapshot at the 1-second + mark which is a reasonable fallback for short clips. + """ + offset = max(1.0, (duration_seconds or 20.0) * 0.05) + tmp = self.images_root / "thumbs" / "_tmp" / f"{sha256_hex}.png" + tmp.parent.mkdir(parents=True, exist_ok=True) + try: + cmd = [ + "ffmpeg", + "-ss", + f"{offset:.2f}", + "-i", + str(source), + "-frames:v", + "1", + "-vf", + f"scale='if(gt(iw,ih),{THUMB_MAX_PX},-2)':'if(gt(iw,ih),-2,{THUMB_MAX_PX})'", + "-y", + str(tmp), + ] + subprocess.run( + cmd, + check=True, + capture_output=True, + timeout=FFMPEG_TIMEOUT_SECONDS, + ) + return self.generate_image_thumbnail(tmp, sha256_hex) + finally: + if tmp.exists(): + tmp.unlink() diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py new file mode 100644 index 0000000..c8a11bb --- /dev/null +++ b/backend/app/tasks/__init__.py @@ -0,0 +1 @@ +"""Celery tasks. Submodules are auto-discovered via celery_app's include list.""" diff --git a/backend/app/tasks/download.py b/backend/app/tasks/download.py new file mode 100644 index 0000000..a6416ab --- /dev/null +++ b/backend/app/tasks/download.py @@ -0,0 +1,76 @@ +"""download_source Celery task — runs DownloadService for one source.""" + +import asyncio +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from ..celery_app import celery +from ..config import get_config +from ..models import ImportSettings +from ..services.credential_crypto import CredentialCrypto +from ..services.credential_service import CredentialService +from ..services.download_service import DownloadService +from ..services.gallery_dl import GalleryDLService +from ..services.importer import Importer +from ..services.thumbnailer import Thumbnailer +from .import_file import _sync_session_factory + +IMAGES_ROOT = Path("/images") +_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64" + + +def _async_session_factory(): + cfg = get_config() + engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True) + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine + + +@celery.task(name="backend.app.tasks.download.download_source", bind=True, acks_late=True) +def download_source(self, source_id: int) -> int: + """Returns the DownloadEvent.id.""" + + async def _run(): + async_factory, async_engine = _async_session_factory() + SyncFactory = _sync_session_factory() + try: + with SyncFactory() as sync_session: + settings = sync_session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + rate_limit = settings.download_rate_limit_seconds + validate_files = settings.download_validate_files + + gdl = GalleryDLService( + images_root=IMAGES_ROOT, + rate_limit=rate_limit, + validate_files=validate_files, + ) + crypto = CredentialCrypto(_KEY_PATH) + + async with async_factory() as async_session: + cred_service = CredentialService(async_session, crypto) + with SyncFactory() as sync_session: + sync_settings = sync_session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=sync_session, + images_root=IMAGES_ROOT, + import_root=IMAGES_ROOT, + thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), + settings=sync_settings, + ) + svc = DownloadService( + async_session=async_session, + sync_session=sync_session, + gdl=gdl, + importer=importer, + cred_service=cred_service, + ) + return await svc.download_source(source_id) + finally: + await async_engine.dispose() + + return asyncio.run(_run()) diff --git a/backend/app/tasks/import_file.py b/backend/app/tasks/import_file.py new file mode 100644 index 0000000..d53de74 --- /dev/null +++ b/backend/app/tasks/import_file.py @@ -0,0 +1,149 @@ +"""import_media_file task: imports one file via the Importer service and +updates the ImportTask state machine + ImportBatch counters atomically. +""" + +from datetime import UTC, datetime +from pathlib import Path + +from sqlalchemy import create_engine, select, update +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImportBatch, ImportSettings, ImportTask +from ..services.importer import Importer +from ..services.thumbnailer import Thumbnailer + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +IMAGES_ROOT = Path("/images") + + +def _map_result_to_status(result): + """(ImportTask.status, should_requeue_ml_and_thumb) for an ImportResult. + 'superseded' = the kept row's file/ML changed → complete + re-derive. + 'attached' = a non-art file preserved → complete, no ML/thumb.""" + if result.status in ("imported", "superseded"): + return ("complete", True) + if result.status == "attached": + return ("complete", False) + if result.status == "skipped": + return ("skipped", False) + return ("failed", False) + + +@celery.task(name="backend.app.tasks.import_file.import_media_file", bind=True) +def import_media_file(self, import_task_id: int) -> dict: + """Returns a dict so the eager-mode tests can assert without DB.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + task = session.get(ImportTask, import_task_id) + if task is None: + return {"status": "missing", "task_id": import_task_id} + + task.status = "processing" + task.started_at = datetime.now(UTC) + session.add(task) + session.commit() + + settings = session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + import_root = Path(settings.import_scan_path) + batch = session.get(ImportBatch, task.batch_id) + deep = bool(batch and batch.scan_mode == "deep") + importer = Importer( + session=session, + images_root=IMAGES_ROOT, + import_root=import_root, + thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), + settings=settings, + deep=deep, + ) + + try: + result = importer.import_one(Path(task.source_path)) + except Exception as exc: # pragma: no cover — pipeline crash + task.status = "failed" + task.error = f"{type(exc).__name__}: {exc}" + task.finished_at = datetime.now(UTC) + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values(failed=ImportBatch.failed + 1) + ) + session.add(task) + session.commit() + raise + + if result.status in ("imported", "superseded"): + task.status = "complete" + task.result_image_id = result.image_id + counter_col_name = "imported" + counter_col = ImportBatch.imported + elif result.status == "attached": + task.status = "complete" + counter_col_name = "attachments" + counter_col = ImportBatch.attachments + elif result.status == "skipped": + task.status = "skipped" + task.error = ( + f"{result.skip_reason.value}: {result.error}" + if result.skip_reason + else result.error + ) + task.result_image_id = result.image_id + counter_col_name = "skipped" + counter_col = ImportBatch.skipped + else: + task.status = "failed" + task.error = result.error + counter_col_name = "failed" + counter_col = ImportBatch.failed + + task.finished_at = datetime.now(UTC) + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values({counter_col_name: counter_col + 1}) + ) + session.add(task) + session.commit() + + # Enqueue thumbnail + ML for newly imported AND superseded images + # (a superseded row has cleared ML + no thumbnail). + if result.status in ("imported", "superseded"): + from .ml import tag_and_embed + from .thumbnail import generate_thumbnail + + ids = list(result.member_image_ids) + if result.image_id is not None and result.image_id not in ids: + ids.append(result.image_id) + for img_id in ids: + generate_thumbnail.delay(img_id) + tag_and_embed.delay(img_id) + + # If this was the last task in the batch, mark the batch complete. + remaining = session.execute( + select(ImportTask.id) + .where(ImportTask.batch_id == task.batch_id) + .where(ImportTask.status.in_(["pending", "queued", "processing"])) + .limit(1) + ).scalar_one_or_none() + if remaining is None: + session.execute( + update(ImportBatch) + .where(ImportBatch.id == task.batch_id) + .values( + status="complete", + finished_at=datetime.now(UTC), + ) + ) + session.commit() + + return {"task_id": import_task_id, "status": task.status} diff --git a/backend/app/tasks/maintenance.py b/backend/app/tasks/maintenance.py new file mode 100644 index 0000000..ced907f --- /dev/null +++ b/backend/app/tasks/maintenance.py @@ -0,0 +1,224 @@ +"""Periodic maintenance: recover stuck import tasks, garbage-collect old finished tasks.""" + +import logging +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from PIL import Image +from sqlalchemy import create_engine, delete, select, update +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import DownloadEvent, ImageRecord, ImportSettings, ImportTask +from ..utils.phash import compute_phash + +log = logging.getLogger(__name__) + +STUCK_THRESHOLD_MINUTES = 30 +OLD_TASK_DAYS = 7 +PHASH_PAGE = 500 +VERIFY_PAGE = 200 +FFPROBE_TIMEOUT_SECONDS = 10 + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks") +def recover_interrupted_tasks() -> int: + """Find ImportTask rows stuck in 'processing' for >30 min and re-queue them. + + Why 30 min: large videos can legitimately take many minutes to import; + 30 is a safe gate that catches actual crashes (which leave the row stuck + forever) without resetting slow-but-still-running jobs. + """ + SessionLocal = _sync_session_factory() + cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES) + with SessionLocal() as session: + stuck_ids = session.execute( + select(ImportTask.id) + .where(ImportTask.status == "processing") + .where(ImportTask.started_at < cutoff) + ).scalars().all() + + if not stuck_ids: + return 0 + + session.execute( + update(ImportTask) + .where(ImportTask.id.in_(stuck_ids)) + .values(status="queued", started_at=None, error="recovered from stuck state") + ) + session.commit() + + from .import_file import import_media_file + for tid in stuck_ids: + import_media_file.delay(tid) + + return len(stuck_ids) + + +@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks") +def cleanup_old_tasks() -> int: + """Delete completed/skipped/failed ImportTask rows older than 7 days. + + Why 7 days: long enough to debug an issue an operator only notices days + later; short enough that the task table stays a useful operational view + rather than an archive. Matches IR's default. + """ + SessionLocal = _sync_session_factory() + cutoff = datetime.now(UTC) - timedelta(days=OLD_TASK_DAYS) + with SessionLocal() as session: + result = session.execute( + delete(ImportTask) + .where(ImportTask.status.in_(["complete", "skipped", "failed"])) + .where(ImportTask.finished_at < cutoff) + ) + session.commit() + return result.rowcount or 0 + + +@celery.task(name="backend.app.tasks.maintenance.backfill_phash") +def backfill_phash() -> int: + """Recompute phash for stored images that have none (imported before + FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill, + idempotent. Videos legitimately keep phash NULL. A missing/unreadable + file is logged and left NULL — never fails the task.""" + SessionLocal = _sync_session_factory() + updated = 0 + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord) + .where(ImageRecord.id > last_id) + .where(ImageRecord.phash.is_(None)) + .where(ImageRecord.mime.like("image/%")) + .order_by(ImageRecord.id.asc()) + .limit(PHASH_PAGE) + ).scalars().all() + if not rows: + break + for rec in rows: + try: + with Image.open(rec.path) as im: + ph = compute_phash(im) + except Exception as exc: + log.warning( + "backfill_phash: unreadable %s: %s", rec.path, exc + ) + ph = None + if ph is not None and rec.phash is None: + rec.phash = ph + updated += 1 + session.commit() + last_id = rows[-1].id + return updated + + +def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str: + """Compute the integrity verdict for one file. Status precedence: + failed_verification (can't run) > corrupt (sha mismatch / decode + fails) > ok (passes both). Never raises.""" + try: + if not path.is_file(): + return "failed_verification" + try: + actual_sha = sha_fn(path) + except (OSError, PermissionError): + return "failed_verification" + if actual_sha != expected_sha: + return "corrupt" + if mime and mime.startswith("image/"): + try: + with Image.open(path) as im: + im.verify() + except Exception: + return "corrupt" + return "ok" + if mime and mime.startswith("video/"): + try: + proc = subprocess.run( + ["ffprobe", "-v", "error", "-i", str(path)], + capture_output=True, + timeout=FFPROBE_TIMEOUT_SECONDS, + ) + except FileNotFoundError: + # ffprobe binary missing — environment problem, not file. + return "failed_verification" + except subprocess.TimeoutExpired: + return "corrupt" + return "ok" if proc.returncode == 0 else "corrupt" + # Unknown mime — sha matched already; trust that. + return "ok" + except Exception as exc: + log.warning("verify_integrity unexpected error for %s: %s", path, exc) + return "failed_verification" + + +@celery.task(name="backend.app.tasks.maintenance.verify_integrity") +def verify_integrity() -> int: + """Verify every ImageRecord file: sha256 recompute + decode/probe + (PIL for images; ffprobe for videos). Writes integrity_status + (always — the column reflects the most recent verdict). + Keyset-paginated, fail-soft per row, idempotent. Returns the total + count verified.""" + from ..services.importer import _sha256_of # reuse the importer's helper + + SessionLocal = _sync_session_factory() + total = 0 + counts = {"ok": 0, "corrupt": 0, "failed_verification": 0} + last_id = 0 + with SessionLocal() as session: + while True: + rows = session.execute( + select(ImageRecord) + .where(ImageRecord.id > last_id) + .order_by(ImageRecord.id.asc()) + .limit(VERIFY_PAGE) + ).scalars().all() + if not rows: + break + for rec in rows: + rec.integrity_status = _verify_one( + Path(rec.path), rec.sha256, rec.mime, _sha256_of + ) + counts[rec.integrity_status] = ( + counts.get(rec.integrity_status, 0) + 1 + ) + total += 1 + session.commit() + last_id = rows[-1].id + log.info("verify_integrity verdicts: %s (total %d)", counts, total) + return total + + +@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events") +def cleanup_old_download_events() -> int: + """FC-3d: delete terminal DownloadEvent rows older than the configured + retention window. Never touches pending/running rows. + + Why terminal-only: pending/running rows represent in-flight work whose + owning task may still be alive; deleting them would orphan the task. + Retention days comes from ImportSettings.download_event_retention_days + so the operator can tune without a code change. + """ + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + settings = session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + retention_days = settings.download_event_retention_days + cutoff = datetime.now(UTC) - timedelta(days=retention_days) + result = session.execute( + delete(DownloadEvent) + .where(DownloadEvent.status.in_(["ok", "error", "skipped"])) + .where(DownloadEvent.started_at < cutoff) + ) + session.commit() + return result.rowcount or 0 diff --git a/backend/app/tasks/migration.py b/backend/app/tasks/migration.py new file mode 100644 index 0000000..e8427ce --- /dev/null +++ b/backend/app/tasks/migration.py @@ -0,0 +1,180 @@ +"""FC-5 run_migration Celery task. + +Dispatches to the right migrator based on `kind`. Updates MigrationRun +row's status/counts/finished_at as it runs. Failures set status='error' +with the error message preserved. + +kinds: backup, gs_ingest, ir_ingest, tag_apply, ml_queue, verify, rollback +""" +from __future__ import annotations + +import asyncio +import logging +from datetime import UTC, datetime +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from ..celery_app import celery +from ..config import get_config +from ..models import MigrationRun +from ..services.credential_crypto import CredentialCrypto +from ..services.migrators import backup as backup_mod +from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify +from ..services.migrators import rollback as rollback_mod + +log = logging.getLogger(__name__) + +IMAGES_ROOT = Path("/images") +_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64" + + +def _async_session_factory(): + cfg = get_config() + engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True) + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine + + +async def _update_run( + db: AsyncSession, run_id: int, *, + status: str | None = None, counts: dict | None = None, + error: str | None = None, finished_at: datetime | None = None, + metadata_patch: dict | None = None, +) -> None: + run = (await db.execute( + select(MigrationRun).where(MigrationRun.id == run_id) + )).scalar_one() + if status is not None: + run.status = status + if counts is not None: + run.counts = counts + if error is not None: + run.error = error + if finished_at is not None: + run.finished_at = finished_at + if metadata_patch: + run.metadata_ = {**(run.metadata_ or {}), **metadata_patch} + await db.commit() + + +async def _run_async(run_id: int, kind: str, params: dict) -> dict: + factory, engine = _async_session_factory() + try: + async with factory() as db: + await _update_run(db, run_id, status="running") + try: + if kind == "backup": + manifest = backup_mod.create_backup( + db_url=get_config().database_url_sync, + images_root=IMAGES_ROOT, + tag=params.get("tag", "manual"), + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": 0, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + metadata_patch={"manifest": manifest}, + ) + return manifest + + elif kind == "gs_ingest": + fc_crypto = CredentialCrypto(_KEY_PATH) + counts = await gs_ingest.migrate_async( + db, data=params["data"], + fc_crypto=fc_crypto, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", counts=counts, + finished_at=datetime.now(UTC), + ) + return counts + + elif kind == "ir_ingest": + counts = await ir_ingest.migrate_async( + db, data=params["data"], + images_root=IMAGES_ROOT, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", counts=counts, + finished_at=datetime.now(UTC), + ) + return counts + + elif kind == "tag_apply": + result = await tag_apply.apply_async( + db, images_root=IMAGES_ROOT, + dry_run=params.get("dry_run", False), + ) + await _update_run( + db, run_id, status="ok", + counts=result["counts"], + finished_at=datetime.now(UTC), + metadata_patch={"unmatched": result["unmatched"]}, + ) + return result + + elif kind == "ml_queue": + count = await ml_queue.queue_all_unprocessed_async(db) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": count, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + ) + return {"queued": count} + + elif kind == "verify": + checks = await verify.verify_async(db, expected=params.get("expected")) + sample = await verify.verify_sha256_sample( + db, sample_size=params.get("sample_size", 20), + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": sample["sample_size"], + "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, + "conflicts": sample["mismatched"] + sample["missing"]}, + finished_at=datetime.now(UTC), + metadata_patch={"checks": checks, "sample": sample}, + ) + return {"checks": checks, "sample": sample} + + elif kind == "rollback": + result = rollback_mod.rollback_to_pre_migration( + db_url=get_config().database_url_sync, + images_root=IMAGES_ROOT, + ) + await _update_run( + db, run_id, status="ok", + counts={"rows_processed": 0, "rows_inserted": 0, + "rows_skipped": 0, "files_copied": 0, + "bytes_copied": 0, "conflicts": 0}, + finished_at=datetime.now(UTC), + metadata_patch={"rollback_result": result}, + ) + return result + + else: + raise ValueError(f"unknown kind: {kind}") + + except Exception as exc: + log.exception("migration kind=%s failed", kind) + await _update_run( + db, run_id, status="error", error=str(exc), + finished_at=datetime.now(UTC), + ) + raise + finally: + await engine.dispose() + + +@celery.task(name="backend.app.tasks.migration.run_migration", bind=True, acks_late=True) +def run_migration(self, run_id: int, kind: str, params: dict) -> dict: + """FC-5: dispatch a migration kind. Updates MigrationRun row as it goes.""" + return asyncio.run(_run_async(run_id, kind, params)) diff --git a/backend/app/tasks/ml.py b/backend/app/tasks/ml.py new file mode 100644 index 0000000..13d4394 --- /dev/null +++ b/backend/app/tasks/ml.py @@ -0,0 +1,336 @@ +"""ML Celery tasks: per-image inference, backfill discovery, centroid +recompute, allowlist auto-apply, model self-heal. + +All run on the ml-worker (queue 'ml') except recompute_centroids and +apply_allowlist_tags sweeps which are 'maintenance' lane. Sync sessions +(Celery workers are sync processes), same pattern as FC-2a tasks. +""" + +from pathlib import Path + +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImageRecord, MLSettings + +IMAGES_ROOT = Path("/images") +VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"} + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +def _is_video(path: Path) -> bool: + return path.suffix.lower() in VIDEO_EXTS + + +@celery.task(name="backend.app.tasks.ml.tag_and_embed", bind=True) +def tag_and_embed(self, image_id: int) -> dict: + """Run Camie + SigLIP on one image; store predictions + embedding; + then enqueue per-image allowlist application. + + Video: sample frames between 10% and 90% of duration (VIDEO_ML_FRAMES, + default 10). Max-pool tagger confidences across frames, mean-pool the + SigLIP embeddings. On no-frames returns status='no_frames' (not an error). + """ + import os + + from ..services.ml.embedder import get_embedder + from ..services.ml.tagger import get_tagger + + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + record = session.get(ImageRecord, image_id) + if record is None: + return {"status": "missing", "image_id": image_id} + settings = session.execute( + select(MLSettings).where(MLSettings.id == 1) + ).scalar_one() + + src = Path(record.path) + if not src.is_file(): + return {"status": "file_missing", "image_id": image_id} + + tagger = get_tagger() + embedder = get_embedder() + + if _is_video(src): + frames = _sample_video_frames( + src, int(os.environ.get("VIDEO_ML_FRAMES", "10")) + ) + if not frames: + return {"status": "no_frames", "image_id": image_id} + preds = _maxpool_predictions([tagger.infer(f) for f in frames]) + import numpy as np + + embedding = np.mean( + [embedder.infer(f) for f in frames], axis=0 + ).astype("float32") + for f in frames: + f.unlink(missing_ok=True) + else: + raw = tagger.infer(src) + preds = { + name: {"category": p.category, "confidence": p.confidence} + for name, p in raw.items() + } + embedding = embedder.infer(src) + + record.tagger_predictions = preds + record.tagger_model_version = settings.tagger_model_version + record.siglip_embedding = embedding.tolist() + record.siglip_model_version = settings.embedder_model_version + session.add(record) + session.commit() + + apply_allowlist_tags.delay(image_id=image_id) + return {"status": "ok", "image_id": image_id, "tags": len(preds)} + + +def _sample_video_frames(src: Path, n: int) -> list[Path]: + """Extract n frames evenly between 10% and 90% of duration via ffmpeg. + Returns temp file paths (caller deletes). Empty list on failure.""" + import json + import subprocess + import tempfile + + try: + probe = subprocess.run( + [ + "ffprobe", "-v", "quiet", "-print_format", "json", + "-show_format", str(src), + ], + check=True, capture_output=True, timeout=30, + ) + duration = float(json.loads(probe.stdout)["format"]["duration"]) + except Exception: + return [] + if duration <= 0: + return [] + + start, end = duration * 0.10, duration * 0.90 + step = (end - start) / max(n - 1, 1) + out: list[Path] = [] + tmpdir = Path(tempfile.mkdtemp(prefix="fc_vid_")) + for i in range(n): + ts = start + i * step + dest = tmpdir / f"frame_{i:02d}.jpg" + try: + subprocess.run( + [ + "ffmpeg", "-ss", f"{ts:.2f}", "-i", str(src), + "-frames:v", "1", "-q:v", "3", "-y", str(dest), + ], + check=True, capture_output=True, timeout=60, + ) + if dest.is_file(): + out.append(dest) + except Exception: + continue + return out + + +def _maxpool_predictions(per_frame: list[dict]) -> dict: + """Aggregate per-frame {name: TagPrediction} dicts by max confidence.""" + merged: dict[str, dict] = {} + for frame_preds in per_frame: + for name, p in frame_preds.items(): + cur = merged.get(name) + if cur is None or p.confidence > cur["confidence"]: + merged[name] = { + "category": p.category, + "confidence": p.confidence, + } + return merged + + +@celery.task(name="backend.app.tasks.ml.backfill", bind=True) +def backfill(self) -> int: + """Enqueue tag_and_embed for images missing predictions/embeddings for + the current model versions. Keyset pagination by id ASC (restart-safe). + """ + SessionLocal = _sync_session_factory() + enqueued = 0 + last_id = 0 + with SessionLocal() as session: + settings = session.execute( + select(MLSettings).where(MLSettings.id == 1) + ).scalar_one() + while True: + rows = session.execute( + select(ImageRecord.id) + .where(ImageRecord.id > last_id) + .where( + (ImageRecord.tagger_predictions.is_(None)) + | ( + ImageRecord.tagger_model_version + != settings.tagger_model_version + ) + | (ImageRecord.siglip_embedding.is_(None)) + | ( + ImageRecord.siglip_model_version + != settings.embedder_model_version + ) + ) + .order_by(ImageRecord.id.asc()) + .limit(500) + ).scalars().all() + if not rows: + break + for image_id in rows: + tag_and_embed.delay(image_id) + enqueued += 1 + last_id = rows[-1] + return enqueued + + +@celery.task(name="backend.app.tasks.ml.apply_allowlist_tags", bind=True) +def apply_allowlist_tags(self, tag_id: int | None = None, + image_id: int | None = None) -> int: + """Retroactively apply allowlisted tags. + + Modes: + - tag_id only : scan all images for this tag. + - image_id only : scan all allowlisted tags for this image. + - both : just the (image, tag) pair. + - neither : full sweep (daily beat). + + Skips: already-applied, rejected (tag_suggestion_rejection), or + confidence below the tag's allowlist min_confidence. Applied with + source='ml_auto'. + """ + from sqlalchemy import and_ + from sqlalchemy import select as sa_select + from sqlalchemy.dialects.postgresql import insert as pg_insert + + from ..models import TagAllowlist, TagSuggestionRejection + from ..models.tag import image_tag + + SessionLocal = _sync_session_factory() + applied = 0 + with SessionLocal() as session: + allow_rows = session.execute( + sa_select(TagAllowlist.tag_id, TagAllowlist.min_confidence) + if tag_id is None + else sa_select( + TagAllowlist.tag_id, TagAllowlist.min_confidence + ).where(TagAllowlist.tag_id == tag_id) + ).all() + allow = {r[0]: r[1] for r in allow_rows} + if not allow: + return 0 + + img_query = sa_select( + ImageRecord.id, ImageRecord.tagger_predictions + ).where(ImageRecord.tagger_predictions.is_not(None)) + if image_id is not None: + img_query = img_query.where(ImageRecord.id == image_id) + + for img_id, preds in session.execute(img_query).all(): + preds = preds or {} + for a_tag_id, min_conf in allow.items(): + exists = session.execute( + sa_select(image_tag.c.tag_id).where( + and_( + image_tag.c.image_record_id == img_id, + image_tag.c.tag_id == a_tag_id, + ) + ) + ).scalar_one_or_none() + if exists is not None: + continue + rej = session.get( + TagSuggestionRejection, (img_id, a_tag_id) + ) + if rej is not None: + continue + from ..models import Tag + + tag = session.get(Tag, a_tag_id) + if tag is None: + continue + conf = _confidence_for_tag(session, tag, preds) + if conf is None or conf < min_conf: + continue + stmt = pg_insert(image_tag).values( + image_record_id=img_id, + tag_id=a_tag_id, + source="ml_auto", + ) + stmt = stmt.on_conflict_do_nothing( + index_elements=["image_record_id", "tag_id"] + ) + session.execute(stmt) + applied += 1 + session.commit() + return applied + + +def _confidence_for_tag(session, tag, preds: dict) -> float | None: + """Highest confidence among predictions that resolve to `tag` — + either the prediction name equals the tag name, or an alias maps + (prediction name, category) -> tag.id. + """ + from sqlalchemy import select as sa_select + + from ..models import TagAlias + + best: float | None = None + direct = preds.get(tag.name) + if direct is not None: + best = float(direct.get("confidence", 0.0)) + alias_rows = session.execute( + sa_select(TagAlias.alias_string, TagAlias.alias_category).where( + TagAlias.canonical_tag_id == tag.id + ) + ).all() + for alias_string, alias_category in alias_rows: + p = preds.get(alias_string) + if p is None: + continue + if p.get("category") != alias_category: + continue + c = float(p.get("confidence", 0.0)) + if best is None or c > best: + best = c + return best + + +@celery.task(name="backend.app.tasks.ml.recompute_centroid", bind=True) +def recompute_centroid(self, tag_id: int) -> bool: + import asyncio + + from ..extensions import get_session + from ..services.ml.centroids import CentroidService + + async def _run() -> bool: + async with get_session() as session: + svc = CentroidService(session) + result = await svc.recompute_for_tag(tag_id) + await session.commit() + return result + + return asyncio.run(_run()) + + +@celery.task(name="backend.app.tasks.ml.recompute_centroids", bind=True) +def recompute_centroids(self) -> int: + """Daily: find drifted centroids, enqueue recompute_centroid for each.""" + import asyncio + + from ..extensions import get_session + from ..services.ml.centroids import CentroidService + + async def _list() -> list[int]: + async with get_session() as session: + return await CentroidService(session).list_drifted() + + drifted = asyncio.run(_list()) + for tid in drifted: + recompute_centroid.delay(tid) + return len(drifted) diff --git a/backend/app/tasks/scan.py b/backend/app/tasks/scan.py new file mode 100644 index 0000000..8c7a400 --- /dev/null +++ b/backend/app/tasks/scan.py @@ -0,0 +1,161 @@ +"""scan tasks: + +* ``scan_directory`` — walks the configured import path, creates an + ImportBatch, enumerates files into ImportTasks, and enqueues + import_media_file for each. +* ``tick_due_sources`` (FC-3d) — periodic Beat-fired scan that fires + ``download_source.delay()`` for sources due for a check. +""" + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +from sqlalchemy import create_engine, select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import DownloadEvent, ImportBatch, ImportSettings, ImportTask +from ..services.scheduler_service import select_due_sources + + +def _iter_import_files(import_root: Path): + """Every regular file except sidecar .json, dotfiles, and .partial + temp files. The Importer dispatches by kind (media / archive / + other) — scan no longer filters to media (FC-2d-iii).""" + for entry in import_root.rglob("*"): + if not entry.is_file(): + continue + if entry.name.startswith("."): + continue + if entry.suffix.lower() in (".json", ".partial"): + continue + yield entry + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True) +def scan_directory(self, triggered_by: str = "manual", + mode: str = "quick") -> int: + """Walks the import root and creates ImportTasks. `mode` is 'quick' + or 'deep'; deep also enqueues a library-wide phash backfill and (via + Importer.deep) re-derives on already-imported files. Returns the + batch id.""" + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + settings = session.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + import_root = Path(settings.import_scan_path) + + batch = ImportBatch( + triggered_by=triggered_by, + source_path=str(import_root), + scan_mode=mode, + status="running", + ) + session.add(batch) + session.flush() + batch_id = batch.id + + # Walk and enumerate. + files_seen = 0 + for entry in _iter_import_files(import_root): + try: + size = entry.stat().st_size + except OSError: + size = None + task = ImportTask( + batch_id=batch_id, + source_path=str(entry), + task_type="media", + status="pending", + size_bytes=size, + ) + session.add(task) + files_seen += 1 + + batch.total_files = files_seen + session.commit() + + # Now enqueue import_media_file for each pending task. + for task in session.execute( + select(ImportTask).where(ImportTask.batch_id == batch_id) + ).scalars(): + task.status = "queued" + session.add(task) + from .import_file import import_media_file + + import_media_file.delay(task.id) + session.commit() + + if mode == "deep": + from .maintenance import backfill_phash + + backfill_phash.delay() + + return batch_id + + +# --- FC-3d: periodic source-check tick ------------------------------------ + + +def _async_session_factory(): + cfg = get_config() + engine = create_async_engine(cfg.database_url, future=True, pool_pre_ping=True) + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False), engine + + +async def _tick_due_sources_async() -> dict: + factory, engine = _async_session_factory() + try: + async with factory() as session: + due = await select_due_sources(session) + if not due: + return { + "due": 0, "queued": 0, "skipped_in_flight": 0, + "tick_at": datetime.now(UTC).isoformat(), + } + due_ids = [s.id for s in due] + in_flight_rows = (await session.execute( + select(DownloadEvent.source_id).where( + DownloadEvent.source_id.in_(due_ids), + DownloadEvent.status.in_(["pending", "running"]), + ) + )).scalars().all() + in_flight = set(in_flight_rows) + + # Local import: keep top of module light, avoid circular import + # at celery task discovery time. + from .download import download_source + + queued = 0 + for s in due: + if s.id in in_flight: + continue + session.add(DownloadEvent(source_id=s.id, status="pending")) + await session.commit() + download_source.delay(s.id) + queued += 1 + return { + "due": len(due), + "queued": queued, + "skipped_in_flight": len(in_flight), + "tick_at": datetime.now(UTC).isoformat(), + } + finally: + await engine.dispose() + + +@celery.task(name="backend.app.tasks.scan.tick_due_sources", bind=True) +def tick_due_sources(self) -> dict: + """FC-3d: scan for sources due for a periodic check and fire + ``download_source.delay()`` for each.""" + return asyncio.run(_tick_due_sources_async()) diff --git a/backend/app/tasks/smoke.py b/backend/app/tasks/smoke.py new file mode 100644 index 0000000..ad452fb --- /dev/null +++ b/backend/app/tasks/smoke.py @@ -0,0 +1,8 @@ +"""Smoke task used during FC-1 to confirm Celery routing works end-to-end.""" + +from ..celery_app import celery + + +@celery.task(name="backend.app.tasks.smoke.ping") +def ping(value: str = "pong") -> str: + return value diff --git a/backend/app/tasks/thumbnail.py b/backend/app/tasks/thumbnail.py new file mode 100644 index 0000000..68fa665 --- /dev/null +++ b/backend/app/tasks/thumbnail.py @@ -0,0 +1,49 @@ +"""generate_thumbnail task: PIL/ffmpeg thumbnail generation on the thumbnail queue. + +Lives separately from import_file because thumbnails can be regenerated en +masse (FC-2c adds the 'regenerate all' admin action) and they're CPU-bound +so they deserve their own queue lane. +""" + +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from ..celery_app import celery +from ..config import get_config +from ..models import ImageRecord +from ..services.importer import is_video +from ..services.thumbnailer import Thumbnailer + +IMAGES_ROOT = Path("/images") + + +def _sync_session_factory(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False) + + +@celery.task(name="backend.app.tasks.thumbnail.generate_thumbnail", bind=True) +def generate_thumbnail(self, image_id: int) -> dict: + SessionLocal = _sync_session_factory() + with SessionLocal() as session: + record = session.get(ImageRecord, image_id) + if record is None: + return {"status": "missing", "image_id": image_id} + + thumbnailer = Thumbnailer(images_root=IMAGES_ROOT) + source = Path(record.path) + try: + if is_video(source): + result = thumbnailer.generate_video_thumbnail(source, record.sha256) + else: + result = thumbnailer.generate_image_thumbnail(source, record.sha256) + except Exception as exc: # pragma: no cover — thumbnail failure is non-fatal + return {"status": "failed", "image_id": image_id, "error": f"{type(exc).__name__}: {exc}"} + + record.thumbnail_path = str(result.path) + session.add(record) + session.commit() + return {"status": "ok", "image_id": image_id, "path": str(result.path)} diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000..9be5cae --- /dev/null +++ b/backend/app/utils/__init__.py @@ -0,0 +1 @@ +"""Pure-function utilities shared across services and tasks.""" diff --git a/backend/app/utils/artist_backfill.py b/backend/app/utils/artist_backfill.py new file mode 100644 index 0000000..3979b24 --- /dev/null +++ b/backend/app/utils/artist_backfill.py @@ -0,0 +1,44 @@ +"""Literal SQL for the FC-2d-vii-c artist backfill / artist-tag delete. + +Intentionally pure string constants — NO model/slug imports, NO logic — +so migration 0008 and its test share one drift-proof source of truth. +Backfill steps are ordered primary -> provenance -> artist-tag and each +only touches rows still NULL (idempotent, first match wins). The +artist-tag step matches Artist.name = Tag.name: the importer always +created both from the same artist_name string. +""" + +BACKFILL_PRIMARY_SQL = """ +UPDATE image_record AS ir +SET artist_id = s.artist_id +FROM post p +JOIN source s ON s.id = p.source_id +WHERE ir.primary_post_id = p.id + AND ir.artist_id IS NULL +""" + +BACKFILL_PROVENANCE_SQL = """ +UPDATE image_record AS ir +SET artist_id = s.artist_id +FROM ( + SELECT DISTINCT ON (ip.image_record_id) + ip.image_record_id, src.artist_id + FROM image_provenance ip + JOIN source src ON src.id = ip.source_id + ORDER BY ip.image_record_id, ip.id +) AS s +WHERE ir.id = s.image_record_id + AND ir.artist_id IS NULL +""" + +BACKFILL_TAG_SQL = """ +UPDATE image_record AS ir +SET artist_id = a.id +FROM image_tag it +JOIN tag t ON t.id = it.tag_id AND t.kind = 'artist' +JOIN artist a ON a.name = t.name +WHERE it.image_record_id = ir.id + AND ir.artist_id IS NULL +""" + +DELETE_ARTIST_TAGS_SQL = "DELETE FROM tag WHERE kind = 'artist'" diff --git a/backend/app/utils/html_sanitize.py b/backend/app/utils/html_sanitize.py new file mode 100644 index 0000000..d660a5c --- /dev/null +++ b/backend/app/utils/html_sanitize.py @@ -0,0 +1,39 @@ +"""Backend HTML sanitization for scraped Post.description. + +Provenance descriptions come from gallery-dl (untrusted third-party HTML). +We render them via v-html in the UI sub-slice, so they MUST be sanitized +server-side to a tight allowlist. nh3 (Rust/ammonia bindings) is used; +bleach is deprecated upstream and intentionally not used. +""" + +import nh3 + +ALLOWED_TAGS = { + "p", "a", "br", "em", "strong", "b", "i", "ul", "ol", "li", "blockquote", +} +ALLOWED_ATTRIBUTES = {"a": {"href", "title"}} +ALLOWED_URL_SCHEMES = {"http", "https"} + + +def sanitize_post_html(raw: str | None) -> str | None: + """Return sanitized HTML, or None for null/empty/whitespace input. + + - keeps only ALLOWED_TAGS / ALLOWED_ATTRIBUTES + - only http/https hrefs survive (javascript: etc. dropped) + - + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..d68122c --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,31 @@ +{ + "name": "fabledcurator-frontend", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "test:unit": "vitest run", + "check": "vue-tsc --noEmit" + }, + "dependencies": { + "vue": "^3.4.0", + "vue-router": "^4.3.0", + "pinia": "^2.1.0", + "vuetify": "^3.5.0", + "@mdi/font": "^7.4.0" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^5.0.0", + "vite": "^5.2.0", + "vue-tsc": "^2.0.0", + "vite-plugin-vuetify": "^2.0.0", + "sass": "^1.71.0", + "vitest": "^2.1.0" + } +} diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg new file mode 100644 index 0000000..6b07fb4 --- /dev/null +++ b/frontend/public/favicon.svg @@ -0,0 +1,5 @@ + + + + diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..a4b0086 --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,21 @@ + + + diff --git a/frontend/src/components/AppShell.vue b/frontend/src/components/AppShell.vue new file mode 100644 index 0000000..3546e40 --- /dev/null +++ b/frontend/src/components/AppShell.vue @@ -0,0 +1,22 @@ + + + + + diff --git a/frontend/src/components/AppSnackbar.vue b/frontend/src/components/AppSnackbar.vue new file mode 100644 index 0000000..cf45f16 --- /dev/null +++ b/frontend/src/components/AppSnackbar.vue @@ -0,0 +1,26 @@ + + + diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue new file mode 100644 index 0000000..4493050 --- /dev/null +++ b/frontend/src/components/TopNav.vue @@ -0,0 +1,133 @@ + + + + + diff --git a/frontend/src/components/credentials/CredentialUploadDialog.vue b/frontend/src/components/credentials/CredentialUploadDialog.vue new file mode 100644 index 0000000..8b2edae --- /dev/null +++ b/frontend/src/components/credentials/CredentialUploadDialog.vue @@ -0,0 +1,76 @@ + + + diff --git a/frontend/src/components/credentials/ExtensionKeyBar.vue b/frontend/src/components/credentials/ExtensionKeyBar.vue new file mode 100644 index 0000000..cf56b3c --- /dev/null +++ b/frontend/src/components/credentials/ExtensionKeyBar.vue @@ -0,0 +1,75 @@ + + + + + diff --git a/frontend/src/components/credentials/PlatformCredentialRow.vue b/frontend/src/components/credentials/PlatformCredentialRow.vue new file mode 100644 index 0000000..07225d8 --- /dev/null +++ b/frontend/src/components/credentials/PlatformCredentialRow.vue @@ -0,0 +1,68 @@ + + + + + diff --git a/frontend/src/components/discovery/ArtistCard.vue b/frontend/src/components/discovery/ArtistCard.vue new file mode 100644 index 0000000..55564cb --- /dev/null +++ b/frontend/src/components/discovery/ArtistCard.vue @@ -0,0 +1,59 @@ + + + + + diff --git a/frontend/src/components/discovery/MasonryGrid.vue b/frontend/src/components/discovery/MasonryGrid.vue new file mode 100644 index 0000000..3673448 --- /dev/null +++ b/frontend/src/components/discovery/MasonryGrid.vue @@ -0,0 +1,84 @@ + + + + + diff --git a/frontend/src/components/discovery/MergeConfirmDialog.vue b/frontend/src/components/discovery/MergeConfirmDialog.vue new file mode 100644 index 0000000..c253746 --- /dev/null +++ b/frontend/src/components/discovery/MergeConfirmDialog.vue @@ -0,0 +1,40 @@ + + + diff --git a/frontend/src/components/discovery/TagCard.vue b/frontend/src/components/discovery/TagCard.vue new file mode 100644 index 0000000..acb3905 --- /dev/null +++ b/frontend/src/components/discovery/TagCard.vue @@ -0,0 +1,109 @@ + + + + + diff --git a/frontend/src/components/downloads/DownloadDetailModal.vue b/frontend/src/components/downloads/DownloadDetailModal.vue new file mode 100644 index 0000000..5ee4823 --- /dev/null +++ b/frontend/src/components/downloads/DownloadDetailModal.vue @@ -0,0 +1,117 @@ + + + + + diff --git a/frontend/src/components/downloads/DownloadEventRow.vue b/frontend/src/components/downloads/DownloadEventRow.vue new file mode 100644 index 0000000..a4360c0 --- /dev/null +++ b/frontend/src/components/downloads/DownloadEventRow.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/frontend/src/components/downloads/FilterPills.vue b/frontend/src/components/downloads/FilterPills.vue new file mode 100644 index 0000000..20ed1bd --- /dev/null +++ b/frontend/src/components/downloads/FilterPills.vue @@ -0,0 +1,22 @@ + + + diff --git a/frontend/src/components/gallery/BulkEditorPanel.vue b/frontend/src/components/gallery/BulkEditorPanel.vue new file mode 100644 index 0000000..d7775eb --- /dev/null +++ b/frontend/src/components/gallery/BulkEditorPanel.vue @@ -0,0 +1,150 @@ + + + + + diff --git a/frontend/src/components/gallery/EmptyState.vue b/frontend/src/components/gallery/EmptyState.vue new file mode 100644 index 0000000..355133f --- /dev/null +++ b/frontend/src/components/gallery/EmptyState.vue @@ -0,0 +1,27 @@ + + + diff --git a/frontend/src/components/gallery/GalleryGrid.vue b/frontend/src/components/gallery/GalleryGrid.vue new file mode 100644 index 0000000..111cf61 --- /dev/null +++ b/frontend/src/components/gallery/GalleryGrid.vue @@ -0,0 +1,131 @@ + + + + + diff --git a/frontend/src/components/gallery/GalleryItem.vue b/frontend/src/components/gallery/GalleryItem.vue new file mode 100644 index 0000000..8d9ba13 --- /dev/null +++ b/frontend/src/components/gallery/GalleryItem.vue @@ -0,0 +1,134 @@ + + + + + diff --git a/frontend/src/components/gallery/PostInfoHeader.vue b/frontend/src/components/gallery/PostInfoHeader.vue new file mode 100644 index 0000000..eb5613d --- /dev/null +++ b/frontend/src/components/gallery/PostInfoHeader.vue @@ -0,0 +1,102 @@ + + + + + diff --git a/frontend/src/components/gallery/TimelineSidebar.vue b/frontend/src/components/gallery/TimelineSidebar.vue new file mode 100644 index 0000000..8ea5c24 --- /dev/null +++ b/frontend/src/components/gallery/TimelineSidebar.vue @@ -0,0 +1,88 @@ + + + + + diff --git a/frontend/src/components/modal/AliasPickerDialog.vue b/frontend/src/components/modal/AliasPickerDialog.vue new file mode 100644 index 0000000..2552d31 --- /dev/null +++ b/frontend/src/components/modal/AliasPickerDialog.vue @@ -0,0 +1,63 @@ + + + diff --git a/frontend/src/components/modal/FandomPicker.vue b/frontend/src/components/modal/FandomPicker.vue new file mode 100644 index 0000000..31189ab --- /dev/null +++ b/frontend/src/components/modal/FandomPicker.vue @@ -0,0 +1,48 @@ + + + diff --git a/frontend/src/components/modal/ImageCanvas.vue b/frontend/src/components/modal/ImageCanvas.vue new file mode 100644 index 0000000..52cbb17 --- /dev/null +++ b/frontend/src/components/modal/ImageCanvas.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue new file mode 100644 index 0000000..99ef9e8 --- /dev/null +++ b/frontend/src/components/modal/ImageViewer.vue @@ -0,0 +1,174 @@ + + + + + diff --git a/frontend/src/components/modal/ProvenancePanel.vue b/frontend/src/components/modal/ProvenancePanel.vue new file mode 100644 index 0000000..9e428c2 --- /dev/null +++ b/frontend/src/components/modal/ProvenancePanel.vue @@ -0,0 +1,150 @@ + + + + + diff --git a/frontend/src/components/modal/SuggestionItem.vue b/frontend/src/components/modal/SuggestionItem.vue new file mode 100644 index 0000000..2045f57 --- /dev/null +++ b/frontend/src/components/modal/SuggestionItem.vue @@ -0,0 +1,54 @@ + + + + + diff --git a/frontend/src/components/modal/SuggestionsCategoryGroup.vue b/frontend/src/components/modal/SuggestionsCategoryGroup.vue new file mode 100644 index 0000000..c6c6611 --- /dev/null +++ b/frontend/src/components/modal/SuggestionsCategoryGroup.vue @@ -0,0 +1,55 @@ + + + + + diff --git a/frontend/src/components/modal/SuggestionsPanel.vue b/frontend/src/components/modal/SuggestionsPanel.vue new file mode 100644 index 0000000..a5f8353 --- /dev/null +++ b/frontend/src/components/modal/SuggestionsPanel.vue @@ -0,0 +1,104 @@ + + + + + diff --git a/frontend/src/components/modal/TagAutocomplete.vue b/frontend/src/components/modal/TagAutocomplete.vue new file mode 100644 index 0000000..cdc81ce --- /dev/null +++ b/frontend/src/components/modal/TagAutocomplete.vue @@ -0,0 +1,135 @@ + + + + + diff --git a/frontend/src/components/modal/TagPanel.vue b/frontend/src/components/modal/TagPanel.vue new file mode 100644 index 0000000..da79fe5 --- /dev/null +++ b/frontend/src/components/modal/TagPanel.vue @@ -0,0 +1,116 @@ + + + + + diff --git a/frontend/src/components/modal/TagRenameDialog.vue b/frontend/src/components/modal/TagRenameDialog.vue new file mode 100644 index 0000000..1bd8c08 --- /dev/null +++ b/frontend/src/components/modal/TagRenameDialog.vue @@ -0,0 +1,58 @@ + + + diff --git a/frontend/src/components/modal/VideoCanvas.vue b/frontend/src/components/modal/VideoCanvas.vue new file mode 100644 index 0000000..e79e6da --- /dev/null +++ b/frontend/src/components/modal/VideoCanvas.vue @@ -0,0 +1,59 @@ + + + + + diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue new file mode 100644 index 0000000..68b537e --- /dev/null +++ b/frontend/src/components/posts/PostCard.vue @@ -0,0 +1,204 @@ + + + + + diff --git a/frontend/src/components/posts/PostsFilterBar.vue b/frontend/src/components/posts/PostsFilterBar.vue new file mode 100644 index 0000000..34e0c01 --- /dev/null +++ b/frontend/src/components/posts/PostsFilterBar.vue @@ -0,0 +1,140 @@ + + + + + diff --git a/frontend/src/components/settings/AliasTable.vue b/frontend/src/components/settings/AliasTable.vue new file mode 100644 index 0000000..d21f08d --- /dev/null +++ b/frontend/src/components/settings/AliasTable.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/frontend/src/components/settings/AllowlistTable.vue b/frontend/src/components/settings/AllowlistTable.vue new file mode 100644 index 0000000..9448125 --- /dev/null +++ b/frontend/src/components/settings/AllowlistTable.vue @@ -0,0 +1,48 @@ + + + diff --git a/frontend/src/components/settings/CentroidRecomputeCard.vue b/frontend/src/components/settings/CentroidRecomputeCard.vue new file mode 100644 index 0000000..22bf4bd --- /dev/null +++ b/frontend/src/components/settings/CentroidRecomputeCard.vue @@ -0,0 +1,30 @@ + + + diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue new file mode 100644 index 0000000..2628309 --- /dev/null +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -0,0 +1,156 @@ + + + + + diff --git a/frontend/src/components/settings/ImportTaskList.vue b/frontend/src/components/settings/ImportTaskList.vue new file mode 100644 index 0000000..a78562d --- /dev/null +++ b/frontend/src/components/settings/ImportTaskList.vue @@ -0,0 +1,141 @@ + + + diff --git a/frontend/src/components/settings/ImportTriggerPanel.vue b/frontend/src/components/settings/ImportTriggerPanel.vue new file mode 100644 index 0000000..232df86 --- /dev/null +++ b/frontend/src/components/settings/ImportTriggerPanel.vue @@ -0,0 +1,45 @@ + + + diff --git a/frontend/src/components/settings/LegacyMigrationCard.vue b/frontend/src/components/settings/LegacyMigrationCard.vue new file mode 100644 index 0000000..be2dfd1 --- /dev/null +++ b/frontend/src/components/settings/LegacyMigrationCard.vue @@ -0,0 +1,245 @@ + + + + + diff --git a/frontend/src/components/settings/MLBackfillCard.vue b/frontend/src/components/settings/MLBackfillCard.vue new file mode 100644 index 0000000..4380971 --- /dev/null +++ b/frontend/src/components/settings/MLBackfillCard.vue @@ -0,0 +1,29 @@ + + + diff --git a/frontend/src/components/settings/MLThresholdSliders.vue b/frontend/src/components/settings/MLThresholdSliders.vue new file mode 100644 index 0000000..34304d3 --- /dev/null +++ b/frontend/src/components/settings/MLThresholdSliders.vue @@ -0,0 +1,40 @@ + + + diff --git a/frontend/src/components/settings/MaintenancePanel.vue b/frontend/src/components/settings/MaintenancePanel.vue new file mode 100644 index 0000000..9639edf --- /dev/null +++ b/frontend/src/components/settings/MaintenancePanel.vue @@ -0,0 +1,34 @@ + + + + + diff --git a/frontend/src/components/settings/SystemStatsCards.vue b/frontend/src/components/settings/SystemStatsCards.vue new file mode 100644 index 0000000..92d9ba7 --- /dev/null +++ b/frontend/src/components/settings/SystemStatsCards.vue @@ -0,0 +1,63 @@ + + + + + diff --git a/frontend/src/components/subscriptions/ArtistCreateDialog.vue b/frontend/src/components/subscriptions/ArtistCreateDialog.vue new file mode 100644 index 0000000..8efde7d --- /dev/null +++ b/frontend/src/components/subscriptions/ArtistCreateDialog.vue @@ -0,0 +1,58 @@ + + + diff --git a/frontend/src/components/subscriptions/ArtistSection.vue b/frontend/src/components/subscriptions/ArtistSection.vue new file mode 100644 index 0000000..148d813 --- /dev/null +++ b/frontend/src/components/subscriptions/ArtistSection.vue @@ -0,0 +1,64 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceFormDialog.vue b/frontend/src/components/subscriptions/SourceFormDialog.vue new file mode 100644 index 0000000..15b2e63 --- /dev/null +++ b/frontend/src/components/subscriptions/SourceFormDialog.vue @@ -0,0 +1,219 @@ + + + diff --git a/frontend/src/components/subscriptions/SourceHealthDot.vue b/frontend/src/components/subscriptions/SourceHealthDot.vue new file mode 100644 index 0000000..44333ef --- /dev/null +++ b/frontend/src/components/subscriptions/SourceHealthDot.vue @@ -0,0 +1,93 @@ + + + + + diff --git a/frontend/src/components/subscriptions/SourceRow.vue b/frontend/src/components/subscriptions/SourceRow.vue new file mode 100644 index 0000000..cb7a551 --- /dev/null +++ b/frontend/src/components/subscriptions/SourceRow.vue @@ -0,0 +1,55 @@ + + + + + diff --git a/frontend/src/composables/useApi.js b/frontend/src/composables/useApi.js new file mode 100644 index 0000000..cc88c0d --- /dev/null +++ b/frontend/src/composables/useApi.js @@ -0,0 +1,50 @@ +// Thin fetch wrapper. Centralized so every call has consistent error handling +// and json parsing. Errors throw an ApiError with the response status + body. + +export class ApiError extends Error { + constructor(message, status, body) { + super(message) + this.name = 'ApiError' + this.status = status + this.body = body + } +} + +async function request(method, url, { body, params, signal } = {}) { + let fullUrl = url + if (params) { + const search = new URLSearchParams() + for (const [k, v] of Object.entries(params)) { + if (v !== undefined && v !== null) search.set(k, String(v)) + } + const qs = search.toString() + if (qs) fullUrl += (url.includes('?') ? '&' : '?') + qs + } + + const headers = {} + const init = { method, headers, signal } + if (body !== undefined) { + headers['Content-Type'] = 'application/json' + init.body = JSON.stringify(body) + } + + const r = await fetch(fullUrl, init) + const text = await r.text() + let parsed + try { parsed = text ? JSON.parse(text) : null } catch { parsed = text } + + if (!r.ok) { + const msg = (parsed && parsed.error) || `${r.status} ${r.statusText}` + throw new ApiError(msg, r.status, parsed) + } + return parsed +} + +export function useApi() { + return { + get: (url, opts) => request('GET', url, opts), + post: (url, opts) => request('POST', url, opts), + patch: (url, opts) => request('PATCH', url, opts), + delete: (url, opts) => request('DELETE', url, opts) + } +} diff --git a/frontend/src/composables/usePanZoom.js b/frontend/src/composables/usePanZoom.js new file mode 100644 index 0000000..b05cc37 --- /dev/null +++ b/frontend/src/composables/usePanZoom.js @@ -0,0 +1,81 @@ +// Click-to-toggle zoom + drag-to-pan when zoomed + wheel-to-zoom. +// Returns reactive state + handlers a component can spread onto its +// root element. + +import { reactive } from 'vue' + +const MIN_SCALE = 1 +const MAX_SCALE = 4 +const WHEEL_SENSITIVITY = 0.0015 + +export function usePanZoom() { + const state = reactive({ + scale: 1, + x: 0, + y: 0, + panning: false + }) + + let dragOriginX = 0 + let dragOriginY = 0 + let panOriginX = 0 + let panOriginY = 0 + + function reset() { + state.scale = 1 + state.x = 0 + state.y = 0 + } + + function toggleZoom(ev) { + if (state.scale > 1) { reset(); return } + state.scale = 2 + // Zoom centered on click point + if (ev && ev.currentTarget) { + const rect = ev.currentTarget.getBoundingClientRect() + const cx = ev.clientX - rect.left - rect.width / 2 + const cy = ev.clientY - rect.top - rect.height / 2 + state.x = -cx + state.y = -cy + } + } + + function onWheel(ev) { + ev.preventDefault() + const delta = -ev.deltaY * WHEEL_SENSITIVITY + const next = Math.min(MAX_SCALE, Math.max(MIN_SCALE, state.scale + delta * state.scale)) + state.scale = next + if (next === MIN_SCALE) { state.x = 0; state.y = 0 } + } + + function onPointerDown(ev) { + if (state.scale === 1) return + state.panning = true + dragOriginX = ev.clientX + dragOriginY = ev.clientY + panOriginX = state.x + panOriginY = state.y + ev.currentTarget.setPointerCapture(ev.pointerId) + } + function onPointerMove(ev) { + if (!state.panning) return + state.x = panOriginX + (ev.clientX - dragOriginX) + state.y = panOriginY + (ev.clientY - dragOriginY) + } + function onPointerUp(ev) { + state.panning = false + try { ev.currentTarget.releasePointerCapture(ev.pointerId) } catch {} + } + + return { + state, + handlers: { + onWheel, + onPointerDown, + onPointerMove, + onPointerUp, + onClick: toggleZoom + }, + reset + } +} diff --git a/frontend/src/composables/usePolyMasonry.js b/frontend/src/composables/usePolyMasonry.js new file mode 100644 index 0000000..210fc94 --- /dev/null +++ b/frontend/src/composables/usePolyMasonry.js @@ -0,0 +1,57 @@ +// Shortest-column masonry distribution (ported from ImageRepo). +// +// Predictive relative height = height / width. Using the aspect ratio means +// columns are balanced before thumbnails load, so there is no reflow when +// images arrive. Missing or zero dimensions fall back to a square (1.0). + +import { ref, onMounted, onUnmounted } from 'vue' + +function relativeHeight(item) { + const w = Number(item.width) + const h = Number(item.height) + if (!w || !h) return 1 + return h / w +} + +export function distributeIntoColumns(items, columnCount) { + const n = Math.max(1, columnCount | 0) + const columns = Array.from({ length: n }, () => []) + const heights = new Array(n).fill(0) + for (const item of items) { + let shortest = 0 + for (let c = 1; c < n; c++) { + if (heights[c] < heights[shortest]) shortest = c + } + columns[shortest].push(item) + heights[shortest] += relativeHeight(item) + } + return columns +} + +// Reactive column count from container width. Breakpoints mirror the +// gallery grid intent: ~260px target column width, min 1 column. +export function columnCountForWidth(width) { + if (!width || width < 0) return 1 + return Math.max(1, Math.floor(width / 260)) +} + +export function usePolyMasonry(containerRef) { + const columnCount = ref(1) + let ro = null + + function measure() { + const el = containerRef.value + if (el) columnCount.value = columnCountForWidth(el.clientWidth) + } + + onMounted(() => { + measure() + if (typeof ResizeObserver !== 'undefined' && containerRef.value) { + ro = new ResizeObserver(measure) + ro.observe(containerRef.value) + } + }) + onUnmounted(() => ro && ro.disconnect()) + + return { columnCount, distribute: distributeIntoColumns } +} diff --git a/frontend/src/main.js b/frontend/src/main.js new file mode 100644 index 0000000..0882de9 --- /dev/null +++ b/frontend/src/main.js @@ -0,0 +1,26 @@ +import { createApp } from 'vue' +import { createPinia } from 'pinia' +import { createVuetify } from 'vuetify' +import 'vuetify/styles' +import '@mdi/font/css/materialdesignicons.css' + +import App from './App.vue' +import router from './router.js' +import { fabledCuratorTheme } from './theme/vuetify-theme.js' + +const vuetify = createVuetify({ + theme: { + defaultTheme: 'fabledCurator', + themes: { fabledCurator: fabledCuratorTheme } + }, + defaults: { + // FabledDesignSystem: pill shape for buttons + VBtn: { rounded: 'pill' } + } +}) + +createApp(App) + .use(createPinia()) + .use(router) + .use(vuetify) + .mount('#app') diff --git a/frontend/src/router.js b/frontend/src/router.js new file mode 100644 index 0000000..b51f37f --- /dev/null +++ b/frontend/src/router.js @@ -0,0 +1,53 @@ +import { createRouter, createWebHistory, createMemoryHistory } from 'vue-router' +import SettingsView from './views/SettingsView.vue' +import GalleryView from './views/GalleryView.vue' +import ShowcaseView from './views/ShowcaseView.vue' +import TagsView from './views/TagsView.vue' +import ArtistView from './views/ArtistView.vue' +import SeriesManageView from './views/SeriesManageView.vue' +import SeriesReaderView from './views/SeriesReaderView.vue' +import SubscriptionsView from './views/SubscriptionsView.vue' +import CredentialsView from './views/CredentialsView.vue' +import DownloadsView from './views/DownloadsView.vue' +import PostsView from './views/PostsView.vue' +import ArtistsView from './views/ArtistsView.vue' + +// The application's front door. `/` redirects here. Changing the front door +// is a one-line edit (e.g. '/gallery' or '/tags'). +export const FRONT_DOOR = '/showcase' + +const routes = [ + // Root is a redirect only — no meta.title so it stays out of the nav. + { path: '/', redirect: FRONT_DOOR }, + + // FC-2: image backbone + { path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase' } }, + { path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery' } }, + { path: '/artists', name: 'artists', component: ArtistsView, meta: { title: 'Artists' } }, + { path: '/tags', name: 'tags', component: TagsView, meta: { title: 'Tags' } }, + // Artist detail — no meta.title (reached by clicking an artist, not nav). + { path: '/artist/:slug', name: 'artist', component: ArtistView }, + // Series management — no meta.title (reached from a series tag card). + { path: '/series/:tagId', name: 'series-manage', component: SeriesManageView }, + // Series reader — immersive (no top nav, no meta.title). + { path: '/series/:tagId/read', name: 'series-read', component: SeriesReaderView, meta: { immersive: true } }, + { path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings' } }, + + // FC-3: subscription backbone + { path: '/posts', name: 'posts', component: PostsView, meta: { title: 'Posts' } }, + { path: '/subscriptions', name: 'subscriptions', component: SubscriptionsView, meta: { title: 'Subscriptions' } }, + { path: '/credentials', name: 'credentials', component: CredentialsView, meta: { title: 'Credentials' } }, + { path: '/downloads', name: 'downloads', component: DownloadsView, meta: { title: 'Downloads' } } +] + +// Browser uses HTML5 history; non-browser (Vitest/SSR) falls back to memory +// history so the module is importable without `window`. +const history = + typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory() + +const router = createRouter({ + history, + routes +}) + +export default router diff --git a/frontend/src/stores/aliases.js b/frontend/src/stores/aliases.js new file mode 100644 index 0000000..5872af7 --- /dev/null +++ b/frontend/src/stores/aliases.js @@ -0,0 +1,26 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useAliasesStore = defineStore('aliases', () => { + const api = useApi() + const rows = ref([]) + const loading = ref(false) + + async function load() { + loading.value = true + try { rows.value = await api.get('/api/aliases') } + finally { loading.value = false } + } + + async function remove(aliasString, aliasCategory) { + await api.delete( + `/api/aliases/${encodeURIComponent(aliasString)}/${encodeURIComponent(aliasCategory)}` + ) + rows.value = rows.value.filter( + r => !(r.alias_string === aliasString && r.alias_category === aliasCategory) + ) + } + + return { rows, loading, load, remove } +}) diff --git a/frontend/src/stores/allowlist.js b/frontend/src/stores/allowlist.js new file mode 100644 index 0000000..78db284 --- /dev/null +++ b/frontend/src/stores/allowlist.js @@ -0,0 +1,30 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useAllowlistStore = defineStore('allowlist', () => { + const api = useApi() + const rows = ref([]) + const loading = ref(false) + + async function load() { + loading.value = true + try { rows.value = await api.get('/api/allowlist') } + finally { loading.value = false } + } + + async function updateThreshold(tagId, minConfidence) { + await api.patch(`/api/tags/${tagId}/allowlist`, { + body: { min_confidence: minConfidence } + }) + const r = rows.value.find(x => x.tag_id === tagId) + if (r) r.min_confidence = minConfidence + } + + async function remove(tagId) { + await api.delete(`/api/tags/${tagId}/allowlist`) + rows.value = rows.value.filter(x => x.tag_id !== tagId) + } + + return { rows, loading, load, updateThreshold, remove } +}) diff --git a/frontend/src/stores/artist.js b/frontend/src/stores/artist.js new file mode 100644 index 0000000..a8e250e --- /dev/null +++ b/frontend/src/stores/artist.js @@ -0,0 +1,63 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useArtistStore = defineStore('artist', () => { + const api = useApi() + const overview = ref(null) + const images = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const imagesLoading = ref(false) + const error = ref(null) + const notFound = ref(false) + let started = false + + async function load(slug) { + overview.value = null + images.value = [] + nextCursor.value = null + notFound.value = false + started = false + error.value = null + loading.value = true + try { + overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`) + await loadMoreImages(slug) + } catch (e) { + if (e.status === 404) notFound.value = true + else error.value = e.message + } finally { + loading.value = false + } + } + + async function loadMoreImages(slug) { + if (imagesLoading.value) return + if (started && nextCursor.value === null) return + imagesLoading.value = true + try { + const params = { limit: PAGE } + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get( + `/api/artist/${encodeURIComponent(slug)}/images`, { params } + ) + images.value.push(...body.images) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + imagesLoading.value = false + } + } + + const hasMoreImages = computed(() => !started || nextCursor.value !== null) + + return { + overview, images, loading, imagesLoading, error, notFound, + hasMoreImages, load, loadMoreImages + } +}) diff --git a/frontend/src/stores/artistDirectory.js b/frontend/src/stores/artistDirectory.js new file mode 100644 index 0000000..7550c36 --- /dev/null +++ b/frontend/src/stores/artistDirectory.js @@ -0,0 +1,57 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useArtistDirectoryStore = defineStore('artistDirectory', () => { + const api = useApi() + const cards = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const q = ref('') + const platform = ref(null) + let started = false + + async function loadMore() { + if (loading.value) return + if (started && nextCursor.value === null) return + loading.value = true + error.value = null + try { + const params = { limit: PAGE } + if (q.value) params.q = q.value + if (platform.value) params.platform = platform.value + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/artists/directory', { params }) + cards.value.push(...body.cards) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function reset() { + cards.value = [] + nextCursor.value = null + started = false + await loadMore() + } + + function setQuery(text) { q.value = text; reset() } + function setPlatform(p) { platform.value = p; reset() } + + const hasMore = computed(() => !started || nextCursor.value !== null) + const isEmpty = computed( + () => !loading.value && cards.value.length === 0 && error.value === null + ) + + return { + cards, loading, error, q, platform, hasMore, isEmpty, + loadMore, reset, setQuery, setPlatform, + } +}) diff --git a/frontend/src/stores/credentials.js b/frontend/src/stores/credentials.js new file mode 100644 index 0000000..33f9274 --- /dev/null +++ b/frontend/src/stores/credentials.js @@ -0,0 +1,55 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useCredentialsStore = defineStore('credentials', () => { + const api = useApi() + + const byPlatform = ref(new Map()) + const extensionKey = ref(null) + const loading = ref(false) + const error = ref(null) + + async function loadAll() { + loading.value = true + error.value = null + try { + const arr = await api.get('/api/credentials') + const m = new Map() + for (const c of arr) m.set(c.platform, c) + byPlatform.value = m + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + async function upload(platform, credential_type, data) { + const rec = await api.post('/api/credentials', { + body: { platform, credential_type, data }, + }) + byPlatform.value.delete(platform) + return rec + } + + async function remove(platform) { + await api.delete(`/api/credentials/${platform}`) + byPlatform.value.delete(platform) + } + + async function loadKey() { + const body = await api.get('/api/settings/extension_api_key') + extensionKey.value = body.key + } + + async function rotateKey() { + const body = await api.post('/api/settings/extension_api_key/rotate') + extensionKey.value = body.key + } + + return { + byPlatform, extensionKey, loading, error, + loadAll, upload, remove, loadKey, rotateKey, + } +}) diff --git a/frontend/src/stores/downloads.js b/frontend/src/stores/downloads.js new file mode 100644 index 0000000..d95f633 --- /dev/null +++ b/frontend/src/stores/downloads.js @@ -0,0 +1,72 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useDownloadsStore = defineStore('downloads', () => { + const api = useApi() + + const events = ref([]) + const cursor = ref(null) + const hasMore = ref(true) + const filter = ref({ status: null, source_id: null, artist_id: null }) + const selected = ref(null) + const loading = ref(false) + const error = ref(null) + + function _params(extra = {}) { + const out = { limit: 50, ...extra } + if (filter.value.status) out.status = filter.value.status + if (filter.value.source_id != null) out.source_id = filter.value.source_id + if (filter.value.artist_id != null) out.artist_id = filter.value.artist_id + return out + } + + async function loadFirst() { + loading.value = true + error.value = null + try { + const body = await api.get('/api/downloads', { params: _params() }) + events.value = body + cursor.value = body.length ? body[body.length - 1].id : null + hasMore.value = body.length === 50 + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + async function loadMore() { + if (!hasMore.value || cursor.value == null) return + loading.value = true + try { + const body = await api.get('/api/downloads', { params: _params({ before: cursor.value }) }) + events.value.push(...body) + cursor.value = body.length ? body[body.length - 1].id : cursor.value + hasMore.value = body.length === 50 + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + async function loadOne(id) { + selected.value = await api.get(`/api/downloads/${id}`) + return selected.value + } + + async function applyFilter(patch) { + filter.value = { ...filter.value, ...patch } + await loadFirst() + } + + function closeDetail() { + selected.value = null + } + + return { + events, cursor, hasMore, filter, selected, loading, error, + loadFirst, loadMore, loadOne, applyFilter, closeDetail, + } +}) diff --git a/frontend/src/stores/gallery.js b/frontend/src/stores/gallery.js new file mode 100644 index 0000000..0c4ca77 --- /dev/null +++ b/frontend/src/stores/gallery.js @@ -0,0 +1,110 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useGalleryStore = defineStore('gallery', () => { + const api = useApi() + + const images = ref([]) + const dateGroups = ref([]) // [{year, month, image_ids}] + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const filter = ref({ tag_id: null, post_id: null }) + + const timelineBuckets = ref([]) + const timelineLoading = ref(false) + + let inflightId = 0 + + async function loadInitial() { + images.value = [] + dateGroups.value = [] + nextCursor.value = null + await loadMore() + } + + async function loadMore() { + if (loading.value) return + loading.value = true + error.value = null + const myId = ++inflightId + try { + const params = { limit: 50, ...activeFilterParam() } + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/gallery/scroll', { params }) + if (myId !== inflightId) return // stale response + images.value.push(...body.images) + dateGroups.value = mergeGroups(dateGroups.value, body.date_groups) + nextCursor.value = body.next_cursor + } catch (e) { + error.value = e.message + } finally { + if (myId === inflightId) loading.value = false + } + } + + async function loadTimeline() { + timelineLoading.value = true + try { + const params = activeFilterParam() + timelineBuckets.value = await api.get('/api/gallery/timeline', { params }) + } finally { + timelineLoading.value = false + } + } + + async function jumpTo(year, month) { + const params = { year, month, ...activeFilterParam() } + const body = await api.get('/api/gallery/jump', { params }) + if (body.cursor) { + images.value = [] + dateGroups.value = [] + nextCursor.value = body.cursor + await loadMore() + } + } + + function activeFilterParam() { + if (filter.value.tag_id) return { tag_id: filter.value.tag_id } + if (filter.value.post_id) return { post_id: filter.value.post_id } + return {} + } + + function setTagFilter(tagId) { + filter.value.tag_id = tagId + filter.value.post_id = null + loadInitial() + loadTimeline() + } + + function setPostFilter(postId) { + filter.value.post_id = postId + filter.value.tag_id = null + loadInitial() + loadTimeline() + } + + const hasMore = computed(() => nextCursor.value !== null) + const isEmpty = computed(() => !loading.value && images.value.length === 0 && error.value === null) + + return { + images, dateGroups, hasMore, isEmpty, loading, error, + filter, timelineBuckets, timelineLoading, + loadInitial, loadMore, loadTimeline, jumpTo, setTagFilter, setPostFilter + } +}) + +function mergeGroups(existing, incoming) { + // Merge sequential groups with the same (year, month) instead of duplicating. + const merged = [...existing] + for (const g of incoming) { + const tail = merged[merged.length - 1] + if (tail && tail.year === g.year && tail.month === g.month) { + tail.image_ids = [...tail.image_ids, ...g.image_ids] + } else { + merged.push({ ...g, image_ids: [...g.image_ids] }) + } + } + return merged +} diff --git a/frontend/src/stores/gallerySelection.js b/frontend/src/stores/gallerySelection.js new file mode 100644 index 0000000..daf31a0 --- /dev/null +++ b/frontend/src/stores/gallerySelection.js @@ -0,0 +1,76 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useGallerySelectionStore = defineStore('gallerySelection', () => { + const api = useApi() + + const isSelectMode = ref(false) + const order = ref([]) // image ids, selection order = source of truth + const commonTags = ref([]) + const consensus = ref({}) // { category: [ {canonical_tag_id,...} ] } + + const count = computed(() => order.value.length) + function isSelected(id) { return order.value.includes(id) } + + function toggle(id) { + const i = order.value.indexOf(id) + if (i === -1) order.value.push(id) + else order.value.splice(i, 1) + } + function clear() { + order.value = [] + commonTags.value = [] + consensus.value = {} + } + function enterSelectMode() { isSelectMode.value = true } + function exitSelectMode() { isSelectMode.value = false; clear() } + + async function loadCommonTags() { + if (order.value.length === 0) { commonTags.value = []; return } + const body = await api.post('/api/images/common-tags', { + body: { image_ids: order.value } + }) + commonTags.value = body.tags + } + + async function loadConsensus() { + if (order.value.length === 0) { consensus.value = {}; return } + const body = await api.post('/api/suggestions/bulk', { + body: { image_ids: order.value } + }) + consensus.value = body.suggestions + } + + async function refresh() { + await Promise.all([loadCommonTags(), loadConsensus()]) + } + + async function bulkAdd(tagId, source = 'manual') { + const body = await api.post('/api/images/bulk/tags', { + body: { image_ids: order.value, tag_id: tagId, source } + }) + await refresh() + globalThis.window?.__fcToast?.({ + text: `Added to ${body.added_count} image(s)`, type: 'success' + }) + return body + } + + async function bulkRemove(tagId) { + const body = await api.post('/api/images/bulk/tags/remove', { + body: { image_ids: order.value, tag_id: tagId } + }) + await refresh() + globalThis.window?.__fcToast?.({ + text: `Removed from ${body.removed_count} image(s)`, type: 'success' + }) + return body + } + + return { + isSelectMode, order, commonTags, consensus, count, + isSelected, toggle, clear, enterSelectMode, exitSelectMode, + loadCommonTags, loadConsensus, refresh, bulkAdd, bulkRemove + } +}) diff --git a/frontend/src/stores/import.js b/frontend/src/stores/import.js new file mode 100644 index 0000000..861db25 --- /dev/null +++ b/frontend/src/stores/import.js @@ -0,0 +1,106 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useImportStore = defineStore('import', () => { + const api = useApi() + + const settings = ref(null) + const settingsLoading = ref(false) + const settingsError = ref(null) + + const activeBatch = ref(null) + const statusLoading = ref(false) + + const tasks = ref([]) + const tasksNextCursor = ref(null) + const tasksLoading = ref(false) + const tasksFilter = ref({ status: null, limit: 50 }) + + const triggerError = ref(null) + + async function loadSettings() { + settingsLoading.value = true + settingsError.value = null + try { + settings.value = await api.get('/api/settings/import') + } catch (e) { + settingsError.value = e.message + } finally { + settingsLoading.value = false + } + } + + async function patchSettings(patch) { + settingsError.value = null + try { + settings.value = await api.patch('/api/settings/import', { body: patch }) + } catch (e) { + settingsError.value = e.message + window.__fcToast?.({ text: `Settings save failed: ${e.message}`, type: 'error' }) + throw e + } + } + + async function refreshStatus() { + statusLoading.value = true + try { + const body = await api.get('/api/import/status') + activeBatch.value = body.active_batch + } finally { + statusLoading.value = false + } + } + + async function triggerScan() { + triggerError.value = null + try { + await api.post('/api/import/trigger', { body: { mode: 'quick' } }) + await refreshStatus() + } catch (e) { + triggerError.value = e.message + window.__fcToast?.({ text: `Scan failed: ${e.message}`, type: 'error' }) + throw e + } + } + + async function loadTasks(reset = true) { + tasksLoading.value = true + try { + const params = { limit: tasksFilter.value.limit } + if (tasksFilter.value.status) params.status = tasksFilter.value.status + if (!reset && tasksNextCursor.value) params.cursor = tasksNextCursor.value + const body = await api.get('/api/import/tasks', { params }) + tasks.value = reset ? body.tasks : [...tasks.value, ...body.tasks] + tasksNextCursor.value = body.next_cursor + } finally { + tasksLoading.value = false + } + } + + function setStatusFilter(status) { + tasksFilter.value.status = status + } + + async function retryFailed() { + await api.post('/api/import/retry-failed') + await loadTasks(true) + } + + async function clearCompleted(ageDays = 0) { + await api.post('/api/import/clear-completed', { body: { age_days: ageDays } }) + await loadTasks(true) + } + + const hasMore = computed(() => tasksNextCursor.value !== null) + + return { + settings, settingsLoading, settingsError, + activeBatch, statusLoading, + tasks, tasksLoading, tasksFilter, hasMore, + triggerError, + loadSettings, patchSettings, + refreshStatus, triggerScan, + loadTasks, setStatusFilter, retryFailed, clearCompleted + } +}) diff --git a/frontend/src/stores/migration.js b/frontend/src/stores/migration.js new file mode 100644 index 0000000..5bdebd1 --- /dev/null +++ b/frontend/src/stores/migration.js @@ -0,0 +1,83 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const POLL_MS = 2000 + +export const useMigrationStore = defineStore('migration', () => { + const api = useApi() + + const activeRun = ref(null) + const recentRuns = ref([]) + const error = ref(null) + const loading = ref(false) + + let pollHandle = null + + async function trigger(kind, params, file = null) { + loading.value = true + error.value = null + try { + let body + if (file) { + // Multipart upload for ingest kinds. + const form = new FormData() + form.append('export_file', file) + if (params && params.dry_run) form.append('dry_run', 'true') + if (params && params.force) form.append('force', 'true') + body = await api.post(`/api/migrate/${kind}`, { body: form }) + } else { + body = await api.post(`/api/migrate/${kind}`, { body: params || {} }) + } + await loadRun(body.run_id) + pollActive(body.run_id) + return body + } catch (e) { + error.value = e + throw e + } finally { + loading.value = false + } + } + + async function loadRun(runId) { + activeRun.value = await api.get(`/api/migrate/runs/${runId}`) + } + + async function loadRecent() { + recentRuns.value = await api.get('/api/migrate/runs', { params: { limit: 10 } }) + } + + function pollActive(runId) { + stopPolling() + pollHandle = setInterval(async () => { + try { + const run = await api.get(`/api/migrate/runs/${runId}`) + activeRun.value = run + if (run.status === 'ok' || run.status === 'error') { + stopPolling() + await loadRecent() + } + } catch (e) { + error.value = e + stopPolling() + } + }, POLL_MS) + } + + function stopPolling() { + if (pollHandle) { + clearInterval(pollHandle) + pollHandle = null + } + } + + const isRunning = computed( + () => activeRun.value && activeRun.value.status === 'running', + ) + + return { + activeRun, recentRuns, error, loading, isRunning, + trigger, loadRun, loadRecent, pollActive, stopPolling, + } +}) diff --git a/frontend/src/stores/ml.js b/frontend/src/stores/ml.js new file mode 100644 index 0000000..1323268 --- /dev/null +++ b/frontend/src/stores/ml.js @@ -0,0 +1,39 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useMLStore = defineStore('ml', () => { + const api = useApi() + const settings = ref(null) + const loading = ref(false) + const error = ref(null) + + async function loadSettings() { + loading.value = true + error.value = null + try { + settings.value = await api.get('/api/ml/settings') + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function patchSettings(patch) { + settings.value = await api.patch('/api/ml/settings', { body: patch }) + } + + async function triggerBackfill() { + await api.post('/api/ml/backfill') + } + + async function triggerRecomputeCentroids() { + await api.post('/api/ml/recompute-centroids') + } + + return { + settings, loading, error, + loadSettings, patchSettings, triggerBackfill, triggerRecomputeCentroids + } +}) diff --git a/frontend/src/stores/modal.js b/frontend/src/stores/modal.js new file mode 100644 index 0000000..1998c97 --- /dev/null +++ b/frontend/src/stores/modal.js @@ -0,0 +1,91 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useModalStore = defineStore('modal', () => { + const api = useApi() + + const currentImageId = ref(null) + const current = ref(null) // full image detail from API + const loading = ref(false) + const error = ref(null) + + async function open(id) { + currentImageId.value = id + loading.value = true + error.value = null + try { + current.value = await api.get(`/api/gallery/image/${id}`) + } catch (e) { + error.value = e.message + current.value = null + } finally { + loading.value = false + } + } + + async function close() { + currentImageId.value = null + current.value = null + error.value = null + } + + async function goPrev() { + if (current.value && current.value.neighbors.prev_id) { + await open(current.value.neighbors.prev_id) + } + } + async function goNext() { + if (current.value && current.value.neighbors.next_id) { + await open(current.value.neighbors.next_id) + } + } + + async function reloadTags() { + if (!currentImageId.value) return + const tags = await api.get(`/api/images/${currentImageId.value}/tags`) + if (current.value) current.value.tags = tags + } + + async function removeTag(tagId) { + if (!currentImageId.value) return + // Optimistic UI: remove locally first, restore on error. + const prev = current.value.tags + current.value.tags = current.value.tags.filter(t => t.id !== tagId) + try { + // FC-2b: removal also records a per-image rejection (suggestions/dismiss + // is the rejection-recording endpoint) so the allowlist maintenance + // task won't re-apply this tag to this image. + await api.delete(`/api/images/${currentImageId.value}/tags/${tagId}`) + await api.post(`/api/images/${currentImageId.value}/suggestions/dismiss`, { + body: { tag_id: tagId } + }) + } catch (e) { + current.value.tags = prev + window.__fcToast?.({ text: `Failed to remove tag: ${e.message}`, type: 'error' }) + throw e + } + } + + async function addExistingTag(tagId) { + if (!currentImageId.value) return + await api.post(`/api/images/${currentImageId.value}/tags`, { + body: { tag_id: tagId, source: 'manual' } + }) + await reloadTags() + } + + async function createAndAdd({ name, kind, fandom_id = null }) { + const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } }) + await addExistingTag(tag.id) + } + + const isOpen = computed(() => currentImageId.value !== null) + const canPrev = computed(() => current.value?.neighbors?.prev_id != null) + const canNext = computed(() => current.value?.neighbors?.next_id != null) + + return { + currentImageId, current, loading, error, isOpen, canPrev, canNext, + open, close, goPrev, goNext, reloadTags, removeTag, addExistingTag, createAndAdd + } +}) diff --git a/frontend/src/stores/platforms.js b/frontend/src/stores/platforms.js new file mode 100644 index 0000000..e996c4b --- /dev/null +++ b/frontend/src/stores/platforms.js @@ -0,0 +1,25 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const usePlatformsStore = defineStore('platforms', () => { + const api = useApi() + + const byKey = ref(new Map()) + const loaded = ref(false) + + const list = computed(() => Array.from(byKey.value.values())) + + async function loadAll() { + if (loaded.value) return + const body = await api.get('/api/platforms') + const m = new Map() + for (const [k, v] of Object.entries(body.platforms || {})) { + m.set(k, v) + } + byKey.value = m + loaded.value = true + } + + return { byKey, loaded, list, loadAll } +}) diff --git a/frontend/src/stores/posts.js b/frontend/src/stores/posts.js new file mode 100644 index 0000000..d5b18c7 --- /dev/null +++ b/frontend/src/stores/posts.js @@ -0,0 +1,64 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const usePostsStore = defineStore('posts', () => { + const api = useApi() + + const items = ref([]) + const cursor = ref(null) + const loading = ref(false) + const done = ref(false) + const error = ref(null) + const filters = ref({ artist_id: null, platform: null }) + + function _qs() { + const q = {} + if (cursor.value) q.cursor = cursor.value + if (filters.value.artist_id != null) q.artist_id = filters.value.artist_id + if (filters.value.platform) q.platform = filters.value.platform + return q + } + + function _reset() { + items.value = [] + cursor.value = null + done.value = false + error.value = null + } + + async function loadInitial(newFilters) { + filters.value = { + artist_id: newFilters?.artist_id ?? null, + platform: newFilters?.platform ?? null, + } + _reset() + await loadMore() + } + + async function loadMore() { + if (loading.value || done.value) return + loading.value = true + error.value = null + try { + const body = await api.get('/api/posts', { params: _qs() }) + items.value.push(...body.items) + cursor.value = body.next_cursor + if (body.next_cursor == null) done.value = true + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + async function getPostFull(id) { + // Used by PostCard's "Show more" expand to fetch the full description. + return await api.get(`/api/posts/${id}`) + } + + return { + items, cursor, loading, done, error, filters, + loadInitial, loadMore, getPostFull, + } +}) diff --git a/frontend/src/stores/provenance.js b/frontend/src/stores/provenance.js new file mode 100644 index 0000000..97b28f8 --- /dev/null +++ b/frontend/src/stores/provenance.js @@ -0,0 +1,47 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +// Provenance is its own system — this store is deliberately independent +// of the modal/tags stores (see project_provenance_separation). +export const useProvenanceStore = defineStore('provenance', () => { + const api = useApi() + + const imageCache = ref({}) // id -> { loading, error, entries|null } + const postCache = ref({}) // id -> { loading, error, data|null } + + async function loadForImage(id) { + const cur = imageCache.value[id] + if (cur && (cur.loading || cur.entries !== null)) return + imageCache.value[id] = { loading: true, error: null, entries: null } + try { + const body = await api.get(`/api/provenance/image/${id}`) + imageCache.value[id] = { + loading: false, error: null, + entries: body.provenance, + attachments: body.attachments || [], + } + } catch (e) { + imageCache.value[id] = + { loading: false, error: e.message, entries: null } + } + } + + async function loadForPost(id) { + const cur = postCache.value[id] + if (cur && (cur.loading || cur.data !== null)) return + postCache.value[id] = { loading: true, error: null, data: null } + try { + const body = await api.get(`/api/provenance/post/${id}`) + postCache.value[id] = { loading: false, error: null, data: body } + } catch (e) { + postCache.value[id] = { loading: false, error: e.message, data: null } + } + } + + function imageProv(id) { return imageCache.value[id] || null } + function postInfo(id) { return postCache.value[id] || null } + + return { imageCache, postCache, loadForImage, loadForPost, + imageProv, postInfo } +}) diff --git a/frontend/src/stores/seriesManage.js b/frontend/src/stores/seriesManage.js new file mode 100644 index 0000000..17f84ee --- /dev/null +++ b/frontend/src/stores/seriesManage.js @@ -0,0 +1,91 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +// Pure, unit-tested: move arr[from] to index `to`, returning a new array. +export function moveItem(arr, from, to) { + const next = arr.slice() + const [x] = next.splice(from, 1) + next.splice(to, 0, x) + return next +} + +export const useSeriesManageStore = defineStore('seriesManage', () => { + const api = useApi() + + const tagId = ref(null) + const series = ref(null) + const pages = ref([]) // [{image_id, page_number, thumbnail_url}] + const picker = ref([]) // gallery scroll results + const pickerCursor = ref(null) + const pickerSelection = ref([]) // image ids + const loading = ref(false) + + async function load(id) { + tagId.value = id + loading.value = true + try { + const body = await api.get(`/api/series/${id}/pages`) + series.value = body.series + pages.value = body.pages + } finally { + loading.value = false + } + } + + async function refresh() { + if (tagId.value != null) await load(tagId.value) + } + + async function loadPicker(reset = false) { + if (reset) { picker.value = []; pickerCursor.value = null } + const params = { limit: 50 } + if (pickerCursor.value) params.cursor = pickerCursor.value + const body = await api.get('/api/gallery/scroll', { params }) + picker.value.push(...body.images) + pickerCursor.value = body.next_cursor + } + + function togglePick(imageId) { + const i = pickerSelection.value.indexOf(imageId) + if (i === -1) pickerSelection.value.push(imageId) + else pickerSelection.value.splice(i, 1) + } + + async function addSelected() { + if (pickerSelection.value.length === 0) return + await api.post(`/api/series/${tagId.value}/pages`, { + body: { image_ids: pickerSelection.value } + }) + pickerSelection.value = [] + await refresh() + globalThis.window?.__fcToast?.({ text: 'Added to series', type: 'success' }) + } + + async function remove(imageId) { + await api.post(`/api/series/${tagId.value}/pages/remove`, { + body: { image_ids: [imageId] } + }) + await refresh() + } + + async function reorder(orderedImageIds) { + await api.post(`/api/series/${tagId.value}/reorder`, { + body: { image_ids: orderedImageIds } + }) + await refresh() + } + + async function setCover(imageId) { + await api.post(`/api/series/${tagId.value}/cover`, { + body: { image_id: imageId } + }) + await refresh() + } + + return { + tagId, series, pages, picker, pickerCursor, pickerSelection, loading, + load, refresh, loadPicker, togglePick, addSelected, remove, + reorder, setCover + } +}) diff --git a/frontend/src/stores/seriesReader.js b/frontend/src/stores/seriesReader.js new file mode 100644 index 0000000..f79b8d6 --- /dev/null +++ b/frontend/src/stores/seriesReader.js @@ -0,0 +1,56 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +// IR reader.js "viewport-center-in-third" rule, pure for unit tests. +// metrics: [{page_number, top, height}] in page order. +export function currentPageFromScroll(metrics, scrollTop, viewportH) { + if (!metrics.length) return 1 + const target = scrollTop + viewportH / 3 + let cur = metrics[0].page_number + for (const m of metrics) { + if (target >= m.top) cur = m.page_number + } + return cur +} + +export function progressPct(scrollTop, scrollHeight, clientHeight) { + const denom = scrollHeight - clientHeight + if (denom <= 0) return 0 + const pct = (scrollTop / denom) * 100 + return Math.min(100, Math.max(0, pct)) +} + +export function clampPage(n, total) { + if (!total || total < 1) return 1 + if (!Number.isFinite(n) || n < 1) return 1 + if (n > total) return total + return Math.floor(n) +} + +export const useSeriesReaderStore = defineStore('seriesReader', () => { + const api = useApi() + + const series = ref(null) + const pages = ref([]) + const loading = ref(false) + const error = ref(null) + + async function load(tagId) { + loading.value = true + error.value = null + try { + const body = await api.get(`/api/series/${tagId}/pages`) + series.value = body.series + pages.value = body.pages + } catch (e) { + error.value = e.message + series.value = null + pages.value = [] + } finally { + loading.value = false + } + } + + return { series, pages, loading, error, load } +}) diff --git a/frontend/src/stores/showcase.js b/frontend/src/stores/showcase.js new file mode 100644 index 0000000..7e72488 --- /dev/null +++ b/frontend/src/stores/showcase.js @@ -0,0 +1,45 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useShowcaseStore = defineStore('showcase', () => { + const api = useApi() + const images = ref([]) + const loading = ref(false) + const error = ref(null) + const exhausted = ref(false) + const seen = new Set() + + async function fetchPage() { + if (loading.value) return + loading.value = true + error.value = null + try { + const body = await api.get('/api/showcase', { params: { limit: PAGE } }) + const fresh = body.images.filter(i => !seen.has(i.id)) + if (fresh.length === 0) { exhausted.value = true; return } + for (const i of fresh) seen.add(i.id) + images.value.push(...fresh) + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function shuffle() { + images.value = [] + seen.clear() + exhausted.value = false + await fetchPage() + } + + const hasMore = computed(() => !exhausted.value) + const isEmpty = computed( + () => !loading.value && images.value.length === 0 && error.value === null + ) + + return { images, loading, error, hasMore, isEmpty, fetchPage, shuffle } +}) diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js new file mode 100644 index 0000000..1032edb --- /dev/null +++ b/frontend/src/stores/sources.js @@ -0,0 +1,116 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useSourcesStore = defineStore('sources', () => { + const api = useApi() + + // keyed cache: null => all sources, number => artist_id filtered + const byArtist = ref(new Map()) + const loading = ref(false) + const error = ref(null) + + const allSources = computed(() => byArtist.value.get(null) ?? []) + + async function loadAll() { + loading.value = true + error.value = null + try { + const body = await api.get('/api/sources') + byArtist.value.set(null, body) + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + async function loadForArtist(artistId) { + loading.value = true + error.value = null + try { + const body = await api.get('/api/sources', { params: { artist_id: artistId } }) + byArtist.value.set(artistId, body) + } catch (e) { + error.value = e + } finally { + loading.value = false + } + } + + function _invalidate(artistId) { + byArtist.value.delete(null) + if (artistId != null) byArtist.value.delete(artistId) + } + + async function create(payload) { + const body = await api.post('/api/sources', { body: payload }) + _invalidate(payload.artist_id) + return body + } + + async function update(id, patch, artistIdHint = null) { + const body = await api.patch(`/api/sources/${id}`, { body: patch }) + _invalidate(artistIdHint ?? body.artist_id) + return body + } + + async function remove(id, artistIdHint = null) { + await api.delete(`/api/sources/${id}`) + _invalidate(artistIdHint) + } + + async function findOrCreateArtist(name) { + const body = await api.post('/api/artists', { body: { name } }) + return { artist: { id: body.id, name: body.name, slug: body.slug }, created: body.created } + } + + async function autocompleteArtist(query, limit = 20) { + if (!query || !query.trim()) return [] + return await api.get('/api/artists/autocomplete', { params: { q: query, limit } }) + } + + // FC-3c: trigger a download for one source. Returns {download_event_id,status}. + const checkingIds = ref(new Set()) + + async function checkNow(id) { + checkingIds.value = new Set(checkingIds.value).add(id) + try { + return await api.post(`/api/sources/${id}/check`) + } finally { + const next = new Set(checkingIds.value) + next.delete(id) + checkingIds.value = next + } + } + + function sourcesByArtistGrouped() { + // returns [{artist: {id,name,slug}, sources: [...]}, ...] + const arr = byArtist.value.get(null) ?? [] + const groups = new Map() + for (const s of arr) { + const key = s.artist_id + if (!groups.has(key)) { + groups.set(key, { + artist: { id: s.artist_id, name: s.artist_name, slug: s.artist_slug }, + sources: [], + }) + } + groups.get(key).sources.push(s) + } + return Array.from(groups.values()).sort( + (a, b) => a.artist.name.localeCompare(b.artist.name) + ) + } + + return { + byArtist, loading, error, + allSources, + checkingIds, + loadAll, loadForArtist, + create, update, remove, + checkNow, + findOrCreateArtist, autocompleteArtist, + sourcesByArtistGrouped, + } +}) diff --git a/frontend/src/stores/suggestions.js b/frontend/src/stores/suggestions.js new file mode 100644 index 0000000..d9914e4 --- /dev/null +++ b/frontend/src/stores/suggestions.js @@ -0,0 +1,94 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +// Category display order: people/sources first, general last. +export const CATEGORY_ORDER = ['artist', 'character', 'copyright', 'general'] +export const CATEGORY_LABELS = { + artist: 'Artist', + character: 'Character', + copyright: 'Copyright', + general: 'General' +} + +export const useSuggestionsStore = defineStore('suggestions', () => { + const api = useApi() + const byCategory = ref({}) // { category: [suggestion, ...] } + const loading = ref(false) + const error = ref(null) + let currentImageId = null + + async function load(imageId) { + currentImageId = imageId + loading.value = true + error.value = null + try { + const body = await api.get(`/api/images/${imageId}/suggestions`) + byCategory.value = body.by_category || {} + } catch (e) { + error.value = e.message + byCategory.value = {} + } finally { + loading.value = false + } + } + + function _drop(category, predicate) { + const list = byCategory.value[category] + if (!list) return + byCategory.value[category] = list.filter(s => !predicate(s)) + } + + async function accept(suggestion) { + // Raw tags (creates_new_tag) have no canonical_tag_id; the backend's + // accept endpoint needs a tag_id, so for raw tags we create the tag + // first via the existing /api/tags endpoint, then accept by id. + let tagId = suggestion.canonical_tag_id + if (tagId == null) { + const created = await api.post('/api/tags', { + body: { name: suggestion.display_name, kind: suggestion.category } + }) + tagId = created.id + } + await api.post(`/api/images/${currentImageId}/suggestions/accept`, { + body: { tag_id: tagId } + }) + _drop(suggestion.category, s => s === suggestion) + window.__fcToast?.({ + text: `Tagged: ${suggestion.display_name}`, + type: 'success' + }) + } + + async function aliasAccept(suggestion, canonicalTagId) { + await api.post(`/api/images/${currentImageId}/suggestions/alias`, { + body: { + alias_string: suggestion.display_name, + alias_category: suggestion.category, + canonical_tag_id: canonicalTagId + } + }) + _drop(suggestion.category, s => s === suggestion) + window.__fcToast?.({ + text: `Aliased & tagged: ${suggestion.display_name}`, + type: 'success' + }) + } + + async function dismiss(suggestion) { + // Dismiss needs a tag_id; raw tags have none, so dismissing a raw + // suggestion just hides it client-side (nothing to persist a rejection + // against until the tag exists). + if (suggestion.canonical_tag_id != null) { + await api.post(`/api/images/${currentImageId}/suggestions/dismiss`, { + body: { tag_id: suggestion.canonical_tag_id } + }) + } + _drop(suggestion.category, s => s === suggestion) + } + + return { + byCategory, loading, error, + load, accept, aliasAccept, dismiss + } +}) diff --git a/frontend/src/stores/system.js b/frontend/src/stores/system.js new file mode 100644 index 0000000..b3f772f --- /dev/null +++ b/frontend/src/stores/system.js @@ -0,0 +1,30 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +export const useSystemStore = defineStore('system', () => { + const api = useApi() + const healthy = ref(null) // null=unknown, true=ok, false=down + const stats = ref(null) + const statsLoading = ref(false) + + async function refreshHealth() { + try { + const body = await api.get('/api/health') + healthy.value = body.status === 'ok' + } catch { + healthy.value = false + } + } + + async function refreshStats() { + statsLoading.value = true + try { + stats.value = await api.get('/api/system/stats') + } finally { + statsLoading.value = false + } + } + + return { healthy, stats, statsLoading, refreshHealth, refreshStats } +}) diff --git a/frontend/src/stores/tagDirectory.js b/frontend/src/stores/tagDirectory.js new file mode 100644 index 0000000..4528a17 --- /dev/null +++ b/frontend/src/stores/tagDirectory.js @@ -0,0 +1,89 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { useApi } from '../composables/useApi.js' + +const PAGE = 60 + +export const useTagDirectoryStore = defineStore('tagDirectory', () => { + const api = useApi() + const cards = ref([]) + const nextCursor = ref(null) + const loading = ref(false) + const error = ref(null) + const kind = ref(null) + const q = ref('') + let started = false + + async function loadMore() { + if (loading.value) return + if (started && nextCursor.value === null) return + loading.value = true + error.value = null + try { + const params = { limit: PAGE } + if (kind.value) params.kind = kind.value + if (q.value) params.q = q.value + if (nextCursor.value) params.cursor = nextCursor.value + const body = await api.get('/api/tags/directory', { params }) + cards.value.push(...body.cards) + nextCursor.value = body.next_cursor + started = true + } catch (e) { + error.value = e.message + } finally { + loading.value = false + } + } + + async function reset() { + cards.value = [] + nextCursor.value = null + started = false + await loadMore() + } + + function setKind(k) { kind.value = k; reset() } + function setQuery(text) { q.value = text; reset() } + + async function rename(id, name) { + try { + const body = await api.patch(`/api/tags/${id}`, { body: { name } }) + const card = cards.value.find((c) => c.id === id) + if (card) card.name = body.name + return { tag: body } + } catch (e) { + if (e.status === 409 && e.body && e.body.target) { + const card = cards.value.find((c) => c.id === id) + return { + collision: { + sourceId: id, + sourceName: card ? card.name : '', + target: e.body.target, + sourceImageCount: e.body.source_image_count, + willAlias: e.body.will_alias + } + } + } + throw e + } + } + + async function merge(sourceId, targetId) { + const result = await api.post(`/api/tags/${sourceId}/merge`, { + body: { target_id: targetId } + }) + const idx = cards.value.findIndex((c) => c.id === sourceId) + if (idx !== -1) cards.value.splice(idx, 1) + return result + } + + const hasMore = computed(() => !started || nextCursor.value !== null) + const isEmpty = computed( + () => !loading.value && cards.value.length === 0 && error.value === null + ) + + return { + cards, loading, error, kind, q, hasMore, isEmpty, + loadMore, reset, setKind, setQuery, rename, merge + } +}) diff --git a/frontend/src/stores/tags.js b/frontend/src/stores/tags.js new file mode 100644 index 0000000..fbb871a --- /dev/null +++ b/frontend/src/stores/tags.js @@ -0,0 +1,58 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useApi } from '../composables/useApi.js' + +const KIND_OPTIONS = [ + { value: 'general', label: 'General', icon: 'mdi-tag' }, + { value: 'artist', label: 'Artist', icon: 'mdi-palette' }, + { value: 'character', label: 'Character', icon: 'mdi-account-circle' }, + { value: 'fandom', label: 'Fandom', icon: 'mdi-book-open-page-variant' }, + { value: 'series', label: 'Series', icon: 'mdi-bookshelf' }, + { value: 'meta', label: 'Meta', icon: 'mdi-cog-outline' }, + { value: 'rating', label: 'Rating', icon: 'mdi-shield-check-outline' } +] + +const KIND_COLOR = { + artist: 'accent', + character: 'info', + fandom: 'secondary', + series: 'warning', + general: 'on-surface', + meta: 'on-surface', + rating: 'on-surface', + archive: 'on-surface', + post: 'on-surface' +} + +export const useTagStore = defineStore('tags', () => { + const api = useApi() + const fandomCache = ref([]) + + async function autocomplete(q, kind = null, limit = 20) { + if (!q || !q.trim()) return [] + const params = { q, limit } + if (kind) params.kind = kind + return await api.get('/api/tags/autocomplete', { params }) + } + + async function loadFandoms() { + fandomCache.value = await api.get('/api/tags/autocomplete', { + params: { q: ' ', kind: 'fandom', limit: 200 } + }) + return fandomCache.value + } + + async function createFandom(name) { + const fandom = await api.post('/api/tags', { body: { name, kind: 'fandom' } }) + fandomCache.value = [...fandomCache.value, { + id: fandom.id, name: fandom.name, kind: 'fandom', + fandom_id: null, fandom_name: null, image_count: 0 + }] + return fandom + } + + function kindOptions() { return KIND_OPTIONS } + function colorFor(kind) { return KIND_COLOR[kind] || 'on-surface' } + + return { fandomCache, autocomplete, loadFandoms, createFandom, kindOptions, colorFor } +}) diff --git a/frontend/src/theme/fabled-tokens.js b/frontend/src/theme/fabled-tokens.js new file mode 100644 index 0000000..4102088 --- /dev/null +++ b/frontend/src/theme/fabled-tokens.js @@ -0,0 +1,41 @@ +// FabledDesignSystem tokens — mirrored from FabledDesignSystem-new/design-system.md. +// Do not hand-edit hexes; update the source-of-truth doc and re-mirror here. + +export const surfaces = { + obsidian: '#14171A', // page background + iron: '#1E2228', // cards + slate: '#2C313A', // hovered surfaces + pewter: '#3F4651' // borders, ghost outlines +} + +export const text = { + parchment: '#E8E4D8', // primary on dark + vellum: '#C2BFB4', // secondary + ash: '#9C9A92' // tertiary +} + +export const action = { + moss: '#4A5D3F', // primary action + bronze: '#8B7355', // secondary action + pewter: '#3F4651', // tertiary/ghost + destructive: '#6B2118' // delete/irreversible +} + +export const semantic = { + success: '#4A5D3F', + warning: '#8B6F1E', + error: '#C04A1F', + info: '#3D5A6E' +} + +// Per-app signature accent. Curator's accent was added to FabledDesignSystem +// during FC-1 (2026-05-14). Aged amber — manuscript ink, curated archive. +export const accent = { + curator: '#A87338' +} + +export const typography = { + display: 'Fraunces, Georgia, serif', + body: 'Inter, system-ui, sans-serif', + mono: '"JetBrains Mono", monospace' +} diff --git a/frontend/src/theme/vuetify-theme.js b/frontend/src/theme/vuetify-theme.js new file mode 100644 index 0000000..4cc7b9b --- /dev/null +++ b/frontend/src/theme/vuetify-theme.js @@ -0,0 +1,34 @@ +import { surfaces, text, action, semantic, accent } from './fabled-tokens.js' + +export const fabledCuratorTheme = { + dark: true, + colors: { + background: surfaces.obsidian, + surface: surfaces.iron, + 'surface-bright': surfaces.slate, + 'surface-light': surfaces.pewter, + + primary: action.moss, + secondary: action.bronze, + tertiary: action.pewter, + + 'on-background': text.parchment, + 'on-surface': text.parchment, + 'on-primary': text.parchment, + 'on-secondary': text.parchment, + + // Secondary/muted body text (empty states, captions, Vuetify hints). + // Maps to the design system's "secondary on dark" so it stays readable + // on obsidian instead of an undefined-token low-contrast fallback. + 'on-surface-variant': text.vellum, + + success: semantic.success, + warning: semantic.warning, + error: semantic.error, + info: semantic.info, + + // Curator signature accent — used for nav-active state, wordmark, selection. + // Not on action buttons (see design-system.md "Accent usage rules"). + accent: accent.curator + } +} diff --git a/frontend/src/utils/date.js b/frontend/src/utils/date.js new file mode 100644 index 0000000..f0ade54 --- /dev/null +++ b/frontend/src/utils/date.js @@ -0,0 +1,15 @@ +// Locale-independent post-date formatting. Uses a fixed month table and +// UTC getters so the rendered string never varies by browser/CI locale +// or timezone (toLocaleDateString would make tests flaky). + +const MONTHS = [ + 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' +] + +export function formatPostDate(iso) { + if (!iso) return null + const d = new Date(iso) + if (isNaN(d.getTime())) return null + return `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}, ${d.getUTCFullYear()}` +} diff --git a/frontend/src/views/ArtistView.vue b/frontend/src/views/ArtistView.vue new file mode 100644 index 0000000..7a363f2 --- /dev/null +++ b/frontend/src/views/ArtistView.vue @@ -0,0 +1,178 @@ + + + + + diff --git a/frontend/src/views/ArtistsView.vue b/frontend/src/views/ArtistsView.vue new file mode 100644 index 0000000..dc32f82 --- /dev/null +++ b/frontend/src/views/ArtistsView.vue @@ -0,0 +1,93 @@ + + + + + diff --git a/frontend/src/views/CredentialsView.vue b/frontend/src/views/CredentialsView.vue new file mode 100644 index 0000000..5501d1d --- /dev/null +++ b/frontend/src/views/CredentialsView.vue @@ -0,0 +1,79 @@ + + + diff --git a/frontend/src/views/DownloadsView.vue b/frontend/src/views/DownloadsView.vue new file mode 100644 index 0000000..0979c98 --- /dev/null +++ b/frontend/src/views/DownloadsView.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/frontend/src/views/GalleryView.vue b/frontend/src/views/GalleryView.vue new file mode 100644 index 0000000..2e2933a --- /dev/null +++ b/frontend/src/views/GalleryView.vue @@ -0,0 +1,101 @@ + + + + + diff --git a/frontend/src/views/PlaceholderView.vue b/frontend/src/views/PlaceholderView.vue new file mode 100644 index 0000000..5c53509 --- /dev/null +++ b/frontend/src/views/PlaceholderView.vue @@ -0,0 +1,34 @@ + + + + + diff --git a/frontend/src/views/PostsView.vue b/frontend/src/views/PostsView.vue new file mode 100644 index 0000000..56a7cef --- /dev/null +++ b/frontend/src/views/PostsView.vue @@ -0,0 +1,106 @@ + + + + + diff --git a/frontend/src/views/SeriesManageView.vue b/frontend/src/views/SeriesManageView.vue new file mode 100644 index 0000000..510b5aa --- /dev/null +++ b/frontend/src/views/SeriesManageView.vue @@ -0,0 +1,143 @@ + + + + + diff --git a/frontend/src/views/SeriesReaderView.vue b/frontend/src/views/SeriesReaderView.vue new file mode 100644 index 0000000..48c43cf --- /dev/null +++ b/frontend/src/views/SeriesReaderView.vue @@ -0,0 +1,307 @@ + + + + + diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue new file mode 100644 index 0000000..a9ccab1 --- /dev/null +++ b/frontend/src/views/SettingsView.vue @@ -0,0 +1,85 @@ + + + diff --git a/frontend/src/views/ShowcaseView.vue b/frontend/src/views/ShowcaseView.vue new file mode 100644 index 0000000..7a4fa64 --- /dev/null +++ b/frontend/src/views/ShowcaseView.vue @@ -0,0 +1,41 @@ + + + diff --git a/frontend/src/views/SubscriptionsView.vue b/frontend/src/views/SubscriptionsView.vue new file mode 100644 index 0000000..c5f2a18 --- /dev/null +++ b/frontend/src/views/SubscriptionsView.vue @@ -0,0 +1,172 @@ + + + + + diff --git a/frontend/src/views/TagsView.vue b/frontend/src/views/TagsView.vue new file mode 100644 index 0000000..212d43a --- /dev/null +++ b/frontend/src/views/TagsView.vue @@ -0,0 +1,124 @@ + + + + + diff --git a/frontend/test/credentials.spec.js b/frontend/test/credentials.spec.js new file mode 100644 index 0000000..8437726 --- /dev/null +++ b/frontend/test/credentials.spec.js @@ -0,0 +1,59 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useCredentialsStore } from '../src/stores/credentials.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)), + } + }) +} + +describe('credentials store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('loadAll populates byPlatform map', async () => { + const s = useCredentialsStore() + stubFetch(() => ({ + status: 200, + body: [{ platform: 'patreon', credential_type: 'cookies', captured_at: '2026-05-20T00:00:00+00:00', expires_at: null, last_verified: null }], + })) + await s.loadAll() + expect(s.byPlatform.get('patreon').credential_type).toBe('cookies') + }) + + it('upload invalidates the cache', async () => { + const s = useCredentialsStore() + s.byPlatform.set('patreon', { platform: 'patreon' }) + stubFetch(() => ({ status: 201, body: { platform: 'patreon', credential_type: 'cookies' } })) + await s.upload('patreon', 'cookies', 'NETSCAPE_CONTENT') + expect(s.byPlatform.has('patreon')).toBe(false) + }) + + it('remove deletes and invalidates the cache', async () => { + const s = useCredentialsStore() + s.byPlatform.set('patreon', { platform: 'patreon' }) + stubFetch(() => ({ status: 204, body: null })) + await s.remove('patreon') + expect(s.byPlatform.has('patreon')).toBe(false) + }) + + it('loadKey + rotateKey', async () => { + const s = useCredentialsStore() + const seen = [] + stubFetch((url) => { + seen.push(url) + if (url.endsWith('/rotate')) return { status: 200, body: { key: 'NEW' } } + return { status: 200, body: { key: 'OLD' } } + }) + await s.loadKey() + expect(s.extensionKey).toBe('OLD') + await s.rotateKey() + expect(s.extensionKey).toBe('NEW') + }) +}) diff --git a/frontend/test/date.spec.js b/frontend/test/date.spec.js new file mode 100644 index 0000000..60415b5 --- /dev/null +++ b/frontend/test/date.spec.js @@ -0,0 +1,15 @@ +import { describe, it, expect } from 'vitest' +import { formatPostDate } from '../src/utils/date.js' + +describe('formatPostDate', () => { + it('formats an ISO datetime as "Mon D, YYYY" in UTC', () => { + expect(formatPostDate('2023-08-01T00:00:00+00:00')).toBe('Aug 1, 2023') + expect(formatPostDate('2023-12-25T23:59:59Z')).toBe('Dec 25, 2023') + }) + + it('returns null for null/empty/invalid', () => { + expect(formatPostDate(null)).toBe(null) + expect(formatPostDate('')).toBe(null) + expect(formatPostDate('not-a-date')).toBe(null) + }) +}) diff --git a/frontend/test/downloads.spec.js b/frontend/test/downloads.spec.js new file mode 100644 index 0000000..a76fa90 --- /dev/null +++ b/frontend/test/downloads.spec.js @@ -0,0 +1,75 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useDownloadsStore } from '../src/stores/downloads.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)), + } + }) +} + +describe('downloads store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('loadFirst populates events and cursor', async () => { + const s = useDownloadsStore() + stubFetch(() => ({ + status: 200, + body: [ + { id: 10, status: 'ok', summary: {} }, + { id: 9, status: 'error', summary: {} }, + ], + })) + await s.loadFirst() + expect(s.events.map(e => e.id)).toEqual([10, 9]) + expect(s.cursor).toBe(9) + expect(s.hasMore).toBe(false) + }) + + it('loadMore appends and updates cursor', async () => { + const s = useDownloadsStore() + s.events = [{ id: 10, summary: {} }, { id: 9, summary: {} }] + s.cursor = 9 + s.hasMore = true + stubFetch(() => ({ + status: 200, + body: [{ id: 8, status: 'ok', summary: {} }], + })) + await s.loadMore() + expect(s.events.map(e => e.id)).toEqual([10, 9, 8]) + expect(s.cursor).toBe(8) + }) + + it('applyFilter merges and reloads', async () => { + const s = useDownloadsStore() + const urls = [] + stubFetch((url) => { + urls.push(url) + return { status: 200, body: [] } + }) + await s.applyFilter({ status: 'error' }) + expect(s.filter.status).toBe('error') + expect(urls[0]).toContain('status=error') + }) + + it('loadOne populates selected', async () => { + const s = useDownloadsStore() + stubFetch(() => ({ status: 200, body: { id: 42, metadata: { run_stats: {} } } })) + const out = await s.loadOne(42) + expect(s.selected.id).toBe(42) + expect(out.id).toBe(42) + }) + + it('closeDetail clears selected', async () => { + const s = useDownloadsStore() + s.selected = { id: 1 } + s.closeDetail() + expect(s.selected).toBe(null) + }) +}) diff --git a/frontend/test/gallery.spec.js b/frontend/test/gallery.spec.js new file mode 100644 index 0000000..4fb866a --- /dev/null +++ b/frontend/test/gallery.spec.js @@ -0,0 +1,52 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useGalleryStore } from '../src/stores/gallery.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, + statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +const EMPTY = { images: [], date_groups: [], next_cursor: null } + +describe('gallery store: tag/post filter exclusivity', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('setPostFilter sets post_id and clears tag_id', () => { + const s = useGalleryStore() + stubFetch(() => ({ status: 200, body: EMPTY })) + s.setTagFilter(3) + expect(s.filter.tag_id).toBe(3) + s.setPostFilter(7) + expect(s.filter.post_id).toBe(7) + expect(s.filter.tag_id).toBe(null) + }) + + it('setTagFilter clears post_id', () => { + const s = useGalleryStore() + stubFetch(() => ({ status: 200, body: EMPTY })) + s.setPostFilter(7) + s.setTagFilter(3) + expect(s.filter.tag_id).toBe(3) + expect(s.filter.post_id).toBe(null) + }) + + it('loadMore sends exactly the active filter param', async () => { + const s = useGalleryStore() + const urls = [] + stubFetch((url) => { urls.push(url); return { status: 200, body: EMPTY } }) + s.setPostFilter(7) + await s.loadMore() + const scrollCall = urls.filter(u => u.includes('/api/gallery/scroll')).pop() + expect(scrollCall).toContain('post_id=7') + expect(scrollCall).not.toContain('tag_id=') + }) +}) diff --git a/frontend/test/gallerySelection.spec.js b/frontend/test/gallerySelection.spec.js new file mode 100644 index 0000000..7641692 --- /dev/null +++ b/frontend/test/gallerySelection.spec.js @@ -0,0 +1,64 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useGallerySelectionStore } from '../src/stores/gallerySelection.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, + statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +describe('gallerySelection store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('toggle adds/removes preserving order; count/isSelected track it', () => { + const s = useGallerySelectionStore() + s.toggle(5); s.toggle(9); s.toggle(2) + expect(s.order).toEqual([5, 9, 2]) + expect(s.isSelected(9)).toBe(true) + expect(s.count).toBe(3) + s.toggle(9) + expect(s.order).toEqual([5, 2]) + expect(s.isSelected(9)).toBe(false) + }) + + it('exitSelectMode clears selection and mode', () => { + const s = useGallerySelectionStore() + s.enterSelectMode(); s.toggle(1) + expect(s.isSelectMode).toBe(true) + s.exitSelectMode() + expect(s.isSelectMode).toBe(false) + expect(s.order).toEqual([]) + }) + + it('bulkAdd posts the id list + source and refreshes', async () => { + const s = useGallerySelectionStore() + s.toggle(1); s.toggle(2) + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: JSON.parse(init.body) }) + if (url.includes('/bulk/tags')) return { status: 200, body: { added_count: 2, total: 2 } } + if (url.includes('common-tags')) return { status: 200, body: { tags: [] } } + if (url.includes('suggestions/bulk')) return { status: 200, body: { suggestions: {}, evaluated: 2, threshold: 0.8 } } + return { status: 200, body: {} } + }) + await s.bulkAdd(42, 'ml_accepted') + const add = calls.find(c => c.url.includes('/api/images/bulk/tags')) + expect(add.body).toEqual({ image_ids: [1, 2], tag_id: 42, source: 'ml_accepted' }) + }) + + it('loadCommonTags stores returned tags', async () => { + const s = useGallerySelectionStore() + s.toggle(3) + stubFetch(() => ({ status: 200, body: { tags: [{ id: 7, name: 'x', kind: 'general' }] } })) + await s.loadCommonTags() + expect(s.commonTags).toEqual([{ id: 7, name: 'x', kind: 'general' }]) + }) +}) diff --git a/frontend/test/platforms.spec.js b/frontend/test/platforms.spec.js new file mode 100644 index 0000000..6667673 --- /dev/null +++ b/frontend/test/platforms.spec.js @@ -0,0 +1,42 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { usePlatformsStore } from '../src/stores/platforms.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)), + } + }) +} + +describe('platforms store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('loadAll fetches /api/platforms and populates list+byKey', async () => { + const s = usePlatformsStore() + stubFetch(() => ({ + status: 200, + body: { platforms: { + patreon: { key: 'patreon', name: 'Patreon', auth_type: 'cookies' }, + discord: { key: 'discord', name: 'Discord', auth_type: 'token' }, + }}, + })) + await s.loadAll() + expect(s.list.map(p => p.key).sort()).toEqual(['discord', 'patreon']) + expect(s.byKey.get('patreon').auth_type).toBe('cookies') + }) + + it('loadAll is cached on subsequent calls', async () => { + const s = usePlatformsStore() + let n = 0 + stubFetch(() => { n++; return { status: 200, body: { platforms: {} } } }) + await s.loadAll() + await s.loadAll() + expect(n).toBe(1) + }) +}) diff --git a/frontend/test/provenance.spec.js b/frontend/test/provenance.spec.js new file mode 100644 index 0000000..c5142d6 --- /dev/null +++ b/frontend/test/provenance.spec.js @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useProvenanceStore } from '../src/stores/provenance.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, + statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +describe('provenance store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('loadForImage caches: repeat same id does not refetch', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ status: 200, body: { image_id: 5, provenance: [] } })) + await store.loadForImage(5) + await store.loadForImage(5) + expect(globalThis.fetch.mock.calls.length).toBe(1) + expect(store.imageProv(5).entries).toEqual([]) + }) + + it('loadForImage fetches distinct ids separately', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ + status: 200, + body: { image_id: 1, provenance: [{ provenance_id: 9 }] } + })) + await store.loadForImage(1) + await store.loadForImage(2) + expect(globalThis.fetch.mock.calls.length).toBe(2) + expect(store.imageProv(1).entries.length).toBe(1) + }) + + it('loadForImage records error without throwing', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ status: 404, body: { error: 'not found' } })) + await store.loadForImage(7) + expect(store.imageProv(7).error).toBeTruthy() + expect(store.imageProv(7).entries).toBe(null) + }) + + it('loadForPost caches and stores post/source/artist', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ + status: 200, + body: { post: { id: 3 }, source: { platform: 'patreon' }, + artist: { name: 'A', slug: 'a' } } + })) + await store.loadForPost(3) + await store.loadForPost(3) + expect(globalThis.fetch.mock.calls.length).toBe(1) + expect(store.postInfo(3).data.artist.slug).toBe('a') + }) + + it('loadForImage keeps attachments from the payload', async () => { + const store = useProvenanceStore() + stubFetch(() => ({ + status: 200, + body: { + image_id: 1, provenance: [], + attachments: [{ id: 9, original_filename: 'p.zip', size_bytes: 5, + ext: '.zip', download_url: '/api/attachments/9/download' }] + } + })) + await store.loadForImage(1) + expect(store.imageProv(1).attachments[0].id).toBe(9) + }) +}) diff --git a/frontend/test/router.spec.js b/frontend/test/router.spec.js new file mode 100644 index 0000000..c976fd3 --- /dev/null +++ b/frontend/test/router.spec.js @@ -0,0 +1,32 @@ +import { describe, it, expect } from 'vitest' +import router, { FRONT_DOOR } from '../src/router.js' + +describe('router', () => { + it('FRONT_DOOR defaults to /showcase', () => { + expect(FRONT_DOOR).toBe('/showcase') + }) + + it('/ redirects to FRONT_DOOR', async () => { + // resolve() does not follow redirects (they apply on navigation), so + // actually navigate and assert the resulting route. + await router.push('/') + expect(router.currentRoute.value.path).toBe(FRONT_DOOR) + }) + + it('gallery is reachable at /gallery', () => { + expect(router.resolve('/gallery').name).toBe('gallery') + }) + + it('showcase, tags, artist resolve to named routes', () => { + expect(router.resolve('/showcase').name).toBe('showcase') + expect(router.resolve('/tags').name).toBe('tags') + expect(router.resolve('/artist/some-slug').name).toBe('artist') + expect(router.resolve('/artist/some-slug').params.slug).toBe('some-slug') + }) + + it('series-read is an immersive route', () => { + const r = router.resolve('/series/5/read') + expect(r.name).toBe('series-read') + expect(r.meta.immersive).toBe(true) + }) +}) diff --git a/frontend/test/seriesManage.spec.js b/frontend/test/seriesManage.spec.js new file mode 100644 index 0000000..a094741 --- /dev/null +++ b/frontend/test/seriesManage.spec.js @@ -0,0 +1,69 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useSeriesManageStore, moveItem } from '../src/stores/seriesManage.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +describe('seriesManage', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('moveItem reorders immutably', () => { + expect(moveItem([1, 2, 3, 4], 3, 0)).toEqual([4, 1, 2, 3]) + expect(moveItem([1, 2, 3], 0, 2)).toEqual([2, 3, 1]) + }) + + it('load fetches pages + series', async () => { + const s = useSeriesManageStore() + stubFetch(() => ({ + status: 200, + body: { series: { id: 7, name: 'V' }, + pages: [{ image_id: 1, page_number: 1, thumbnail_url: 't' }] } + })) + await s.load(7) + expect(s.series).toEqual({ id: 7, name: 'V' }) + expect(s.pages.map(p => p.image_id)).toEqual([1]) + }) + + it('reorder posts the full ordered id list', async () => { + const s = useSeriesManageStore() + s.tagId = 7 + s.pages = [{ image_id: 1 }, { image_id: 2 }, { image_id: 3 }] + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) + if (url.includes('/reorder')) return { status: 200, body: { ok: true } } + return { status: 200, body: { series: { id: 7, name: 'V' }, pages: [] } } + }) + await s.reorder([3, 1, 2]) + const r = calls.find(c => c.url.includes('/reorder')) + expect(r.url).toContain('/api/series/7/reorder') + expect(r.body).toEqual({ image_ids: [3, 1, 2] }) + }) + + it('addSelected posts picker selection then reloads', async () => { + const s = useSeriesManageStore() + s.tagId = 7 + s.pickerSelection = [9, 10] + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) + if (url.includes('/pages') && init.method === 'POST') + return { status: 200, body: { added_count: 2 } } + return { status: 200, body: { series: { id: 7, name: 'V' }, pages: [] } } + }) + await s.addSelected() + const add = calls.find(c => c.url.endsWith('/api/series/7/pages')) + expect(add.body).toEqual({ image_ids: [9, 10] }) + expect(s.pickerSelection).toEqual([]) + }) +}) diff --git a/frontend/test/seriesReader.spec.js b/frontend/test/seriesReader.spec.js new file mode 100644 index 0000000..de4d61e --- /dev/null +++ b/frontend/test/seriesReader.spec.js @@ -0,0 +1,81 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { + useSeriesReaderStore, + currentPageFromScroll, + progressPct, + clampPage +} from '../src/stores/seriesReader.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +const M = [ + { page_number: 1, top: 0, height: 100 }, + { page_number: 2, top: 100, height: 100 }, + { page_number: 3, top: 200, height: 100 } +] + +describe('seriesReader pure helpers', () => { + it('currentPageFromScroll: top of doc → page 1', () => { + expect(currentPageFromScroll(M, 0, 90)).toBe(1) // target 30 + }) + it('currentPageFromScroll: boundary crosses into page 2', () => { + expect(currentPageFromScroll(M, 80, 90)).toBe(2) // target 110 + }) + it('currentPageFromScroll: scrolled past last → last page', () => { + expect(currentPageFromScroll(M, 5000, 300)).toBe(3) + }) + it('currentPageFromScroll: empty metrics → 1', () => { + expect(currentPageFromScroll([], 0, 90)).toBe(1) + }) + it('progressPct: 0 at top, ~50 mid, clamped 100, zero-guard', () => { + expect(progressPct(0, 1000, 500)).toBe(0) + expect(progressPct(250, 1000, 500)).toBe(50) + expect(progressPct(9999, 1000, 500)).toBe(100) + expect(progressPct(0, 100, 100)).toBe(0) // denom 0 → 0 + }) + it('clampPage: below→1, above→total, valid passthrough, NaN→1', () => { + expect(clampPage(0, 5)).toBe(1) + expect(clampPage(99, 5)).toBe(5) + expect(clampPage(3, 5)).toBe(3) + expect(clampPage(NaN, 5)).toBe(1) + expect(clampPage(2, 0)).toBe(1) + }) +}) + +describe('seriesReader store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('load() stores series + pages', async () => { + const s = useSeriesReaderStore() + stubFetch(() => ({ + status: 200, + body: { + series: { id: 7, name: 'V' }, + pages: [{ image_id: 1, page_number: 1, thumbnail_url: 't', image_url: 'i' }] + } + })) + await s.load(7) + expect(s.series).toEqual({ id: 7, name: 'V' }) + expect(s.pages[0].image_url).toBe('i') + expect(s.error).toBe(null) + }) + + it('load() on 404 sets error, no throw', async () => { + const s = useSeriesReaderStore() + stubFetch(() => ({ status: 404, body: { error: 'not a series' } })) + await s.load(9) + expect(s.error).toBe('not a series') + expect(s.pages).toEqual([]) + }) +}) diff --git a/frontend/test/sources.spec.js b/frontend/test/sources.spec.js new file mode 100644 index 0000000..395ede5 --- /dev/null +++ b/frontend/test/sources.spec.js @@ -0,0 +1,106 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useSourcesStore } from '../src/stores/sources.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)), + } + }) +} + +describe('sources store', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => vi.restoreAllMocks()) + + it('loadAll caches under the "all" key', async () => { + const s = useSourcesStore() + stubFetch(() => ({ + status: 200, + body: [{ id: 1, artist_id: 5, artist_name: 'Alice', artist_slug: 'alice', + platform: 'patreon', url: 'https://p/a', enabled: true, + config_overrides: null, last_checked_at: null, + last_error: null, check_interval_override: null }], + })) + await s.loadAll() + expect(s.allSources.map(r => r.id)).toEqual([1]) + }) + + it('loadForArtist filters by artist_id', async () => { + const s = useSourcesStore() + const calls = [] + stubFetch((url) => { + calls.push(url) + return { status: 200, body: [] } + }) + await s.loadForArtist(7) + expect(calls[0]).toContain('artist_id=7') + }) + + it('create invalidates the all-cache and the artist-cache', async () => { + const s = useSourcesStore() + s.byArtist.set(null, []) + s.byArtist.set(5, []) + stubFetch(() => ({ + status: 201, + body: { id: 1, artist_id: 5, artist_name: 'Alice', artist_slug: 'alice', + platform: 'patreon', url: 'https://p/a', enabled: true, + config_overrides: null, last_checked_at: null, + last_error: null, check_interval_override: null }, + })) + await s.create({ artist_id: 5, platform: 'patreon', url: 'https://p/a' }) + expect(s.byArtist.has(null)).toBe(false) + expect(s.byArtist.has(5)).toBe(false) + }) + + it('findOrCreateArtist posts to /api/artists', async () => { + const s = useSourcesStore() + const calls = [] + stubFetch((url, init) => { + calls.push({ url, body: init.body ? JSON.parse(init.body) : null }) + return { status: 201, body: { id: 1, name: 'Alice', slug: 'alice', created: true } } + }) + const out = await s.findOrCreateArtist('Alice') + expect(out.artist.name).toBe('Alice') + expect(out.created).toBe(true) + expect(calls[0].body).toEqual({ name: 'Alice' }) + }) + + it('autocompleteArtist hits /api/artists/autocomplete', async () => { + const s = useSourcesStore() + const urls = [] + stubFetch((url) => { urls.push(url); return { status: 200, body: [] } }) + await s.autocompleteArtist('al') + expect(urls[0]).toContain('/api/artists/autocomplete') + expect(urls[0]).toContain('q=al') + }) + + it('checkNow posts to /api/sources//check', async () => { + const s = useSourcesStore() + const calls = [] + stubFetch((url, init) => { + calls.push({ url, method: init?.method, body: init?.body ? JSON.parse(init.body) : null }) + return { status: 202, body: { download_event_id: 7, status: 'pending' } } + }) + const out = await s.checkNow(5) + expect(calls[0].url).toBe('/api/sources/5/check') + expect(calls[0].method).toBe('POST') + expect(out.download_event_id).toBe(7) + }) + + it('checkNow exposes the 409 download_event_id via error.body', async () => { + const s = useSourcesStore() + stubFetch(() => ({ status: 409, body: { download_event_id: 12, status: 'already_running' } })) + try { + await s.checkNow(5) + } catch (e) { + expect(e.body?.download_event_id).toBe(12) + return + } + throw new Error('Expected checkNow to throw on 409') + }) +}) diff --git a/frontend/test/tagDirectory.spec.js b/frontend/test/tagDirectory.spec.js new file mode 100644 index 0000000..8fc44e2 --- /dev/null +++ b/frontend/test/tagDirectory.spec.js @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setActivePinia, createPinia } from 'pinia' +import { useTagDirectoryStore } from '../src/stores/tagDirectory.js' + +function stubFetch(handler) { + globalThis.fetch = vi.fn(async (url, init) => { + const { status, body } = handler(url, init) + return { + ok: status >= 200 && status < 300, + status, + statusText: String(status), + text: async () => (body == null ? '' : JSON.stringify(body)) + } + }) +} + +describe('tagDirectory store: rename / merge', () => { + beforeEach(() => setActivePinia(createPinia())) + afterEach(() => { vi.restoreAllMocks() }) + + it('rename success patches the card name in place', async () => { + const store = useTagDirectoryStore() + store.cards.push({ id: 1, name: 'old', kind: 'general' }) + stubFetch(() => ({ status: 200, body: { id: 1, name: 'new', kind: 'general' } })) + const res = await store.rename(1, 'new') + expect(res.collision).toBeUndefined() + expect(store.cards[0].name).toBe('new') + }) + + it('rename 409 resolves a collision object (does not throw)', async () => { + const store = useTagDirectoryStore() + store.cards.push({ id: 7, name: 'dupe', kind: 'general' }) + stubFetch(() => ({ + status: 409, + body: { + error: 'exists', + target: { id: 2, name: 'Canon' }, + source_image_count: 5, + will_alias: true + } + })) + const res = await store.rename(7, 'Canon') + expect(res.collision).toEqual({ + sourceId: 7, + sourceName: 'dupe', + target: { id: 2, name: 'Canon' }, + sourceImageCount: 5, + willAlias: true + }) + expect(store.cards[0].name).toBe('dupe') // unchanged + }) + + it('merge removes the source card optimistically', async () => { + const store = useTagDirectoryStore() + store.cards.push({ id: 7, name: 'dupe' }, { id: 2, name: 'Canon' }) + stubFetch(() => ({ + status: 200, + body: { + target: { id: 2, name: 'Canon', kind: 'general' }, + merged_count: 3, + alias_created: true, + source_deleted: true + } + })) + const res = await store.merge(7, 2) + expect(store.cards.map(c => c.id)).toEqual([2]) + expect(res.merged_count).toBe(3) + }) + + it('rename non-409 error rethrows', async () => { + const store = useTagDirectoryStore() + store.cards.push({ id: 1, name: 'x' }) + stubFetch(() => ({ status: 400, body: { error: 'bad' } })) + await expect(store.rename(1, '')).rejects.toThrow() + }) +}) diff --git a/frontend/test/usePolyMasonry.spec.js b/frontend/test/usePolyMasonry.spec.js new file mode 100644 index 0000000..f71d1d7 --- /dev/null +++ b/frontend/test/usePolyMasonry.spec.js @@ -0,0 +1,39 @@ +import { describe, it, expect } from 'vitest' +import { distributeIntoColumns } from '../src/composables/usePolyMasonry.js' + +describe('distributeIntoColumns', () => { + it('returns the requested number of columns', () => { + const cols = distributeIntoColumns([], 3) + expect(cols.length).toBe(3) + expect(cols.every(c => c.length === 0)).toBe(true) + }) + + it('places each item in the then-shortest column', () => { + // 3 square items (relative height 1 each), 3 columns -> one per column. + const items = [ + { id: 1, width: 100, height: 100 }, + { id: 2, width: 100, height: 100 }, + { id: 3, width: 100, height: 100 } + ] + const cols = distributeIntoColumns(items, 3) + expect(cols.map(c => c.map(i => i.id))).toEqual([[1], [2], [3]]) + }) + + it('routes a 4th item to a balanced column', () => { + const items = [ + { id: 1, width: 100, height: 300 }, // tall -> col0 height 3 + { id: 2, width: 100, height: 100 }, // col1 height 1 + { id: 3, width: 100, height: 100 }, // col2 height 1 + { id: 4, width: 100, height: 100 } // shortest is col1 (1) -> col1 + ] + const cols = distributeIntoColumns(items, 3) + expect(cols[0].map(i => i.id)).toEqual([1]) + expect(cols[1].map(i => i.id)).toEqual([2, 4]) + expect(cols[2].map(i => i.id)).toEqual([3]) + }) + + it('treats missing/zero dimensions as a square (height 1)', () => { + const cols = distributeIntoColumns([{ id: 9 }], 1) + expect(cols[0]).toEqual([{ id: 9 }]) + }) +}) diff --git a/frontend/vite.config.js b/frontend/vite.config.js new file mode 100644 index 0000000..b506803 --- /dev/null +++ b/frontend/vite.config.js @@ -0,0 +1,23 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' +import vuetify from 'vite-plugin-vuetify' + +export default defineConfig({ + plugins: [ + vue(), + vuetify({ autoImport: true }) + ], + build: { + outDir: 'dist', + emptyOutDir: true, + sourcemap: false + }, + server: { + port: 5173, + proxy: { + // Dev server proxies API to local Quart + '/api': 'http://localhost:8080', + '/ws': { target: 'ws://localhost:8080', ws: true } + } + } +}) diff --git a/frontend/vitest.config.js b/frontend/vitest.config.js new file mode 100644 index 0000000..3df5841 --- /dev/null +++ b/frontend/vitest.config.js @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config' +import vue from '@vitejs/plugin-vue' + +// plugin-vue is required so specs that import router.js (which imports .vue +// SFCs) can transform those single-file components. No component is mounted +// in the suite (router.resolve / pure-logic only), so jsdom is unnecessary. +export default defineConfig({ + plugins: [vue()], + test: { + environment: 'node', + include: ['test/**/*.spec.js'], + passWithNoTests: true + } +}) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..18206b0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "fabledcurator" +version = "0.1.0" +description = "FabledSword family — self-hosted media curation, gallery, ML tagging, and subscription-driven downloads." +requires-python = ">=3.14" + +[tool.setuptools.packages.find] +where = ["."] +include = ["backend*"] + +[tool.pytest.ini_options] +# auto mode: async test fns and async fixtures don't need explicit +# @pytest.mark.asyncio / @pytest_asyncio.fixture decoration. Required +# because the test_api_* fixtures use plain @pytest.fixture on async +# app/client; strict mode errors on those under pytest-asyncio 1.x. +asyncio_mode = "auto" +# The fast backend job runs `-m "not integration"`; the dedicated CI +# `integration` job (pgvector Postgres + Redis service containers) runs +# `-m integration`. Both also runnable locally via docker-compose. +markers = [ + "integration: tests that require a real Postgres/Redis. Run by the CI integration job and locally via docker-compose.", +] diff --git a/requirements-ml.txt b/requirements-ml.txt new file mode 100644 index 0000000..af14624 --- /dev/null +++ b/requirements-ml.txt @@ -0,0 +1,21 @@ +-r requirements.txt + +# ML stack — versions current as of 2026-05-14 with Python 3.14 wheel coverage. + +# torch + torchvision are NOT listed here: they are installed CPU-only from +# the PyTorch CPU index in Dockerfile.ml. The default PyPI torch wheel bundles +# the NVIDIA CUDA runtime (a ~5.6GB image layer); this pipeline is CPU-only, +# so Dockerfile.ml uses the +cpu wheels from +# https://download.pytorch.org/whl/cpu instead. +# +# IMPORTANT: torchvision 0.27 declares requires_python "!=3.14.1,>=3.10" — +# Python 3.14.1 specifically is excluded due to a known incompatibility. +# The python-ci runner pulls python:3.14-bookworm (latest patch); if that +# resolves to 3.14.1 the install will fail. Pin a specific Python patch in +# the runner image (CI-Runner/CI-python/Dockerfile) if this becomes a +# blocker. 3.14.0 and 3.14.2+ are fine. + +transformers>=5.8,<6.0 +onnxruntime>=1.26,<2.0 +huggingface-hub>=1.14,<2.0 +opencv-python-headless>=4.13,<5.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a6e009b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,35 @@ +# Web +quart>=0.20,<0.21 +hypercorn>=0.18,<0.19 + +# DB +sqlalchemy[asyncio]>=2.0,<2.1 +asyncpg>=0.31,<0.32 +psycopg[binary]>=3.3,<3.4 +alembic>=1.18,<1.19 +pgvector>=0.4,<0.5 + +# Task queue +celery>=5.6,<5.7 +redis>=7.4,<8.0 + +# Crypto for credential storage (lands in FC-3, but pinned now for stability) +cryptography>=48,<49 + +# Image handling (lands in FC-2) +pillow>=12,<13 +imagehash>=4.3,<4.4 + +# Gallery-dl wrapper (lands in FC-3) +gallery-dl>=1.32,<1.33 + +# Utilities +python-dotenv>=1.2,<2.0 +structlog>=25.5,<26.0 + +# HTML sanitization for scraped post descriptions (FC-2d provenance) +nh3>=0.2,<0.3 + +# Archive extraction (FC-2d-iii filesystem-import aid) +rarfile>=4.2,<5 +py7zr>=1,<2 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..5ded561 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,28 @@ +target-version = "py314" +line-length = 100 + +[lint] +select = [ + "E", "W", # pycodestyle + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # comprehensions + "ASYNC", # async correctness +] +ignore = [ + "E501", # line length, handled by formatter +] + +[lint.per-file-ignores] +"alembic/versions/*.py" = ["E", "F", "I", "UP"] +"tests/*" = ["F401"] + +[lint.isort] +# Tell ruff that `backend` is our first-party module so imports get +# grouped correctly: stdlib, third-party, first-party, then local relative. +known-first-party = ["backend"] + +[format] +quote-style = "double" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..00a8127 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,149 @@ +"""Shared pytest fixtures. + +The async db fixture provides an AsyncSession bound to a transaction that +gets rolled back after each test. CI provisions a real Postgres + pgvector +(see .forgejo/workflows/ci.yml), so tests exercise the actual schema and +migration code paths. +""" + +import os + +import pytest +import pytest_asyncio +from sqlalchemy import create_engine +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) +from sqlalchemy.orm import sessionmaker + +from backend.app.models import Base + + +def _async_database_url() -> str: + user = os.environ.get("DB_USER", "fabledcurator") + password = os.environ["DB_PASSWORD"] + host = os.environ.get("DB_HOST", "postgres") + port = os.environ.get("DB_PORT", "5432") + name = os.environ.get("DB_NAME", "fabledcurator_test") + return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}" + + +def _sync_database_url() -> str: + user = os.environ.get("DB_USER", "fabledcurator") + password = os.environ["DB_PASSWORD"] + host = os.environ.get("DB_HOST", "postgres") + port = os.environ.get("DB_PORT", "5432") + name = os.environ.get("DB_NAME", "fabledcurator_test") + return f"postgresql+psycopg://{user}:{password}@{host}:{port}/{name}" + + +@pytest_asyncio.fixture +async def engine(): + e = create_async_engine(_async_database_url(), future=True) + yield e + await e.dispose() + + +@pytest_asyncio.fixture +async def db(engine) -> AsyncSession: + Session = async_sessionmaker(engine, expire_on_commit=False) + async with Session() as session: + await session.begin() + try: + yield session + finally: + await session.rollback() + + +@pytest.fixture +def sync_engine(): + e = create_engine(_sync_database_url(), future=True) + yield e + e.dispose() + + +@pytest.fixture +def db_sync(sync_engine): + """Synchronous Session bound to a savepoint — used by Importer tests + (the Importer is sync-only by design).""" + SyncSession = sessionmaker(sync_engine, expire_on_commit=False) + with SyncSession() as session: + session.begin() + try: + yield session + finally: + session.rollback() + + +# Migration-seeded baseline (singleton config like import_settings / +# ml_settings), captured lazily at the FIRST integration test's setup — +# when the CI integration job has just run `alembic upgrade head` so the DB +# is pristine. Cached module-wide. Never populated in the DB-less unit job +# because only integration-marked tests trigger the connection. +_SEED_SNAPSHOT: dict[str, list[dict]] | None = None + + +@pytest.fixture(autouse=True) +def _reset_db_after_integration(request): + """Integration tests exercise app/celery code that commits on its own + connection, which the rollback fixtures above can't undo — so data + would leak between tests. Capture the seeded baseline at the first + integration test, then after each integration-marked test truncate + every model table and restore that baseline. Connects to the DB ONLY + for integration-marked tests, so the no-DB fast unit job is untouched. + """ + global _SEED_SNAPSHOT + is_integration = request.node.get_closest_marker("integration") is not None + + if is_integration and _SEED_SNAPSHOT is None: + eng = create_engine(_sync_database_url(), future=True) + snap: dict[str, list[dict]] = {} + try: + with eng.connect() as conn: + for t in Base.metadata.sorted_tables: + rows = [ + dict(m) + for m in conn.execute(t.select()).mappings().all() + ] + if rows: + snap[t.name] = rows + finally: + eng.dispose() + _SEED_SNAPSHOT = snap + + yield + + if not is_integration: + return + tables = ", ".join(t.name for t in Base.metadata.sorted_tables) + if not tables: + return + eng = create_engine(_sync_database_url(), future=True) + try: + with eng.begin() as conn: + conn.exec_driver_sql( + f"TRUNCATE {tables} RESTART IDENTITY CASCADE" + ) + for t in Base.metadata.sorted_tables: + rows = (_SEED_SNAPSHOT or {}).get(t.name) + if rows: + conn.execute(t.insert(), rows) + finally: + eng.dispose() + + +@pytest_asyncio.fixture(autouse=True) +async def _dispose_app_engine(request): + """The app's shared async engine (extensions.get_engine) is a process + singleton, but pytest-asyncio gives each test a fresh event loop; + asyncpg connections are loop-bound, so reuse across tests raises + 'attached to a different loop'. Dispose after each integration test so + the next test rebuilds the engine on its own loop. + """ + yield + if request.node.get_closest_marker("integration") is None: + return + from backend.app import extensions + await extensions.dispose_engine() diff --git a/tests/test_api_aliases.py b/tests/test_api_aliases.py new file mode 100644 index 0000000..123f8ef --- /dev/null +++ b/tests/test_api_aliases.py @@ -0,0 +1,51 @@ +import pytest + +from backend.app import create_app +from backend.app.models import TagKind +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_create_list_delete(client, db): + tag = await TagService(db).find_or_create("Canon", TagKind.character) + await db.commit() + + resp = await client.post( + "/api/aliases", + json={ + "alias_string": "model_name", + "alias_category": "character", + "canonical_tag_id": tag.id, + }, + ) + assert resp.status_code == 201 + + resp = await client.get("/api/aliases") + body = await resp.get_json() + assert any( + a["alias_string"] == "model_name" + and a["canonical_tag_name"] == "Canon" + for a in body + ) + + resp = await client.delete("/api/aliases/model_name/character") + assert resp.status_code == 204 + + +@pytest.mark.asyncio +async def test_create_requires_fields(client): + resp = await client.post("/api/aliases", json={"alias_string": "x"}) + assert resp.status_code == 400 diff --git a/tests/test_api_allowlist.py b/tests/test_api_allowlist.py new file mode 100644 index 0000000..4505b7d --- /dev/null +++ b/tests/test_api_allowlist.py @@ -0,0 +1,53 @@ +import pytest + +from backend.app import create_app +from backend.app.models import TagAllowlist, TagKind +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_list_and_patch_and_delete(client, db): + tag = await TagService(db).find_or_create("AL", TagKind.character) + db.add(TagAllowlist(tag_id=tag.id, min_confidence=0.95)) + await db.commit() + + resp = await client.get("/api/allowlist") + assert resp.status_code == 200 + assert any(r["tag_id"] == tag.id for r in await resp.get_json()) + + resp = await client.patch( + f"/api/tags/{tag.id}/allowlist", json={"min_confidence": 0.80} + ) + assert resp.status_code == 204 + + resp = await client.get(f"/api/tags/{tag.id}/allowlist") + assert (await resp.get_json())["min_confidence"] == pytest.approx(0.80) + + resp = await client.delete(f"/api/tags/{tag.id}/allowlist") + assert resp.status_code == 204 + resp = await client.get(f"/api/tags/{tag.id}/allowlist") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_patch_rejects_out_of_range(client, db): + tag = await TagService(db).find_or_create("AL2", TagKind.character) + db.add(TagAllowlist(tag_id=tag.id)) + await db.commit() + resp = await client.patch( + f"/api/tags/{tag.id}/allowlist", json={"min_confidence": 1.5} + ) + assert resp.status_code == 400 diff --git a/tests/test_api_artist.py b/tests/test_api_artist.py new file mode 100644 index 0000000..599093f --- /dev/null +++ b/tests/test_api_artist.py @@ -0,0 +1,37 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_artist_404(client): + resp = await client.get("/api/artist/nope") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_artist_overview_ok(client, db): + db.add(Artist(name="Mira", slug="mira")) + await db.flush() + await db.commit() + resp = await client.get("/api/artist/mira") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["name"] == "Mira" + assert body["image_count"] == 0 + + +@pytest.mark.asyncio +async def test_artist_images_404(client): + resp = await client.get("/api/artist/nope/images") + assert resp.status_code == 404 diff --git a/tests/test_api_artists_create.py b/tests/test_api_artists_create.py new file mode 100644 index 0000000..ccf1030 --- /dev/null +++ b/tests/test_api_artists_create.py @@ -0,0 +1,62 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_create_artist_returns_201(client): + resp = await client.post("/api/artists", json={"name": "Alice"}) + assert resp.status_code == 201 + body = await resp.get_json() + assert body["name"] == "Alice" + assert body["slug"] == "alice" + assert body["created"] is True + + +@pytest.mark.asyncio +async def test_create_artist_idempotent(client): + a = await client.post("/api/artists", json={"name": "Bob"}) + b = await client.post("/api/artists", json={"name": "Bob"}) + assert (await a.get_json())["created"] is True + body_b = await b.get_json() + assert body_b["created"] is False + assert body_b["id"] == (await a.get_json())["id"] + + +@pytest.mark.asyncio +async def test_create_artist_rejects_empty(client): + resp = await client.post("/api/artists", json={"name": " "}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_autocomplete_returns_matches(client): + await client.post("/api/artists", json={"name": "Alice"}) + await client.post("/api/artists", json={"name": "Alice Cooper"}) + await client.post("/api/artists", json={"name": "Bob"}) + resp = await client.get("/api/artists/autocomplete?q=alic") + assert resp.status_code == 200 + names = [a["name"] for a in await resp.get_json()] + assert "Alice" in names + assert "Alice Cooper" in names + assert "Bob" not in names + + +@pytest.mark.asyncio +async def test_autocomplete_empty_query(client): + resp = await client.get("/api/artists/autocomplete?q=") + assert resp.status_code == 200 + assert await resp.get_json() == [] diff --git a/tests/test_api_artists_directory.py b/tests/test_api_artists_directory.py new file mode 100644 index 0000000..37459fc --- /dev/null +++ b/tests/test_api_artists_directory.py @@ -0,0 +1,104 @@ +"""FC-3f: /api/artists/directory API tests.""" + +import pytest + +from backend.app import create_app +from backend.app.models import Artist, ImageRecord, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def seeded(db): + alice = Artist(name="alice-api", slug="alice-api", is_subscription=True) + bob = Artist(name="bob-api", slug="bob-api", is_subscription=False) + db.add(alice) + db.add(bob) + await db.flush() + db.add(Source( + artist_id=alice.id, platform="patreon", + url="https://p/alice-api", enabled=True, + )) + db.add(ImageRecord( + path="/images/api/alice.jpg", + sha256=("a" * 60) + "0001", + size_bytes=10, mime="image/jpeg", width=10, height=10, + origin="downloaded", artist_id=alice.id, + )) + await db.commit() + return alice, bob + + +@pytest.mark.asyncio +async def test_directory_returns_cards_and_cursor_keys(client, seeded): + resp = await client.get("/api/artists/directory") + assert resp.status_code == 200 + body = await resp.get_json() + assert set(body.keys()) == {"cards", "next_cursor"} + assert isinstance(body["cards"], list) + + +@pytest.mark.asyncio +async def test_directory_card_shape(client, seeded): + resp = await client.get("/api/artists/directory?q=alice-api") + body = await resp.get_json() + card = next(c for c in body["cards"] if c["name"] == "alice-api") + assert set(card.keys()) == { + "id", "name", "slug", "is_subscription", "image_count", "preview_thumbnails", + } + assert card["is_subscription"] is True + assert card["image_count"] == 1 + assert len(card["preview_thumbnails"]) == 1 + + +@pytest.mark.asyncio +async def test_directory_rejects_malformed_cursor(client): + resp = await client.get("/api/artists/directory?cursor=garbage!!!") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "invalid_cursor" + + +@pytest.mark.asyncio +async def test_directory_rejects_unknown_platform(client): + resp = await client.get("/api/artists/directory?platform=myspace") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_platform" + + +@pytest.mark.asyncio +async def test_directory_rejects_limit_out_of_range(client): + resp = await client.get("/api/artists/directory?limit=0") + assert resp.status_code == 400 + resp = await client.get("/api/artists/directory?limit=500") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_directory_q_propagates(client, seeded): + resp = await client.get("/api/artists/directory?q=alice-api") + body = await resp.get_json() + names = [c["name"] for c in body["cards"]] + assert "alice-api" in names + assert "bob-api" not in names + + +@pytest.mark.asyncio +async def test_directory_platform_propagates(client, seeded): + resp = await client.get("/api/artists/directory?platform=patreon") + body = await resp.get_json() + names = [c["name"] for c in body["cards"]] + assert "alice-api" in names + assert "bob-api" not in names # no patreon source diff --git a/tests/test_api_attachments.py b/tests/test_api_attachments.py new file mode 100644 index 0000000..5516438 --- /dev/null +++ b/tests/test_api_attachments.py @@ -0,0 +1,47 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist, PostAttachment + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_download_streams_with_disposition(client, db, tmp_path): + blob = tmp_path / "pack.zip" + blob.write_bytes(b"PK\x03\x04binarypayload") + a = Artist(name="Q", slug="q") + db.add(a) + await db.flush() + att = PostAttachment( + post_id=None, artist_id=a.id, sha256="q" + "0" * 63, + path=str(blob), original_filename="pack.zip", ext=".zip", + mime="application/zip", size_bytes=blob.stat().st_size, + ) + db.add(att) + await db.flush() + await db.commit() + + resp = await client.get(f"/api/attachments/{att.id}/download") + assert resp.status_code == 200 + disp = resp.headers.get("Content-Disposition", "") + assert "attachment" in disp + assert "pack.zip" in disp + assert (await resp.get_data()) == b"PK\x03\x04binarypayload" + + +@pytest.mark.asyncio +async def test_download_404(client): + resp = await client.get("/api/attachments/999999/download") + assert resp.status_code == 404 diff --git a/tests/test_api_bulk_tags.py b/tests/test_api_bulk_tags.py new file mode 100644 index 0000000..1917734 --- /dev/null +++ b/tests/test_api_bulk_tags.py @@ -0,0 +1,102 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _mk_tag(client, name, kind="general"): + r = await client.post("/api/tags", json={"name": name, "kind": kind}) + return (await r.get_json())["id"] + + +@pytest.mark.asyncio +async def test_common_tags_rejects_empty_list(client): + # Spec guard: image_ids must be a non-empty list (the frontend never + # calls this with an empty selection). + resp = await client.post("/api/images/common-tags", json={"image_ids": []}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_common_tags_requires_list(client): + resp = await client.post("/api/images/common-tags", json={}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_common_tags_ok_shape(client): + # Unknown image ids are valid input; result is just an empty tag list. + resp = await client.post( + "/api/images/common-tags", json={"image_ids": [999999]} + ) + assert resp.status_code == 200 + assert (await resp.get_json())["tags"] == [] + + +@pytest.mark.asyncio +async def test_bulk_add_route_happy(client): + tag = await _mk_tag(client, "RouteAdd") + resp = await client.post( + "/api/images/bulk/tags", + json={"image_ids": [1, 2, 3], "tag_id": tag}, + ) + # images 1..3 may not exist; FK means added_count reflects real images. + assert resp.status_code in (200, 400) + + +@pytest.mark.asyncio +async def test_bulk_add_rejects_bad_source(client): + tag = await _mk_tag(client, "RouteAddSrc") + resp = await client.post( + "/api/images/bulk/tags", + json={"image_ids": [1], "tag_id": tag, "source": "evil"}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_bulk_add_missing_tag_404(client): + resp = await client.post( + "/api/images/bulk/tags", + json={"image_ids": [1], "tag_id": 999999}, + ) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_bulk_add_over_cap_400(client): + tag = await _mk_tag(client, "RouteAddCap") + resp = await client.post( + "/api/images/bulk/tags", + json={"image_ids": list(range(1, 202)), "tag_id": tag}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_bulk_remove_route_missing_tag_404(client): + resp = await client.post( + "/api/images/bulk/tags/remove", + json={"image_ids": [1], "tag_id": 999999}, + ) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_bulk_remove_route_requires_ids(client): + resp = await client.post( + "/api/images/bulk/tags/remove", json={"tag_id": 1} + ) + assert resp.status_code == 400 diff --git a/tests/test_api_credentials.py b/tests/test_api_credentials.py new file mode 100644 index 0000000..7c5b5a0 --- /dev/null +++ b/tests/test_api_credentials.py @@ -0,0 +1,145 @@ +import pytest + +from backend.app import create_app +from backend.app.models import AppSetting + +pytestmark = pytest.mark.integration + + +_NETSCAPE = ( + "# Netscape HTTP Cookie File\n" + ".patreon.com\tTRUE\t/\tTRUE\t1700000000\tsession_id\tabc\n" +) + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def ext_key(db): + # Seed an extension API key for tests that want to assert the + # X-Extension-Key path explicitly. + db.add(AppSetting(key="extension_api_key", value="test-ext-key")) + await db.commit() + return "test-ext-key" + + +@pytest.mark.asyncio +async def test_list_empty(client): + resp = await client.get("/api/credentials") + assert resp.status_code == 200 + assert await resp.get_json() == [] + + +@pytest.mark.asyncio +async def test_create_and_get(client): + resp = await client.post("/api/credentials", json={ + "platform": "patreon", + "credential_type": "cookies", + "data": _NETSCAPE, + }) + assert resp.status_code == 201 + body = await resp.get_json() + assert body["platform"] == "patreon" + assert body["credential_type"] == "cookies" + assert "data" not in body # never echoed + assert "encrypted_blob" not in body + + one = await client.get("/api/credentials/patreon") + assert one.status_code == 200 + assert (await one.get_json())["platform"] == "patreon" + + +@pytest.mark.asyncio +async def test_create_updates_existing(client): + a = await client.post("/api/credentials", json={ + "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE, + }) + assert a.status_code == 201 + b = await client.post("/api/credentials", json={ + "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE + "x", + }) + # On update, our convention is 200 not 201 + assert b.status_code == 200 + listing = await (await client.get("/api/credentials")).get_json() + assert len(listing) == 1 + + +@pytest.mark.asyncio +async def test_unknown_platform_400(client): + resp = await client.post("/api/credentials", json={ + "platform": "fanbox", "credential_type": "cookies", "data": "x", + }) + assert resp.status_code == 400 + assert (await resp.get_json())["error"] == "unknown_platform" + + +@pytest.mark.asyncio +async def test_wrong_auth_type_400(client): + resp = await client.post("/api/credentials", json={ + "platform": "discord", "credential_type": "cookies", "data": "x", + }) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "wrong_auth_type" + assert body["expected"] == "token" + + +@pytest.mark.asyncio +async def test_empty_data_400(client): + resp = await client.post("/api/credentials", json={ + "platform": "patreon", "credential_type": "cookies", "data": " ", + }) + assert resp.status_code == 400 + assert (await resp.get_json())["error"] == "empty_data" + + +@pytest.mark.asyncio +async def test_invalid_body_400(client): + resp = await client.post("/api/credentials", json=[1, 2, 3]) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_get_404(client): + resp = await client.get("/api/credentials/patreon") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_delete_204_then_404(client): + await client.post("/api/credentials", json={ + "platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE, + }) + dele = await client.delete("/api/credentials/patreon") + assert dele.status_code == 204 + nope = await client.delete("/api/credentials/patreon") + assert nope.status_code == 404 + + +@pytest.mark.asyncio +async def test_extension_key_correct_accepts(client, ext_key): + resp = await client.post( + "/api/credentials", + json={"platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE}, + headers={"X-Extension-Key": ext_key}, + ) + assert resp.status_code == 201 + + +@pytest.mark.asyncio +async def test_extension_key_wrong_rejects(client, ext_key): + resp = await client.post( + "/api/credentials", + json={"platform": "patreon", "credential_type": "cookies", "data": _NETSCAPE}, + headers={"X-Extension-Key": "WRONG"}, + ) + assert resp.status_code == 401 diff --git a/tests/test_api_downloads.py b/tests/test_api_downloads.py new file mode 100644 index 0000000..d64e649 --- /dev/null +++ b/tests/test_api_downloads.py @@ -0,0 +1,109 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist, DownloadEvent, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def seed(db): + artist = Artist(name="Alice", slug="alice") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", enabled=True, + config_overrides={}, + ) + db.add(source) + await db.flush() + events = [ + DownloadEvent( + source_id=source.id, status="ok", + files_count=3, bytes_downloaded=100000, + metadata_={ + "run_stats": {"downloaded_count": 3, "skipped_count": 1, "quarantined_count": 0}, + "duration_seconds": 10.5, "error_type": None, + }, + ), + DownloadEvent( + source_id=source.id, status="error", error="auth failed", + metadata_={ + "run_stats": {"downloaded_count": 0, "skipped_count": 0, "quarantined_count": 0}, + "duration_seconds": 2.1, "error_type": "auth_error", + }, + ), + ] + db.add_all(events) + await db.commit() + return artist, source, events + + +@pytest.mark.asyncio +async def test_list_returns_newest_first(client, seed): + resp = await client.get("/api/downloads") + assert resp.status_code == 200 + body = await resp.get_json() + assert len(body) == 2 + assert body[0]["id"] > body[1]["id"] + assert body[0]["summary"]["error_type"] == "auth_error" + assert body[0]["artist_name"] == "Alice" + + +@pytest.mark.asyncio +async def test_list_filter_by_status(client, seed): + resp = await client.get("/api/downloads?status=ok") + body = await resp.get_json() + assert all(r["status"] == "ok" for r in body) + + +@pytest.mark.asyncio +async def test_list_filter_by_source_id(client, seed): + _, source, _ = seed + resp = await client.get(f"/api/downloads?source_id={source.id}") + body = await resp.get_json() + assert all(r["source_id"] == source.id for r in body) + + +@pytest.mark.asyncio +async def test_list_rejects_invalid_status(client): + resp = await client.get("/api/downloads?status=bogus") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_list_before_keyset(client, seed): + _, _, events = seed + middle_id = events[0].id + resp = await client.get(f"/api/downloads?before={middle_id}") + body = await resp.get_json() + assert all(r["id"] < middle_id for r in body) + + +@pytest.mark.asyncio +async def test_detail_returns_full_metadata(client, seed): + _, _, events = seed + target = events[1] + resp = await client.get(f"/api/downloads/{target.id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["metadata"]["error_type"] == "auth_error" + assert body["metadata"]["duration_seconds"] == 2.1 + + +@pytest.mark.asyncio +async def test_detail_404(client): + resp = await client.get("/api/downloads/99999") + assert resp.status_code == 404 diff --git a/tests/test_api_extension_api_key.py b/tests/test_api_extension_api_key.py new file mode 100644 index 0000000..222afc5 --- /dev/null +++ b/tests/test_api_extension_api_key.py @@ -0,0 +1,53 @@ +import pytest +from sqlalchemy import select + +from backend.app import create_app +from backend.app.models import AppSetting + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_get_seeds_a_value(client, db): + # First call seeds the row (idempotent on repeat calls). + resp = await client.get("/api/settings/extension_api_key") + assert resp.status_code == 200 + body = await resp.get_json() + assert isinstance(body["key"], str) and len(body["key"]) >= 16 + + row = (await db.execute( + select(AppSetting.value).where(AppSetting.key == "extension_api_key") + )).scalar_one() + assert row == body["key"] + + +@pytest.mark.asyncio +async def test_get_is_idempotent(client): + a = (await (await client.get("/api/settings/extension_api_key")).get_json())["key"] + b = (await (await client.get("/api/settings/extension_api_key")).get_json())["key"] + assert a == b + + +@pytest.mark.asyncio +async def test_rotate_changes_the_value(client, db): + before = (await (await client.get("/api/settings/extension_api_key")).get_json())["key"] + rotated = await client.post("/api/settings/extension_api_key/rotate") + assert rotated.status_code == 200 + after = (await rotated.get_json())["key"] + assert after != before + + row = (await db.execute( + select(AppSetting.value).where(AppSetting.key == "extension_api_key") + )).scalar_one() + assert row == after diff --git a/tests/test_api_gallery.py b/tests/test_api_gallery.py new file mode 100644 index 0000000..6dadd45 --- /dev/null +++ b/tests/test_api_gallery.py @@ -0,0 +1,72 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app import create_app +from backend.app.models import ImageRecord + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _seed(db, count: int = 3): + base = datetime.now(UTC) + for i in range(count): + r = ImageRecord( + path=f"/images/x/{i}.jpg", + sha256=f"x{i:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + r.created_at = base - timedelta(minutes=i) + db.add(r) + await db.flush() + await db.commit() + + +@pytest.mark.asyncio +async def test_scroll_returns_images(client, db): + await _seed(db) + resp = await client.get("/api/gallery/scroll?limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert len(body["images"]) == 3 + assert "next_cursor" in body + assert "date_groups" in body + + +@pytest.mark.asyncio +async def test_scroll_rejects_bad_limit(client): + resp = await client.get("/api/gallery/scroll?limit=notanumber") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_timeline_endpoint(client, db): + await _seed(db) + resp = await client.get("/api/gallery/timeline") + assert resp.status_code == 200 + body = await resp.get_json() + assert sum(b["count"] for b in body) == 3 + + +@pytest.mark.asyncio +async def test_jump_requires_year_month(client): + resp = await client.get("/api/gallery/jump") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_image_detail_404_when_missing(client): + resp = await client.get("/api/gallery/image/99999") + assert resp.status_code == 404 diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py new file mode 100644 index 0000000..7e15f4b --- /dev/null +++ b/tests/test_api_import_admin.py @@ -0,0 +1,139 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app import create_app +from backend.app.celery_app import celery +from backend.app.models import ImportBatch, ImportTask + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_status_when_idle(client): + resp = await client.get("/api/import/status") + body = await resp.get_json() + assert body["active_batch"] is None + + +@pytest.mark.asyncio +async def test_list_tasks_with_status_filter(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + for status in ("complete", "failed", "complete"): + db.add(ImportTask( + batch_id=batch.id, source_path="/x", task_type="media", status=status, + finished_at=datetime.now(UTC), + )) + await db.commit() + + resp = await client.get("/api/import/tasks?status=failed") + body = await resp.get_json() + assert len(body["tasks"]) == 1 + assert body["tasks"][0]["status"] == "failed" + + +@pytest.mark.asyncio +async def test_retry_failed(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + db.add(ImportTask(batch_id=batch.id, source_path="/x", task_type="media", status="failed")) + await db.commit() + + resp = await client.post("/api/import/retry-failed") + body = await resp.get_json() + assert body["retried"] == 1 + + +@pytest.mark.asyncio +async def test_clear_completed(client, db): + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + long_ago = datetime.now(UTC) - timedelta(days=30) + db.add(ImportTask( + batch_id=batch.id, source_path="/x", task_type="media", + status="complete", finished_at=long_ago, + )) + db.add(ImportTask( + batch_id=batch.id, source_path="/y", task_type="media", + status="complete", finished_at=datetime.now(UTC), + )) + await db.commit() + + resp = await client.post("/api/import/clear-completed", json={"age_days": 7}) + body = await resp.get_json() + assert body["deleted"] == 1 + + +@pytest.mark.asyncio +async def test_trigger_accepts_deep(client, monkeypatch): + # Stub the task dispatch — assert the API accepts 'deep' and forwards + # mode, without running the real (eager) scan_directory. + from backend.app.tasks import scan as scan_mod + + seen = {} + + class _Res: + id = "celery-deep" + + def _fake_delay(*, triggered_by, mode): + seen["triggered_by"] = triggered_by + seen["mode"] = mode + return _Res() + + monkeypatch.setattr(scan_mod.scan_directory, "delay", _fake_delay) + resp = await client.post("/api/import/trigger", json={"mode": "deep"}) + assert resp.status_code == 202 + body = await resp.get_json() + assert body["mode"] == "deep" + assert seen == {"triggered_by": "manual", "mode": "deep"} + + +@pytest.mark.asyncio +async def test_trigger_still_rejects_unknown_mode(client): + resp = await client.post("/api/import/trigger", json={"mode": "wat"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_trigger_accepts_verify(client, monkeypatch): + # Stub the verify task's dispatch so the API contract is asserted + # without running the real (eager) verify_integrity end-to-end. + from backend.app.tasks import maintenance as m + + seen = {} + + class _Res: + id = "celery-verify" + + def _fake_delay(): + seen["called"] = True + return _Res() + + monkeypatch.setattr(m.verify_integrity, "delay", _fake_delay) + resp = await client.post("/api/import/trigger", json={"mode": "verify"}) + assert resp.status_code == 202 + body = await resp.get_json() + assert body["mode"] == "verify" + assert seen == {"called": True} diff --git a/tests/test_api_migrate.py b/tests/test_api_migrate.py new file mode 100644 index 0000000..dffb529 --- /dev/null +++ b/tests/test_api_migrate.py @@ -0,0 +1,135 @@ +"""FC-5: /api/migrate API tests.""" +import io +import json + +import pytest +from werkzeug.datastructures import FileStorage + +import backend.app.tasks.migration # noqa: F401 — register celery task +from backend.app import create_app +from backend.app.celery_app import celery + +pytestmark = pytest.mark.integration + + +def _file_storage(payload: dict, filename: str = "export.json") -> FileStorage: + return FileStorage( + stream=io.BytesIO(json.dumps(payload).encode("utf-8")), + filename=filename, + content_type="application/json", + ) + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_post_backup_returns_202_and_id(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + resp = await client.post("/api/migrate/backup", json={}) + assert resp.status_code == 202 + body = await resp.get_json() + assert "run_id" in body + assert body["status"] == "pending" + + +@pytest.mark.asyncio +async def test_post_rejects_unknown_kind(client): + resp = await client.post("/api/migrate/destroy", json={}) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_kind" + + +@pytest.mark.asyncio +async def test_post_ingest_rejects_missing_file(client): + resp = await client.post("/api/migrate/gs_ingest", form={}) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "missing_export_file" + + +@pytest.mark.asyncio +async def test_post_ingest_accepts_multipart_file(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: True, # pretend backup exists + ) + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + resp = await client.post( + "/api/migrate/gs_ingest", + form={"dry_run": "false"}, + files={"export_file": _file_storage(export, "gs-export.json")}, + ) + assert resp.status_code == 202 + + +@pytest.mark.asyncio +async def test_post_apply_without_backup_rejected(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: False, + ) + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + resp = await client.post( + "/api/migrate/gs_ingest", + form={"dry_run": "false"}, + files={"export_file": _file_storage(export, "gs-export.json")}, + ) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "no_backup" + + +@pytest.mark.asyncio +async def test_post_dry_run_allowed_without_backup(client, monkeypatch): + monkeypatch.setattr( + "backend.app.api.migrate._has_recent_pre_migration_backup", + lambda: False, + ) + monkeypatch.setattr( + "backend.app.api.migrate.run_migration", + type("F", (), {"delay": lambda self, *a, **k: None})(), + ) + export = { + "source_app": "gallerysubscriber", "schema_version": 1, + "subscriptions": [], "credentials": [], + } + resp = await client.post( + "/api/migrate/gs_ingest", + form={"dry_run": "true"}, + files={"export_file": _file_storage(export, "gs-export.json")}, + ) + assert resp.status_code == 202 + + +@pytest.mark.asyncio +async def test_get_run_404_unknown(client): + resp = await client.get("/api/migrate/runs/9999999") + assert resp.status_code == 404 + + +def test_run_migration_task_registered(): + assert "backend.app.tasks.migration.run_migration" in celery.tasks diff --git a/tests/test_api_ml_admin.py b/tests/test_api_ml_admin.py new file mode 100644 index 0000000..cb99630 --- /dev/null +++ b/tests/test_api_ml_admin.py @@ -0,0 +1,67 @@ +import pytest + +from backend.app import create_app +from backend.app.celery_app import celery +from backend.app.models import TagKind +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_get_and_patch_settings(client): + resp = await client.get("/api/ml/settings") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["suggestion_threshold_general"] == pytest.approx(0.95) + + resp = await client.patch( + "/api/ml/settings", json={"suggestion_threshold_general": 0.90} + ) + assert resp.status_code == 200 + assert (await resp.get_json())["suggestion_threshold_general"] == pytest.approx(0.90) + + +@pytest.mark.asyncio +async def test_backfill_and_recompute_trigger(client): + r1 = await client.post("/api/ml/backfill") + assert r1.status_code == 202 + r2 = await client.post("/api/ml/recompute-centroids") + assert r2.status_code == 202 + + +@pytest.mark.asyncio +async def test_rename_tag(client, db): + tag = await TagService(db).find_or_create("oldname", TagKind.character) + await db.commit() + resp = await client.patch(f"/api/tags/{tag.id}", json={"name": "New Name"}) + assert resp.status_code == 200 + assert (await resp.get_json())["name"] == "New Name" + + +@pytest.mark.asyncio +async def test_rename_collision_409(client, db): + svc = TagService(db) + await svc.find_or_create("Taken", TagKind.character) + other = await svc.find_or_create("Mover", TagKind.character) + await db.commit() + resp = await client.patch(f"/api/tags/{other.id}", json={"name": "Taken"}) + assert resp.status_code == 409 diff --git a/tests/test_api_platforms.py b/tests/test_api_platforms.py new file mode 100644 index 0000000..3f6c624 --- /dev/null +++ b/tests/test_api_platforms.py @@ -0,0 +1,53 @@ +import re + +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_platforms_returns_gs_six(client): + resp = await client.get("/api/platforms") + assert resp.status_code == 200 + body = await resp.get_json() + platforms = body["platforms"] + assert set(platforms.keys()) == { + "patreon", "subscribestar", "hentaifoundry", + "discord", "pixiv", "deviantart", + } + assert "fanbox" not in platforms + + +@pytest.mark.asyncio +async def test_platforms_record_shape(client): + body = await (await client.get("/api/platforms")).get_json() + patreon = body["platforms"]["patreon"] + for key in ( + "key", "name", "description", "auth_type", "requires_auth", + "url_pattern", "url_examples", "default_config", "notes", + ): + assert key in patreon + assert patreon["auth_type"] == "cookies" + re.compile(patreon["url_pattern"]) + assert patreon["default_config"]["sleep"] == 3.0 + + +@pytest.mark.asyncio +async def test_platform_auth_types_match_gs(client): + body = await (await client.get("/api/platforms")).get_json() + assert body["platforms"]["discord"]["auth_type"] == "token" + assert body["platforms"]["pixiv"]["auth_type"] == "token" + assert body["platforms"]["patreon"]["auth_type"] == "cookies" diff --git a/tests/test_api_posts.py b/tests/test_api_posts.py new file mode 100644 index 0000000..a820ad8 --- /dev/null +++ b/tests/test_api_posts.py @@ -0,0 +1,119 @@ +"""FC-3e: /api/posts API tests. + +Validates list shape, cursor handling, filter validation, and detail +endpoint. Service-level fixtures are exercised by test_post_feed_service +— here we focus on the HTTP surface (validation, status codes, dict shape). +""" +from datetime import UTC, datetime + +import pytest + +from backend.app import create_app +from backend.app.models import Artist, Post, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def seeded_post(db): + artist = Artist(name="alice-api", slug="alice-api") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://p/alice-api", enabled=True, + ) + db.add(source) + await db.flush() + post = Post( + source_id=source.id, external_post_id="API1", + post_title="Hello", post_url="https://p/alice-api/1", + post_date=datetime.now(UTC), + description="

hi

", + ) + db.add(post) + await db.commit() + return artist, source, post + + +@pytest.mark.asyncio +async def test_list_returns_items_and_cursor_keys(client, seeded_post): + resp = await client.get("/api/posts") + assert resp.status_code == 200 + body = await resp.get_json() + assert set(body.keys()) == {"items", "next_cursor"} + assert isinstance(body["items"], list) + assert body["items"][0]["post_title"] == "Hello" + assert body["items"][0]["description_plain"] == "hi" + + +@pytest.mark.asyncio +async def test_list_rejects_malformed_cursor(client): + resp = await client.get("/api/posts?cursor=garbage!!!") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "invalid_cursor" + + +@pytest.mark.asyncio +async def test_list_rejects_unknown_platform(client): + resp = await client.get("/api/posts?platform=myspace") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_platform" + + +@pytest.mark.asyncio +async def test_list_rejects_non_int_artist_id(client): + resp = await client.get("/api/posts?artist_id=notanint") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "invalid_artist_id" + + +@pytest.mark.asyncio +async def test_list_rejects_limit_out_of_range(client): + resp = await client.get("/api/posts?limit=0") + assert resp.status_code == 400 + resp = await client.get("/api/posts?limit=500") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_list_filter_propagates_artist(client, seeded_post): + artist, _, post = seeded_post + resp = await client.get(f"/api/posts?artist_id={artist.id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert len(body["items"]) == 1 + assert body["items"][0]["id"] == post.id + + +@pytest.mark.asyncio +async def test_detail_200_for_known(client, seeded_post): + _, _, post = seeded_post + resp = await client.get(f"/api/posts/{post.id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["id"] == post.id + assert "description_full" in body + assert body["description_full"] == "hi" + + +@pytest.mark.asyncio +async def test_detail_404_for_unknown(client): + resp = await client.get("/api/posts/999999") + assert resp.status_code == 404 + body = await resp.get_json() + assert body["error"] == "not_found" diff --git a/tests/test_api_provenance.py b/tests/test_api_provenance.py new file mode 100644 index 0000000..328739e --- /dev/null +++ b/tests/test_api_provenance.py @@ -0,0 +1,84 @@ +from datetime import UTC, datetime + +import pytest + +from backend.app import create_app +from backend.app.models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, +) + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _seed_full(db): + rec = ImageRecord( + path="/images/p/1.jpg", sha256="p" + "0" * 63, + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + artist = Artist(name="Alice", slug="alice") + db.add_all([rec, artist]) + await db.flush() + source = Source(artist_id=artist.id, platform="patreon", + url="https://patreon.test/alice") + db.add(source) + await db.flush() + post = Post(source_id=source.id, external_post_id="555", + post_url="https://patreon.test/p/555", post_title="Set 1", + post_date=datetime(2023, 8, 1, tzinfo=UTC), + description="

hi

", attachment_count=2) + db.add(post) + await db.flush() + db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, + source_id=source.id)) + await db.flush() + await db.commit() + return rec.id, post.id + + +@pytest.mark.asyncio +async def test_image_provenance_endpoint(client, db): + image_id, post_id = await _seed_full(db) + resp = await client.get(f"/api/provenance/image/{image_id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["image_id"] == image_id + assert len(body["provenance"]) == 1 + assert body["provenance"][0]["post"]["id"] == post_id + + +@pytest.mark.asyncio +async def test_image_provenance_404(client): + resp = await client.get("/api/provenance/image/999999") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_post_provenance_endpoint(client, db): + _, post_id = await _seed_full(db) + resp = await client.get(f"/api/provenance/post/{post_id}") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["post"]["id"] == post_id + assert body["artist"]["slug"] == "alice" + + +@pytest.mark.asyncio +async def test_post_provenance_404(client): + resp = await client.get("/api/provenance/post/999999") + assert resp.status_code == 404 diff --git a/tests/test_api_series.py b/tests/test_api_series.py new file mode 100644 index 0000000..5f70f6b --- /dev/null +++ b/tests/test_api_series.py @@ -0,0 +1,72 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _series(client, name="Vol"): + r = await client.post("/api/tags", json={"name": name, "kind": "series"}) + return (await r.get_json())["id"] + + +@pytest.mark.asyncio +async def test_pages_404_for_non_series(client): + r = await client.post("/api/tags", json={"name": "g", "kind": "general"}) + gid = (await r.get_json())["id"] + resp = await client.get(f"/api/series/{gid}/pages") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_pages_empty_shape(client): + sid = await _series(client, "Empty") + resp = await client.get(f"/api/series/{sid}/pages") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["series"]["id"] == sid + assert body["pages"] == [] + + +@pytest.mark.asyncio +async def test_add_requires_ids(client): + sid = await _series(client, "NoIds") + resp = await client.post(f"/api/series/{sid}/pages", json={}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_add_over_cap_400(client): + sid = await _series(client, "Cap") + resp = await client.post( + f"/api/series/{sid}/pages", + json={"image_ids": list(range(1, 502))}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_reorder_mismatch_400(client): + sid = await _series(client, "Re") + resp = await client.post( + f"/api/series/{sid}/reorder", json={"image_ids": [1, 2, 3]} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_cover_requires_image_id(client): + sid = await _series(client, "Cov") + resp = await client.post(f"/api/series/{sid}/cover", json={}) + assert resp.status_code == 400 diff --git a/tests/test_api_settings.py b/tests/test_api_settings.py new file mode 100644 index 0000000..cea2466 --- /dev/null +++ b/tests/test_api_settings.py @@ -0,0 +1,169 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_get_import_settings_defaults(client): + resp = await client.get("/api/settings/import") + assert resp.status_code == 200 + body = await resp.get_json() + # Defaults from the migration: + assert body["min_width"] == 0 + assert body["min_height"] == 0 + assert body["skip_transparent"] is False + assert body["transparency_threshold"] == 0.9 + + +@pytest.mark.asyncio +async def test_patch_import_settings(client): + resp = await client.patch( + "/api/settings/import", + json={"min_width": 500, "skip_transparent": True, "transparency_threshold": 0.7}, + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["min_width"] == 500 + assert body["skip_transparent"] is True + assert body["transparency_threshold"] == 0.7 + + +@pytest.mark.asyncio +async def test_patch_rejects_non_object(client): + resp = await client.patch("/api/settings/import", json=[1, 2, 3]) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_import_settings_phash_threshold_default(client): + resp = await client.get("/api/settings/import") + assert resp.status_code == 200 + assert (await resp.get_json())["phash_threshold"] == 10 + + +@pytest.mark.asyncio +async def test_patch_phash_threshold(client): + resp = await client.patch( + "/api/settings/import", json={"phash_threshold": 4} + ) + assert resp.status_code == 200 + assert (await resp.get_json())["phash_threshold"] == 4 + + +@pytest.mark.asyncio +async def test_patch_phash_threshold_rejects_negative(client): + resp = await client.patch( + "/api/settings/import", json={"phash_threshold": -1} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_system_stats_shape(client): + resp = await client.get("/api/system/stats") + assert resp.status_code == 200 + body = await resp.get_json() + for key in ("total_images", "total_tags", "storage_bytes", "tasks", "active_batch"): + assert key in body + for status in ("pending", "queued", "processing", "complete", "skipped", "failed"): + assert status in body["tasks"] + + +# --- FC-3d: scheduling knobs ---------------------------------------------- + + +@pytest.mark.asyncio +async def test_get_includes_scheduling_defaults(client): + resp = await client.get("/api/settings/import") + body = await resp.get_json() + assert body["download_schedule_default_seconds"] == 28800 + assert body["download_event_retention_days"] == 90 + assert body["download_failure_warning_threshold"] == 5 + + +@pytest.mark.asyncio +async def test_patch_scheduling_defaults(client): + resp = await client.patch("/api/settings/import", json={ + "download_schedule_default_seconds": 7200, + "download_event_retention_days": 30, + "download_failure_warning_threshold": 3, + }) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["download_schedule_default_seconds"] == 7200 + assert body["download_event_retention_days"] == 30 + assert body["download_failure_warning_threshold"] == 3 + + +@pytest.mark.asyncio +async def test_patch_rejects_below_min_interval(client): + resp = await client.patch( + "/api/settings/import", + json={"download_schedule_default_seconds": 30}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_patch_rejects_above_max_interval(client): + resp = await client.patch( + "/api/settings/import", + json={"download_schedule_default_seconds": 90000}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_patch_rejects_zero_retention_days(client): + resp = await client.patch( + "/api/settings/import", + json={"download_event_retention_days": 0}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_patch_rejects_zero_failure_threshold(client): + resp = await client.patch( + "/api/settings/import", + json={"download_failure_warning_threshold": 0}, + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_system_stats_integrity_counts(client, db): + from backend.app.models import ImageRecord + + base = "stats_" + "0" * 56 + for i, status in enumerate( + ["ok", "ok", "corrupt", "failed_verification"] + ): + db.add(ImageRecord( + path=f"/images/i/{i}.jpg", sha256=base + f"{i:02d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status=status, + )) + await db.commit() + + resp = await client.get("/api/system/stats") + assert resp.status_code == 200 + body = await resp.get_json() + integ = body["integrity"] + assert integ["ok"] >= 2 + assert integ["corrupt"] >= 1 + assert integ["failed_verification"] >= 1 + assert "unknown" in integ diff --git a/tests/test_api_settings_downloader.py b/tests/test_api_settings_downloader.py new file mode 100644 index 0000000..814dec8 --- /dev/null +++ b/tests/test_api_settings_downloader.py @@ -0,0 +1,59 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_get_includes_downloader_fields(client): + resp = await client.get("/api/settings/import") + assert resp.status_code == 200 + body = await resp.get_json() + assert body["download_rate_limit_seconds"] == 3.0 + assert body["download_validate_files"] is True + + +@pytest.mark.asyncio +async def test_patch_download_rate_limit(client): + resp = await client.patch( + "/api/settings/import", json={"download_rate_limit_seconds": 1.5} + ) + assert resp.status_code == 200 + assert (await resp.get_json())["download_rate_limit_seconds"] == 1.5 + + +@pytest.mark.asyncio +async def test_patch_download_validate_files_false(client): + resp = await client.patch( + "/api/settings/import", json={"download_validate_files": False} + ) + assert resp.status_code == 200 + assert (await resp.get_json())["download_validate_files"] is False + + +@pytest.mark.asyncio +async def test_patch_rejects_negative_rate_limit(client): + resp = await client.patch( + "/api/settings/import", json={"download_rate_limit_seconds": -1.0} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_patch_rejects_non_bool_validate(client): + resp = await client.patch( + "/api/settings/import", json={"download_validate_files": "yes"} + ) + assert resp.status_code == 400 diff --git a/tests/test_api_showcase.py b/tests/test_api_showcase.py new file mode 100644 index 0000000..d61e793 --- /dev/null +++ b/tests/test_api_showcase.py @@ -0,0 +1,35 @@ +import pytest + +from backend.app import create_app +from backend.app.models import ImageRecord + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_showcase_returns_images(client, db): + for i in range(8): + db.add(ImageRecord( + path=f"/images/sc/{i}.jpg", sha256=f"c{i:063d}", + size_bytes=1, mime="image/jpeg", width=10, height=10, + origin="imported_filesystem", integrity_status="unknown", + )) + await db.flush() + await db.commit() + resp = await client.get("/api/showcase?limit=4") + assert resp.status_code == 200 + body = await resp.get_json() + assert "images" in body and len(body["images"]) <= 4 + + +@pytest.mark.asyncio +async def test_showcase_rejects_bad_limit(client): + resp = await client.get("/api/showcase?limit=oops") + assert resp.status_code == 400 diff --git a/tests/test_api_sources.py b/tests/test_api_sources.py new file mode 100644 index 0000000..9c58430 --- /dev/null +++ b/tests/test_api_sources.py @@ -0,0 +1,173 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def artist(db): + a = Artist(name="Alice", slug="alice") + db.add(a) + await db.commit() + return a + + +@pytest.mark.asyncio +async def test_create_list_get_delete(client, artist): + create = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", + "url": "https://patreon.com/alice", + }) + assert create.status_code == 201 + created = await create.get_json() + assert created["artist_name"] == "Alice" + sid = created["id"] + + listing = await client.get("/api/sources") + assert listing.status_code == 200 + assert len(await listing.get_json()) == 1 + + filtered = await client.get(f"/api/sources?artist_id={artist.id}") + assert filtered.status_code == 200 + assert len(await filtered.get_json()) == 1 + + one = await client.get(f"/api/sources/{sid}") + assert one.status_code == 200 + assert (await one.get_json())["id"] == sid + + deleted = await client.delete(f"/api/sources/{sid}") + assert deleted.status_code == 204 + assert (await client.get(f"/api/sources/{sid}")).status_code == 404 + + +@pytest.mark.asyncio +async def test_create_rejects_unknown_platform(client, artist): + resp = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "myspace", "url": "https://m/x", + }) + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "unknown_platform" + assert "known" in body + + +@pytest.mark.asyncio +async def test_create_rejects_bad_config(client, artist): + resp = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", + "url": "https://p/a", "config_overrides": [1, 2, 3], + }) + assert resp.status_code == 400 + assert (await resp.get_json())["error"] == "invalid_config" + + +@pytest.mark.asyncio +async def test_create_rejects_empty_url(client, artist): + resp = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", "url": " ", + }) + assert resp.status_code == 400 + assert (await resp.get_json())["error"] == "empty_url" + + +@pytest.mark.asyncio +async def test_create_rejects_unknown_artist(client): + resp = await client.post("/api/sources", json={ + "artist_id": 99999, "platform": "patreon", "url": "https://p/a", + }) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_create_duplicate_returns_409_with_existing_id(client, artist): + a = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", "url": "https://p/a", + }) + first = (await a.get_json())["id"] + b = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", "url": "https://p/a", + }) + assert b.status_code == 409 + body = await b.get_json() + assert body["error"] == "duplicate" + assert body["existing_id"] == first + + +@pytest.mark.asyncio +async def test_patch_updates_fields(client, artist): + create = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", "url": "https://p/a", + }) + sid = (await create.get_json())["id"] + patch = await client.patch(f"/api/sources/{sid}", json={"enabled": False}) + assert patch.status_code == 200 + assert (await patch.get_json())["enabled"] is False + + +@pytest.mark.asyncio +async def test_get_404(client): + assert (await client.get("/api/sources/99999")).status_code == 404 + + +@pytest.mark.asyncio +async def test_patch_404(client): + assert (await client.patch("/api/sources/99999", json={"enabled": False})).status_code == 404 + + +@pytest.mark.asyncio +async def test_delete_404(client): + assert (await client.delete("/api/sources/99999")).status_code == 404 + + +# --- FC-3d: SourceRecord surfaces consecutive_failures + next_check_at ---- + + +@pytest.mark.asyncio +async def test_create_response_includes_fc3d_fields(client, artist): + resp = await client.post("/api/sources", json={ + "artist_id": artist.id, "platform": "patreon", + "url": "https://patreon.com/alice-fc3d-new", + }) + assert resp.status_code == 201 + body = await resp.get_json() + assert body["consecutive_failures"] == 0 + assert body["next_check_at"] is None # never checked yet + + +@pytest.mark.asyncio +async def test_list_derives_next_check_at_when_last_checked_set( + client, artist, db, +): + from datetime import UTC, datetime + + from backend.app.models import Source + + src = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice-fc3d-checked", enabled=True, + consecutive_failures=0, + last_checked_at=datetime.now(UTC), + ) + db.add(src) + await db.commit() + + resp = await client.get("/api/sources") + body = await resp.get_json() + target = next( + r for r in body if r["url"] == "https://patreon.com/alice-fc3d-checked" + ) + assert target["next_check_at"] is not None + assert "T" in target["next_check_at"] # ISO 8601 diff --git a/tests/test_api_sources_check.py b/tests/test_api_sources_check.py new file mode 100644 index 0000000..c532738 --- /dev/null +++ b/tests/test_api_sources_check.py @@ -0,0 +1,77 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Artist, DownloadEvent, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.fixture +async def seed(db): + artist = Artist(name="Alice", slug="alice") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", enabled=True, + config_overrides={}, + ) + db.add(source) + await db.commit() + return source + + +@pytest.mark.asyncio +async def test_check_202_creates_pending_event(client, seed, monkeypatch): + monkeypatch.setattr( + "backend.app.tasks.download.download_source.delay", + lambda *a, **k: None, + ) + resp = await client.post(f"/api/sources/{seed.id}/check") + assert resp.status_code == 202 + body = await resp.get_json() + assert body["status"] == "pending" + assert isinstance(body["download_event_id"], int) + + +@pytest.mark.asyncio +async def test_check_404_for_unknown_source(client): + resp = await client.post("/api/sources/99999/check") + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_check_400_for_disabled_source(client, db, seed): + seed.enabled = False + await db.commit() + resp = await client.post(f"/api/sources/{seed.id}/check") + assert resp.status_code == 400 + body = await resp.get_json() + assert body["error"] == "source_disabled" + + +@pytest.mark.asyncio +async def test_check_409_when_already_running(client, db, seed, monkeypatch): + monkeypatch.setattr( + "backend.app.tasks.download.download_source.delay", + lambda *a, **k: None, + ) + existing = DownloadEvent(source_id=seed.id, status="running") + db.add(existing) + await db.commit() + await db.refresh(existing) + resp = await client.post(f"/api/sources/{seed.id}/check") + assert resp.status_code == 409 + body = await resp.get_json() + assert body["download_event_id"] == existing.id diff --git a/tests/test_api_suggestions.py b/tests/test_api_suggestions.py new file mode 100644 index 0000000..50c7704 --- /dev/null +++ b/tests/test_api_suggestions.py @@ -0,0 +1,89 @@ +import pytest + +from backend.app import create_app +from backend.app.celery_app import celery +from backend.app.models import ImageRecord, TagKind +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +@pytest.fixture(autouse=True) +def eager(): + celery.conf.task_always_eager = True + yield + celery.conf.task_always_eager = False + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _img(db, preds): + img = ImageRecord( + path="/images/s.jpg", sha256="s" * 64, size_bytes=1, + mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + tagger_predictions=preds, + ) + db.add(img) + await db.commit() + return img + + +@pytest.mark.asyncio +async def test_get_suggestions(client, db): + img = await _img( + db, {"sword": {"category": "general", "confidence": 0.97}} + ) + resp = await client.get(f"/api/images/{img.id}/suggestions") + assert resp.status_code == 200 + body = await resp.get_json() + assert "general" in body["by_category"] + + +@pytest.mark.asyncio +async def test_accept_requires_tag_id(client, db): + img = await _img(db, {}) + resp = await client.post( + f"/api/images/{img.id}/suggestions/accept", json={} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_accept_then_applied(client, db): + img = await _img(db, {}) + tag = await TagService(db).find_or_create("AcceptMe", TagKind.character) + await db.commit() + resp = await client.post( + f"/api/images/{img.id}/suggestions/accept", json={"tag_id": tag.id} + ) + assert resp.status_code == 204 + + +@pytest.mark.asyncio +async def test_dismiss(client, db): + img = await _img(db, {}) + tag = await TagService(db).find_or_create("DismissMe", TagKind.general) + await db.commit() + resp = await client.post( + f"/api/images/{img.id}/suggestions/dismiss", json={"tag_id": tag.id} + ) + assert resp.status_code == 204 + + +@pytest.mark.asyncio +async def test_alias_requires_fields(client, db): + img = await _img(db, {}) + resp = await client.post( + f"/api/images/{img.id}/suggestions/alias", json={"alias_string": "x"} + ) + assert resp.status_code == 400 diff --git a/tests/test_api_tag_merge.py b/tests/test_api_tag_merge.py new file mode 100644 index 0000000..8b8dd15 --- /dev/null +++ b/tests/test_api_tag_merge.py @@ -0,0 +1,112 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _mk(client, name, kind): + r = await client.post("/api/tags", json={"name": name, "kind": kind}) + return (await r.get_json())["id"] + + +@pytest.mark.asyncio +async def test_rename_collision_returns_rich_409(client): + tgt = await _mk(client, "Canonical", "general") + src = await _mk(client, "Duplicate", "general") + resp = await client.patch(f"/api/tags/{src}", json={"name": "Canonical"}) + assert resp.status_code == 409 + body = await resp.get_json() + assert body["target"]["id"] == tgt + assert body["target"]["name"] == "Canonical" + assert body["source_image_count"] == 0 + assert body["will_alias"] is False + + +@pytest.mark.asyncio +async def test_merge_endpoint_moves_and_deletes(client, monkeypatch): + calls = [] + from backend.app.tasks import ml as ml_tasks + + monkeypatch.setattr( + ml_tasks.apply_allowlist_tags, + "delay", + lambda **kw: calls.append(kw), + ) + tgt = await _mk(client, "Keep", "general") + src = await _mk(client, "Gone", "general") + resp = await client.post( + f"/api/tags/{src}/merge", json={"target_id": tgt} + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["target"]["id"] == tgt + assert body["source_deleted"] is True + # source is gone: renaming something else onto its old name no longer collides + other = await _mk(client, "Other", "general") + r2 = await client.patch(f"/api/tags/{other}", json={"name": "Gone"}) + assert r2.status_code == 200 + + +@pytest.mark.asyncio +async def test_merge_enqueues_backfill_when_target_allowlisted( + client, monkeypatch +): + calls = [] + from backend.app.tasks import ml as ml_tasks + + monkeypatch.setattr( + ml_tasks.apply_allowlist_tags, + "delay", + lambda **kw: calls.append(kw), + ) + tgt = await _mk(client, "AllowTgt", "general") + src = await _mk(client, "AllowSrc", "general") + # No public route adds a tag to the allowlist (it happens via + # accept-suggestion); set the row directly through the app session. + from backend.app.extensions import get_session + from backend.app.models.tag_allowlist import TagAllowlist + + async with get_session() as s: + s.add(TagAllowlist(tag_id=tgt)) + await s.commit() + + resp = await client.post( + f"/api/tags/{src}/merge", json={"target_id": tgt} + ) + assert resp.status_code == 200 + assert calls == [{"tag_id": tgt}] + + +@pytest.mark.asyncio +async def test_merge_self_is_400(client): + t = await _mk(client, "Selfie", "general") + resp = await client.post(f"/api/tags/{t}/merge", json={"target_id": t}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_merge_missing_target_is_404(client): + t = await _mk(client, "Lonely", "general") + resp = await client.post( + f"/api/tags/{t}/merge", json={"target_id": 999999} + ) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_merge_requires_target_id(client): + t = await _mk(client, "NoBody", "general") + resp = await client.post(f"/api/tags/{t}/merge", json={}) + assert resp.status_code == 400 diff --git a/tests/test_api_tags.py b/tests/test_api_tags.py new file mode 100644 index 0000000..39b45a9 --- /dev/null +++ b/tests/test_api_tags.py @@ -0,0 +1,60 @@ +import pytest + +from backend.app import create_app + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_autocomplete_empty(client): + resp = await client.get("/api/tags/autocomplete?q=") + assert resp.status_code == 200 + assert await resp.get_json() == [] + + +@pytest.mark.asyncio +async def test_create_tag(client): + resp = await client.post("/api/tags", json={"name": "Bob", "kind": "artist"}) + assert resp.status_code == 201 + body = await resp.get_json() + assert body["name"] == "Bob" + assert body["kind"] == "artist" + + +@pytest.mark.asyncio +async def test_create_tag_rejects_invalid_kind(client): + resp = await client.post("/api/tags", json={"name": "Bob", "kind": "notakind"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_create_character_without_fandom_ok(client): + resp = await client.post("/api/tags", json={"name": "Alice", "kind": "character"}) + assert resp.status_code == 201 + + +@pytest.mark.asyncio +async def test_create_character_with_bad_fandom_id(client): + artist_resp = await client.post("/api/tags", json={"name": "X", "kind": "artist"}) + artist_id = (await artist_resp.get_json())["id"] + resp = await client.post( + "/api/tags", json={"name": "Y", "kind": "character", "fandom_id": artist_id} + ) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_create_tag_missing_required(client): + resp = await client.post("/api/tags", json={"name": "Bob"}) + assert resp.status_code == 400 diff --git a/tests/test_api_tags_directory.py b/tests/test_api_tags_directory.py new file mode 100644 index 0000000..1865482 --- /dev/null +++ b/tests/test_api_tags_directory.py @@ -0,0 +1,49 @@ +import pytest + +from backend.app import create_app +from backend.app.models import Tag, TagKind + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_directory_endpoint(client, db): + db.add(Tag(name="api_tag", kind=TagKind.general)) + await db.flush() + await db.commit() + resp = await client.get("/api/tags/directory?limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert "cards" in body and "next_cursor" in body + + +@pytest.mark.asyncio +async def test_directory_bad_limit(client): + resp = await client.get("/api/tags/directory?limit=nan") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_directory_invalid_kind_is_400_not_500(client): + # 'copyright' is not a TagKind member; the route must reject it + # cleanly (400) instead of letting the enum cast 500. + resp = await client.get("/api/tags/directory?kind=copyright") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_directory_valid_kind_ok(client, db): + db.add(Tag(name="a_fandom", kind=TagKind.fandom)) + await db.flush() + await db.commit() + resp = await client.get("/api/tags/directory?kind=fandom&limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + assert any(c["name"] == "a_fandom" for c in body["cards"]) diff --git a/tests/test_archive_extractor.py b/tests/test_archive_extractor.py new file mode 100644 index 0000000..24e3a95 --- /dev/null +++ b/tests/test_archive_extractor.py @@ -0,0 +1,64 @@ +import zipfile +from pathlib import Path + +import pytest + +from backend.app.services.archive_extractor import extract_archive, is_archive + + +def test_is_archive(): + assert is_archive(Path("a.zip")) + assert is_archive(Path("a.CBZ")) + assert is_archive(Path("a.rar")) + assert is_archive(Path("a.7z")) + assert not is_archive(Path("a.jpg")) + + +def _zip(path, entries): + with zipfile.ZipFile(path, "w") as zf: + for name, data in entries.items(): + zf.writestr(name, data) + + +def test_zip_members_enumerated(tmp_path): + z = tmp_path / "set.zip" + _zip(z, {"a.jpg": b"img", "sub/b.png": b"img2", "notes.txt": b"hi"}) + with extract_archive(z) as members: + names = sorted(n for n, _ in members) + assert names == ["a.jpg", "notes.txt", "sub/b.png"] + for _, p in members: + assert p.is_file() + for _, p in members: + assert not p.exists() # temp cleaned on exit + + +def test_cbz_alias(tmp_path): + z = tmp_path / "set.cbz" + _zip(z, {"p1.jpg": b"i"}) + with extract_archive(z) as members: + assert [n for n, _ in members] == ["p1.jpg"] + + +def test_corrupt_archive_yields_nothing(tmp_path): + bad = tmp_path / "broken.zip" + bad.write_bytes(b"not a zip at all") + with extract_archive(bad) as members: + assert members == [] + + +def test_7z_roundtrip(tmp_path): + py7zr = pytest.importorskip("py7zr") + a = tmp_path / "s.7z" + (tmp_path / "x.jpg").write_bytes(b"img") + with py7zr.SevenZipFile(a, "w") as zf: + zf.write(tmp_path / "x.jpg", "x.jpg") + with extract_archive(a) as members: + assert [n for n, _ in members] == ["x.jpg"] + + +def test_rar_corrupt_is_failsoft(tmp_path): + pytest.importorskip("rarfile") + bad = tmp_path / "b.rar" + bad.write_bytes(b"Rar! not really") + with extract_archive(bad) as members: + assert members == [] # invalid / no unrar → fail-soft, nothing lost diff --git a/tests/test_artist_directory_service.py b/tests/test_artist_directory_service.py new file mode 100644 index 0000000..fbfbf2f --- /dev/null +++ b/tests/test_artist_directory_service.py @@ -0,0 +1,259 @@ +"""FC-3f: ArtistDirectoryService unit tests. + +Mirrors TagDirectoryService's test shape — covers cursor encode/decode, +ordering by (name ASC, id ASC), substring q filter, platform EXISTS +filter (with the row-duplication trap), pagination, image_count via +LEFT JOIN, and the window-function preview helper. +""" +import pytest + +from backend.app.models import Artist, ImageRecord, Source +from backend.app.services.artist_directory_service import ( + ArtistDirectoryService, + _decode, + _encode, +) + +pytestmark = pytest.mark.integration + + +# --- Cursor round-trip ---------------------------------------------------- + + +def test_cursor_round_trip(): + c = _encode("alice", 5) + assert _decode(c) == ("alice", 5) + + +def test_decode_cursor_rejects_garbage(): + with pytest.raises(ValueError): + _decode("not-base64!!!") + + +def test_cursor_round_trip_handles_pipes_in_name(): + # Names can theoretically contain '|'; rsplit on the LAST '|' must + # still recover (name, id) correctly. + c = _encode("a|b|c", 7) + assert _decode(c) == ("a|b|c", 7) + + +# --- Fixture builders ----------------------------------------------------- + + +async def _seed_artist(db, name: str, *, is_subscription: bool = False): + a = Artist( + name=name, slug=name.lower().replace(" ", "-"), + is_subscription=is_subscription, + ) + db.add(a) + await db.flush() + return a + + +async def _seed_source(db, artist_id: int, platform: str, url: str): + s = Source(artist_id=artist_id, platform=platform, url=url, enabled=True) + db.add(s) + await db.flush() + return s + + +async def _seed_image(db, artist_id: int, suffix: str): + db.add(ImageRecord( + path=f"/images/{suffix}.jpg", + sha256=("0" * 60) + f"{abs(hash(suffix)) % 10000:04d}", + size_bytes=10, mime="image/jpeg", width=10, height=10, + origin="downloaded", artist_id=artist_id, + )) + await db.flush() + + +# --- Ordering + pagination ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_orders_by_name_asc(db): + await _seed_artist(db, "zara-order") + await _seed_artist(db, "alice-order") + await _seed_artist(db, "miles-order") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-order", platform=None, cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-order", "miles-order", "zara-order"] + + +@pytest.mark.asyncio +async def test_list_pagination_across_two_pages(db): + artists = [] + for letter in "abcde": + a = await _seed_artist(db, f"{letter}-page") + artists.append(a) + await db.commit() + + page1 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=None, limit=2, + ) + assert [c["name"] for c in page1.cards] == ["a-page", "b-page"] + assert page1.next_cursor is not None + + page2 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=page1.next_cursor, limit=2, + ) + assert [c["name"] for c in page2.cards] == ["c-page", "d-page"] + assert page2.next_cursor is not None + + page3 = await ArtistDirectoryService(db).list_artists( + q="-page", platform=None, cursor=page2.next_cursor, limit=2, + ) + assert [c["name"] for c in page3.cards] == ["e-page"] + assert page3.next_cursor is None + + +# --- q filter ------------------------------------------------------------ + + +@pytest.mark.asyncio +async def test_list_q_filter_case_insensitive_substring(db): + await _seed_artist(db, "Alice-Q") + await _seed_artist(db, "Bob-Q") + await _seed_artist(db, "Charlie-Q") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="ALICE", platform=None, cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["Alice-Q"] + + +# --- platform filter (correlated EXISTS, not JOIN) ----------------------- + + +@pytest.mark.asyncio +async def test_list_platform_filter_excludes_no_source(db): + a = await _seed_artist(db, "alice-noplat") + await _seed_source(db, a.id, "patreon", "https://p/alice-np") + await _seed_artist(db, "bob-noplat") # no sources + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-noplat", platform="patreon", cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-noplat"] + + +@pytest.mark.asyncio +async def test_list_platform_filter_excludes_wrong_platform(db): + a = await _seed_artist(db, "alice-wplat") + await _seed_source(db, a.id, "deviantart", "https://d/alice-wp") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-wplat", platform="patreon", cursor=None, limit=60, + ) + assert page.cards == [] + + +@pytest.mark.asyncio +async def test_list_platform_filter_does_not_duplicate_artist_with_multiple_sources(db): + """The EXISTS subquery prevents JOIN-style row duplication when an + artist has multiple sources on the requested platform.""" + a = await _seed_artist(db, "alice-multi") + await _seed_source(db, a.id, "patreon", "https://p/alice-m1") + await _seed_source(db, a.id, "patreon", "https://p/alice-m2") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-multi", platform="patreon", cursor=None, limit=60, + ) + assert len(page.cards) == 1 + assert page.cards[0]["name"] == "alice-multi" + + +# --- image_count via LEFT JOIN ------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_image_count_reflects_artist_id_links(db): + a = await _seed_artist(db, "alice-count") + for i in range(4): + await _seed_image(db, a.id, f"count-{i}") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-count", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-count") + assert target["image_count"] == 4 + + +@pytest.mark.asyncio +async def test_list_zero_image_artist_preserved_by_left_join(db): + """LEFT JOIN ImageRecord keeps the artist row even with no images.""" + await _seed_artist(db, "alice-zero") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-zero", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-zero") + assert target["image_count"] == 0 + assert target["preview_thumbnails"] == [] + + +# --- Preview thumbnails (window function) -------------------------------- + + +@pytest.mark.asyncio +async def test_list_previews_returns_top_three_by_image_id_desc(db): + a = await _seed_artist(db, "alice-prev") + for i in range(5): + await _seed_image(db, a.id, f"prev-{i}") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-prev", platform=None, cursor=None, limit=60, + ) + target = next(c for c in page.cards if c["name"] == "alice-prev") + assert len(target["preview_thumbnails"]) == 3 + for url in target["preview_thumbnails"]: + assert url.startswith("/images/thumbs/") + + +# --- is_subscription field plumbed through ------------------------------- + + +@pytest.mark.asyncio +async def test_list_is_subscription_field_present(db): + await _seed_artist(db, "alice-sub", is_subscription=True) + await _seed_artist(db, "bob-sub", is_subscription=False) + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="-sub", platform=None, cursor=None, limit=60, + ) + alice = next(c for c in page.cards if c["name"] == "alice-sub") + bob = next(c for c in page.cards if c["name"] == "bob-sub") + assert alice["is_subscription"] is True + assert bob["is_subscription"] is False + + +# --- Combined q + platform ----------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_combined_q_and_platform(db): + alice = await _seed_artist(db, "alice-combo-f") + await _seed_source(db, alice.id, "patreon", "https://p/alice-combo") + bob = await _seed_artist(db, "bob-combo-f") + await _seed_source(db, bob.id, "patreon", "https://p/bob-combo") + await db.commit() + + page = await ArtistDirectoryService(db).list_artists( + q="alice", platform="patreon", cursor=None, limit=60, + ) + names = [c["name"] for c in page.cards] + assert names == ["alice-combo-f"] diff --git a/tests/test_artist_service.py b/tests/test_artist_service.py new file mode 100644 index 0000000..9858af9 --- /dev/null +++ b/tests/test_artist_service.py @@ -0,0 +1,109 @@ +from datetime import UTC, datetime + +import pytest + +from backend.app.models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, + Tag, + TagKind, +) +from backend.app.models.tag import image_tag +from backend.app.services.artist_service import ArtistService + +pytestmark = pytest.mark.integration + + +async def _fixture(db): + artist = Artist(name="Nadia", slug="nadia") + db.add(artist) + await db.flush() + src = Source(artist_id=artist.id, platform="web", url="http://x") + db.add(src) + await db.flush() + post = Post( + source_id=src.id, external_post_id="p1", + post_date=datetime(2026, 3, 1, tzinfo=UTC), + ) + db.add(post) + await db.flush() + img = ImageRecord( + path="/images/a/1.jpg", sha256="a" + "0" * 63, + size_bytes=1, mime="image/jpeg", width=4, height=8, + origin="downloaded", integrity_status="unknown", + # FC-2d-vii-c: provenance images also carry the canonical + # artist_id (set by importer/migration); ArtistService reads it. + artist_id=artist.id, + ) + db.add(img) + await db.flush() + db.add(ImageProvenance( + image_record_id=img.id, post_id=post.id, source_id=src.id)) + tag = Tag(name="forest", kind=TagKind.general) + db.add(tag) + await db.flush() + await db.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=tag.id, source="manual")) + await db.flush() + return artist, src, img + + +@pytest.mark.asyncio +async def test_overview_aggregates(db): + artist, src, img = await _fixture(db) + svc = ArtistService(db) + ov = await svc.overview("nadia") + assert ov["name"] == "Nadia" + assert ov["image_count"] == 1 + assert ov["date_range"]["min"].startswith("2026-03-01") + assert ov["date_range"]["max"].startswith("2026-03-01") + assert any(t["name"] == "forest" for t in ov["cooccurring_tags"]) + assert ov["sources"][0]["image_count"] == 1 + assert ov["activity"][0]["count"] == 1 + + +@pytest.mark.asyncio +async def test_overview_unknown_slug_returns_none(db): + svc = ArtistService(db) + assert await svc.overview("ghost") is None + + +@pytest.mark.asyncio +async def test_paged_images(db): + artist, src, img = await _fixture(db) + svc = ArtistService(db) + page = await svc.images("nadia", cursor=None, limit=10) + assert page is not None + assert len(page.images) == 1 + assert page.images[0]["thumbnail_url"].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_paged_images_unknown_slug_none(db): + svc = ArtistService(db) + assert await svc.images("ghost", cursor=None, limit=10) is None + + +@pytest.mark.asyncio +async def test_overview_and_images_include_artist_id_only(db): + # FC-2d-vii-c: a folder-style image has artist_id but no provenance. + a = Artist(name="Solo", slug="solo") + db.add(a) + await db.flush() + rec = ImageRecord( + path="/images/s/1.jpg", sha256="s" + "0" * 63, + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + artist_id=a.id, + ) + db.add(rec) + await db.flush() + + svc = ArtistService(db) + ov = await svc.overview("solo") + assert ov["image_count"] == 1 + page = await svc.images("solo", cursor=None, limit=10) + assert [i["id"] for i in page.images] == [rec.id] diff --git a/tests/test_artist_service_find_or_create.py b/tests/test_artist_service_find_or_create.py new file mode 100644 index 0000000..4bc3959 --- /dev/null +++ b/tests/test_artist_service_find_or_create.py @@ -0,0 +1,56 @@ +import pytest + +from backend.app.models import Artist +from backend.app.services.artist_service import ArtistService + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_find_or_create_creates_new(db): + svc = ArtistService(db) + artist, created = await svc.find_or_create("Brand New") + assert created is True + assert artist.id is not None + assert artist.slug == "brand-new" + + +@pytest.mark.asyncio +async def test_find_or_create_idempotent_on_same_name(db): + svc = ArtistService(db) + a1, c1 = await svc.find_or_create("Alice") + a2, c2 = await svc.find_or_create("Alice") + assert c1 is True + assert c2 is False + assert a1.id == a2.id + + +@pytest.mark.asyncio +async def test_find_or_create_rejects_empty_name(db): + svc = ArtistService(db) + with pytest.raises(ValueError): + await svc.find_or_create(" ") + + +@pytest.mark.asyncio +async def test_autocomplete_ranks_exact_prefix_substring(db): + db.add_all([ + Artist(name="Alice", slug="alice"), + Artist(name="Alice Cooper", slug="alice-cooper"), + Artist(name="Malice", slug="malice"), + Artist(name="Bob", slug="bob"), + ]) + await db.flush() + svc = ArtistService(db) + rows = await svc.autocomplete("alice") + names = [r.name for r in rows] + assert names[0] == "Alice" # exact first + assert names[1] == "Alice Cooper" # then prefix + assert "Malice" in names # then substring + assert "Bob" not in names + + +@pytest.mark.asyncio +async def test_autocomplete_empty_query_returns_empty(db): + svc = ArtistService(db) + assert await svc.autocomplete("") == [] diff --git a/tests/test_attachment_store.py b/tests/test_attachment_store.py new file mode 100644 index 0000000..f10ad3a --- /dev/null +++ b/tests/test_attachment_store.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from backend.app.services.attachment_store import AttachmentStore + + +def test_store_layout_and_dedup(tmp_path): + src = tmp_path / "pack.zip" + src.write_bytes(b"\x00\x01binary\xff") + store = AttachmentStore(images_root=tmp_path / "images") + sha = "ab" + "c" * 62 + + p1 = store.store(src, sha) + assert p1.endswith(f"attachments/{sha[:3]}/{sha}.zip") + assert Path(p1).read_bytes() == b"\x00\x01binary\xff" + + # second store of same sha: returns same path, no rewrite + before = Path(p1).stat().st_mtime_ns + p2 = store.store(src, sha) + assert p2 == p1 + assert Path(p1).stat().st_mtime_ns == before + + +def test_store_preserves_extension_case_insensitive(tmp_path): + src = tmp_path / "Doc.PDF" + src.write_bytes(b"%PDF-1.4") + store = AttachmentStore(images_root=tmp_path / "images") + p = store.store(src, "f" * 64) + assert p.endswith(".pdf") diff --git a/tests/test_backfill_phash.py b/tests/test_backfill_phash.py new file mode 100644 index 0000000..b16d1ac --- /dev/null +++ b/tests/test_backfill_phash.py @@ -0,0 +1,81 @@ +"""FC-2d-vi: library-wide phash backfill (keyset, idempotent).""" + +import pytest +from PIL import Image + +from backend.app.models import ImageRecord +from backend.app.tasks.maintenance import backfill_phash + +pytestmark = pytest.mark.integration + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + """sessionmaker-like returning the test's bound session, so the task + runs against the same transaction / sees the seeded rows.""" + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _img(path, color): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (64, 64), color).save(path, "JPEG") + + +def _rec(db_sync, path, *, sha, mime="image/jpeg", phash=None): + rec = ImageRecord( + path=str(path), sha256=sha, phash=phash, size_bytes=1, + mime=mime, width=64, height=64, origin="imported_filesystem", + integrity_status="unknown", + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def test_backfill_fills_null_image_phash(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + p = tmp_path / "a.jpg" + _img(p, (10, 120, 200)) + pre = tmp_path / "preset.jpg" + _img(pre, (5, 5, 5)) + + rec = _rec(db_sync, p, sha="a" + "0" * 63, phash=None) + vid = _rec( + db_sync, tmp_path / "v.mp4", sha="v" + "0" * 63, + mime="video/mp4", phash=None, + ) + missing = _rec( + db_sync, tmp_path / "gone.jpg", sha="g" + "0" * 63, phash=None, + ) + preset = _rec( + db_sync, pre, sha="p" + "0" * 63, phash="deadbeefdeadbeef", + ) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + updated = backfill_phash() + db_sync.expire_all() + assert updated == 1 + assert db_sync.get(ImageRecord, rec.id).phash is not None + assert db_sync.get(ImageRecord, vid.id).phash is None + assert db_sync.get(ImageRecord, missing.id).phash is None + assert db_sync.get(ImageRecord, preset.id).phash == "deadbeefdeadbeef" + + # idempotent: a second run touches nothing + assert backfill_phash() == 0 diff --git a/tests/test_bulk_tag_service.py b/tests/test_bulk_tag_service.py new file mode 100644 index 0000000..cb8d1c5 --- /dev/null +++ b/tests/test_bulk_tag_service.py @@ -0,0 +1,122 @@ +import pytest +from sqlalchemy import func, select + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.tag import image_tag +from backend.app.models.tag_suggestion_rejection import TagSuggestionRejection +from backend.app.services.bulk_tag_service import BulkTagService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _img(db): + global _SEQ + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_bulk_{_SEQ}.png", + sha256=f"{_SEQ:064d}", + size_bytes=1, + mime="image/png", + origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +@pytest.mark.asyncio +async def test_common_tags_only_returns_tags_on_every_image(db): + tags = TagService(db) + t_all = await tags.find_or_create("OnAll", TagKind.general) + t_some = await tags.find_or_create("OnSome", TagKind.general) + i1, i2 = await _img(db), await _img(db) + await tags.add_to_image(i1, t_all.id) + await tags.add_to_image(i2, t_all.id) + await tags.add_to_image(i1, t_some.id) # only one image + svc = BulkTagService(db) + result = await svc.common_tags([i1, i2]) + assert [t["id"] for t in result] == [t_all.id] + assert result[0]["name"] == "OnAll" + assert result[0]["kind"] == "general" + + +@pytest.mark.asyncio +async def test_common_tags_empty_when_no_images(db): + svc = BulkTagService(db) + assert await svc.common_tags([]) == [] + + +@pytest.mark.asyncio +async def test_bulk_add_applies_and_counts_idempotent(db): + tags = TagService(db) + tag = await tags.find_or_create("BulkAdd", TagKind.general) + i1, i2 = await _img(db), await _img(db) + svc = BulkTagService(db) + added = await svc.bulk_add([i1, i2], tag.id) + assert added == 2 + # Idempotent: re-adding adds nothing. + again = await svc.bulk_add([i1, i2], tag.id) + assert again == 0 + rows = ( + await db.execute( + select(image_tag.c.image_record_id, image_tag.c.source).where( + image_tag.c.tag_id == tag.id + ) + ) + ).all() + assert {r.image_record_id for r in rows} == {i1, i2} + assert all(r.source == "manual" for r in rows) + + +@pytest.mark.asyncio +async def test_bulk_add_records_explicit_source(db): + tags = TagService(db) + tag = await tags.find_or_create("BulkAddML", TagKind.general) + i1 = await _img(db) + svc = BulkTagService(db) + await svc.bulk_add([i1], tag.id, source="ml_accepted") + src = await db.scalar( + select(image_tag.c.source).where(image_tag.c.tag_id == tag.id) + ) + assert src == "ml_accepted" + + +@pytest.mark.asyncio +async def test_bulk_remove_counts_and_records_rejections(db): + tags = TagService(db) + tag = await tags.find_or_create("BulkRem", TagKind.general) + i1, i2 = await _img(db), await _img(db) + svc = BulkTagService(db) + await svc.bulk_add([i1, i2], tag.id) + removed = await svc.bulk_remove([i1, i2], tag.id) + assert removed == 2 + remaining = await db.scalar( + select(func.count()) + .select_from(image_tag) + .where(image_tag.c.tag_id == tag.id) + ) + assert remaining == 0 + rej = ( + await db.execute( + select(TagSuggestionRejection.image_record_id).where( + TagSuggestionRejection.tag_id == tag.id + ) + ) + ).scalars().all() + assert set(rej) == {i1, i2} + + +@pytest.mark.asyncio +async def test_bulk_remove_rejection_is_idempotent(db): + tags = TagService(db) + tag = await tags.find_or_create("BulkRem2", TagKind.general) + i1 = await _img(db) + svc = BulkTagService(db) + await svc.bulk_add([i1], tag.id) + await svc.bulk_remove([i1], tag.id) + # Second remove: nothing to delete, rejection already present, no error. + removed = await svc.bulk_remove([i1], tag.id) + assert removed == 0 diff --git a/tests/test_celery_smoke.py b/tests/test_celery_smoke.py new file mode 100644 index 0000000..40b0054 --- /dev/null +++ b/tests/test_celery_smoke.py @@ -0,0 +1,24 @@ +"""Smoke test — celery runs the ping task in eager mode (no broker required).""" + +import pytest + +from backend.app.celery_app import celery +from backend.app.tasks.smoke import ping + + +@pytest.fixture(autouse=True) +def celery_eager(): + celery.conf.task_always_eager = True + celery.conf.task_eager_propagates = True + yield + celery.conf.task_always_eager = False + + +def test_ping_returns_pong(): + result = ping.delay() + assert result.get(timeout=1) == "pong" + + +def test_ping_with_value(): + result = ping.delay("hello") + assert result.get(timeout=1) == "hello" diff --git a/tests/test_cleanup_old_download_events.py b/tests/test_cleanup_old_download_events.py new file mode 100644 index 0000000..cba325d --- /dev/null +++ b/tests/test_cleanup_old_download_events.py @@ -0,0 +1,108 @@ +"""FC-3d: cleanup_old_download_events task tests. + +Asserts: (1) old terminal events are deleted, (2) recent terminal +events survive, (3) pending/running events survive regardless of age. +""" +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +import backend.app.tasks.maintenance # noqa: F401 — side-effect: register task +from backend.app.celery_app import celery +from backend.app.config import get_config +from backend.app.models import Artist, DownloadEvent, ImportSettings, Source + +pytestmark = pytest.mark.integration + + +def _sync_session(): + cfg = get_config() + engine = create_engine(cfg.database_url_sync, future=True, pool_pre_ping=True) + return sessionmaker(engine, expire_on_commit=False)() + + +def test_cleanup_deletes_old_terminal_events_only(): + """The maintenance task opens its own sync session and commits, so we + seed via a separate sync session and rely on the autouse + _reset_db_after_integration TRUNCATE to clean up afterward. + """ + from backend.app.tasks.maintenance import cleanup_old_download_events + + s = _sync_session() + try: + # Force retention to 30 days for the test. + settings = s.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + settings.download_event_retention_days = 30 + s.commit() + + artist = Artist(name="cleanup-a", slug="cleanup-a") + s.add(artist) + s.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://cleanup-x", enabled=True, + ) + s.add(source) + s.flush() + + old_cutoff = datetime.now(UTC) - timedelta(days=60) + recent = datetime.now(UTC) - timedelta(days=5) + + old_ok = DownloadEvent( + source_id=source.id, status="ok", + started_at=old_cutoff, finished_at=old_cutoff, + ) + old_err = DownloadEvent( + source_id=source.id, status="error", + started_at=old_cutoff, finished_at=old_cutoff, + error="x", + ) + old_skip = DownloadEvent( + source_id=source.id, status="skipped", + started_at=old_cutoff, finished_at=old_cutoff, + ) + old_pending = DownloadEvent( + source_id=source.id, status="pending", + started_at=old_cutoff, + ) + old_running = DownloadEvent( + source_id=source.id, status="running", + started_at=old_cutoff, + ) + recent_ok = DownloadEvent( + source_id=source.id, status="ok", + started_at=recent, finished_at=recent, + ) + for ev in (old_ok, old_err, old_skip, old_pending, old_running, recent_ok): + s.add(ev) + s.commit() + ids = { + "old_ok": old_ok.id, + "old_err": old_err.id, + "old_skip": old_skip.id, + "old_pending": old_pending.id, + "old_running": old_running.id, + "recent_ok": recent_ok.id, + } + finally: + s.close() + + deleted = cleanup_old_download_events() + assert deleted == 3 + + s2 = _sync_session() + try: + survivors = set(s2.execute( + select(DownloadEvent.id).where(DownloadEvent.id.in_(list(ids.values()))) + ).scalars().all()) + finally: + s2.close() + assert survivors == {ids["old_pending"], ids["old_running"], ids["recent_ok"]} + + +def test_cleanup_task_is_registered(): + assert "backend.app.tasks.maintenance.cleanup_old_download_events" in celery.tasks diff --git a/tests/test_credential_crypto.py b/tests/test_credential_crypto.py new file mode 100644 index 0000000..199b838 --- /dev/null +++ b/tests/test_credential_crypto.py @@ -0,0 +1,52 @@ +import os +import stat +from pathlib import Path + +import pytest + +from backend.app.services.credential_crypto import ( + CredentialCrypto, + InvalidCredentialBlob, +) + +pytestmark = pytest.mark.integration + + +def test_load_or_create_writes_key_with_mode_0600(tmp_path): + key_path = tmp_path / "secrets" / "credential_key.b64" + CredentialCrypto(key_path) + assert key_path.exists() + mode = stat.S_IMODE(os.stat(key_path).st_mode) + assert mode == 0o600 + # parent dir is 0o700 + parent_mode = stat.S_IMODE(os.stat(key_path.parent).st_mode) + assert parent_mode == 0o700 + + +def test_load_existing_key_is_idempotent(tmp_path): + key_path = tmp_path / "credential_key.b64" + crypto1 = CredentialCrypto(key_path) + contents_after_first = key_path.read_bytes() + crypto2 = CredentialCrypto(key_path) + contents_after_second = key_path.read_bytes() + assert contents_after_first == contents_after_second + # And both crypto instances decrypt each other's ciphertext + ct = crypto1.encrypt("hello") + assert crypto2.decrypt(ct) == "hello" + + +def test_encrypt_decrypt_round_trip(tmp_path): + crypto = CredentialCrypto(tmp_path / "k") + plaintext = "domain.com\tTRUE\t/\tTRUE\t1700000000\tname\tvalue" + ct = crypto.encrypt(plaintext) + assert isinstance(ct, bytes) + assert ct != plaintext.encode() + assert crypto.decrypt(ct) == plaintext + + +def test_decrypt_with_wrong_key_raises(tmp_path): + crypto_a = CredentialCrypto(tmp_path / "a") + crypto_b = CredentialCrypto(tmp_path / "b") + ct = crypto_a.encrypt("secret") + with pytest.raises(InvalidCredentialBlob): + crypto_b.decrypt(ct) diff --git a/tests/test_credential_service.py b/tests/test_credential_service.py new file mode 100644 index 0000000..4d85f41 --- /dev/null +++ b/tests/test_credential_service.py @@ -0,0 +1,124 @@ +import pytest +from sqlalchemy import select + +from backend.app.models import Credential +from backend.app.services.credential_crypto import CredentialCrypto +from backend.app.services.credential_service import ( + CredentialService, + EmptyDataError, + UnknownPlatformError, + WrongAuthTypeError, +) + +pytestmark = pytest.mark.integration + + +_NETSCAPE = ( + "# Netscape HTTP Cookie File\n" + ".patreon.com\tTRUE\t/\tTRUE\t1700000000\tsession_id\tabc\n" +) + + +@pytest.fixture +def crypto(tmp_path): + return CredentialCrypto(tmp_path / "k") + + +@pytest.mark.asyncio +async def test_upsert_creates_then_updates(db, crypto): + svc = CredentialService(db, crypto) + rec = await svc.upsert( + platform="patreon", credential_type="cookies", data=_NETSCAPE, + ) + assert rec.platform == "patreon" + assert rec.credential_type == "cookies" + again = await svc.upsert( + platform="patreon", credential_type="cookies", data=_NETSCAPE + "x", + ) + assert again.platform == "patreon" + rows = (await db.execute( + select(Credential).where(Credential.platform == "patreon") + )).scalars().all() + assert len(rows) == 1 + + +@pytest.mark.asyncio +async def test_upsert_rejects_unknown_platform(db, crypto): + svc = CredentialService(db, crypto) + with pytest.raises(UnknownPlatformError): + await svc.upsert(platform="fanbox", credential_type="cookies", data="x") + + +@pytest.mark.asyncio +async def test_upsert_rejects_wrong_auth_type(db, crypto): + svc = CredentialService(db, crypto) + # discord is token-only + with pytest.raises(WrongAuthTypeError): + await svc.upsert(platform="discord", credential_type="cookies", data="x") + # patreon is cookies-only + with pytest.raises(WrongAuthTypeError): + await svc.upsert(platform="patreon", credential_type="token", data="x") + + +@pytest.mark.asyncio +async def test_upsert_rejects_empty_data(db, crypto): + svc = CredentialService(db, crypto) + with pytest.raises(EmptyDataError): + await svc.upsert(platform="patreon", credential_type="cookies", data=" ") + + +@pytest.mark.asyncio +async def test_list_excludes_blob(db, crypto): + svc = CredentialService(db, crypto) + await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) + records = await svc.list() + assert len(records) == 1 + d = records[0].to_dict() + assert "data" not in d + assert "encrypted_blob" not in d + + +@pytest.mark.asyncio +async def test_delete(db, crypto): + svc = CredentialService(db, crypto) + await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) + await svc.delete("patreon") + assert await svc.get("patreon") is None + + +@pytest.mark.asyncio +async def test_get_cookies_path_writes_usable_file(db, crypto, tmp_path): + svc = CredentialService(db, crypto, cookies_dir=tmp_path / "cookies") + await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) + path = await svc.get_cookies_path("patreon") + assert path is not None and path.exists() + contents = path.read_text() + assert ".patreon.com" in contents + assert "session_id" in contents + + +@pytest.mark.asyncio +async def test_get_cookies_path_none_for_missing(db, crypto, tmp_path): + svc = CredentialService(db, crypto, cookies_dir=tmp_path) + assert await svc.get_cookies_path("patreon") is None + + +@pytest.mark.asyncio +async def test_get_cookies_path_none_for_token_kind(db, crypto, tmp_path): + svc = CredentialService(db, crypto, cookies_dir=tmp_path) + await svc.upsert(platform="discord", credential_type="token", data="tok") + assert await svc.get_cookies_path("discord") is None + + +@pytest.mark.asyncio +async def test_get_token_decrypts(db, crypto): + svc = CredentialService(db, crypto) + await svc.upsert(platform="discord", credential_type="token", data="my-token") + assert await svc.get_token("discord") == "my-token" + + +@pytest.mark.asyncio +async def test_get_token_none_for_cookies_kind(db, crypto): + svc = CredentialService(db, crypto) + await svc.upsert(platform="patreon", credential_type="cookies", data=_NETSCAPE) + assert await svc.get_token("patreon") is None diff --git a/tests/test_download_models.py b/tests/test_download_models.py new file mode 100644 index 0000000..edd13be --- /dev/null +++ b/tests/test_download_models.py @@ -0,0 +1,45 @@ +"""download_models tests. No network in CI; we test the 'already present → +skip' short-circuit by faking the expected files, and that main() wires +both ensure_* calls. +""" + +from unittest.mock import patch + +from backend.app.scripts import download_models as dm + + +def test_ensure_camie_skips_when_present(tmp_path, monkeypatch): + monkeypatch.setattr(dm, "MODEL_ROOT", tmp_path) + camie = tmp_path / "camie" + camie.mkdir(parents=True) + (camie / "model.onnx").write_bytes(b"x") + (camie / "selected_tags.csv").write_text("tag_id,name,category,count\n") + with patch.object(dm, "_snapshot") as snap: + dm.ensure_camie() + snap.assert_not_called() + + +def test_ensure_camie_downloads_when_missing(tmp_path, monkeypatch): + monkeypatch.setattr(dm, "MODEL_ROOT", tmp_path) + with patch.object(dm, "_snapshot") as snap: + dm.ensure_camie() + snap.assert_called_once() + + +def test_ensure_siglip_skips_when_present(tmp_path, monkeypatch): + monkeypatch.setattr(dm, "MODEL_ROOT", tmp_path) + sig = tmp_path / "siglip" + sig.mkdir(parents=True) + (sig / "config.json").write_text("{}") + (sig / "model.safetensors").write_bytes(b"x") + with patch.object(dm, "_snapshot") as snap: + dm.ensure_siglip() + snap.assert_not_called() + + +def test_main_calls_both(monkeypatch): + calls = [] + monkeypatch.setattr(dm, "ensure_camie", lambda: calls.append("camie")) + monkeypatch.setattr(dm, "ensure_siglip", lambda: calls.append("siglip")) + assert dm.main() == 0 + assert calls == ["camie", "siglip"] diff --git a/tests/test_download_service.py b/tests/test_download_service.py new file mode 100644 index 0000000..15ad337 --- /dev/null +++ b/tests/test_download_service.py @@ -0,0 +1,372 @@ +"""DownloadService orchestrator tests. + +GalleryDLService is mocked to return a canned DownloadResult; tests +provide on-disk files for the importer to attach_in_place. +""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from sqlalchemy import select + +from backend.app.models import Artist, DownloadEvent, ImportSettings, Source +from backend.app.services.credential_crypto import CredentialCrypto +from backend.app.services.credential_service import CredentialService +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def seed_artist_and_source(db, db_sync): + artist = Artist(name="Alice", slug="alice") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", enabled=True, + config_overrides={}, + ) + db.add(source) + await db.commit() + return artist, source + + +def _make_jpg(path: Path, split: str = "h"): + """Write a structured JPEG so phash isn't degenerate. + + Solid-color images all phash-collapse (DCT of a flat image is zero). + Use `split='h'` (top/bottom halves) or `split='v'` (left/right) to + give each fixture file a distinct phash. + """ + from PIL import Image + path.parent.mkdir(parents=True, exist_ok=True) + im = Image.new("RGB", (64, 64), (255, 255, 255)) + px = im.load() + if split == "h": + for y in range(32): + for x in range(64): + px[x, y] = (0, 0, 0) + else: # 'v' + for y in range(64): + for x in range(32): + px[x, y] = (0, 0, 0) + im.save(path, "JPEG") + + +def _make_fake_dl_result( + *, success=True, written_paths=None, quarantined_paths=None, + files_downloaded=0, error_type=None, error_message=None, + stdout="", stderr="", +): + return SimpleNamespace( + success=success, + url="https://patreon.com/alice", + artist_slug="alice", + platform="patreon", + files_downloaded=files_downloaded, + files_quarantined=len(quarantined_paths or []), + quarantined_paths=quarantined_paths or [], + written_paths=written_paths or [], + stdout=stdout, + stderr=stderr, + return_code=0 if success else 1, + error_type=error_type, + error_message=error_message, + duration_seconds=1.23, + started_at="2026-05-20T14:00:00+00:00", + completed_at="2026-05-20T14:01:00+00:00", + ) + + +def _fake_gdl_with_result(result): + fake = MagicMock() + fake.download = AsyncMock(return_value=result) + fake._compute_run_stats = lambda *a, **k: { + "exit_code": 0, "downloaded_count": len(result.written_paths), + "skipped_count": 0, "per_item_failures": 0, + "warning_count": 0, "tier_gated_count": 0, + } + fake._extract_errors_warnings = lambda *a, **k: "" + fake._truncate_log = lambda x, **k: x + return fake + + +@pytest.mark.asyncio +async def test_download_source_attaches_written_files( + db, db_sync, tmp_path, seed_artist_and_source, +): + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + + images_root = tmp_path / "images" + f1 = images_root / "alice" / "patreon" / "post" / "a.jpg" + f2 = images_root / "alice" / "patreon" / "post" / "b.jpg" + # Structured h/v split → distinct sha256 AND distinct phash; + # also pin phash_threshold=0 so even tiny perceptual similarity + # between the two doesn't trigger duplicate_phash. + _make_jpg(f1, split="h") + _make_jpg(f2, split="v") + + fake_gdl = _fake_gdl_with_result(_make_fake_dl_result( + success=True, + written_paths=[str(f1), str(f2)], + files_downloaded=2, + stdout=f"{f1}\n{f2}\n", + )) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + sync_settings.phash_threshold = 0 # only exact phash collisions dedup + + importer = Importer( + session=db_sync, images_root=images_root, + import_root=images_root, + thumbnailer=Thumbnailer(images_root=images_root), + settings=sync_settings, + ) + crypto = CredentialCrypto(tmp_path / "key.b64") + cred_service = CredentialService(db, crypto) + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + event_id = await svc.download_source(source.id) + + ev = (await db.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + assert ev.status == "ok" + assert ev.files_count == 2 + assert ev.metadata_["import_summary"]["attached"] == 2 + assert ev.metadata_["run_stats"]["downloaded_count"] == 2 + + +@pytest.mark.asyncio +async def test_download_source_skipped_when_disabled( + db, db_sync, tmp_path, seed_artist_and_source, +): + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + source.enabled = False + await db.commit() + + fake_gdl = MagicMock() + fake_gdl.download = AsyncMock() + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path, + import_root=tmp_path, + thumbnailer=Thumbnailer(images_root=tmp_path), + settings=sync_settings, + ) + cred_service = CredentialService(db, MagicMock()) + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + event_id = await svc.download_source(source.id) + + ev = (await db.execute( + select(DownloadEvent).where(DownloadEvent.id == event_id) + )).scalar_one() + assert ev.status == "skipped" + assert ev.error == "source disabled" + fake_gdl.download.assert_not_called() + + +@pytest.mark.asyncio +async def test_in_flight_idempotency_returns_existing_event( + db, db_sync, tmp_path, seed_artist_and_source, +): + from backend.app.services.download_service import DownloadService + + _artist, source = seed_artist_and_source + running = DownloadEvent(source_id=source.id, status="running") + db.add(running) + await db.commit() + await db.refresh(running) + + fake_gdl = MagicMock() + fake_gdl.download = AsyncMock() + importer_stub = MagicMock() + cred_service = CredentialService(db, MagicMock()) + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer_stub, cred_service=cred_service, + ) + event_id = await svc.download_source(source.id) + assert event_id == running.id + fake_gdl.download.assert_not_called() + + +@pytest.mark.asyncio +async def test_patreon_retry_caches_campaign_id( + db, db_sync, tmp_path, seed_artist_and_source, monkeypatch, +): + from backend.app.services.download_service import DownloadService + from backend.app.services.importer import Importer + + _artist, source = seed_artist_and_source + + failed = _make_fake_dl_result( + success=False, written_paths=[], stdout="", + stderr="[patreon][error] Failed to extract campaign ID for alice\n", + ) + succeeded = _make_fake_dl_result(success=True, written_paths=[]) + + download_calls = [] + + async def fake_download(url, **k): + download_calls.append(url) + return failed if len(download_calls) == 1 else succeeded + + fake_gdl = MagicMock() + fake_gdl.download = fake_download + fake_gdl._compute_run_stats = lambda *a, **k: { + "exit_code": 0, "downloaded_count": 0, "skipped_count": 0, + "per_item_failures": 0, "warning_count": 0, "tier_gated_count": 0, + } + fake_gdl._extract_errors_warnings = lambda *a, **k: "" + fake_gdl._truncate_log = lambda x, **k: x + + monkeypatch.setattr( + "backend.app.services.download_service.resolve_campaign_id", + AsyncMock(return_value="99"), + ) + + sync_settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + importer = Importer( + session=db_sync, images_root=tmp_path, + import_root=tmp_path, + thumbnailer=Thumbnailer(images_root=tmp_path), + settings=sync_settings, + ) + cred_service = CredentialService(db, MagicMock()) + + svc = DownloadService( + async_session=db, sync_session=db_sync, + gdl=fake_gdl, importer=importer, cred_service=cred_service, + ) + await svc.download_source(source.id) + + assert len(download_calls) == 2 + assert "id:99" in download_calls[1] + + overrides = db_sync.execute( + select(Source.config_overrides).where(Source.id == source.id) + ).scalar_one() + assert overrides.get("patreon_campaign_id") == "99" + + +# --- FC-3d: finalize hook updates Source health columns ------------------- + + +async def _seed_source_with_health(db, *, failures=0, last_error=None, suffix="fz"): + artist = Artist(name=f"alice-{suffix}", slug=f"alice-{suffix}") + db.add(artist) + await db.flush() + source = Source( + artist_id=artist.id, platform="patreon", + url=f"https://patreon.com/alice-{suffix}", enabled=True, + consecutive_failures=failures, last_error=last_error, + ) + db.add(source) + await db.flush() + event = DownloadEvent(source_id=source.id, status="running") + db.add(event) + await db.commit() + return source.id, event.id + + +@pytest.mark.asyncio +async def test_finalize_ok_resets_failures_and_clears_error(db): + from backend.app.services.download_service import DownloadService + + source_id, event_id = await _seed_source_with_health( + db, failures=3, last_error="prev", suffix="ok", + ) + svc = DownloadService( + async_session=db, sync_session=None, + gdl=None, importer=None, cred_service=None, + ) + await svc._finalize_event_and_source( + event_id=event_id, source_id=source_id, status="ok", + error_message=None, + ) + + row = (await db.execute( + select( + Source.consecutive_failures, Source.last_error, Source.last_checked_at, + ).where(Source.id == source_id) + )).one() + assert row.consecutive_failures == 0 + assert row.last_error is None + assert row.last_checked_at is not None + + +@pytest.mark.asyncio +async def test_finalize_error_increments_failures_and_sets_error(db): + from backend.app.services.download_service import DownloadService + + source_id, event_id = await _seed_source_with_health( + db, failures=2, last_error=None, suffix="err", + ) + svc = DownloadService( + async_session=db, sync_session=None, + gdl=None, importer=None, cred_service=None, + ) + await svc._finalize_event_and_source( + event_id=event_id, source_id=source_id, status="error", + error_message="boom", + ) + + row = (await db.execute( + select( + Source.consecutive_failures, Source.last_error, Source.last_checked_at, + ).where(Source.id == source_id) + )).one() + assert row.consecutive_failures == 3 + assert row.last_error == "boom" + assert row.last_checked_at is not None + + +@pytest.mark.asyncio +async def test_finalize_skipped_preserves_failures_clears_error(db): + from backend.app.services.download_service import DownloadService + + source_id, event_id = await _seed_source_with_health( + db, failures=4, last_error="stale", suffix="skip", + ) + svc = DownloadService( + async_session=db, sync_session=None, + gdl=None, importer=None, cred_service=None, + ) + await svc._finalize_event_and_source( + event_id=event_id, source_id=source_id, status="skipped", + error_message=None, + ) + + row = (await db.execute( + select( + Source.consecutive_failures, Source.last_error, Source.last_checked_at, + ).where(Source.id == source_id) + )).one() + assert row.consecutive_failures == 4 + assert row.last_error is None + assert row.last_checked_at is not None diff --git a/tests/test_download_source_task.py b/tests/test_download_source_task.py new file mode 100644 index 0000000..2a1a946 --- /dev/null +++ b/tests/test_download_source_task.py @@ -0,0 +1,20 @@ +"""Smoke test that the FC-3c Celery task is registered and routed correctly. + +Mirrors test_tasks_register.py — Celery's `include=[...]` is lazy, so +the task module must be imported explicitly to trigger registration. +""" + +# Side-effect import: the @celery.task decorator on download_source fires +# at module import time and registers the task with the global instance. +import backend.app.tasks.download # noqa: F401 +from backend.app.celery_app import celery + + +def test_download_source_is_registered(): + assert "backend.app.tasks.download.download_source" in celery.tasks + + +def test_download_source_routes_to_download_queue(): + routes = celery.conf.task_routes + assert "backend.app.tasks.download.*" in routes + assert routes["backend.app.tasks.download.*"]["queue"] == "download" diff --git a/tests/test_extensions_session.py b/tests/test_extensions_session.py new file mode 100644 index 0000000..8243f8d --- /dev/null +++ b/tests/test_extensions_session.py @@ -0,0 +1,31 @@ +"""Unit: the shared engine is a memoized singleton; dispose resets it. + +create_async_engine does NOT open a connection, so identity checks need no +DB and run in CI (not integration-marked). +""" + +import pytest + +from backend.app import extensions + + +@pytest.mark.asyncio +async def test_get_engine_is_singleton(): + a = extensions.get_engine() + b = extensions.get_engine() + assert a is b + await extensions.dispose_engine() + + +@pytest.mark.asyncio +async def test_dispose_engine_allows_fresh_engine(): + first = extensions.get_engine() + await extensions.dispose_engine() + second = extensions.get_engine() + assert first is not second + await extensions.dispose_engine() + + +def test_get_session_returns_async_context_manager(): + cm = extensions.get_session() + assert hasattr(cm, "__aenter__") and hasattr(cm, "__aexit__") diff --git a/tests/test_file_validator.py b/tests/test_file_validator.py new file mode 100644 index 0000000..5a36ed2 --- /dev/null +++ b/tests/test_file_validator.py @@ -0,0 +1,85 @@ +from backend.app.services.file_validator import ( + is_validatable, + validate_file, +) + + +def test_is_validatable_only_known_extensions(tmp_path): + assert is_validatable(tmp_path / "x.jpg") + assert is_validatable(tmp_path / "x.JPEG") + assert is_validatable(tmp_path / "x.png") + assert is_validatable(tmp_path / "x.gif") + assert is_validatable(tmp_path / "x.webp") + assert not is_validatable(tmp_path / "x.json") + assert not is_validatable(tmp_path / "x.mp4") + + +def test_valid_jpeg_passes(tmp_path): + from PIL import Image + p = tmp_path / "good.jpg" + Image.new("RGB", (32, 32), (200, 50, 50)).save(p, "JPEG") + r = validate_file(p) + assert r.ok is True + assert r.format == "jpeg" + assert r.size > 0 + + +def test_truncated_jpeg_fails(tmp_path): + from PIL import Image + p = tmp_path / "bad.jpg" + Image.new("RGB", (32, 32), (200, 50, 50)).save(p, "JPEG") + raw = p.read_bytes() + p.write_bytes(raw[:-16]) + r = validate_file(p) + assert r.ok is False + assert r.format == "jpeg" + assert "EOI" in r.reason + + +def test_missing_jpeg_soi_fails(tmp_path): + p = tmp_path / "junk.jpg" + p.write_bytes(b"\x00" * 64) + r = validate_file(p) + assert r.ok is False + assert "SOI" in r.reason + + +def test_truncated_png_fails(tmp_path): + from PIL import Image + p = tmp_path / "bad.png" + Image.new("RGB", (32, 32), (10, 10, 10)).save(p, "PNG") + raw = p.read_bytes() + p.write_bytes(raw[:-12]) + r = validate_file(p) + assert r.ok is False + assert r.format == "png" + + +def test_gif_signature_and_trailer(tmp_path): + p = tmp_path / "bad.gif" + p.write_bytes(b"GIF89a" + b"\x00" * 64) + r = validate_file(p) + assert r.ok is False + assert r.format == "gif" + + +def test_unknown_extension_passes(tmp_path): + p = tmp_path / "x.bin" + p.write_bytes(b"\x00" * 64) + r = validate_file(p) + assert r.ok is True + assert r.format is None + + +def test_missing_file_fails_cleanly(tmp_path): + r = validate_file(tmp_path / "nope.jpg") + assert r.ok is False + assert "not found" in r.reason + + +def test_too_small_fails(tmp_path): + p = tmp_path / "tiny.jpg" + p.write_bytes(b"\xff\xd8\xff") + r = validate_file(p) + assert r.ok is False + assert r.reason == "too small" diff --git a/tests/test_gallery_artist_payload.py b/tests/test_gallery_artist_payload.py new file mode 100644 index 0000000..c3c3bbe --- /dev/null +++ b/tests/test_gallery_artist_payload.py @@ -0,0 +1,69 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app import create_app +from backend.app.models import Artist, ImageRecord, Source + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _img(db, n): + rec = ImageRecord( + path=f"/images/ap/{n}.jpg", sha256=f"ap{n:062d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + rec.created_at = datetime.now(UTC) - timedelta(minutes=n) + db.add(rec) + await db.flush() + return rec + + +async def _artist_source(db, name, slug): + a = Artist(name=name, slug=slug) + db.add(a) + await db.flush() + s = Source(artist_id=a.id, platform="patreon", + url=f"https://patreon.test/{slug}") + db.add(s) + await db.flush() + return a, s + + +@pytest.mark.asyncio +async def test_scroll_artist_null_when_none(client, db): + rec = await _img(db, 1) + await db.commit() + resp = await client.get("/api/gallery/scroll?limit=10") + body = await resp.get_json() + item = next(i for i in body["images"] if i["id"] == rec.id) + assert item["artist"] is None + + +@pytest.mark.asyncio +async def test_scroll_artist_via_artist_id(client, db): + # FC-2d-vii-c: the canonical resolution path is image_record.artist_id + # (covers folder-style images with no provenance too). + rec = await _img(db, 1) + a, _ = await _artist_source(db, "Folder", "folder") + rec.artist_id = a.id + await db.flush() + await db.commit() + + resp = await client.get("/api/gallery/scroll?limit=10") + assert resp.status_code == 200 + body = await resp.get_json() + item = next(i for i in body["images"] if i["id"] == rec.id) + assert item["artist"] == {"name": "Folder", "slug": "folder"} diff --git a/tests/test_gallery_dl_service.py b/tests/test_gallery_dl_service.py new file mode 100644 index 0000000..f172643 --- /dev/null +++ b/tests/test_gallery_dl_service.py @@ -0,0 +1,151 @@ +"""GalleryDLService tests — subprocess is mocked so no real gallery-dl runs.""" + +from types import SimpleNamespace + +import pytest + +from backend.app.services.gallery_dl import ( + ErrorType, + GalleryDLService, + SourceConfig, +) + + +@pytest.fixture +def gdl(tmp_path): + return GalleryDLService( + images_root=tmp_path / "images", + rate_limit=3.0, + validate_files=False, + ) + + +def _proc(stdout="", stderr="", returncode=0): + return SimpleNamespace(stdout=stdout, stderr=stderr, returncode=returncode) + + +@pytest.mark.asyncio +async def test_download_happy_path(gdl, monkeypatch): + out_path = gdl.images_root / "alice" / "patreon" / "post1" / "img.jpg" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 32 + b"\xff\xd9") + + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout=str(out_path) + "\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", + artist_slug="alice", + platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is True + assert result.files_downloaded == 1 + assert result.error_type is None + assert str(out_path) in result.written_paths + + +@pytest.mark.asyncio +async def test_download_no_new_content(gdl, monkeypatch): + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout="# /images/alice/patreon/old.jpg\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is True + assert result.files_downloaded == 0 + + +@pytest.mark.asyncio +async def test_download_auth_error_categorization(gdl, monkeypatch): + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc( + stdout="", stderr='[patreon][error] "GET /api/foo HTTP/1.1" 401 None\n', + returncode=1, + ), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.success is False + assert result.error_type == ErrorType.AUTH_ERROR + + +@pytest.mark.asyncio +async def test_download_timeout(gdl, monkeypatch): + import subprocess as sp + + def _raise(*a, **k): + raise sp.TimeoutExpired(cmd="gallery-dl", timeout=10) + + monkeypatch.setattr("backend.app.services.gallery_dl.subprocess.run", _raise) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(timeout=10), + ) + assert result.success is False + assert result.error_type == ErrorType.TIMEOUT + + +@pytest.mark.asyncio +async def test_validation_quarantines_truncated_files(tmp_path, monkeypatch): + gdl = GalleryDLService( + images_root=tmp_path / "images", rate_limit=3.0, validate_files=True, + ) + bad = gdl.images_root / "alice" / "patreon" / "post" / "bad.jpg" + bad.parent.mkdir(parents=True, exist_ok=True) + bad.write_bytes(b"\xff\xd8\xff" + b"\x00" * 100) # SOI but no EOI + + monkeypatch.setattr( + "backend.app.services.gallery_dl.subprocess.run", + lambda *a, **k: _proc(stdout=str(bad) + "\n", returncode=0), + ) + result = await gdl.download( + url="https://patreon.com/alice", artist_slug="alice", platform="patreon", + source_config=SourceConfig(), + ) + assert result.files_quarantined == 1 + assert len(result.quarantined_paths) == 1 + assert not bad.exists() + assert result.error_type == ErrorType.VALIDATION_FAILED + + +def test_compute_run_stats(gdl): + stats = gdl._compute_run_stats( + return_code=0, + stdout="/path/a.jpg\n/path/b.jpg\n# /path/c.jpg\n", + stderr="[patreon][warning] foo\n[patreon][download][error] failed to download /x\n", + ) + assert stats["downloaded_count"] == 2 + assert stats["skipped_count"] == 1 + assert stats["warning_count"] >= 1 + assert stats["per_item_failures"] == 1 + + +def test_extract_errors_warnings(gdl): + stderr = ( + "[patreon][debug] foo\n[patreon][error] bar\n" + "[urllib3][debug] baz\n[patreon][warning] qux\n" + ) + result = gdl._extract_errors_warnings(stderr) + assert "[error]" in result + assert "[warning]" in result + assert "[debug]" not in result + + +def test_truncate_log_caps_large_text(gdl): + huge = "line\n" * 200_000 + result = gdl._truncate_log(huge, max_bytes=10_000) + assert len(result.encode()) < 11_000 + assert "lines elided" in result + + +def test_truncate_log_passes_short_text(gdl): + text = "small\n" + assert gdl._truncate_log(text) == text diff --git a/tests/test_gallery_provenance_filter.py b/tests/test_gallery_provenance_filter.py new file mode 100644 index 0000000..a7f2ba4 --- /dev/null +++ b/tests/test_gallery_provenance_filter.py @@ -0,0 +1,133 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app import create_app +from backend.app.models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, +) +from backend.app.services.gallery_service import GalleryService + +pytestmark = pytest.mark.integration + + +@pytest.fixture +async def app(): + return create_app() + + +@pytest.fixture +async def client(app): + async with app.test_client() as c: + yield c + + +async def _img(db, n): + base = datetime.now(UTC) + rec = ImageRecord( + path=f"/images/f/{n}.jpg", sha256=f"f{n:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + rec.created_at = base - timedelta(minutes=n) + db.add(rec) + await db.flush() + return rec + + +async def _post(db, artist_name, slug, ext): + a = Artist(name=artist_name, slug=slug) + db.add(a) + await db.flush() + s = Source(artist_id=a.id, platform="patreon", + url=f"https://patreon.test/{slug}") + db.add(s) + await db.flush() + p = Post(source_id=s.id, external_post_id=ext) + db.add(p) + await db.flush() + return a, s, p + + +@pytest.mark.asyncio +async def test_scroll_post_id_filter(db): + i1 = await _img(db, 1) + i2 = await _img(db, 2) + await _img(db, 3) # unmatched image — proves the filter excludes it + _, s, p = await _post(db, "A", "a", "10") + db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id, + source_id=s.id)) + db.add(ImageProvenance(image_record_id=i2.id, post_id=p.id, + source_id=s.id)) + await db.flush() + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10, post_id=p.id) + assert {x.id for x in page.images} == {i1.id, i2.id} + + +@pytest.mark.asyncio +async def test_scroll_post_id_dedups_multi_rows(db): + i1 = await _img(db, 1) + _, s, p = await _post(db, "A", "a", "10") + # two provenance rows, same image+post (enrich-on-duplicate shape) + db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id, + source_id=s.id)) + await db.flush() + db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id, + source_id=s.id)) + await db.flush() + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10, post_id=p.id) + assert [x.id for x in page.images] == [i1.id] # appears once + + +@pytest.mark.asyncio +async def test_scroll_artist_id_filter(db): + i1 = await _img(db, 1) + await _img(db, 2) # unmatched image — proves the filter excludes it + a, s, p = await _post(db, "A", "a", "10") + db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id, + source_id=s.id)) + await db.flush() + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10, artist_id=a.id) + assert [x.id for x in page.images] == [i1.id] + + +@pytest.mark.asyncio +async def test_mutually_exclusive_filters_raise(db): + svc = GalleryService(db) + with pytest.raises(ValueError): + await svc.scroll(cursor=None, limit=10, tag_id=1, post_id=2) + + +@pytest.mark.asyncio +async def test_timeline_and_jump_accept_post_id(db): + i1 = await _img(db, 1) + _, s, p = await _post(db, "A", "a", "10") + db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id, + source_id=s.id)) + await db.flush() + svc = GalleryService(db) + buckets = await svc.timeline(post_id=p.id) + assert sum(b.count for b in buckets) == 1 + cur = await svc.jump_cursor( + year=i1.created_at.year, month=i1.created_at.month, post_id=p.id + ) + assert cur is not None + + +@pytest.mark.asyncio +async def test_api_scroll_rejects_combined_filters(client): + resp = await client.get("/api/gallery/scroll?tag_id=1&post_id=2") + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_api_timeline_rejects_combined_filters(client): + resp = await client.get("/api/gallery/timeline?post_id=1&artist_id=2") + assert resp.status_code == 400 diff --git a/tests/test_gallery_service.py b/tests/test_gallery_service.py new file mode 100644 index 0000000..ecba6bb --- /dev/null +++ b/tests/test_gallery_service.py @@ -0,0 +1,135 @@ +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.gallery_service import ( + GalleryService, + decode_cursor, + encode_cursor, +) + +pytestmark = pytest.mark.integration + + +def _now(): + return datetime.now(UTC) + + +async def _seed_images(db, count: int, sha_prefix: str = "0") -> list[ImageRecord]: + base = _now() + records = [] + for i in range(count): + r = ImageRecord( + path=f"/images/test/{i}.jpg", + sha256=f"{sha_prefix}{i:063d}", + size_bytes=1000, + mime="image/jpeg", + width=100, + height=100, + origin="imported_filesystem", + integrity_status="unknown", + ) + r.created_at = base - timedelta(minutes=i) + db.add(r) + records.append(r) + await db.flush() + return records + + +def test_cursor_roundtrip(): + ts = datetime(2026, 5, 14, 12, 30, 0, tzinfo=UTC) + encoded = encode_cursor(ts, 42) + back_ts, back_id = decode_cursor(encoded) + assert back_ts == ts + assert back_id == 42 + + +def test_decode_invalid_cursor_raises(): + with pytest.raises(ValueError): + decode_cursor("not-base64!!!") + + +@pytest.mark.asyncio +async def test_scroll_returns_newest_first(db): + await _seed_images(db, 5) + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10) + assert len(page.images) == 5 + assert page.images[0].created_at > page.images[-1].created_at + + +@pytest.mark.asyncio +async def test_scroll_pagination(db): + await _seed_images(db, 5) + svc = GalleryService(db) + first = await svc.scroll(cursor=None, limit=2) + assert len(first.images) == 2 + assert first.next_cursor is not None + second = await svc.scroll(cursor=first.next_cursor, limit=2) + assert len(second.images) == 2 + assert {i.id for i in first.images}.isdisjoint({i.id for i in second.images}) + + +@pytest.mark.asyncio +async def test_scroll_with_tag_filter(db): + images = await _seed_images(db, 5) + tag = Tag(name="filterme", kind=TagKind.general) + db.add(tag) + await db.flush() + await db.execute( + image_tag.insert().values( + image_record_id=images[0].id, tag_id=tag.id, source="manual" + ) + ) + + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10, tag_id=tag.id) + assert len(page.images) == 1 + assert page.images[0].id == images[0].id + + +@pytest.mark.asyncio +async def test_timeline_groups_by_month(db): + await _seed_images(db, 3) + svc = GalleryService(db) + buckets = await svc.timeline() + assert sum(b.count for b in buckets) == 3 + + +@pytest.mark.asyncio +async def test_date_groups_in_page(db): + await _seed_images(db, 3) + svc = GalleryService(db) + page = await svc.scroll(cursor=None, limit=10) + assert len(page.date_groups) == 1 + y, m, ids = page.date_groups[0] + assert len(ids) == 3 + + +@pytest.mark.asyncio +async def test_neighbors(db): + images = await _seed_images(db, 3) + svc = GalleryService(db) + middle = images[1] + payload = await svc.get_image_with_tags(middle.id) + assert payload["neighbors"]["prev_id"] == images[0].id + assert payload["neighbors"]["next_id"] == images[2].id + + +@pytest.mark.asyncio +async def test_get_image_with_tags_returns_none_for_missing(db): + svc = GalleryService(db) + assert await svc.get_image_with_tags(99999) is None + + +@pytest.mark.asyncio +async def test_get_image_with_tags_includes_integrity_status(db): + images = await _seed_images(db, 1) + img = images[0] + img.integrity_status = "ok" + await db.flush() + svc = GalleryService(db) + payload = await svc.get_image_with_tags(img.id) + assert payload["integrity_status"] == "ok" diff --git a/tests/test_gs_ingest.py b/tests/test_gs_ingest.py new file mode 100644 index 0000000..189bbef --- /dev/null +++ b/tests/test_gs_ingest.py @@ -0,0 +1,122 @@ +"""FC-5: gs_ingest unit tests.""" +from datetime import UTC, datetime + +import pytest +from sqlalchemy import select + +from backend.app.models import Artist, Credential, Source +from backend.app.services.credential_crypto import CredentialCrypto +from backend.app.services.migrators import gs_ingest + +pytestmark = pytest.mark.integration + + +def _gs_export(subscriptions=None, credentials=None) -> dict: + return { + "source_app": "gallerysubscriber", + "schema_version": 1, + "exported_at": datetime.now(UTC).isoformat(), + "subscriptions": subscriptions or [], + "credentials": credentials or [], + } + + +@pytest.mark.asyncio +async def test_subscription_creates_artist_with_slug(db): + data = _gs_export(subscriptions=[ + {"name": "Alice Author", "enabled": True, "metadata": {}, "sources": []}, + ]) + counts = await gs_ingest.migrate_async(db, data=data, dry_run=False) + assert counts["rows_inserted"] >= 1 + + artist = (await db.execute( + select(Artist).where(Artist.name == "Alice Author") + )).scalar_one() + assert artist.slug == "alice-author" + assert artist.is_subscription is True + + +@pytest.mark.asyncio +async def test_nested_source_attached_to_artist(db): + data = _gs_export(subscriptions=[{ + "name": "bob-author", + "enabled": True, + "metadata": {}, + "sources": [{ + "platform": "patreon", + "url": "https://patreon.com/bob", + "enabled": True, + "check_interval": 28800, + "metadata": {"k": "v"}, + }], + }]) + await gs_ingest.migrate_async(db, data=data, dry_run=False) + + src = (await db.execute( + select(Source).where(Source.url == "https://patreon.com/bob") + )).scalar_one() + artist = (await db.execute( + select(Artist).where(Artist.id == src.artist_id) + )).scalar_one() + assert artist.name == "bob-author" + assert src.platform == "patreon" + assert src.check_interval_override == 28800 + assert src.config_overrides == {"k": "v"} + + +@pytest.mark.asyncio +async def test_credential_re_encrypted_with_fc_key(db, tmp_path): + fc_key_path = tmp_path / "credential_key.b64" + fc_crypto = CredentialCrypto(fc_key_path) + + data = _gs_export(credentials=[{ + "platform": "patreon", + "credential_type": "cookies", + "plaintext": "session_cookie=abcdef", + "expires_at": None, + }]) + await gs_ingest.migrate_async(db, data=data, fc_crypto=fc_crypto, dry_run=False) + + cred = (await db.execute( + select(Credential).where(Credential.platform == "patreon") + )).scalar_one() + assert fc_crypto.decrypt(cred.encrypted_blob) == "session_cookie=abcdef" + assert cred.credential_type == "cookies" + + +@pytest.mark.asyncio +async def test_dry_run_makes_no_changes(db): + data = _gs_export(subscriptions=[ + {"name": "dave", "enabled": True, "metadata": {}, "sources": []}, + ]) + counts = await gs_ingest.migrate_async(db, data=data, dry_run=True) + assert counts["rows_processed"] >= 1 + + after = (await db.execute( + select(Artist).where(Artist.name == "dave") + )).scalar_one_or_none() + assert after is None + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db): + data = _gs_export(subscriptions=[ + {"name": "eve", "enabled": True, "metadata": {}, "sources": []}, + ]) + await gs_ingest.migrate_async(db, data=data, dry_run=False) + counts2 = await gs_ingest.migrate_async(db, data=data, dry_run=False) + assert counts2["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_rejects_wrong_source_app(db): + bad = {"source_app": "wrong", "schema_version": 1, "subscriptions": [], "credentials": []} + with pytest.raises(ValueError): + await gs_ingest.migrate_async(db, data=bad, dry_run=False) + + +@pytest.mark.asyncio +async def test_rejects_unsupported_schema_version(db): + bad = {"source_app": "gallerysubscriber", "schema_version": 99, "subscriptions": [], "credentials": []} + with pytest.raises(ValueError): + await gs_ingest.migrate_async(db, data=bad, dry_run=False) diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..b8f9b2a --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,21 @@ +"""Health endpoint smoke test.""" + +import pytest +import pytest_asyncio + +from backend.app import create_app + + +@pytest_asyncio.fixture +async def client(): + app = create_app() + async with app.test_client() as c: + yield c + + +@pytest.mark.asyncio +async def test_health_returns_ok(client): + response = await client.get("/api/health") + assert response.status_code == 200 + body = await response.get_json() + assert body == {"status": "ok"} diff --git a/tests/test_html_sanitize.py b/tests/test_html_sanitize.py new file mode 100644 index 0000000..f6a9139 --- /dev/null +++ b/tests/test_html_sanitize.py @@ -0,0 +1,38 @@ +from backend.app.utils.html_sanitize import sanitize_post_html + + +def test_none_and_blank_return_none(): + assert sanitize_post_html(None) is None + assert sanitize_post_html("") is None + assert sanitize_post_html(" \n ") is None + + +def test_allowed_tags_survive(): + out = sanitize_post_html("

hi there
x

") + assert "

" in out and "" in out and "
" in out and "" in out + + +def test_disallowed_tags_and_attrs_stripped_text_kept(): + out = sanitize_post_html('

body

') + assert "", + count=2, + ) + db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, + source_id=source.id)) + await db.flush() + + svc = ProvenanceService(db) + payload = await svc.for_image(rec.id) + assert payload["image_id"] == rec.id + assert len(payload["provenance"]) == 1 + e = payload["provenance"][0] + assert e["post"]["id"] == post.id + assert e["post"]["external_post_id"] == "555" + assert e["post"]["title"] == "Set 1" + assert e["post"]["attachment_count"] == 2 + assert e["post"]["description_html"] == "

hi

" # script removed + assert e["post"]["url"] == "https://patreon.test/p/555" + assert e["post"]["date"].startswith("2023-08-01") + assert e["source"] == {"id": source.id, "platform": "patreon", + "url": source.url} + assert e["artist"] == {"id": artist.id, "name": "Alice", + "slug": "alice"} + assert e["provenance_id"] is not None + assert e["captured_at"] is not None + + +@pytest.mark.asyncio +async def test_for_image_multiple_provenance_peer_ordering(db): + rec = await _seed_image(db) + _, s1, p1 = await _seed_post(db, artist_name="A1", slug="a1", + platform="patreon", ext_id="1") + _, s2, p2 = await _seed_post(db, artist_name="A2", slug="a2", + platform="fanbox", ext_id="2") + ip1 = ImageProvenance(image_record_id=rec.id, post_id=p1.id, + source_id=s1.id) + ip2 = ImageProvenance(image_record_id=rec.id, post_id=p2.id, + source_id=s2.id) + db.add(ip1) + await db.flush() + db.add(ip2) + await db.flush() + + svc = ProvenanceService(db) + payload = await svc.for_image(rec.id) + ids = [e["provenance_id"] for e in payload["provenance"]] + assert ids == sorted(ids) # captured_at, id ascending → insertion order + assert len(payload["provenance"]) == 2 + + +@pytest.mark.asyncio +async def test_for_image_null_post_fields_serialize_null(db): + rec = await _seed_image(db) + _, source, post = await _seed_post( + db, artist_name="Bob", slug="bob", platform="x", ext_id="9", + ) # title/desc/count/post_url default-ish + db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, + source_id=source.id)) + await db.flush() + svc = ProvenanceService(db) + e = (await svc.for_image(rec.id))["provenance"][0] + assert e["post"]["title"] is None + assert e["post"]["description_html"] is None + assert e["post"]["attachment_count"] is None + + +@pytest.mark.asyncio +async def test_for_post_missing_returns_none(db): + svc = ProvenanceService(db) + assert await svc.for_post(999999) is None + + +@pytest.mark.asyncio +async def test_for_post_returns_post_source_artist(db): + artist, source, post = await _seed_post( + db, artist_name="Carol", slug="carol", platform="patreon", + ext_id="77", title="T", desc="

d

", count=3, + ) + svc = ProvenanceService(db) + payload = await svc.for_post(post.id) + assert payload["post"]["id"] == post.id + assert payload["post"]["title"] == "T" + assert payload["post"]["description_html"] == "

d

" + assert payload["post"]["attachment_count"] == 3 + assert payload["source"] == {"id": source.id, "platform": "patreon", + "url": source.url} + assert payload["artist"] == {"id": artist.id, "name": "Carol", + "slug": "carol"} + + +@pytest.mark.asyncio +async def test_for_post_includes_attachments(db): + artist, source, post = await _seed_post( + db, artist_name="Att", slug="att", platform="patreon", + ext_id="55", + ) + db.add(PostAttachment( + post_id=post.id, artist_id=artist.id, sha256="t" + "0" * 63, + path="/images/attachments/t00/t.zip", original_filename="t.zip", + ext=".zip", mime="application/zip", size_bytes=9, + )) + await db.flush() + svc = ProvenanceService(db) + payload = await svc.for_post(post.id) + assert len(payload["attachments"]) == 1 + att = payload["attachments"][0] + assert att["original_filename"] == "t.zip" + assert att["download_url"].endswith( + f"/api/attachments/{att['id']}/download" + ) diff --git a/tests/test_scan_enumerates_all.py b/tests/test_scan_enumerates_all.py new file mode 100644 index 0000000..86abd55 --- /dev/null +++ b/tests/test_scan_enumerates_all.py @@ -0,0 +1,20 @@ +"""FC-2d-iii: scan no longer filters to media-only.""" + +import pytest + +pytestmark = pytest.mark.integration + + +def test_scan_includes_non_media(tmp_path): + from backend.app.tasks import scan as scan_mod + + root = tmp_path / "imp" + (root / "Alice").mkdir(parents=True) + (root / "Alice" / "pic.jpg").write_bytes(b"x") + (root / "Alice" / "pack.zip").write_bytes(b"x") + (root / "Alice" / "doc.pdf").write_bytes(b"x") + (root / "Alice" / "meta.json").write_text("{}") + (root / "Alice" / ".hidden").write_text("x") + + seen = sorted(p.name for p in scan_mod._iter_import_files(root)) + assert seen == ["doc.pdf", "pack.zip", "pic.jpg"] # no json/dotfile diff --git a/tests/test_scheduler_service.py b/tests/test_scheduler_service.py new file mode 100644 index 0000000..4c5e298 --- /dev/null +++ b/tests/test_scheduler_service.py @@ -0,0 +1,149 @@ +"""FC-3d: scheduler_service unit tests. + +Covers compute_effective_interval (override/artist/default precedence, +backoff exponent at failure counts 0/1/3/6/10, floor and ceiling clamp) +and select_due_sources (auto_check off, never-checked, enabled=False, +due/not-due boundary). +""" +from datetime import UTC, datetime, timedelta + +import pytest + +from backend.app.models import Artist, ImportSettings, Source +from backend.app.services.scheduler_service import ( + compute_effective_interval, + select_due_sources, +) + +pytestmark = pytest.mark.integration + + +# --- compute_effective_interval (pure, no DB) ----------------------------- + + +def _src(override=None, failures=0): + return Source( + artist_id=1, platform="patreon", url="https://x", enabled=True, + check_interval_override=override, consecutive_failures=failures, + ) + + +def _art(interval=None, auto=True): + return Artist(name="A", slug="a", auto_check=auto, check_interval_seconds=interval) + + +def _settings(default=28800): + return ImportSettings(id=1, download_schedule_default_seconds=default) + + +def test_interval_uses_source_override_when_set(): + assert compute_effective_interval(_src(override=600), _art(interval=3600), _settings()) == 600 + + +def test_interval_falls_back_to_artist_interval(): + assert compute_effective_interval(_src(), _art(interval=3600), _settings()) == 3600 + + +def test_interval_falls_back_to_global_default(): + assert compute_effective_interval(_src(), _art(), _settings(default=900)) == 900 + + +def test_interval_backoff_zero_failures_is_identity(): + assert compute_effective_interval(_src(override=600), _art(), _settings()) == 600 + + +def test_interval_backoff_one_failure_doubles(): + assert compute_effective_interval(_src(override=600, failures=1), _art(), _settings()) == 1200 + + +def test_interval_backoff_three_failures_x8(): + assert compute_effective_interval(_src(override=600, failures=3), _art(), _settings()) == 4800 + + +def test_interval_backoff_caps_at_exponent_six(): + # 600 * 2^6 = 38400; 2^7 would be 76800 but exponent caps at 6. + assert compute_effective_interval(_src(override=600, failures=6), _art(), _settings()) == 38400 + assert compute_effective_interval(_src(override=600, failures=10), _art(), _settings()) == 38400 + + +def test_interval_floor_at_60_seconds(): + # base 30 with 0 failures would be 30s; floor lifts to 60. + assert compute_effective_interval(_src(override=30), _art(), _settings()) == 60 + + +def test_interval_ceiling_at_86400_seconds(): + # base 28800 * 2^6 = 1843200; ceiling caps at 86400. + assert compute_effective_interval(_src(failures=6), _art(), _settings(default=28800)) == 86400 + + +# --- select_due_sources (real session) ------------------------------------ + + +async def _seed_artist(db, *, auto=True, interval=None, name="A"): + a = Artist(name=name, slug=name.lower(), auto_check=auto, check_interval_seconds=interval) + db.add(a) + await db.flush() + return a + + +@pytest.mark.asyncio +async def test_select_skips_disabled_sources(db): + artist = await _seed_artist(db, name="sel-disabled") + db.add(Source( + artist_id=artist.id, platform="patreon", url="https://sel-disabled", + enabled=False, consecutive_failures=0, + )) + await db.commit() + due = await select_due_sources(db) + assert all(s.url != "https://sel-disabled" for s in due) + + +@pytest.mark.asyncio +async def test_select_skips_artist_auto_check_false(db): + artist = await _seed_artist(db, auto=False, name="sel-noauto") + db.add(Source( + artist_id=artist.id, platform="patreon", url="https://sel-noauto", + enabled=True, consecutive_failures=0, + )) + await db.commit() + due = await select_due_sources(db) + assert all(s.url != "https://sel-noauto" for s in due) + + +@pytest.mark.asyncio +async def test_select_includes_never_checked(db): + artist = await _seed_artist(db, name="sel-never") + src = Source( + artist_id=artist.id, platform="patreon", url="https://sel-never", + enabled=True, consecutive_failures=0, + ) + db.add(src) + await db.commit() + due = await select_due_sources(db) + assert any(s.url == "https://sel-never" for s in due) + + +@pytest.mark.asyncio +async def test_select_excludes_not_yet_due(db): + artist = await _seed_artist(db, interval=3600, name="sel-recent") + db.add(Source( + artist_id=artist.id, platform="patreon", url="https://sel-recent", + enabled=True, consecutive_failures=0, + last_checked_at=datetime.now(UTC) - timedelta(seconds=60), + )) + await db.commit() + due = await select_due_sources(db) + assert all(s.url != "https://sel-recent" for s in due) + + +@pytest.mark.asyncio +async def test_select_includes_past_due(db): + artist = await _seed_artist(db, interval=60, name="sel-past") + db.add(Source( + artist_id=artist.id, platform="patreon", url="https://sel-past", + enabled=True, consecutive_failures=0, + last_checked_at=datetime.now(UTC) - timedelta(seconds=3600), + )) + await db.commit() + due = await select_due_sources(db) + assert any(s.url == "https://sel-past" for s in due) diff --git a/tests/test_series_model.py b/tests/test_series_model.py new file mode 100644 index 0000000..ff382c5 --- /dev/null +++ b/tests/test_series_model.py @@ -0,0 +1,40 @@ +import pytest +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_page import SeriesPage +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _img(db): + global _SEQ + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_sp_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +@pytest.mark.asyncio +async def test_series_page_roundtrip_and_unique_image(db): + s = await TagService(db).find_or_create("Vol1", TagKind.series) + i1 = await _img(db) + db.add(SeriesPage(series_tag_id=s.id, image_id=i1, page_number=1)) + await db.flush() + got = await db.scalar( + select(SeriesPage.page_number).where(SeriesPage.image_id == i1) + ) + assert got == 1 + # UNIQUE image_id: same image can't be a page twice (any series). + s2 = await TagService(db).find_or_create("Vol2", TagKind.series) + db.add(SeriesPage(series_tag_id=s2.id, image_id=i1, page_number=1)) + with pytest.raises(IntegrityError): + await db.flush() diff --git a/tests/test_series_service.py b/tests/test_series_service.py new file mode 100644 index 0000000..00847b1 --- /dev/null +++ b/tests/test_series_service.py @@ -0,0 +1,128 @@ +import pytest +from sqlalchemy import func, select + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.series_page import SeriesPage +from backend.app.services.series_service import SeriesError, SeriesService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + +_SEQ = 0 + + +async def _img(db): + global _SEQ + _SEQ += 1 + rec = ImageRecord( + path=f"/tmp/fc_ss_{_SEQ}.png", sha256=f"{_SEQ:064d}", + size_bytes=1, mime="image/png", origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +async def _order(db, sid): + rows = ( + await db.execute( + select(SeriesPage.image_id, SeriesPage.page_number) + .where(SeriesPage.series_tag_id == sid) + .order_by(SeriesPage.page_number) + ) + ).all() + return [r.image_id for r in rows] + + +@pytest.mark.asyncio +async def test_add_appends_in_order(db): + s = await TagService(db).find_or_create("S", TagKind.series) + i1, i2 = await _img(db), await _img(db) + n = await SeriesService(db).add_images(s.id, [i1, i2]) + assert n == 2 + assert await _order(db, s.id) == [i1, i2] + + +@pytest.mark.asyncio +async def test_add_moves_from_other_series(db): + svc = SeriesService(db) + a = await TagService(db).find_or_create("A", TagKind.series) + b = await TagService(db).find_or_create("B", TagKind.series) + i1 = await _img(db) + await svc.add_images(a.id, [i1]) + moved = await svc.add_images(b.id, [i1]) + assert moved == 1 + assert await _order(db, a.id) == [] + assert await _order(db, b.id) == [i1] + cnt = await db.scalar( + select(func.count()) + .select_from(SeriesPage).where(SeriesPage.image_id == i1) + ) + assert cnt == 1 + + +@pytest.mark.asyncio +async def test_add_idempotent_for_same_series(db): + s = await TagService(db).find_or_create("S2", TagKind.series) + i1 = await _img(db) + svc = SeriesService(db) + await svc.add_images(s.id, [i1]) + again = await svc.add_images(s.id, [i1]) + assert again == 0 + assert await _order(db, s.id) == [i1] + + +@pytest.mark.asyncio +async def test_remove(db): + s = await TagService(db).find_or_create("S3", TagKind.series) + i1, i2 = await _img(db), await _img(db) + svc = SeriesService(db) + await svc.add_images(s.id, [i1, i2]) + removed = await svc.remove_images(s.id, [i1]) + assert removed == 1 + assert await _order(db, s.id) == [i2] + + +@pytest.mark.asyncio +async def test_reorder_rewrites_and_rejects_mismatch(db): + s = await TagService(db).find_or_create("S4", TagKind.series) + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + svc = SeriesService(db) + await svc.add_images(s.id, [i1, i2, i3]) + await svc.reorder(s.id, [i3, i1, i2]) + assert await _order(db, s.id) == [i3, i1, i2] + with pytest.raises(SeriesError): + await svc.reorder(s.id, [i1, i2]) + with pytest.raises(SeriesError): + await svc.reorder(s.id, [i1, i2, i3, 99999]) + + +@pytest.mark.asyncio +async def test_set_cover_makes_page_one_preserving_rest(db): + s = await TagService(db).find_or_create("S5", TagKind.series) + i1, i2, i3 = await _img(db), await _img(db), await _img(db) + svc = SeriesService(db) + await svc.add_images(s.id, [i1, i2, i3]) + await svc.set_cover(s.id, i3) + assert await _order(db, s.id) == [i3, i1, i2] + + +@pytest.mark.asyncio +async def test_list_pages_shape_and_guards(db): + svc = SeriesService(db) + s = await TagService(db).find_or_create("S6", TagKind.series) + i1 = await _img(db) + await svc.add_images(s.id, [i1]) + out = await svc.list_pages(s.id) + assert out["series"]["id"] == s.id + assert out["pages"][0]["image_id"] == i1 + assert out["pages"][0]["page_number"] == 1 + assert out["pages"][0]["thumbnail_url"] + assert out["pages"][0]["image_url"].startswith("/images/") + g = await TagService(db).find_or_create("notser", TagKind.general) + with pytest.raises(SeriesError): + await svc.list_pages(g.id) + with pytest.raises(SeriesError): + await svc.add_images(g.id, [i1]) + with pytest.raises(SeriesError): + await svc.add_images(s.id, list(range(1, 502))) diff --git a/tests/test_showcase_service.py b/tests/test_showcase_service.py new file mode 100644 index 0000000..fac5fa9 --- /dev/null +++ b/tests/test_showcase_service.py @@ -0,0 +1,45 @@ +import pytest + +from backend.app.models import ImageRecord +from backend.app.services.showcase_service import ShowcaseService + +pytestmark = pytest.mark.integration + + +async def _seed(db, n): + for i in range(n): + db.add(ImageRecord( + path=f"/images/s/{i}.jpg", sha256=f"s{i:063d}", + size_bytes=1, mime="image/jpeg", width=100, height=200, + origin="imported_filesystem", integrity_status="unknown", + )) + await db.flush() + await db.commit() + + +@pytest.mark.asyncio +async def test_random_sample_returns_items_with_shape(db): + await _seed(db, 20) + svc = ShowcaseService(db) + items = await svc.random_sample(limit=5) + assert 1 <= len(items) <= 5 + first = items[0] + assert set(first.keys()) == { + "id", "sha256", "mime", "width", "height", "thumbnail_url" + } + assert first["thumbnail_url"].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_random_sample_empty_table_returns_empty(db): + svc = ShowcaseService(db) + assert await svc.random_sample(limit=5) == [] + + +@pytest.mark.asyncio +async def test_random_sample_rejects_bad_limit(db): + svc = ShowcaseService(db) + with pytest.raises(ValueError): + await svc.random_sample(limit=0) + with pytest.raises(ValueError): + await svc.random_sample(limit=201) diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py new file mode 100644 index 0000000..f9ebabc --- /dev/null +++ b/tests/test_sidecar_import.py @@ -0,0 +1,168 @@ +"""FC-2d-v: sidecar → Source/Post/ImageProvenance (sync importer).""" + +import json +from pathlib import Path + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + Source, +) +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +@pytest.fixture +def importer(db_sync, import_layout): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, + images_root=images_root, + import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), + settings=settings, + ) + + +def _split(path: Path, orient, size=(256, 256)): + """Structured image (half/half) — solid colors phash-collapse, which + would route a 2nd image to the supersede path (no sidecar applied).""" + path.parent.mkdir(parents=True, exist_ok=True) + w, h = size + im = Image.new("L", size, 0) + px = im.load() + for y in range(h): + for x in range(w): + if (x / w if orient == "v" else y / h) >= 0.5: + px[x, y] = 255 + im.convert("RGB").save(path, "JPEG") + + +def _sidecar(media: Path, payload: dict): + media.with_suffix(".json").write_text(json.dumps(payload)) + + +def test_sidecar_creates_provenance(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Alice" / "a.jpg" + _split(m, "v") + _sidecar(m, { + "category": "patreon", "id": 555, + "url": "https://patreon.com/posts/555", "title": "Set 1", + "content": "

hi

", "page_count": 2, + "published_at": "2023-08-01T00:00:00Z", + }) + r = importer.import_one(m) + assert r.status == "imported" + rec = importer.session.get(ImageRecord, r.image_id) + src = importer.session.execute(select(Source)).scalar_one() + post = importer.session.execute(select(Post)).scalar_one() + assert src.platform == "patreon" + assert post.external_post_id == "555" + assert post.post_url == "https://patreon.com/posts/555" + assert post.post_title == "Set 1" + assert post.description == "

hi

" + assert post.attachment_count == 2 + assert post.post_date is not None + assert post.raw_metadata["id"] == 555 + prov = importer.session.execute(select(ImageProvenance)).scalar_one() + assert prov.image_record_id == rec.id and prov.post_id == post.id + assert rec.primary_post_id == post.id + + +def test_reimport_same_post_idempotent(importer, import_layout): + import_root, _ = import_layout + # Threshold 0: only an exact phash match collapses. Orthogonal splits + # still sit within the default Hamming-10 at 64-bit, so without this + # the 2nd image would be skipped as a near-dup (phash dedup working). + importer.settings.phash_threshold = 0 + payload = {"category": "patreon", "id": 777, "title": "P"} + m1 = import_root / "Bob" / "p1.jpg" + _split(m1, "v") + _sidecar(m1, payload) + importer.import_one(m1) + m2 = import_root / "Bob" / "p2.jpg" + _split(m2, "h") # distinct phash; threshold 0 → both import + _sidecar(m2, payload) + r2 = importer.import_one(m2) + assert r2.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 1 + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 1 + assert importer.session.execute( + select(func.count()).select_from(ImageProvenance) + ).scalar_one() == 2 + + +def test_garbage_sidecar_still_imports(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Carol" / "c.jpg" + _split(m, "v") + m.with_suffix(".json").write_text("{ not json") + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 0 + + +def test_no_sidecar_unchanged(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Dave" / "d.jpg" + _split(m, "v") + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 0 + + +def test_no_artist_anywhere_skips_provenance(importer, import_layout): + import_root, _ = import_layout + m = import_root / "rootfile.jpg" # no top-level artist folder + _split(m, "v") + _sidecar(m, {"category": "x", "id": 1}) # no artist key + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 0 + + +def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout): + from backend.app.models import Artist + + import_root, _ = import_layout + m = import_root / "e.jpg" # root → no folder artist + _split(m, "v") + _sidecar(m, {"category": "pixiv", "id": 9, "artist": "Yuki"}) + r = importer.import_one(m) + assert r.status == "imported" + a = importer.session.execute( + select(Artist).where(Artist.slug == "yuki") + ).scalar_one() + src = importer.session.execute(select(Source)).scalar_one() + assert src.artist_id == a.id diff --git a/tests/test_sidecar_util.py b/tests/test_sidecar_util.py new file mode 100644 index 0000000..0fd3e40 --- /dev/null +++ b/tests/test_sidecar_util.py @@ -0,0 +1,75 @@ +from backend.app.utils.sidecar import SidecarData, find_sidecar, parse_sidecar + + +def test_find_sidecar_stem_json(tmp_path): + media = tmp_path / "photo.jpg" + media.write_bytes(b"x") + sc = tmp_path / "photo.json" + sc.write_text("{}") + assert find_sidecar(media) == sc + + +def test_find_sidecar_full_name_json(tmp_path): + media = tmp_path / "photo.jpg" + media.write_bytes(b"x") + sc = tmp_path / "photo.jpg.json" + sc.write_text("{}") + assert find_sidecar(media) == sc + + +def test_find_sidecar_none(tmp_path): + media = tmp_path / "photo.jpg" + media.write_bytes(b"x") + assert find_sidecar(media) is None + + +def test_parse_empty_dict_all_none(): + sd = parse_sidecar({}) + assert isinstance(sd, SidecarData) + assert (sd.platform, sd.external_post_id, sd.post_url, sd.post_title, + sd.description, sd.attachment_count, sd.post_date) == ( + None, None, None, None, None, None, None) + assert sd.raw == {} + + +def test_parse_core_fields_and_id_priority(): + sd = parse_sidecar({ + "category": "patreon", + "id": 12345, "post_id": 999, + "url": "https://patreon.com/posts/12345", + "title": " Hello ", + "content": "

body

", + "page_count": 4, + "published_at": "2023-08-01T04:20:02Z", + }) + assert sd.platform == "patreon" + assert sd.external_post_id == "12345" # 'id' wins over 'post_id' + assert sd.post_url == "https://patreon.com/posts/12345" + assert sd.post_title == "Hello" + assert sd.description == "

body

" + assert sd.attachment_count == 4 + assert sd.post_date.year == 2023 and sd.post_date.tzinfo is not None + + +def test_parse_description_precedence_and_images_count(): + sd = parse_sidecar({"description": "d", "caption": "c", + "images": [1, 2, 3]}) + assert sd.description == "d" # content>description>caption + assert sd.attachment_count == 3 # len(images) fallback + + +def test_parse_date_epoch_and_unparseable_and_naive(): + assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None + assert parse_sidecar({"date": "not-a-date"}).post_date is None + naive = parse_sidecar({"date": "2023-08-01T00:00:00"}).post_date + assert naive is not None and naive.utcoffset().total_seconds() == 0 + + +def test_parse_ignores_non_str_junk(): + sd = parse_sidecar({"category": 5, "title": 7, "page_count": "x", + "id": True}) + assert sd.platform is None and sd.post_title is None + assert sd.attachment_count is None + assert sd.external_post_id is None # bool is not a valid id + assert sd.raw == {"category": 5, "title": 7, "page_count": "x", + "id": True} diff --git a/tests/test_slug.py b/tests/test_slug.py new file mode 100644 index 0000000..41e3092 --- /dev/null +++ b/tests/test_slug.py @@ -0,0 +1,25 @@ +from backend.app.utils.slug import slugify + + +def test_slugify_basic(): + assert slugify("Bob Ross") == "bob-ross" + + +def test_slugify_punctuation(): + assert slugify("hello, world!") == "hello-world" + + +def test_slugify_unicode(): + assert slugify("naïve café") == "naive-cafe" + + +def test_slugify_empty(): + assert slugify("") == "untitled" + + +def test_slugify_only_punctuation(): + assert slugify("!!!") == "untitled" + + +def test_slugify_leading_trailing_spaces(): + assert slugify(" Alice ") == "alice" diff --git a/tests/test_source_service.py b/tests/test_source_service.py new file mode 100644 index 0000000..aa310ef --- /dev/null +++ b/tests/test_source_service.py @@ -0,0 +1,135 @@ +import pytest +from sqlalchemy import select + +from backend.app.models import Artist +from backend.app.services.source_service import ( + KNOWN_PLATFORMS, + ArtistNotFoundError, + DuplicateSourceError, + EmptyUrlError, + InvalidConfigError, + SourceService, + UnknownPlatformError, +) + +pytestmark = pytest.mark.integration + + +async def _artist(db, name="Alice"): + a = Artist(name=name, slug=name.lower()) + db.add(a) + await db.flush() + return a + + +@pytest.mark.asyncio +async def test_known_platforms_is_gs_six(db): + assert KNOWN_PLATFORMS == frozenset({ + "patreon", "subscribestar", "hentaifoundry", + "discord", "pixiv", "deviantart", + }) + assert "fanbox" not in KNOWN_PLATFORMS + + +@pytest.mark.asyncio +async def test_create_flips_is_subscription_on_first_source(db): + artist = await _artist(db) + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", + ) + assert rec.id is not None + is_sub = (await db.execute( + select(Artist.is_subscription).where(Artist.id == artist.id) + )).scalar_one() + assert is_sub is True + + +@pytest.mark.asyncio +async def test_delete_last_source_flips_is_subscription_off(db): + artist = await _artist(db) + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", + ) + await svc.delete(rec.id) + is_sub = (await db.execute( + select(Artist.is_subscription).where(Artist.id == artist.id) + )).scalar_one() + assert is_sub is False + + +@pytest.mark.asyncio +async def test_create_rejects_unknown_platform(db): + artist = await _artist(db) + svc = SourceService(db) + with pytest.raises(UnknownPlatformError): + await svc.create( + artist_id=artist.id, platform="myspace", url="https://m/x", + ) + + +@pytest.mark.asyncio +async def test_create_rejects_non_dict_config(db): + artist = await _artist(db) + svc = SourceService(db) + with pytest.raises(InvalidConfigError): + await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", config_overrides=[1, 2, 3], + ) + + +@pytest.mark.asyncio +async def test_create_rejects_empty_url(db): + artist = await _artist(db) + svc = SourceService(db) + with pytest.raises(EmptyUrlError): + await svc.create(artist_id=artist.id, platform="patreon", url=" ") + + +@pytest.mark.asyncio +async def test_create_rejects_unknown_artist(db): + svc = SourceService(db) + with pytest.raises(ArtistNotFoundError): + await svc.create(artist_id=99999, platform="patreon", url="https://x/y") + + +@pytest.mark.asyncio +async def test_create_duplicate_raises_with_existing_id(db): + artist = await _artist(db) + svc = SourceService(db) + first = await svc.create( + artist_id=artist.id, platform="patreon", url="https://patreon.com/alice", + ) + with pytest.raises(DuplicateSourceError) as exc: + await svc.create( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", + ) + assert exc.value.existing_id == first.id + + +@pytest.mark.asyncio +async def test_list_filters_by_artist(db): + a = await _artist(db, "Alice") + b = await _artist(db, "Bob") + svc = SourceService(db) + await svc.create(artist_id=a.id, platform="patreon", url="https://patreon.com/a") + await svc.create(artist_id=b.id, platform="patreon", url="https://patreon.com/b") + only_a = await svc.list(artist_id=a.id) + assert [s.artist_id for s in only_a] == [a.id] + all_rows = await svc.list() + assert len(all_rows) == 2 + + +@pytest.mark.asyncio +async def test_update_changes_fields(db): + artist = await _artist(db) + svc = SourceService(db) + rec = await svc.create( + artist_id=artist.id, platform="patreon", url="https://patreon.com/a", + ) + updated = await svc.update(rec.id, enabled=False, config_overrides={"videos": False}) + assert updated.enabled is False + assert updated.config_overrides == {"videos": False} diff --git a/tests/test_suggestions_bulk.py b/tests/test_suggestions_bulk.py new file mode 100644 index 0000000..2b00649 --- /dev/null +++ b/tests/test_suggestions_bulk.py @@ -0,0 +1,115 @@ +import pytest + +from backend.app.models import ImageRecord, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.ml.suggestions import SuggestionService +from backend.app.services.tag_service import TagService + +pytestmark = pytest.mark.integration + + +def _img(sha: str, predictions: dict) -> ImageRecord: + return ImageRecord( + path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, + mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + tagger_predictions=predictions, + ) + + +@pytest.mark.asyncio +async def test_consensus_includes_tag_over_threshold(db): + tags = TagService(db) + t = await tags.find_or_create("sword", TagKind.general) + a = _img("a" * 64, {"sword": {"category": "general", "confidence": 0.97}}) + b = _img("b" * 64, {"sword": {"category": "general", "confidence": 0.95}}) + db.add_all([a, b]) + await db.flush() + res = await SuggestionService(db).for_selection([a.id, b.id], threshold=0.8) + gen = res["general"] + assert any(s["canonical_tag_id"] == t.id for s in gen) + s = next(s for s in gen if s["canonical_tag_id"] == t.id) + assert s["coverage"] == 1.0 + assert 0.95 <= s["confidence"] <= 0.97 + + +@pytest.mark.asyncio +async def test_consensus_counts_already_applied_for_coverage(db): + tags = TagService(db) + t = await tags.find_or_create("sky", TagKind.general) + a = _img("c" * 64, {"sky": {"category": "general", "confidence": 0.96}}) + b = _img("d" * 64, {}) # no prediction + db.add_all([a, b]) + await db.flush() + # b already has the tag applied -> counts toward coverage, not confidence + await db.execute( + image_tag.insert().values( + image_record_id=b.id, tag_id=t.id, source="manual" + ) + ) + res = await SuggestionService(db).for_selection([a.id, b.id], threshold=0.8) + s = next(s for s in res["general"] if s["canonical_tag_id"] == t.id) + assert s["coverage"] == 1.0 # 1 suggested + 1 applied / 2 + assert s["confidence"] == pytest.approx(0.96, abs=1e-4) + + +@pytest.mark.asyncio +async def test_consensus_excludes_below_threshold(db): + tags = TagService(db) + await tags.find_or_create("rare", TagKind.general) + a = _img("e" * 64, {"rare": {"category": "general", "confidence": 0.96}}) + b = _img("f" * 64, {}) + db.add_all([a, b]) + await db.flush() + res = await SuggestionService(db).for_selection([a.id, b.id], threshold=0.8) + assert all( + s["name"] != "rare" for s in res.get("general", []) + ) # coverage 0.5 < 0.8 + + +@pytest.mark.asyncio +async def test_consensus_skips_creates_new_tag(db): + a = _img("g" * 64, {"neverseen": {"category": "general", "confidence": 0.99}}) + b = _img("h" * 64, {"neverseen": {"category": "general", "confidence": 0.99}}) + db.add_all([a, b]) + await db.flush() + res = await SuggestionService(db).for_selection([a.id, b.id], threshold=0.8) + # 'neverseen' has no Tag row -> creates_new_tag -> excluded from consensus + assert all(s["name"] != "neverseen" for s in res.get("general", [])) + + +@pytest.mark.asyncio +async def test_consensus_threshold_clamped_and_empty_for_no_ids(db): + res = await SuggestionService(db).for_selection([], threshold=5.0) + assert res == {} + + +@pytest.mark.asyncio +async def test_bulk_suggestions_route(db): + from backend.app import create_app + + tags = TagService(db) + await tags.find_or_create("sword", TagKind.general) + a = _img("i" * 64, {"sword": {"category": "general", "confidence": 0.97}}) + db.add(a) + await db.commit() + app = create_app() + async with app.test_client() as c: + resp = await c.post( + "/api/suggestions/bulk", json={"image_ids": [a.id]} + ) + assert resp.status_code == 200 + body = await resp.get_json() + assert body["evaluated"] == 1 + assert body["threshold"] == 0.8 + assert "suggestions" in body + + +@pytest.mark.asyncio +async def test_bulk_suggestions_requires_ids(db): + from backend.app import create_app + + app = create_app() + async with app.test_client() as c: + resp = await c.post("/api/suggestions/bulk", json={}) + assert resp.status_code == 400 diff --git a/tests/test_tag_apply.py b/tests/test_tag_apply.py new file mode 100644 index 0000000..3d9ea6b --- /dev/null +++ b/tests/test_tag_apply.py @@ -0,0 +1,156 @@ +"""FC-5: tag_apply unit tests.""" +import json + +import pytest +from sqlalchemy import select + +from backend.app.models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from backend.app.services.migrators import tag_apply + +pytestmark = pytest.mark.integration + + +async def _seed_image(db, sha: str, *, suffix="x"): + img = ImageRecord( + path=f"/images/imported/aa/{suffix}.jpg", + sha256=sha, size_bytes=10, mime="image/jpeg", + width=10, height=10, origin="imported_filesystem", + ) + db.add(img) + await db.flush() + return img + + +def _write_manifest(tmp_path, *, artist_assignments=None, tag_associations=None, series_pages=None): + d = tmp_path / "_migration_state" + d.mkdir(parents=True, exist_ok=True) + (d / "ir_tag_manifest.json").write_text(json.dumps({ + "schema_version": 1, + "image_artist_assignments": artist_assignments or [], + "image_tag_associations": tag_associations or [], + "series_pages": series_pages or [], + })) + + +@pytest.mark.asyncio +async def test_artist_assignment_sets_image_artist_id(db, tmp_path): + sha = "a" * 64 + await _seed_image(db, sha) + await db.commit() + + _write_manifest(tmp_path, artist_assignments=[ + {"sha256": sha, "artist_name": "BobAuthor"}, + ]) + + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result["counts"]["rows_inserted"] >= 1 + + img = (await db.execute( + select(ImageRecord).where(ImageRecord.sha256 == sha) + )).scalar_one() + artist = (await db.execute( + select(Artist).where(Artist.id == img.artist_id) + )).scalar_one() + assert artist.name == "BobAuthor" + + +@pytest.mark.asyncio +async def test_tag_association_creates_image_tag(db, tmp_path): + sha = "b" * 64 + await _seed_image(db, sha, suffix="b") + # Seed the tag itself (ir_ingest would have created it). + tag = Tag(name="blue_eyes", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + row = (await db.execute( + select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) + )).all() + assert len(row) == 1 + + +@pytest.mark.asyncio +async def test_unmatched_sha_appears_in_unmatched_list(db, tmp_path): + _write_manifest(tmp_path, tag_associations=[ + {"sha256": "z" * 64, "tag_name": "missing_image", "tag_kind": "general", "fandom_name": None}, + ]) + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert any(e["sha256"] == "z" * 64 for e in result["unmatched"]) + + +@pytest.mark.asyncio +async def test_series_page_created(db, tmp_path): + sha = "c" * 64 + await _seed_image(db, sha, suffix="c") + series_tag = Tag(name="MySeries", kind=TagKind.series) + db.add(series_tag) + await db.commit() + + _write_manifest(tmp_path, series_pages=[ + {"sha256": sha, "series_tag_name": "MySeries", "page_number": 1}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + sp = (await db.execute( + select(SeriesPage).where(SeriesPage.image_id == img_id) + )).scalar_one() + assert sp.page_number == 1 + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db, tmp_path): + sha = "d" * 64 + await _seed_image(db, sha, suffix="d") + tag = Tag(name="rerun_tag", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "rerun_tag", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result2["counts"]["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_missing_manifest_raises(db, tmp_path): + with pytest.raises(FileNotFoundError): + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + +@pytest.mark.asyncio +async def test_dry_run_makes_no_changes(db, tmp_path): + sha = "e" * 64 + await _seed_image(db, sha, suffix="e") + tag = Tag(name="dry_tag", kind=TagKind.general) + db.add(tag) + await db.commit() + + _write_manifest(tmp_path, tag_associations=[ + {"sha256": sha, "tag_name": "dry_tag", "tag_kind": "general", "fandom_name": None}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True) + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + rows = (await db.execute( + select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) + )).all() + assert len(rows) == 0 diff --git a/tests/test_tag_directory_service.py b/tests/test_tag_directory_service.py new file mode 100644 index 0000000..b40e6b8 --- /dev/null +++ b/tests/test_tag_directory_service.py @@ -0,0 +1,72 @@ +import pytest + +from backend.app.models import ImageRecord, Tag, TagKind +from backend.app.models.tag import image_tag +from backend.app.services.tag_directory_service import TagDirectoryService + +pytestmark = pytest.mark.integration + + +async def _img(db, i): + r = ImageRecord( + path=f"/images/td/{i}.jpg", sha256=f"d{i:063d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + db.add(r) + await db.flush() + return r + + +@pytest.mark.asyncio +async def test_directory_lists_tags_with_counts_and_previews(db): + t = Tag(name="sunset", kind=TagKind.general) + db.add(t) + await db.flush() + for i in range(4): + img = await _img(db, i) + await db.execute(image_tag.insert().values( + image_record_id=img.id, tag_id=t.id, source="manual")) + await db.flush() + svc = TagDirectoryService(db) + page = await svc.list_tags(kind=None, q=None, cursor=None, limit=10) + card = next(c for c in page.cards if c["name"] == "sunset") + assert card["image_count"] == 4 + assert len(card["preview_thumbnails"]) == 3 + assert card["preview_thumbnails"][0].startswith("/images/thumbs/") + + +@pytest.mark.asyncio +async def test_directory_kind_filter_and_search(db): + db.add_all([ + Tag(name="artist_a", kind=TagKind.artist), + Tag(name="char_b", kind=TagKind.character), + Tag(name="char_zoom", kind=TagKind.character), + ]) + await db.flush() + svc = TagDirectoryService(db) + only_char = await svc.list_tags(kind="character", q=None, cursor=None, limit=10) + assert {c["name"] for c in only_char.cards} == {"char_b", "char_zoom"} + searched = await svc.list_tags(kind=None, q="zoom", cursor=None, limit=10) + assert {c["name"] for c in searched.cards} == {"char_zoom"} + + +@pytest.mark.asyncio +async def test_directory_cursor_pagination(db): + for i in range(5): + db.add(Tag(name=f"t{i:02d}", kind=TagKind.general)) + await db.flush() + svc = TagDirectoryService(db) + first = await svc.list_tags(kind=None, q=None, cursor=None, limit=2) + assert len(first.cards) == 2 + assert first.next_cursor is not None + second = await svc.list_tags(kind=None, q=None, cursor=first.next_cursor, limit=2) + assert len(second.cards) == 2 + assert {c["id"] for c in first.cards}.isdisjoint({c["id"] for c in second.cards}) + + +@pytest.mark.asyncio +async def test_directory_rejects_bad_limit(db): + svc = TagDirectoryService(db) + with pytest.raises(ValueError): + await svc.list_tags(kind=None, q=None, cursor=None, limit=0) diff --git a/tests/test_tag_merge.py b/tests/test_tag_merge.py new file mode 100644 index 0000000..50a839c --- /dev/null +++ b/tests/test_tag_merge.py @@ -0,0 +1,429 @@ +import pytest +from sqlalchemy import func, select + +from backend.app.models import Tag, TagKind, image_tag +from backend.app.models.tag_allowlist import TagAllowlist +from backend.app.services.tag_service import ( + MergeResult, + TagMergeConflict, + TagService, + TagValidationError, +) + +pytestmark = pytest.mark.integration + +_IMG_SEQ = 0 + + +async def _img(db): + """Insert a minimal valid ImageRecord (all NOT NULL cols satisfied: + path, sha256, size_bytes, mime, origin) and return its id.""" + global _IMG_SEQ + _IMG_SEQ += 1 + from backend.app.models import ImageRecord + + rec = ImageRecord( + path=f"/tmp/fc_merge_{_IMG_SEQ}.png", + sha256=f"{_IMG_SEQ:064d}", + size_bytes=1, + mime="image/png", + origin="uploaded", + ) + db.add(rec) + await db.flush() + return rec.id + + +@pytest.mark.asyncio +async def test_rename_collision_raises_merge_conflict_with_payload(db): + svc = TagService(db) + target = await svc.find_or_create("Existing", TagKind.character) + source = await svc.find_or_create("Dupe", TagKind.character) + img = await _img(db) + await svc.add_to_image(img, source.id, source="manual") + + with pytest.raises(TagMergeConflict) as ei: + await svc.rename(source.id, "Existing") + exc = ei.value + assert exc.target_id == target.id + assert exc.target_name == "Existing" + assert exc.source_image_count == 1 + assert exc.will_alias is False # purely manual, ML-unknown + + +@pytest.mark.asyncio +async def test_merge_conflict_is_a_tag_validation_error(db): + assert issubclass(TagMergeConflict, TagValidationError) + + +@pytest.mark.asyncio +async def test_will_alias_true_when_machine_sourced(db): + svc = TagService(db) + await svc.find_or_create("Canon", TagKind.general) + source = await svc.find_or_create("Auto", TagKind.general) + img = await _img(db) + await svc.add_to_image(img, source.id, source="ml_auto") + with pytest.raises(TagMergeConflict) as ei: + await svc.rename(source.id, "Canon") + assert ei.value.will_alias is True + + +@pytest.mark.asyncio +async def test_will_alias_true_when_allowlisted(db): + svc = TagService(db) + await svc.find_or_create("Canon2", TagKind.general) + source = await svc.find_or_create("Allowed", TagKind.general) + db.add(TagAllowlist(tag_id=source.id)) + await db.flush() + with pytest.raises(TagMergeConflict) as ei: + await svc.rename(source.id, "Canon2") + assert ei.value.will_alias is True + + +@pytest.mark.asyncio +async def test_merge_rejects_self_merge(db): + svc = TagService(db) + t = await svc.find_or_create("Solo", TagKind.general) + with pytest.raises(TagValidationError, match="into itself"): + await svc.merge(t.id, t.id) + + +@pytest.mark.asyncio +async def test_merge_rejects_missing_tag(db): + svc = TagService(db) + t = await svc.find_or_create("Real", TagKind.general) + with pytest.raises(TagValidationError, match="not found"): + await svc.merge(t.id, 999999) + + +@pytest.mark.asyncio +async def test_merge_rejects_cross_kind(db): + svc = TagService(db) + a = await svc.find_or_create("NameA", TagKind.general) + b = await svc.find_or_create("NameB", TagKind.artist) + with pytest.raises(TagValidationError, match="same kind"): + await svc.merge(a.id, b.id) + + +@pytest.mark.asyncio +async def test_merge_rejects_cross_fandom(db): + svc = TagService(db) + f1 = await svc.find_or_create("Fandom1", TagKind.fandom) + f2 = await svc.find_or_create("Fandom2", TagKind.fandom) + c1 = await svc.find_or_create("Char", TagKind.character, fandom_id=f1.id) + c2 = await svc.find_or_create("Char2", TagKind.character, fandom_id=f2.id) + with pytest.raises(TagValidationError, match="same kind and fandom"): + await svc.merge(c1.id, c2.id) + + +@pytest.mark.asyncio +async def test_merge_empty_tags_returns_result(db): + svc = TagService(db) + a = await svc.find_or_create("EmptyA", TagKind.general) + b = await svc.find_or_create("EmptyB", TagKind.general) + result = await svc.merge(a.id, b.id) + assert isinstance(result, MergeResult) + assert result.target_id == b.id + assert result.target_name == "EmptyB" + assert result.target_kind == "general" + assert result.merged_count == 0 + assert result.alias_created is False + assert result.source_deleted is True + assert await db.get(Tag, a.id) is None + + +@pytest.mark.asyncio +async def test_merge_moves_image_tags_and_counts(db): + svc = TagService(db) + a = await svc.find_or_create("SrcIT", TagKind.general) + b = await svc.find_or_create("TgtIT", TagKind.general) + i1, i2 = await _img(db), await _img(db) + await svc.add_to_image(i1, a.id, source="manual") + await svc.add_to_image(i2, a.id, source="manual") + result = await svc.merge(a.id, b.id) + assert result.merged_count == 2 + rows = ( + await db.execute( + image_tag.select().where(image_tag.c.tag_id == b.id) + ) + ).all() + assert {r.image_record_id for r in rows} == {i1, i2} + assert ( + await db.execute( + image_tag.select().where(image_tag.c.tag_id == a.id) + ) + ).first() is None + + +@pytest.mark.asyncio +async def test_merge_dedups_image_tag_when_image_has_both(db): + svc = TagService(db) + a = await svc.find_or_create("SrcDup", TagKind.general) + b = await svc.find_or_create("TgtDup", TagKind.general) + img = await _img(db) + await svc.add_to_image(img, a.id, source="manual") + await svc.add_to_image(img, b.id, source="manual") + result = await svc.merge(a.id, b.id) + # The image already had the target → that source row is dropped, not moved. + assert result.merged_count == 0 + rows = ( + await db.execute( + image_tag.select().where( + image_tag.c.image_record_id == img + ) + ) + ).all() + assert [r.tag_id for r in rows] == [b.id] + + +@pytest.mark.asyncio +async def test_merge_dedups_suggestion_rejections(db): + from backend.app.models.tag_suggestion_rejection import ( + TagSuggestionRejection, + ) + + svc = TagService(db) + a = await svc.find_or_create("SrcRej", TagKind.general) + b = await svc.find_or_create("TgtRej", TagKind.general) + i1, i2 = await _img(db), await _img(db) + db.add(TagSuggestionRejection(image_record_id=i1, tag_id=a.id)) + db.add(TagSuggestionRejection(image_record_id=i2, tag_id=a.id)) + db.add(TagSuggestionRejection(image_record_id=i1, tag_id=b.id)) + await db.flush() + await svc.merge(a.id, b.id) + rej = ( + await db.execute( + select(TagSuggestionRejection).where( + TagSuggestionRejection.tag_id == b.id + ) + ) + ).scalars().all() + assert {r.image_record_id for r in rej} == {i1, i2} + assert ( + await db.execute( + select(TagSuggestionRejection).where( + TagSuggestionRejection.tag_id == a.id + ) + ) + ).first() is None + + +@pytest.mark.asyncio +async def test_merge_allowlist_target_has_keeps_target_threshold(db): + svc = TagService(db) + a = await svc.find_or_create("SrcAL", TagKind.general) + b = await svc.find_or_create("TgtAL", TagKind.general) + db.add(TagAllowlist(tag_id=a.id, min_confidence=0.5)) + db.add(TagAllowlist(tag_id=b.id, min_confidence=0.9)) + await db.flush() + await svc.merge(a.id, b.id) + rows = (await db.execute(select(TagAllowlist))).scalars().all() + assert len(rows) == 1 + assert rows[0].tag_id == b.id + assert rows[0].min_confidence == 0.9 + + +@pytest.mark.asyncio +async def test_merge_allowlist_source_only_moves_to_target(db): + svc = TagService(db) + a = await svc.find_or_create("SrcAL2", TagKind.general) + b = await svc.find_or_create("TgtAL2", TagKind.general) + db.add(TagAllowlist(tag_id=a.id, min_confidence=0.42)) + await db.flush() + await svc.merge(a.id, b.id) + rows = (await db.execute(select(TagAllowlist))).scalars().all() + assert len(rows) == 1 + assert rows[0].tag_id == b.id + assert rows[0].min_confidence == 0.42 + + +@pytest.mark.asyncio +async def test_merge_deletes_source_embedding(db): + from backend.app.models.tag_reference_embedding import ( + TagReferenceEmbedding, + ) + + svc = TagService(db) + a = await svc.find_or_create("SrcEmb", TagKind.general) + b = await svc.find_or_create("TgtEmb", TagKind.general) + db.add( + TagReferenceEmbedding( + tag_id=a.id, + embedding=[0.0] * 1152, + reference_count=1, + model_version="v", + ) + ) + await db.flush() + await svc.merge(a.id, b.id) + db.expire_all() # merge() uses Core DML; drop stale identity-map state + remaining = await db.scalar( + select(func.count()) + .select_from(TagReferenceEmbedding) + .where(TagReferenceEmbedding.tag_id == a.id) + ) + assert remaining == 0 + + +@pytest.mark.asyncio +async def test_merge_repoints_existing_aliases(db): + from backend.app.models.tag_alias import TagAlias + + svc = TagService(db) + a = await svc.find_or_create("SrcAlias", TagKind.general) + b = await svc.find_or_create("TgtAlias", TagKind.general) + db.add( + TagAlias( + alias_string="oldpred", + alias_category="general", + canonical_tag_id=a.id, + ) + ) + await db.flush() + await svc.merge(a.id, b.id) + # Read the column value, not the entity: merge() uses Core DML, so an + # ORM instance would be stale and a lazy reload here would emit sync IO + # outside the async greenlet (MissingGreenlet). + cid = await db.scalar( + select(TagAlias.canonical_tag_id).where( + TagAlias.alias_string == "oldpred" + ) + ) + assert cid == b.id + + +@pytest.mark.asyncio +async def test_merge_fandom_reparents_children(db): + svc = TagService(db) + f_src = await svc.find_or_create("FandomSrc", TagKind.fandom) + f_tgt = await svc.find_or_create("FandomTgt", TagKind.fandom) + child = await svc.find_or_create( + "ChildChar", TagKind.character, fandom_id=f_src.id + ) + await svc.merge(f_src.id, f_tgt.id) + fid = await db.scalar(select(Tag.fandom_id).where(Tag.id == child.id)) + assert fid == f_tgt.id + + +@pytest.mark.asyncio +async def test_no_alias_when_purely_manual(db): + from backend.app.models.tag_alias import TagAlias + + svc = TagService(db) + a = await svc.find_or_create("ManualOnly", TagKind.general) + b = await svc.find_or_create("CanonM", TagKind.general) + img = await _img(db) + await svc.add_to_image(img, a.id, source="manual") + result = await svc.merge(a.id, b.id) + assert result.alias_created is False + assert (await db.execute(select(TagAlias))).first() is None + + +@pytest.mark.asyncio +async def test_alias_per_observed_prediction_category(db): + from backend.app.models import ImageRecord + from backend.app.models.tag_alias import TagAlias + + svc = TagService(db) + a = await svc.find_or_create("predname", TagKind.general) + b = await svc.find_or_create("CanonP", TagKind.general) + img = await _img(db) + # mark source machine-known so keep_as_alias is True + await svc.add_to_image(img, a.id, source="ml_auto") + r1 = await db.get(ImageRecord, img) + r1.tagger_predictions = { + "predname": {"category": "general", "confidence": 0.9} + } + i2 = await _img(db) + r2 = await db.get(ImageRecord, i2) + r2.tagger_predictions = { + "predname": {"category": "copyright", "confidence": 0.8} + } + await db.flush() + result = await svc.merge(a.id, b.id) + assert result.alias_created is True + rows = ( + await db.execute( + select( + TagAlias.alias_string, + TagAlias.alias_category, + TagAlias.canonical_tag_id, + ) + ) + ).all() + assert {(r.alias_string, r.alias_category) for r in rows} == { + ("predname", "general"), + ("predname", "copyright"), + } + assert all(r.canonical_tag_id == b.id for r in rows) + + +@pytest.mark.asyncio +async def test_alias_fallback_to_kind_when_no_predictions(db): + from backend.app.models.tag_alias import TagAlias + + svc = TagService(db) + a = await svc.find_or_create("AllowNoPred", TagKind.character) + b = await svc.find_or_create("CanonF", TagKind.character) + db.add(TagAllowlist(tag_id=a.id)) + await db.flush() + result = await svc.merge(a.id, b.id) + assert result.alias_created is True + row = ( + await db.execute( + select( + TagAlias.alias_category, TagAlias.canonical_tag_id + ).where(TagAlias.alias_string == "AllowNoPred") + ) + ).one() + assert row.alias_category == "character" + assert row.canonical_tag_id == b.id + + +@pytest.mark.asyncio +async def test_alias_create_does_not_clobber_existing(db): + from backend.app.models import ImageRecord + from backend.app.models.tag_alias import TagAlias + + svc = TagService(db) + a = await svc.find_or_create("dupalias", TagKind.general) + b = await svc.find_or_create("CanonD", TagKind.general) + other = await svc.find_or_create("OtherCanon", TagKind.general) + db.add( + TagAlias( + alias_string="dupalias", + alias_category="general", + canonical_tag_id=other.id, + ) + ) + img = await _img(db) + await svc.add_to_image(img, a.id, source="ml_auto") + r = await db.get(ImageRecord, img) + r.tagger_predictions = { + "dupalias": {"category": "general", "confidence": 0.9} + } + await db.flush() + await svc.merge(a.id, b.id) + cid = await db.scalar( + select(TagAlias.canonical_tag_id).where( + TagAlias.alias_string == "dupalias" + ) + ) + assert cid == other.id # pre-existing alias untouched + + +@pytest.mark.asyncio +async def test_merge_repoints_series_pages(db): + from backend.app.models.series_page import SeriesPage + + svc = TagService(db) + a = await svc.find_or_create("SerA", TagKind.series) + b = await svc.find_or_create("SerB", TagKind.series) + img = await _img(db) + db.add(SeriesPage(series_tag_id=a.id, image_id=img, page_number=1)) + await db.flush() + await svc.merge(a.id, b.id) + owner = await db.scalar( + select(SeriesPage.series_tag_id).where(SeriesPage.image_id == img) + ) + assert owner == b.id diff --git a/tests/test_tag_rename.py b/tests/test_tag_rename.py new file mode 100644 index 0000000..fcb1ee2 --- /dev/null +++ b/tests/test_tag_rename.py @@ -0,0 +1,41 @@ +import pytest + +from backend.app.models import TagKind +from backend.app.services.tag_service import TagService, TagValidationError + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_rename_success(db): + svc = TagService(db) + t = await svc.find_or_create("uchiha_sasuke", TagKind.character) + renamed = await svc.rename(t.id, "Sasuke Uchiha") + assert renamed.id == t.id + assert renamed.name == "Sasuke Uchiha" + + +@pytest.mark.asyncio +async def test_rename_collision_raises(db): + svc = TagService(db) + await svc.find_or_create("Existing", TagKind.character) + other = await svc.find_or_create("Other", TagKind.character) + with pytest.raises(TagValidationError, match="already exists"): + await svc.rename(other.id, "Existing") + + +@pytest.mark.asyncio +async def test_rename_same_name_different_kind_ok(db): + svc = TagService(db) + await svc.find_or_create("Shared", TagKind.character) + artist = await svc.find_or_create("ArtistTag", TagKind.artist) + renamed = await svc.rename(artist.id, "Shared") + assert renamed.name == "Shared" + + +@pytest.mark.asyncio +async def test_rename_empty_raises(db): + svc = TagService(db) + t = await svc.find_or_create("Whatever", TagKind.general) + with pytest.raises(TagValidationError): + await svc.rename(t.id, " ") diff --git a/tests/test_tag_service.py b/tests/test_tag_service.py new file mode 100644 index 0000000..a8f6b4c --- /dev/null +++ b/tests/test_tag_service.py @@ -0,0 +1,103 @@ +import pytest + +from backend.app.models import TagKind +from backend.app.services.tag_service import TagService, TagValidationError + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_find_or_create_creates(db): + svc = TagService(db) + t = await svc.find_or_create("Alice", TagKind.character, fandom_id=None) + assert t.id is not None + assert t.name == "Alice" + assert t.kind == TagKind.character + + +@pytest.mark.asyncio +async def test_find_or_create_idempotent(db): + svc = TagService(db) + a = await svc.find_or_create("Bob", TagKind.artist) + b = await svc.find_or_create("Bob", TagKind.artist) + assert a.id == b.id + + +@pytest.mark.asyncio +async def test_same_name_different_kind_coexists(db): + svc = TagService(db) + artist = await svc.find_or_create("Forge", TagKind.artist) + series = await svc.find_or_create("Forge", TagKind.series) + assert artist.id != series.id + + +@pytest.mark.asyncio +async def test_fandom_id_requires_character(db): + svc = TagService(db) + fandom = await svc.find_or_create("Naruto", TagKind.fandom) + with pytest.raises(TagValidationError, match="character"): + await svc.find_or_create("Bob", TagKind.artist, fandom_id=fandom.id) + + +@pytest.mark.asyncio +async def test_fandom_id_must_reference_fandom(db): + svc = TagService(db) + artist = await svc.find_or_create("X", TagKind.artist) + with pytest.raises(TagValidationError, match="fandom"): + await svc.find_or_create("Y", TagKind.character, fandom_id=artist.id) + + +@pytest.mark.asyncio +async def test_character_with_fandom(db): + svc = TagService(db) + fandom = await svc.find_or_create("Naruto", TagKind.fandom) + char = await svc.find_or_create("Sasuke", TagKind.character, fandom_id=fandom.id) + assert char.fandom_id == fandom.id + + +@pytest.mark.asyncio +async def test_character_same_name_different_fandoms(db): + svc = TagService(db) + f1 = await svc.find_or_create("Series A", TagKind.fandom) + f2 = await svc.find_or_create("Series B", TagKind.fandom) + c1 = await svc.find_or_create("Alice", TagKind.character, fandom_id=f1.id) + c2 = await svc.find_or_create("Alice", TagKind.character, fandom_id=f2.id) + assert c1.id != c2.id + + +@pytest.mark.asyncio +async def test_empty_name_raises(db): + svc = TagService(db) + with pytest.raises(TagValidationError): + await svc.find_or_create(" ", TagKind.general) + + +@pytest.mark.asyncio +async def test_autocomplete_ranking(db): + svc = TagService(db) + await svc.find_or_create("alice", TagKind.character) + await svc.find_or_create("alicent", TagKind.character) + await svc.find_or_create("malice", TagKind.character) + hits = await svc.autocomplete("alice") + # exact match first, then prefix, then substring + names = [h.name for h in hits] + assert names[0] == "alice" + assert names[1] == "alicent" + assert names[2] == "malice" + + +@pytest.mark.asyncio +async def test_autocomplete_kind_filter(db): + svc = TagService(db) + await svc.find_or_create("Foo", TagKind.artist) + await svc.find_or_create("Foo", TagKind.character) + hits = await svc.autocomplete("Foo", kind=TagKind.artist) + assert len(hits) == 1 + assert hits[0].kind == "artist" + + +@pytest.mark.asyncio +async def test_autocomplete_empty_query_returns_nothing(db): + svc = TagService(db) + await svc.find_or_create("Foo", TagKind.artist) + assert await svc.autocomplete("") == [] diff --git a/tests/test_tasks_ml.py b/tests/test_tasks_ml.py new file mode 100644 index 0000000..e759a2e --- /dev/null +++ b/tests/test_tasks_ml.py @@ -0,0 +1,122 @@ +"""tag_and_embed / backfill task tests. Models aren't in CI, so we test +the pure helpers (_maxpool_predictions, _is_video) as unit tests, and the +DB-touching backfill query as an integration test with monkeypatched +inference. +""" + +from pathlib import Path + +import pytest + +from backend.app.services.ml.tagger import TagPrediction +from backend.app.tasks.ml import _is_video, _maxpool_predictions + + +def test_is_video(): + assert _is_video(Path("a.mp4")) is True + assert _is_video(Path("a.MKV")) is True + assert _is_video(Path("a.jpg")) is False + + +def test_maxpool_predictions(): + f1 = {"smile": TagPrediction("smile", "general", 0.6)} + f2 = { + "smile": TagPrediction("smile", "general", 0.9), + "sword": TagPrediction("sword", "general", 0.7), + } + merged = _maxpool_predictions([f1, f2]) + assert merged["smile"]["confidence"] == 0.9 + assert merged["sword"]["confidence"] == 0.7 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_backfill_enqueues_missing(db, monkeypatch): + from backend.app.models import ImageRecord + from backend.app.tasks import ml as ml_tasks + + calls = [] + monkeypatch.setattr( + ml_tasks.tag_and_embed, "delay", lambda image_id: calls.append(image_id) + ) + + img = ImageRecord( + path="/images/n.jpg", sha256="n" * 64, size_bytes=1, + mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + tagger_predictions=None, siglip_embedding=None, + ) + db.add(img) + await db.commit() + + count = ml_tasks.backfill() + assert count >= 1 + assert img.id in calls + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_apply_allowlist_applies_above_threshold(db): + from sqlalchemy import select + + from backend.app.models import ImageRecord, TagAllowlist, TagKind + from backend.app.models.tag import image_tag + from backend.app.services.tag_service import TagService + from backend.app.tasks import ml as ml_tasks + + tag = await TagService(db).find_or_create("autohero", TagKind.character) + db.add(TagAllowlist(tag_id=tag.id, min_confidence=0.95)) + img = ImageRecord( + path="/images/al.jpg", sha256="al" + "0" * 62, size_bytes=1, + mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + tagger_predictions={ + "autohero": {"category": "character", "confidence": 0.97} + }, + ) + db.add(img) + await db.commit() + + n = ml_tasks.apply_allowlist_tags(tag_id=tag.id) + assert n >= 1 + src = ( + await db.execute( + select(image_tag.c.source) + .where(image_tag.c.image_record_id == img.id) + .where(image_tag.c.tag_id == tag.id) + ) + ).scalar_one() + assert src == "ml_auto" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_apply_allowlist_skips_below_threshold(db): + from sqlalchemy import select + + from backend.app.models import ImageRecord, TagAllowlist, TagKind + from backend.app.models.tag import image_tag + from backend.app.services.tag_service import TagService + from backend.app.tasks import ml as ml_tasks + + tag = await TagService(db).find_or_create("lowconf", TagKind.character) + db.add(TagAllowlist(tag_id=tag.id, min_confidence=0.95)) + img = ImageRecord( + path="/images/lc.jpg", sha256="lc" + "0" * 62, size_bytes=1, + mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + tagger_predictions={ + "lowconf": {"category": "character", "confidence": 0.40} + }, + ) + db.add(img) + await db.commit() + ml_tasks.apply_allowlist_tags(tag_id=tag.id) + applied = ( + await db.execute( + select(image_tag.c.tag_id) + .where(image_tag.c.image_record_id == img.id) + .where(image_tag.c.tag_id == tag.id) + ) + ).scalar_one_or_none() + assert applied is None diff --git a/tests/test_tasks_register.py b/tests/test_tasks_register.py new file mode 100644 index 0000000..a8d4f72 --- /dev/null +++ b/tests/test_tasks_register.py @@ -0,0 +1,25 @@ +"""Confirms every FC-2a task module imports cleanly and registers with Celery. + +Celery's `include=[...]` parameter on the Celery() constructor is lazy — it +only imports those modules when a worker boots. To assert task registration +in a plain pytest run we explicitly import the task modules ourselves. +""" + +# Importing for side-effect: each module's @celery.task decorators register +# the task with the global Celery instance at module import time. +import backend.app.tasks.import_file # noqa: F401 +import backend.app.tasks.scan # noqa: F401 +import backend.app.tasks.thumbnail # noqa: F401 +from backend.app.celery_app import celery + + +def test_scan_task_registered(): + assert "backend.app.tasks.scan.scan_directory" in celery.tasks + + +def test_import_media_file_registered(): + assert "backend.app.tasks.import_file.import_media_file" in celery.tasks + + +def test_generate_thumbnail_registered(): + assert "backend.app.tasks.thumbnail.generate_thumbnail" in celery.tasks diff --git a/tests/test_text_utils.py b/tests/test_text_utils.py new file mode 100644 index 0000000..18bf2bc --- /dev/null +++ b/tests/test_text_utils.py @@ -0,0 +1,56 @@ +"""FC-3e: HTML-strip + word-boundary truncator unit tests.""" + +from backend.app.utils.text import html_to_plain, truncate_at_word + + +def test_html_to_plain_strips_tags(): + assert html_to_plain("

hi

") == "hi" + + +def test_html_to_plain_converts_br_to_newline(): + assert html_to_plain("a
b") == "a\nb" + + +def test_html_to_plain_converts_p_close_to_newline(): + assert html_to_plain("

one

two

") == "one\ntwo" + + +def test_html_to_plain_collapses_runs_of_whitespace(): + assert html_to_plain("a b") == "a b" + + +def test_html_to_plain_preserves_intentional_newlines_between_blocks(): + assert html_to_plain("alpha

beta") == "alpha\n\nbeta" + + +def test_html_to_plain_idempotent_on_plain_text(): + assert html_to_plain("plain text") == "plain text" + + +def test_html_to_plain_none_returns_empty(): + assert html_to_plain(None) == "" + + +def test_html_to_plain_decodes_entities(): + assert html_to_plain("a & b") == "a & b" + + +def test_truncate_at_word_under_limit_no_change(): + assert truncate_at_word("short", 100) == ("short", False) + + +def test_truncate_at_word_breaks_at_whitespace(): + assert truncate_at_word("alpha beta gamma", 8) == ("alpha…", True) + + +def test_truncate_at_word_no_whitespace_hard_cuts(): + # No space to break on within the limit — hard-cut at the limit. + assert truncate_at_word("abcdefghij", 5) == ("abcde…", True) + + +def test_truncate_at_word_handles_none(): + assert truncate_at_word(None, 10) == ("", False) + + +def test_truncate_at_word_handles_empty_string(): + assert truncate_at_word("", 10) == ("", False) diff --git a/tests/test_thumbnailer.py b/tests/test_thumbnailer.py new file mode 100644 index 0000000..851de13 --- /dev/null +++ b/tests/test_thumbnailer.py @@ -0,0 +1,66 @@ +"""Image-thumbnailer smoke tests. Video tests are not included here because +they require a real ffmpeg + a real video on disk; FC-2e's integration suite +will cover those end-to-end against the running container. +""" + +from pathlib import Path + +import pytest +from PIL import Image + +from backend.app.services.thumbnailer import ( + THUMB_MAX_PX, + Thumbnailer, + thumbnail_path_for, +) + + +@pytest.fixture +def thumbnailer(tmp_path): + return Thumbnailer(images_root=tmp_path) + + +def _make_image(path: Path, size: tuple[int, int], mode: str = "RGB"): + im = Image.new(mode, size, color=(200, 100, 50, 255) if mode == "RGBA" else (200, 100, 50)) + path.parent.mkdir(parents=True, exist_ok=True) + im.save(path) + + +def test_thumbnail_path_for_jpeg(): + p = thumbnail_path_for("abcdef" + "0" * 58, alpha=False, root=Path("/images")) + assert p == Path("/images/thumbs/abc/" + "abcdef" + "0" * 58 + ".jpg") + + +def test_thumbnail_path_for_png(): + p = thumbnail_path_for("abcdef" + "0" * 58, alpha=True, root=Path("/images")) + assert p.suffix == ".png" + + +def test_generate_thumbnail_rgb(thumbnailer, tmp_path): + src = tmp_path / "src.jpg" + _make_image(src, (1600, 1200), mode="RGB") + sha = "deadbeef" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.mime == "image/jpeg" + assert result.path.suffix == ".jpg" + assert result.path.exists() + assert max(result.width, result.height) == THUMB_MAX_PX + + +def test_generate_thumbnail_rgba_preserves_alpha(thumbnailer, tmp_path): + src = tmp_path / "src.png" + _make_image(src, (1200, 800), mode="RGBA") + sha = "feedface" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.mime == "image/png" + assert result.path.suffix == ".png" + + +def test_generate_thumbnail_tall_image(thumbnailer, tmp_path): + src = tmp_path / "src.jpg" + _make_image(src, (200, 2000), mode="RGB") + sha = "cafef00d" + "0" * 56 + result = thumbnailer.generate_image_thumbnail(src, sha) + assert result.height == THUMB_MAX_PX + # tall image: width scales down proportionally + assert result.width < THUMB_MAX_PX diff --git a/tests/test_tick_due_sources.py b/tests/test_tick_due_sources.py new file mode 100644 index 0000000..3df93e1 --- /dev/null +++ b/tests/test_tick_due_sources.py @@ -0,0 +1,132 @@ +"""FC-3d: tick_due_sources task tests. + +Verifies (a) due sources are queued, (b) sources with a pending/running +DownloadEvent are skipped, (c) the tick is read-only on Source itself +(no last_checked_at write), and (d) the result dict shape. +""" +import pytest +from sqlalchemy import select + +import backend.app.tasks.scan # noqa: F401 — side-effect: register celery task +from backend.app.celery_app import celery +from backend.app.models import Artist, DownloadEvent, Source + +pytestmark = pytest.mark.integration + + +class _FakeTask: + """Stand-in for the real Celery Task object — only `.delay()` matters.""" + + def __init__(self, sink: list): + self._sink = sink + + def delay(self, source_id: int): + self._sink.append(source_id) + + +@pytest.mark.asyncio +async def test_tick_returns_counts_dict_when_no_due_sources(db, monkeypatch): + """Conftest TRUNCATEs after every integration test, so we start clean.""" + delayed: list[int] = [] + monkeypatch.setattr( + "backend.app.tasks.download.download_source", _FakeTask(delayed), + ) + from backend.app.tasks.scan import _tick_due_sources_async + + result = await _tick_due_sources_async() + assert set(result.keys()) == {"due", "queued", "skipped_in_flight", "tick_at"} + assert result["due"] == 0 + assert result["queued"] == 0 + assert result["skipped_in_flight"] == 0 + assert delayed == [] + + +@pytest.mark.asyncio +async def test_tick_queues_due_source(db, monkeypatch): + artist = Artist(name="tick-q", slug="tick-q", auto_check=True) + db.add(artist) + await db.flush() + src = Source( + artist_id=artist.id, platform="patreon", url="https://tick-q", + enabled=True, consecutive_failures=0, + ) + db.add(src) + await db.commit() + + delayed: list[int] = [] + monkeypatch.setattr( + "backend.app.tasks.download.download_source", _FakeTask(delayed), + ) + from backend.app.tasks.scan import _tick_due_sources_async + result = await _tick_due_sources_async() + assert result["queued"] >= 1 + assert src.id in delayed + + pending_count = (await db.execute( + select(DownloadEvent.id).where( + DownloadEvent.source_id == src.id, + DownloadEvent.status == "pending", + ) + )).scalars().all() + assert len(pending_count) == 1 + + +@pytest.mark.asyncio +async def test_tick_skips_in_flight_source(db, monkeypatch): + artist = Artist(name="tick-inf", slug="tick-inf", auto_check=True) + db.add(artist) + await db.flush() + src = Source( + artist_id=artist.id, platform="patreon", url="https://tick-inf", + enabled=True, consecutive_failures=0, + ) + db.add(src) + await db.flush() + db.add(DownloadEvent(source_id=src.id, status="running")) + await db.commit() + + delayed: list[int] = [] + monkeypatch.setattr( + "backend.app.tasks.download.download_source", _FakeTask(delayed), + ) + from backend.app.tasks.scan import _tick_due_sources_async + result = await _tick_due_sources_async() + assert src.id not in delayed + assert result["skipped_in_flight"] >= 1 + + +@pytest.mark.asyncio +async def test_tick_does_not_modify_source_last_checked_at(db, monkeypatch): + """The tick creates DownloadEvent(pending) but never writes Source columns. + Source.last_checked_at is owned by DownloadService.finalize. + """ + artist = Artist(name="tick-ro", slug="tick-ro", auto_check=True) + db.add(artist) + await db.flush() + src = Source( + artist_id=artist.id, platform="patreon", url="https://tick-ro", + enabled=True, consecutive_failures=0, + ) + db.add(src) + await db.commit() + src_id = src.id + + before = (await db.execute( + select(Source.last_checked_at).where(Source.id == src_id) + )).scalar_one() + + monkeypatch.setattr( + "backend.app.tasks.download.download_source", _FakeTask([]), + ) + from backend.app.tasks.scan import _tick_due_sources_async + await _tick_due_sources_async() + + after = (await db.execute( + select(Source.last_checked_at).where(Source.id == src_id) + )).scalar_one() + assert before == after # still None / unchanged + + +def test_tick_task_is_registered(): + """Side-effect import at the top of this test file forces task module load.""" + assert "backend.app.tasks.scan.tick_due_sources" in celery.tasks diff --git a/tests/test_verify_integrity.py b/tests/test_verify_integrity.py new file mode 100644 index 0000000..cc05e3b --- /dev/null +++ b/tests/test_verify_integrity.py @@ -0,0 +1,141 @@ +"""FC-2e: verify_integrity — sha + decode/probe per ImageRecord.""" + +import subprocess + +import pytest +from PIL import Image + +from backend.app.models import ImageRecord +from backend.app.tasks.maintenance import verify_integrity + +pytestmark = pytest.mark.integration + + +class _Ctx: + def __init__(self, s): + self.s = s + + def __enter__(self): + return self.s + + def __exit__(self, *a): + return False + + +def _sf(db_sync): + class _SM: + def __call__(self): + return _Ctx(db_sync) + + return _SM() + + +def _good_jpeg(path): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (32, 32), (40, 80, 160)).save(path, "JPEG") + + +def _rec(db_sync, path, *, sha, mime="image/jpeg"): + rec = ImageRecord( + path=str(path), sha256=sha, phash=None, size_bytes=1, + mime=mime, width=32, height=32, origin="imported_filesystem", + integrity_status="unknown", + ) + db_sync.add(rec) + db_sync.flush() + return rec + + +def _sha_of(path): + import hashlib + + h = hashlib.sha256() + h.update(path.read_bytes()) + return h.hexdigest() + + +def test_ok_corrupt_failed_paths(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + good = tmp_path / "good.jpg" + _good_jpeg(good) + good2 = tmp_path / "good2.jpg" # distinct path — image_record.path is unique + _good_jpeg(good2) + bad = tmp_path / "bad.jpg" + bad.write_bytes(b"not a real jpeg") + truncated = tmp_path / "trunc.jpg" + _good_jpeg(truncated) + truncated.write_bytes(truncated.read_bytes()[:30]) # break the bitstream + + rec_ok = _rec(db_sync, good, sha=_sha_of(good)) + rec_sha_mismatch = _rec(db_sync, good2, sha="0" * 64) # bytes intact, sha wrong → corrupt + rec_decode_fail = _rec(db_sync, truncated, sha=_sha_of(truncated)) + # 'bad' has not-jpeg bytes; sha matches the bad bytes → decode fails + rec_bad_decode = _rec(db_sync, bad, sha=_sha_of(bad)) + rec_missing = _rec( + db_sync, tmp_path / "gone.jpg", sha="g" + "0" * 63, + ) + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + total = verify_integrity() + db_sync.expire_all() + assert total == 5 + assert db_sync.get(ImageRecord, rec_ok.id).integrity_status == "ok" + assert db_sync.get(ImageRecord, rec_sha_mismatch.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_decode_fail.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_bad_decode.id).integrity_status == "corrupt" + assert db_sync.get(ImageRecord, rec_missing.id).integrity_status == "failed_verification" + + # Idempotent — second run re-derives the same verdict. + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec_ok.id).integrity_status == "ok" + + +def test_video_ffprobe_paths(db_sync, tmp_path, monkeypatch): + from backend.app.tasks import maintenance as m + + vid = tmp_path / "v.mp4" + vid.write_bytes(b"fake but consistent video bytes") + rec = _rec(db_sync, vid, sha=_sha_of(vid), mime="video/mp4") + db_sync.commit() + + monkeypatch.setattr(m, "_sync_session_factory", lambda: _sf(db_sync)) + + # Case A: ffprobe succeeds → ok + def _run_ok(args, **kw): + return subprocess.CompletedProcess(args, 0, b"", b"") + + monkeypatch.setattr(m.subprocess, "run", _run_ok) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "ok" + + # Case B: ffprobe non-zero → corrupt + def _run_bad(args, **kw): + return subprocess.CompletedProcess(args, 1, b"", b"err") + + monkeypatch.setattr(m.subprocess, "run", _run_bad) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "corrupt" + + # Case C: ffprobe times out → corrupt + def _run_timeout(args, **kw): + raise subprocess.TimeoutExpired(cmd=args, timeout=10) + + monkeypatch.setattr(m.subprocess, "run", _run_timeout) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "corrupt" + + # Case D: ffprobe binary missing → failed_verification + def _run_missing(args, **kw): + raise FileNotFoundError("ffprobe") + + monkeypatch.setattr(m.subprocess, "run", _run_missing) + verify_integrity() + db_sync.expire_all() + assert db_sync.get(ImageRecord, rec.id).integrity_status == "failed_verification"