Merge dev: FC-1 → FC-5 v1 build (#1)
First merge of `dev` into `main` for FabledCurator. Brings FC-1 (Foundation) through FC-5 (Migration tooling) onto `main`. See PR #1 body for the full stage rollup.
This commit was merged in pull request #1.
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
# Database
|
||||
DB_USER=fabledcurator
|
||||
DB_PASSWORD=changeme_use_a_real_password
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_NAME=fabledcurator
|
||||
|
||||
# Redis / Celery
|
||||
CELERY_BROKER_URL=redis://redis:6379/0
|
||||
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
|
||||
# App
|
||||
# Generate with: openssl rand -hex 32
|
||||
SECRET_KEY=changeme_32_byte_hex_secret
|
||||
|
||||
# Extension API key — used in FC-3, lands later but reserved now
|
||||
# Generate with: openssl rand -hex 32
|
||||
EXTENSION_API_KEY=
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Deployment posture: plain HTTP (no TLS in the app; reverse proxy if needed)
|
||||
# See docs/superpowers/specs/2026-05-13-fabledcurator-merge-design.md §2.1
|
||||
@@ -0,0 +1,74 @@
|
||||
name: Build images
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
# - write:release (for future release-cutting workflows)
|
||||
# - write:issue (for future issue-management automation)
|
||||
# The injected GITHUB_TOKEN cannot be used — it lacks write:package.
|
||||
|
||||
jobs:
|
||||
build-web:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Determine tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Login to Forgejo registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: git.fabledsword.com
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.RELEASE_TOKEN }}
|
||||
|
||||
- name: Build and push web image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
|
||||
build-ml:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Determine tag
|
||||
id: tag
|
||||
run: |
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Login to Forgejo registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: git.fabledsword.com
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.RELEASE_TOKEN }}
|
||||
|
||||
- name: Build and push ml image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.ml
|
||||
push: true
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
@@ -0,0 +1,124 @@
|
||||
name: CI
|
||||
|
||||
# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
|
||||
# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers.
|
||||
# - frontend-build: vitest unit + vite build.
|
||||
# - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
backend-lint-and-test:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
# DB_PASSWORD and SECRET_KEY are required by config.py at import time
|
||||
# even though unit tests don't actually touch the DB or use the secret.
|
||||
DB_PASSWORD: ci_unit_test_placeholder
|
||||
SECRET_KEY: ci_unit_test_placeholder
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python deps
|
||||
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
||||
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
||||
# versions live on the runner image, not here.
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio
|
||||
|
||||
- name: Ruff lint
|
||||
run: ruff check backend/ tests/ alembic/
|
||||
|
||||
- name: Pytest (unit only — integration runs in the integration job)
|
||||
run: pytest tests/ -v -m "not integration"
|
||||
|
||||
frontend-build:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
||||
# If we want strict lockfile-based reproducibility later, commit a
|
||||
# package-lock.json and flip this back to `npm ci`.
|
||||
- run: npm install --no-audit --no-fund
|
||||
# `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
|
||||
# with no .ts files and no JSDoc annotations, so vue-tsc has nothing
|
||||
# to type-check. Re-enable once we add a tsconfig.json and either
|
||||
# convert to TS or add JSDoc.
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
||||
# default bridge with NO service-name DNS, and publishing fixed host
|
||||
# ports collides with the operator's running docker-compose dev stack on
|
||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
||||
# each service by its bridge IP — discovered at runtime via the mounted
|
||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
||||
# values don't depend on cross-step env passing. Pattern documented in
|
||||
# FabledRulebook/forgejo.md "CI philosophy".
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
# Scope to THIS job's service containers (act_runner names them
|
||||
# ...JOB-integration...); the operator's compose stack uses the
|
||||
# same images but different names, so it won't match.
|
||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration
|
||||
+8
-7
@@ -42,13 +42,14 @@ yarn-error.log*
|
||||
extension/*.xpi
|
||||
extension/web-ext-artifacts/
|
||||
|
||||
# Runtime volumes (must never be tracked)
|
||||
images/
|
||||
import/
|
||||
downloads/
|
||||
models/
|
||||
postgres_data/
|
||||
redis_data/
|
||||
# Runtime volumes (must never be tracked) — anchored to repo root so they
|
||||
# don't accidentally match Python package dirs like backend/app/models/
|
||||
/images/
|
||||
/import/
|
||||
/downloads/
|
||||
/models/
|
||||
/postgres_data/
|
||||
/redis_data/
|
||||
|
||||
# IDE / OS
|
||||
.vscode/
|
||||
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM node:22-alpine AS frontend-builder
|
||||
WORKDIR /build
|
||||
COPY frontend/package.json frontend/package-lock.json* ./
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
# feedback-no-local-runs), so `npm install` instead of `npm ci`. Flip to
|
||||
# `npm ci` once a lockfile is committed.
|
||||
RUN npm install --no-audit --no-fund
|
||||
COPY frontend/ ./
|
||||
RUN npm run build
|
||||
|
||||
FROM python:3.14-slim AS runtime
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
# System deps: ffmpeg (transcode + thumbnails, used in FC-2), unar (archives, FC-2),
|
||||
# libpq for psycopg, image libs.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
unar \
|
||||
libpq5 \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY backend/ ./backend/
|
||||
COPY alembic/ ./alembic/
|
||||
COPY alembic.ini ./
|
||||
COPY entrypoint.sh ./
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
COPY --from=frontend-builder /build/dist ./frontend/dist
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["web"]
|
||||
@@ -0,0 +1,43 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM python:3.14-slim
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
HF_HOME=/models/.huggingface \
|
||||
TRANSFORMERS_CACHE=/models/.huggingface \
|
||||
ML_MODEL_DIR=/models
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libpq5 \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements-ml.txt requirements.txt ./
|
||||
# CPU-only torch: the default PyPI wheel bundles the CUDA runtime (~5.6GB
|
||||
# layer); this pipeline never uses a GPU. --index-url (not --extra-index-url)
|
||||
# guarantees only +cpu wheels are considered, so no nvidia-*-cu12 deps.
|
||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||
"torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
|
||||
RUN pip install -r requirements-ml.txt
|
||||
|
||||
COPY backend/ ./backend/
|
||||
COPY alembic/ ./alembic/
|
||||
COPY alembic.ini ./
|
||||
COPY entrypoint.sh ./
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
# Models self-heal into /models on first start (FC-2 implements this)
|
||||
VOLUME ["/models"]
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["ml-worker"]
|
||||
@@ -10,17 +10,41 @@ Pre-v1. Not yet functional.
|
||||
|
||||
## Quick start
|
||||
|
||||
For local development and testing, just:
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# edit .env: set DB_PASSWORD and SECRET_KEY
|
||||
docker compose up -d
|
||||
# UI: http://localhost:8080
|
||||
```
|
||||
|
||||
That uses sane dev defaults baked into `docker-compose.yml` and the dev
|
||||
override (`docker-compose.override.yml`, auto-merged) — local builds, DEBUG
|
||||
logging, exposed Postgres + Redis ports on the host. No `.env` required.
|
||||
|
||||
For a production-like deployment, override the dev defaults via shell env
|
||||
or a `.env` file (see `.env.example` for the variable names) and use:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml up -d
|
||||
# (skips the override so containers pull registry images)
|
||||
```
|
||||
|
||||
## Deployment posture
|
||||
|
||||
FabledCurator is designed to run inside a self-hosted homelab environment over plain HTTP. If you want TLS, terminate it at your reverse proxy. The app does not generate certificates, redirect to HTTPS, or set HSTS.
|
||||
|
||||
## CI / Forgejo setup
|
||||
|
||||
The repo's workflows expect:
|
||||
|
||||
- **Runner label `python-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. The runner image (`runner-base:python-ci`) is built from `CI-Runner/CI-python/` in the operator's workspace; `make push` from that directory builds and pushes a new image when toolchain pins change.
|
||||
- **Repo secret `RELEASE_TOKEN`** — a Forgejo PAT with the following scopes:
|
||||
- `write:package` + `read:package` — for `docker push` to `git.fabledsword.com`
|
||||
- `write:release` — for future release-cutting workflows
|
||||
- `write:issue` — for future issue-management automation
|
||||
|
||||
Generate at https://git.fabledsword.com/user/settings/applications. The injected `GITHUB_TOKEN` cannot be used because it lacks `write:package`.
|
||||
|
||||
## License
|
||||
|
||||
Personal project; use at your own discretion.
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
version_path_separator = os
|
||||
sqlalchemy.url =
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Alembic environment — reads DATABASE_URL from app config."""
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from alembic import context
|
||||
from backend.app.config import get_config
|
||||
from backend.app.models import Base
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
config.set_main_option("sqlalchemy.url", get_config().database_url_sync)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,277 @@
|
||||
"""initial unified schema
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-05-13
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
revision: str = "0001"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
|
||||
op.create_table(
|
||||
"artist",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("slug", sa.String(length=255), nullable=False),
|
||||
sa.Column("notes", sa.Text(), nullable=True),
|
||||
sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("check_interval_seconds", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_artist"),
|
||||
sa.UniqueConstraint("name", name="uq_artist_name"),
|
||||
sa.UniqueConstraint("slug", name="uq_artist_slug"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"source",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("artist_id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("config_overrides", sa.JSON(), nullable=True),
|
||||
sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("check_interval_override", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_source"),
|
||||
)
|
||||
op.create_index("ix_source_artist_id", "source", ["artist_id"])
|
||||
|
||||
op.create_table(
|
||||
"credential",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="active"),
|
||||
sa.Column(
|
||||
"captured_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_credential"),
|
||||
sa.UniqueConstraint("platform", name="uq_credential_platform"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"post",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("external_post_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("post_url", sa.Text(), nullable=True),
|
||||
sa.Column("post_title", sa.Text(), nullable=True),
|
||||
sa.Column("post_date", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("raw_metadata", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"downloaded_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_post"),
|
||||
sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
)
|
||||
op.create_index("ix_post_source_id", "post", ["source_id"])
|
||||
|
||||
op.create_table(
|
||||
"image_record",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("phash", sa.String(length=32), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("mime", sa.String(length=64), nullable=False),
|
||||
sa.Column("width", sa.Integer(), nullable=True),
|
||||
sa.Column("height", sa.Integer(), nullable=True),
|
||||
sa.Column("thumbnail_path", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"origin",
|
||||
sa.Enum(
|
||||
"downloaded",
|
||||
"imported_filesystem",
|
||||
"uploaded",
|
||||
name="origin_enum",
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("primary_post_id", sa.Integer(), nullable=True),
|
||||
sa.Column("wd14_predictions", sa.JSON(), nullable=True),
|
||||
sa.Column("wd14_model_version", sa.String(length=128), nullable=True),
|
||||
sa.Column("siglip_embedding", Vector(1152), nullable=True),
|
||||
sa.Column("siglip_model_version", sa.String(length=128), nullable=True),
|
||||
sa.Column("centroid_scores", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["primary_post_id"],
|
||||
["post.id"],
|
||||
name="fk_image_record_primary_post_id_post",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_image_record"),
|
||||
sa.UniqueConstraint("path", name="uq_image_record_path"),
|
||||
sa.UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||
)
|
||||
op.create_index("ix_image_record_sha256", "image_record", ["sha256"])
|
||||
op.create_index("ix_image_record_phash", "image_record", ["phash"])
|
||||
op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"])
|
||||
|
||||
op.create_table(
|
||||
"image_provenance",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("post_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("captured_metadata", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"captured_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"],
|
||||
["image_record.id"],
|
||||
name="fk_image_provenance_image_record_id_image_record",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["post_id"],
|
||||
["post.id"],
|
||||
name="fk_image_provenance_post_id_post",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
["source.id"],
|
||||
name="fk_image_provenance_source_id_source",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_image_provenance"),
|
||||
)
|
||||
op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"])
|
||||
op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"])
|
||||
op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"])
|
||||
|
||||
op.create_table(
|
||||
"tag",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("namespace", sa.String(length=64), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tag"),
|
||||
sa.UniqueConstraint("name", name="uq_tag_name"),
|
||||
)
|
||||
op.create_index("ix_tag_name", "tag", ["name"])
|
||||
op.create_index("ix_tag_namespace", "tag", ["namespace"])
|
||||
|
||||
op.create_table(
|
||||
"image_tag",
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"],
|
||||
["image_record.id"],
|
||||
name="fk_image_tag_image_record_id_image_record",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"download_event",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("post_id", sa.Integer(), nullable=True),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
["source.id"],
|
||||
name="fk_download_event_source_id_source",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["post_id"],
|
||||
["post.id"],
|
||||
name="fk_download_event_post_id_post",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_download_event"),
|
||||
)
|
||||
op.create_index("ix_download_event_source_id", "download_event", ["source_id"])
|
||||
op.create_index("ix_download_event_post_id", "download_event", ["post_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("download_event")
|
||||
op.drop_table("image_tag")
|
||||
op.drop_table("tag")
|
||||
op.drop_table("image_provenance")
|
||||
op.drop_table("image_record")
|
||||
op.execute("DROP TYPE IF EXISTS origin_enum")
|
||||
op.drop_table("post")
|
||||
op.drop_table("credential")
|
||||
op.drop_table("source")
|
||||
op.drop_table("artist")
|
||||
op.execute("DROP EXTENSION IF EXISTS vector")
|
||||
@@ -0,0 +1,208 @@
|
||||
"""fc2a: tag kinds, import_task, import_batch, integrity_status
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
Create Date: 2026-05-14
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002"
|
||||
down_revision: Union[str, None] = "0001"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
TAG_KINDS = (
|
||||
"artist",
|
||||
"character",
|
||||
"fandom",
|
||||
"general",
|
||||
"series",
|
||||
"archive",
|
||||
"post",
|
||||
"meta",
|
||||
"rating",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# --- Tag kind enum + fandom_id ---
|
||||
tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind")
|
||||
tag_kind.create(op.get_bind(), checkfirst=True)
|
||||
|
||||
op.add_column(
|
||||
"tag",
|
||||
sa.Column("kind", tag_kind, nullable=False, server_default="general"),
|
||||
)
|
||||
op.add_column(
|
||||
"tag",
|
||||
sa.Column("fandom_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_tag_fandom_id_tag",
|
||||
"tag",
|
||||
"tag",
|
||||
["fandom_id"],
|
||||
["id"],
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
|
||||
# Drop the old global uniqueness on name; add kind+fandom-aware uniqueness.
|
||||
op.drop_constraint("uq_tag_name", "tag", type_="unique")
|
||||
op.drop_index("ix_tag_name", table_name="tag")
|
||||
op.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX uq_tag_name_kind_fandom
|
||||
ON tag (name, kind, COALESCE(fandom_id, 0))
|
||||
"""
|
||||
)
|
||||
|
||||
# CHECK: fandom_id is only allowed for character kind.
|
||||
op.create_check_constraint(
|
||||
"ck_tag_fandom_requires_character",
|
||||
"tag",
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
)
|
||||
|
||||
# Drop the old namespace column — superseded by kind.
|
||||
op.drop_index("ix_tag_namespace", table_name="tag")
|
||||
op.drop_column("tag", "namespace")
|
||||
|
||||
# --- ImportBatch ---
|
||||
op.create_table(
|
||||
"import_batch",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("triggered_by", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_path", sa.Text(), nullable=False),
|
||||
sa.Column("scan_mode", sa.String(length=16), nullable=False),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("imported", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, server_default="running"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_batch"),
|
||||
)
|
||||
op.create_index("ix_import_batch_status", "import_batch", ["status"])
|
||||
|
||||
# --- ImportTask ---
|
||||
op.create_table(
|
||||
"import_task",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("batch_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_path", sa.Text(), nullable=False),
|
||||
sa.Column("task_type", sa.String(length=16), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"),
|
||||
sa.Column("result_image_id", sa.Integer(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["batch_id"],
|
||||
["import_batch.id"],
|
||||
name="fk_import_task_batch_id_import_batch",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["result_image_id"],
|
||||
["image_record.id"],
|
||||
name="fk_import_task_result_image_id_image_record",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_task"),
|
||||
)
|
||||
op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"])
|
||||
op.create_index("ix_import_task_status", "import_task", ["status"])
|
||||
op.create_index(
|
||||
"ix_import_task_created_at_desc",
|
||||
"import_task",
|
||||
[sa.text("created_at DESC")],
|
||||
)
|
||||
|
||||
# --- ImportSettings (single-row table) ---
|
||||
op.create_table(
|
||||
"import_settings",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"),
|
||||
sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||
),
|
||||
sa.Column(
|
||||
"transparency_threshold",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default="0.9",
|
||||
),
|
||||
sa.Column(
|
||||
"skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||
),
|
||||
sa.Column(
|
||||
"single_color_threshold",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default="0.95",
|
||||
),
|
||||
sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_settings"),
|
||||
sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"),
|
||||
)
|
||||
# Seed the single row immediately so callers can always SELECT id=1.
|
||||
op.execute("INSERT INTO import_settings (id) VALUES (1)")
|
||||
|
||||
# --- ImageRecord additions ---
|
||||
op.add_column(
|
||||
"image_record",
|
||||
sa.Column(
|
||||
"integrity_status",
|
||||
sa.String(length=24),
|
||||
nullable=False,
|
||||
server_default="unknown",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_image_record_integrity_status",
|
||||
"image_record",
|
||||
["integrity_status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_image_record_integrity_status", table_name="image_record")
|
||||
op.drop_column("image_record", "integrity_status")
|
||||
|
||||
op.drop_table("import_settings")
|
||||
op.drop_index("ix_import_task_created_at_desc", table_name="import_task")
|
||||
op.drop_index("ix_import_task_status", table_name="import_task")
|
||||
op.drop_index("ix_import_task_batch_id", table_name="import_task")
|
||||
op.drop_table("import_task")
|
||||
op.drop_index("ix_import_batch_status", table_name="import_batch")
|
||||
op.drop_table("import_batch")
|
||||
|
||||
op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check")
|
||||
op.execute("DROP INDEX uq_tag_name_kind_fandom")
|
||||
op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True))
|
||||
op.create_index("ix_tag_namespace", "tag", ["namespace"])
|
||||
op.create_index("ix_tag_name", "tag", ["name"], unique=False)
|
||||
op.create_unique_constraint("uq_tag_name", "tag", ["name"])
|
||||
op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey")
|
||||
op.drop_column("tag", "fandom_id")
|
||||
op.drop_column("tag", "kind")
|
||||
sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""fc2b: ML pipeline — allowlist, aliases, centroids, ml_settings
|
||||
|
||||
Revision ID: 0003
|
||||
Revises: 0002
|
||||
Create Date: 2026-05-15
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
revision: str = "0003"
|
||||
down_revision: Union[str, None] = "0002"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 3.1 rename wd14_* -> tagger_*
|
||||
op.alter_column("image_record", "wd14_predictions", new_column_name="tagger_predictions")
|
||||
op.alter_column(
|
||||
"image_record", "wd14_model_version", new_column_name="tagger_model_version"
|
||||
)
|
||||
|
||||
# 3.2 tag_allowlist
|
||||
op.create_table(
|
||||
"tag_allowlist",
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"min_confidence", sa.Float(), nullable=False, server_default="0.95"
|
||||
),
|
||||
sa.Column(
|
||||
"added_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_tag_allowlist_tag_id_tag",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("tag_id", name="pk_tag_allowlist"),
|
||||
sa.CheckConstraint(
|
||||
"min_confidence > 0 AND min_confidence <= 1",
|
||||
name="ck_tag_allowlist_confidence_range",
|
||||
),
|
||||
)
|
||||
|
||||
# 3.3 tag_suggestion_rejection
|
||||
op.create_table(
|
||||
"tag_suggestion_rejection",
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"rejected_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"], ["image_record.id"],
|
||||
name="fk_tsr_image_record_id_image_record", ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_tsr_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"image_record_id", "tag_id", name="pk_tag_suggestion_rejection"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_tag_suggestion_rejection_tag", "tag_suggestion_rejection", ["tag_id"]
|
||||
)
|
||||
|
||||
# 3.4 tag_alias
|
||||
op.create_table(
|
||||
"tag_alias",
|
||||
sa.Column("alias_string", sa.String(length=255), nullable=False),
|
||||
sa.Column("alias_category", sa.String(length=32), nullable=False),
|
||||
sa.Column("canonical_tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["canonical_tag_id"], ["tag.id"],
|
||||
name="fk_tag_alias_canonical_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"alias_string", "alias_category", name="pk_tag_alias"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_tag_alias_canonical", "tag_alias", ["canonical_tag_id"])
|
||||
|
||||
# 3.5 tag_reference_embedding (centroids)
|
||||
op.create_table(
|
||||
"tag_reference_embedding",
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", Vector(1152), nullable=False),
|
||||
sa.Column("reference_count", sa.Integer(), nullable=False),
|
||||
sa.Column("model_version", sa.String(length=128), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"],
|
||||
name="fk_tag_reference_embedding_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("tag_id", name="pk_tag_reference_embedding"),
|
||||
)
|
||||
|
||||
# 3.6 ml_settings singleton
|
||||
op.create_table(
|
||||
"ml_settings",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"suggestion_threshold_artist", sa.Float(), nullable=False,
|
||||
server_default="0.30",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_character", sa.Float(), nullable=False,
|
||||
server_default="0.50",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_copyright", sa.Float(), nullable=False,
|
||||
server_default="0.50",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_general", sa.Float(), nullable=False,
|
||||
server_default="0.95",
|
||||
),
|
||||
sa.Column(
|
||||
"centroid_similarity_threshold", sa.Float(), nullable=False,
|
||||
server_default="0.55",
|
||||
),
|
||||
sa.Column(
|
||||
"min_reference_images", sa.Integer(), nullable=False,
|
||||
server_default="5",
|
||||
),
|
||||
sa.Column(
|
||||
"tagger_model_version", sa.String(length=128), nullable=False,
|
||||
server_default="camie-tagger-v2",
|
||||
),
|
||||
sa.Column(
|
||||
"embedder_model_version", sa.String(length=128), nullable=False,
|
||||
server_default="siglip-so400m-patch14-384",
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ml_settings"),
|
||||
sa.CheckConstraint("id = 1", name="ck_ml_settings_singleton"),
|
||||
)
|
||||
op.execute("INSERT INTO ml_settings (id) VALUES (1)")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("ml_settings")
|
||||
op.drop_table("tag_reference_embedding")
|
||||
op.drop_index("ix_tag_alias_canonical", table_name="tag_alias")
|
||||
op.drop_table("tag_alias")
|
||||
op.drop_index(
|
||||
"ix_tag_suggestion_rejection_tag", table_name="tag_suggestion_rejection"
|
||||
)
|
||||
op.drop_table("tag_suggestion_rejection")
|
||||
op.drop_table("tag_allowlist")
|
||||
op.alter_column(
|
||||
"image_record", "tagger_model_version", new_column_name="wd14_model_version"
|
||||
)
|
||||
op.alter_column(
|
||||
"image_record", "tagger_predictions", new_column_name="wd14_predictions"
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""fc2c-i: enable tsm_system_rows for scalable random sampling
|
||||
|
||||
Revision ID: 0004
|
||||
Revises: 0003
|
||||
Create Date: 2026-05-15
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004"
|
||||
down_revision: Union[str, None] = "0003"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP EXTENSION IF EXISTS tsm_system_rows")
|
||||
@@ -0,0 +1,50 @@
|
||||
"""fc2c-iii-a: series_page ordered membership
|
||||
|
||||
Revision ID: 0005
|
||||
Revises: 0004
|
||||
Create Date: 2026-05-16
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0005"
|
||||
down_revision: Union[str, None] = "0004"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"series_page",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("image_id", sa.Integer(), nullable=False),
|
||||
sa.Column("page_number", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["series_tag_id"], ["tag.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_id"], ["image_record.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("image_id", name="uq_series_page_image"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_series_page_series_tag_id", "series_page", ["series_tag_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_series_page_series_tag_id", table_name="series_page")
|
||||
op.drop_table("series_page")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""fc2d: import_settings.phash_threshold
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
Create Date: 2026-05-17
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006"
|
||||
down_revision: Union[str, None] = "0005"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"phash_threshold", sa.Integer(),
|
||||
nullable=False, server_default="10",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "phash_threshold")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""fc2d-iv: post.description + post.attachment_count
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
Create Date: 2026-05-18
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007"
|
||||
down_revision: Union[str, None] = "0006"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"post", sa.Column("description", sa.Text(), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("attachment_count", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("post", "attachment_count")
|
||||
op.drop_column("post", "description")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""fc2d-vii-c: image_record.artist_id + backfill + drop artist tags
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
Create Date: 2026-05-18
|
||||
|
||||
Internal forward-correctness migration (the big legacy-import migration
|
||||
stays deferred). downgrade() does NOT recreate deleted artist tags;
|
||||
downgrade is dev-only and the data is reconstructable by re-import.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from backend.app.utils.artist_backfill import (
|
||||
BACKFILL_PRIMARY_SQL,
|
||||
BACKFILL_PROVENANCE_SQL,
|
||||
BACKFILL_TAG_SQL,
|
||||
DELETE_ARTIST_TAGS_SQL,
|
||||
)
|
||||
|
||||
revision: str = "0008"
|
||||
down_revision: Union[str, None] = "0007"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"image_record",
|
||||
sa.Column("artist_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_image_record_artist_id", "image_record", "artist",
|
||||
["artist_id"], ["id"], ondelete="SET NULL",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_image_record_artist_id", "image_record", ["artist_id"],
|
||||
)
|
||||
op.execute(BACKFILL_PRIMARY_SQL)
|
||||
op.execute(BACKFILL_PROVENANCE_SQL)
|
||||
op.execute(BACKFILL_TAG_SQL)
|
||||
op.execute(DELETE_ARTIST_TAGS_SQL)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_image_record_artist_id", table_name="image_record")
|
||||
op.drop_constraint(
|
||||
"fk_image_record_artist_id", "image_record", type_="foreignkey"
|
||||
)
|
||||
op.drop_column("image_record", "artist_id")
|
||||
@@ -0,0 +1,68 @@
|
||||
"""fc2d-iii: post_attachment + import_batch.attachments
|
||||
|
||||
Revision ID: 0009
|
||||
Revises: 0008
|
||||
Create Date: 2026-05-19
|
||||
|
||||
Internal forward-correctness migration (big legacy-import migration
|
||||
stays deferred). No backfill — no attachments exist yet.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0009"
|
||||
down_revision: Union[str, None] = "0008"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"post_attachment",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"post_id", sa.Integer(),
|
||||
sa.ForeignKey("post.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"artist_id", sa.Integer(),
|
||||
sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("original_filename", sa.Text(), nullable=False),
|
||||
sa.Column("ext", sa.String(32), nullable=False),
|
||||
sa.Column("mime", sa.String(128), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column(
|
||||
"captured_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(), nullable=False,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_sha256", "post_attachment", ["sha256"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_post_id", "post_attachment", ["post_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_artist_id", "post_attachment", ["artist_id"],
|
||||
)
|
||||
op.add_column(
|
||||
"import_batch",
|
||||
sa.Column(
|
||||
"attachments", sa.Integer(), nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_batch", "attachments")
|
||||
op.drop_index("ix_post_attachment_artist_id", table_name="post_attachment")
|
||||
op.drop_index("ix_post_attachment_post_id", table_name="post_attachment")
|
||||
op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
|
||||
op.drop_table("post_attachment")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""fc3a: unique(source.artist_id, source.platform, source.url)
|
||||
|
||||
Revision ID: 0010
|
||||
Revises: 0009
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Enforces FC-3a's dedup invariant at the DB level. No backfill — no
|
||||
existing rows are expected to collide; if they do the migration will
|
||||
fail loudly (intended).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0010"
|
||||
down_revision: Union[str, None] = "0009"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_unique_constraint(
|
||||
"uq_source_artist_platform_url",
|
||||
"source",
|
||||
["artist_id", "platform", "url"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
"uq_source_artist_platform_url", "source", type_="unique"
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""fc3b: rename credential.kind -> credential_type, drop status, add last_verified
|
||||
|
||||
Revision ID: 0011
|
||||
Revises: 0010
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Aligns the credential table with the GallerySubscriber wire-field names
|
||||
so the existing browser extension can POST to FC unmodified. Greenfield —
|
||||
no rows exist in production yet, so no data preservation logic is
|
||||
needed; the rename uses ALTER COLUMN rather than copy-then-drop.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0011"
|
||||
down_revision: Union[str, None] = "0010"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("credential", "kind", new_column_name="credential_type")
|
||||
op.drop_column("credential", "status")
|
||||
op.add_column(
|
||||
"credential",
|
||||
sa.Column("last_verified", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("credential", "last_verified")
|
||||
op.add_column(
|
||||
"credential",
|
||||
sa.Column(
|
||||
"status", sa.String(length=32), nullable=False,
|
||||
server_default="active",
|
||||
),
|
||||
)
|
||||
op.alter_column("credential", "credential_type", new_column_name="kind")
|
||||
@@ -0,0 +1,36 @@
|
||||
"""fc3b: app_setting key/value table
|
||||
|
||||
Revision ID: 0012
|
||||
Revises: 0011
|
||||
Create Date: 2026-05-20
|
||||
|
||||
A simple key/value table for small app settings that don't fit
|
||||
ImportSettings. Initially seeds only `extension_api_key` (done in
|
||||
create_app on first boot — not in the migration, to keep it
|
||||
deterministic and independent of randomness).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0012"
|
||||
down_revision: Union[str, None] = "0011"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"app_setting",
|
||||
sa.Column("key", sa.String(length=64), primary_key=True),
|
||||
sa.Column("value", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("app_setting")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""fc3c: download_event.metadata + import_settings downloader fields
|
||||
|
||||
Revision ID: 0013
|
||||
Revises: 0012
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Additive only. download_event.metadata is the rich JSONB blob FC-3c
|
||||
populates per run (run_stats, stdout/stderr, quarantined paths, import
|
||||
summary). import_settings gains two operator-tunable downloader knobs:
|
||||
download_rate_limit_seconds (gallery-dl extractor.sleep) and
|
||||
download_validate_files (toggle the magic-byte validator).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0013"
|
||||
down_revision: Union[str, None] = "0012"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"download_event",
|
||||
sa.Column(
|
||||
"metadata", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_rate_limit_seconds", sa.Float(),
|
||||
nullable=False, server_default="3.0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_validate_files", sa.Boolean(),
|
||||
nullable=False, server_default=sa.true(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "download_validate_files")
|
||||
op.drop_column("import_settings", "download_rate_limit_seconds")
|
||||
op.drop_column("download_event", "metadata")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""fc3d: scheduling + source health columns
|
||||
|
||||
Revision ID: 0014
|
||||
Revises: 0013
|
||||
Create Date: 2026-05-21
|
||||
|
||||
Additive only. source.consecutive_failures (default 0, DownloadService
|
||||
finalize hook owns the writes). import_settings gains the three
|
||||
scheduling knobs (global default interval, event retention, failure
|
||||
warning threshold).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0014"
|
||||
down_revision: Union[str, None] = "0013"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source",
|
||||
sa.Column(
|
||||
"consecutive_failures", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_schedule_default_seconds", sa.Integer(),
|
||||
nullable=False, server_default="28800",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_event_retention_days", sa.Integer(),
|
||||
nullable=False, server_default="90",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_failure_warning_threshold", sa.Integer(),
|
||||
nullable=False, server_default="5",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "download_failure_warning_threshold")
|
||||
op.drop_column("import_settings", "download_event_retention_days")
|
||||
op.drop_column("import_settings", "download_schedule_default_seconds")
|
||||
op.drop_column("source", "consecutive_failures")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""fc5: migration_run table
|
||||
|
||||
Revision ID: 0015
|
||||
Revises: 0014
|
||||
Create Date: 2026-05-22
|
||||
|
||||
Additive only. New table tracks each invocation of the FC-5 migration
|
||||
tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are
|
||||
plain String(32) — values validated at the API layer per the spec, not
|
||||
a Postgres ENUM (so adding kinds later doesn't need a schema migration).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0015"
|
||||
down_revision: Union[str, None] = "0014"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"migration_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kind", sa.String(32), nullable=False, index=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, index=True),
|
||||
sa.Column(
|
||||
"dry_run", sa.Boolean(), nullable=False, server_default=sa.false(),
|
||||
),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"counts", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"metadata", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("migration_run")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Quart app factory."""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Quart
|
||||
|
||||
from .api import all_blueprints
|
||||
from .config import get_config
|
||||
from .frontend import frontend_bp
|
||||
from .services.credential_crypto import CredentialCrypto
|
||||
|
||||
_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||
|
||||
|
||||
def create_app() -> Quart:
|
||||
cfg = get_config()
|
||||
logging.basicConfig(level=cfg.log_level)
|
||||
|
||||
# Bootstrap the credential encryption key file (mode 0600) before
|
||||
# any request can land. FC-3b: file-based system-generated key.
|
||||
CredentialCrypto(_CREDENTIAL_KEY_PATH)
|
||||
|
||||
app = Quart(__name__)
|
||||
app.secret_key = cfg.secret_key
|
||||
|
||||
for bp in all_blueprints():
|
||||
app.register_blueprint(bp)
|
||||
# Registered last so /api/* routes win over the SPA catch-all.
|
||||
app.register_blueprint(frontend_bp)
|
||||
|
||||
@app.after_serving
|
||||
async def _dispose_db_engine() -> None:
|
||||
from .extensions import dispose_engine
|
||||
await dispose_engine()
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,57 @@
|
||||
"""API blueprint registration.
|
||||
|
||||
This module is imported by the Quart app factory; it just exposes the
|
||||
top-level `api_bp` for backward compatibility with FC-1's health route,
|
||||
and ALL_BLUEPRINTS for the factory to register sibling blueprints.
|
||||
"""
|
||||
|
||||
from quart import Blueprint
|
||||
|
||||
from . import health
|
||||
|
||||
api_bp = Blueprint("api", __name__, url_prefix="/api")
|
||||
api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
|
||||
|
||||
|
||||
def all_blueprints() -> list[Blueprint]:
|
||||
from .aliases import aliases_bp
|
||||
from .allowlist import allowlist_bp
|
||||
from .artist import artist_bp
|
||||
from .artists import artists_bp
|
||||
from .attachments import attachments_bp
|
||||
from .credentials import credentials_bp
|
||||
from .downloads import downloads_bp
|
||||
from .gallery import gallery_bp
|
||||
from .import_admin import import_admin_bp
|
||||
from .migrate import migrate_bp
|
||||
from .ml_admin import ml_admin_bp
|
||||
from .platforms import platforms_bp
|
||||
from .posts import posts_bp
|
||||
from .provenance import provenance_bp
|
||||
from .settings import settings_bp
|
||||
from .showcase import showcase_bp
|
||||
from .sources import sources_bp
|
||||
from .suggestions import suggestions_bp
|
||||
from .tags import tags_bp
|
||||
return [
|
||||
api_bp,
|
||||
attachments_bp,
|
||||
gallery_bp,
|
||||
provenance_bp,
|
||||
tags_bp,
|
||||
artist_bp,
|
||||
artists_bp,
|
||||
showcase_bp,
|
||||
settings_bp,
|
||||
import_admin_bp,
|
||||
migrate_bp,
|
||||
suggestions_bp,
|
||||
allowlist_bp,
|
||||
aliases_bp,
|
||||
ml_admin_bp,
|
||||
sources_bp,
|
||||
platforms_bp,
|
||||
posts_bp,
|
||||
credentials_bp,
|
||||
downloads_bp,
|
||||
]
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Aliases API: list, create, remove."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.ml.aliases import AliasService
|
||||
|
||||
aliases_bp = Blueprint("aliases", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
@aliases_bp.route("/aliases", methods=["GET"])
|
||||
async def list_aliases():
|
||||
async with get_session() as session:
|
||||
rows = await AliasService(session).list_all()
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"alias_string": r.alias_string,
|
||||
"alias_category": r.alias_category,
|
||||
"canonical_tag_id": r.canonical_tag_id,
|
||||
"canonical_tag_name": r.canonical_tag_name,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@aliases_bp.route("/aliases", methods=["POST"])
|
||||
async def create_alias():
|
||||
body = await request.get_json()
|
||||
required = {"alias_string", "alias_category", "canonical_tag_id"}
|
||||
if not body or not required.issubset(body):
|
||||
return jsonify({"error": f"required: {sorted(required)}"}), 400
|
||||
async with get_session() as session:
|
||||
await AliasService(session).create(
|
||||
body["alias_string"],
|
||||
body["alias_category"],
|
||||
body["canonical_tag_id"],
|
||||
)
|
||||
await session.commit()
|
||||
return "", 201
|
||||
|
||||
|
||||
@aliases_bp.route(
|
||||
"/aliases/<alias_string>/<alias_category>", methods=["DELETE"]
|
||||
)
|
||||
async def remove_alias(alias_string: str, alias_category: str):
|
||||
async with get_session() as session:
|
||||
await AliasService(session).remove(alias_string, alias_category)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Allowlist API: list, adjust threshold, remove."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import TagAllowlist
|
||||
from ..services.ml.allowlist import AllowlistService
|
||||
|
||||
allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
@allowlist_bp.route("/allowlist", methods=["GET"])
|
||||
async def list_allowlist():
|
||||
async with get_session() as session:
|
||||
rows = await AllowlistService(session).list_all()
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"tag_id": r.tag_id,
|
||||
"tag_name": r.tag_name,
|
||||
"tag_kind": r.tag_kind,
|
||||
"min_confidence": r.min_confidence,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
|
||||
async def get_one(tag_id: int):
|
||||
async with get_session() as session:
|
||||
row = await session.get(TagAllowlist, tag_id)
|
||||
if row is None:
|
||||
return jsonify({"error": "not on allowlist"}), 404
|
||||
return jsonify(
|
||||
{"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()}
|
||||
)
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["PATCH"])
|
||||
async def patch_threshold(tag_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "min_confidence" not in body:
|
||||
return jsonify({"error": "min_confidence required"}), 400
|
||||
mc = float(body["min_confidence"])
|
||||
if not (0 < mc <= 1):
|
||||
return jsonify({"error": "min_confidence must be in (0, 1]"}), 400
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).update_threshold(tag_id, mc)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["DELETE"])
|
||||
async def remove(tag_id: int):
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).remove(tag_id)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Artist API: overview aggregates + paged images."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.artist_service import ArtistService
|
||||
|
||||
artist_bp = Blueprint("artist", __name__, url_prefix="/api/artist")
|
||||
|
||||
|
||||
@artist_bp.route("/<slug>", methods=["GET"])
|
||||
async def overview(slug: str):
|
||||
async with get_session() as session:
|
||||
svc = ArtistService(session)
|
||||
data = await svc.overview(slug)
|
||||
if data is None:
|
||||
return jsonify({"error": "artist not found"}), 404
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
@artist_bp.route("/<slug>/images", methods=["GET"])
|
||||
async def images(slug: str):
|
||||
cursor = request.args.get("cursor") or None
|
||||
try:
|
||||
limit = int(request.args.get("limit", "60"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
async with get_session() as session:
|
||||
svc = ArtistService(session)
|
||||
try:
|
||||
page = await svc.images(slug, cursor=cursor, limit=limit)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
if page is None:
|
||||
return jsonify({"error": "artist not found"}), 404
|
||||
return jsonify({"images": page.images, "next_cursor": page.next_cursor})
|
||||
@@ -0,0 +1,83 @@
|
||||
"""FC-3a: plural /api/artists endpoints — POST (find-or-create) +
|
||||
autocomplete. FC-3f: GET /api/artists/directory (cursor-paginated
|
||||
browse grid). The slug-routed singular /api/artist/<slug> blueprint
|
||||
stays separate and unmodified."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.artist_directory_service import ArtistDirectoryService
|
||||
from ..services.artist_service import ArtistService
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
|
||||
artists_bp = Blueprint("artists", __name__, url_prefix="/api/artists")
|
||||
|
||||
|
||||
@artists_bp.route("", methods=["POST"])
|
||||
async def create_or_get():
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return jsonify({"error": "invalid_body"}), 400
|
||||
name = body.get("name", "")
|
||||
async with get_session() as session:
|
||||
svc = ArtistService(session)
|
||||
try:
|
||||
artist, created = await svc.find_or_create(name)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": "empty_name", "detail": str(exc)}), 400
|
||||
return jsonify({
|
||||
"id": artist.id, "name": artist.name, "slug": artist.slug,
|
||||
"created": created,
|
||||
}), 201
|
||||
|
||||
|
||||
@artists_bp.route("/autocomplete", methods=["GET"])
|
||||
async def autocomplete():
|
||||
q = request.args.get("q") or ""
|
||||
try:
|
||||
limit = int(request.args.get("limit", "20"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
if limit < 1 or limit > 100:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
async with get_session() as session:
|
||||
rows = await ArtistService(session).autocomplete(q, limit=limit)
|
||||
return jsonify([
|
||||
{"id": a.id, "name": a.name, "slug": a.slug} for a in rows
|
||||
])
|
||||
|
||||
|
||||
@artists_bp.route("/directory", methods=["GET"])
|
||||
async def directory():
|
||||
"""FC-3f: cursor-paginated artists directory.
|
||||
|
||||
Mirrors /api/tags/directory shape: { cards: [...], next_cursor }.
|
||||
"""
|
||||
args = request.args
|
||||
|
||||
cursor = args.get("cursor") or None
|
||||
q = args.get("q") or None
|
||||
platform = args.get("platform") or None
|
||||
limit_raw = args.get("limit", "60")
|
||||
|
||||
try:
|
||||
limit = int(limit_raw)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
if limit < 1 or limit > 200:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
|
||||
if platform is not None and platform not in KNOWN_PLATFORMS:
|
||||
return jsonify({"error": "unknown_platform"}), 400
|
||||
|
||||
async with get_session() as session:
|
||||
svc = ArtistDirectoryService(session)
|
||||
try:
|
||||
page = await svc.list_artists(
|
||||
q=q, platform=platform, cursor=cursor, limit=limit,
|
||||
)
|
||||
except ValueError:
|
||||
# Service raises only on bad cursor (limit was validated above).
|
||||
return jsonify({"error": "invalid_cursor"}), 400
|
||||
|
||||
return jsonify({"cards": page.cards, "next_cursor": page.next_cursor})
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Attachment download: streams a preserved non-art post file."""
|
||||
|
||||
from quart import Blueprint, jsonify, send_file
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import PostAttachment
|
||||
|
||||
attachments_bp = Blueprint(
|
||||
"attachments", __name__, url_prefix="/api/attachments"
|
||||
)
|
||||
|
||||
|
||||
@attachments_bp.route("/<int:attachment_id>/download", methods=["GET"])
|
||||
async def download(attachment_id: int):
|
||||
async with get_session() as session:
|
||||
att = await session.get(PostAttachment, attachment_id)
|
||||
if att is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
path = att.path
|
||||
filename = att.original_filename
|
||||
mimetype = att.mime or "application/octet-stream"
|
||||
return await send_file(
|
||||
path,
|
||||
mimetype=mimetype,
|
||||
as_attachment=True,
|
||||
attachment_filename=filename,
|
||||
)
|
||||
@@ -0,0 +1,126 @@
|
||||
"""FC-3b: /api/credentials — CRUD with X-Extension-Key auth.
|
||||
|
||||
Browser requests (no header) are accepted per the homelab posture —
|
||||
there is no user-session model in FC. The X-Extension-Key check
|
||||
exists to give the extension a revocable path: a request that
|
||||
supplies the header must match `app_setting.extension_api_key`.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, Credential
|
||||
from ..services.credential_crypto import CredentialCrypto
|
||||
from ..services.credential_service import (
|
||||
CredentialService,
|
||||
EmptyDataError,
|
||||
UnknownPlatformError,
|
||||
WrongAuthTypeError,
|
||||
)
|
||||
|
||||
credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials")
|
||||
|
||||
|
||||
# Production key path; tests pass `cookies_dir=tmp_path` via the
|
||||
# service constructor (the API uses the default). The Fernet key
|
||||
# file is also at `/images/secrets/credential_key.b64`.
|
||||
_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||
_crypto: CredentialCrypto | None = None
|
||||
|
||||
|
||||
def _get_crypto() -> CredentialCrypto:
|
||||
global _crypto
|
||||
if _crypto is None:
|
||||
_crypto = CredentialCrypto(_KEY_PATH)
|
||||
return _crypto
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
async def _ext_key_ok(session) -> bool:
|
||||
"""If X-Extension-Key is supplied, it must match the stored value.
|
||||
Missing header → True (browser path; accepted per homelab posture).
|
||||
"""
|
||||
supplied = request.headers.get("X-Extension-Key")
|
||||
if supplied is None:
|
||||
return True
|
||||
stored = (await session.execute(
|
||||
select(AppSetting.value).where(AppSetting.key == "extension_api_key")
|
||||
)).scalar_one_or_none()
|
||||
return stored is not None and supplied == stored
|
||||
|
||||
|
||||
@credentials_bp.route("", methods=["GET"])
|
||||
async def list_credentials():
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_ok(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
records = await CredentialService(session, _get_crypto()).list()
|
||||
return jsonify([r.to_dict() for r in records])
|
||||
|
||||
|
||||
@credentials_bp.route("/<platform>", methods=["GET"])
|
||||
async def get_credential(platform: str):
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_ok(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
record = await CredentialService(session, _get_crypto()).get(platform)
|
||||
if record is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@credentials_bp.route("", methods=["POST"])
|
||||
async def upsert_credential():
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
try:
|
||||
platform = body["platform"]
|
||||
credential_type = body["credential_type"]
|
||||
data = body["data"]
|
||||
except KeyError:
|
||||
return _bad("invalid_body", detail="platform, credential_type, data are required")
|
||||
if not isinstance(data, str):
|
||||
return _bad("invalid_body", detail="data must be a string")
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_ok(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
existed = (await session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none() is not None
|
||||
svc = CredentialService(session, _get_crypto())
|
||||
try:
|
||||
record = await svc.upsert(
|
||||
platform=platform, credential_type=credential_type, data=data,
|
||||
)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad("unknown_platform", detail=str(exc))
|
||||
except WrongAuthTypeError as exc:
|
||||
return _bad("wrong_auth_type", detail=str(exc), expected=exc.expected)
|
||||
except EmptyDataError as exc:
|
||||
return _bad("empty_data", detail=str(exc))
|
||||
return jsonify(record.to_dict()), (200 if existed else 201)
|
||||
|
||||
|
||||
@credentials_bp.route("/<platform>", methods=["DELETE"])
|
||||
async def delete_credential(platform: str):
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_ok(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
svc = CredentialService(session, _get_crypto())
|
||||
try:
|
||||
await svc.delete(platform)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
return "", 204
|
||||
@@ -0,0 +1,110 @@
|
||||
"""FC-3c: GET /api/downloads (list) + GET /api/downloads/<id> (detail).
|
||||
|
||||
List view: keyset-paginated, newest first, optional filtering by
|
||||
status/source/artist. Returns slim records.
|
||||
Detail view: full DownloadEvent including the metadata JSONB.
|
||||
"""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
|
||||
downloads_bp = Blueprint("downloads", __name__, url_prefix="/api/downloads")
|
||||
|
||||
_ALLOWED_STATUSES = frozenset({"pending", "running", "ok", "error", "skipped"})
|
||||
|
||||
|
||||
def _summary_from_metadata(metadata: dict | None) -> dict:
|
||||
metadata = metadata or {}
|
||||
run_stats = metadata.get("run_stats") or {}
|
||||
return {
|
||||
"downloaded": int(run_stats.get("downloaded_count", 0)),
|
||||
"skipped": int(run_stats.get("skipped_count", 0)),
|
||||
"quarantined": int(run_stats.get("quarantined_count", 0)),
|
||||
"duration_seconds": metadata.get("duration_seconds"),
|
||||
"error_type": metadata.get("error_type"),
|
||||
}
|
||||
|
||||
|
||||
def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | None) -> dict:
|
||||
return {
|
||||
"id": event.id,
|
||||
"source_id": event.source_id,
|
||||
"artist_name": artist.name if artist else None,
|
||||
"artist_slug": artist.slug if artist else None,
|
||||
"platform": source.platform if source else None,
|
||||
"status": event.status,
|
||||
"started_at": event.started_at.isoformat() if event.started_at else None,
|
||||
"finished_at": event.finished_at.isoformat() if event.finished_at else None,
|
||||
"files_count": event.files_count,
|
||||
"bytes_downloaded": event.bytes_downloaded,
|
||||
"error": event.error,
|
||||
"summary": _summary_from_metadata(event.metadata_),
|
||||
}
|
||||
|
||||
|
||||
def _detail_record(event: DownloadEvent, source: Source | None, artist: Artist | None) -> dict:
|
||||
base = _list_record(event, source, artist)
|
||||
base["metadata"] = event.metadata_ or {}
|
||||
return base
|
||||
|
||||
|
||||
@downloads_bp.route("", methods=["GET"])
|
||||
async def list_downloads():
|
||||
status = request.args.get("status")
|
||||
if status and status not in _ALLOWED_STATUSES:
|
||||
return jsonify({"error": "invalid_status"}), 400
|
||||
try:
|
||||
limit = int(request.args.get("limit", "50"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
limit = max(1, min(200, limit))
|
||||
before_raw = request.args.get("before")
|
||||
before = None
|
||||
if before_raw is not None:
|
||||
try:
|
||||
before = int(before_raw)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_before"}), 400
|
||||
try:
|
||||
source_id = int(request.args["source_id"]) if "source_id" in request.args else None
|
||||
artist_id = int(request.args["artist_id"]) if "artist_id" in request.args else None
|
||||
except (ValueError, KeyError):
|
||||
return jsonify({"error": "invalid_filter"}), 400
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = (
|
||||
select(DownloadEvent, Source, Artist)
|
||||
.outerjoin(Source, Source.id == DownloadEvent.source_id)
|
||||
.outerjoin(Artist, Artist.id == Source.artist_id)
|
||||
.order_by(DownloadEvent.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
if status:
|
||||
stmt = stmt.where(DownloadEvent.status == status)
|
||||
if source_id is not None:
|
||||
stmt = stmt.where(DownloadEvent.source_id == source_id)
|
||||
if artist_id is not None:
|
||||
stmt = stmt.where(Source.artist_id == artist_id)
|
||||
if before is not None:
|
||||
stmt = stmt.where(DownloadEvent.id < before)
|
||||
rows = (await session.execute(stmt)).all()
|
||||
|
||||
return jsonify([_list_record(e, s, a) for e, s, a in rows])
|
||||
|
||||
|
||||
@downloads_bp.route("/<int:event_id>", methods=["GET"])
|
||||
async def get_download(event_id: int):
|
||||
async with get_session() as session:
|
||||
row = (await session.execute(
|
||||
select(DownloadEvent, Source, Artist)
|
||||
.outerjoin(Source, Source.id == DownloadEvent.source_id)
|
||||
.outerjoin(Artist, Artist.id == Source.artist_id)
|
||||
.where(DownloadEvent.id == event_id)
|
||||
)).first()
|
||||
if row is None:
|
||||
return jsonify({"error": "not_found"}), 404
|
||||
event, source, artist = row
|
||||
return jsonify(_detail_record(event, source, artist))
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Gallery API: cursor scroll, timeline, jump, image detail."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.gallery_service import GalleryService
|
||||
|
||||
gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
|
||||
|
||||
|
||||
@gallery_bp.route("/scroll", methods=["GET"])
|
||||
async def scroll():
|
||||
cursor = request.args.get("cursor") or None
|
||||
try:
|
||||
limit = int(request.args.get("limit", "50"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
page = await svc.scroll(
|
||||
cursor=cursor, limit=limit, tag_id=tag_id,
|
||||
post_id=post_id, artist_id=artist_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"id": i.id,
|
||||
"sha256": i.sha256,
|
||||
"mime": i.mime,
|
||||
"width": i.width,
|
||||
"height": i.height,
|
||||
"created_at": i.created_at.isoformat(),
|
||||
"thumbnail_url": i.thumbnail_url,
|
||||
"artist": i.artist,
|
||||
}
|
||||
for i in page.images
|
||||
],
|
||||
"next_cursor": page.next_cursor,
|
||||
"date_groups": [
|
||||
{"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/timeline", methods=["GET"])
|
||||
async def timeline():
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
buckets = await svc.timeline(
|
||||
tag_id=tag_id, post_id=post_id, artist_id=artist_id
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify(
|
||||
[{"year": b.year, "month": b.month, "count": b.count} for b in buckets]
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/jump", methods=["GET"])
|
||||
async def jump():
|
||||
try:
|
||||
year = int(request.args["year"])
|
||||
month = int(request.args["month"])
|
||||
except (KeyError, ValueError):
|
||||
return jsonify({"error": "year and month query params required"}), 400
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
cursor = await svc.jump_cursor(
|
||||
year=year, month=month, tag_id=tag_id,
|
||||
post_id=post_id, artist_id=artist_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify({"cursor": cursor})
|
||||
|
||||
|
||||
@gallery_bp.route("/image/<int:image_id>", methods=["GET"])
|
||||
async def image_detail(image_id: int):
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
payload = await svc.get_image_with_tags(image_id)
|
||||
if payload is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(payload)
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Health endpoint — no DB or Redis touch; just liveness."""
|
||||
|
||||
|
||||
async def get_health():
|
||||
return {"status": "ok"}, 200
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Import admin API: trigger scan, list tasks, retry, clear."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import delete, select, update
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import ImportBatch, ImportTask
|
||||
|
||||
import_admin_bp = Blueprint("import_admin", __name__, url_prefix="/api/import")
|
||||
|
||||
|
||||
@import_admin_bp.route("/trigger", methods=["POST"])
|
||||
async def trigger_scan():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
mode = body.get("mode", "quick")
|
||||
if mode not in ("quick", "deep", "verify"):
|
||||
return jsonify({"error": f"mode {mode!r} not supported; use 'quick', 'deep', or 'verify'"}), 400
|
||||
|
||||
# 'verify' is a library task — short-circuit the import_root walk
|
||||
# (no ImportBatch, no per-file ImportTasks).
|
||||
if mode == "verify":
|
||||
from ..tasks.maintenance import verify_integrity
|
||||
|
||||
async_result = verify_integrity.delay()
|
||||
return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202
|
||||
|
||||
from ..tasks.scan import scan_directory
|
||||
|
||||
async_result = scan_directory.delay(triggered_by="manual", mode=mode)
|
||||
return jsonify({"celery_task_id": async_result.id, "mode": mode}), 202
|
||||
|
||||
|
||||
@import_admin_bp.route("/status", methods=["GET"])
|
||||
async def status():
|
||||
async with get_session() as session:
|
||||
active = (
|
||||
await session.execute(
|
||||
select(ImportBatch)
|
||||
.where(ImportBatch.status == "running")
|
||||
.order_by(ImportBatch.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
payload = {"active_batch": None}
|
||||
if active:
|
||||
payload["active_batch"] = {
|
||||
"id": active.id,
|
||||
"total_files": active.total_files,
|
||||
"imported": active.imported,
|
||||
"skipped": active.skipped,
|
||||
"failed": active.failed,
|
||||
"started_at": active.started_at.isoformat(),
|
||||
}
|
||||
return jsonify(payload)
|
||||
|
||||
|
||||
@import_admin_bp.route("/tasks", methods=["GET"])
|
||||
async def list_tasks():
|
||||
status_filter = request.args.get("status")
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
cursor_raw = request.args.get("cursor")
|
||||
cursor_id = int(cursor_raw) if cursor_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(ImportTask).order_by(ImportTask.created_at.desc(), ImportTask.id.desc())
|
||||
if status_filter:
|
||||
stmt = stmt.where(ImportTask.status == status_filter)
|
||||
if cursor_id is not None:
|
||||
stmt = stmt.where(ImportTask.id < cursor_id)
|
||||
stmt = stmt.limit(limit + 1)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return jsonify({
|
||||
"tasks": [
|
||||
{
|
||||
"id": t.id,
|
||||
"batch_id": t.batch_id,
|
||||
"source_path": t.source_path,
|
||||
"task_type": t.task_type,
|
||||
"status": t.status,
|
||||
"result_image_id": t.result_image_id,
|
||||
"error": t.error,
|
||||
"size_bytes": t.size_bytes,
|
||||
"created_at": t.created_at.isoformat(),
|
||||
"started_at": t.started_at.isoformat() if t.started_at else None,
|
||||
"finished_at": t.finished_at.isoformat() if t.finished_at else None,
|
||||
}
|
||||
for t in rows
|
||||
],
|
||||
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||
})
|
||||
|
||||
|
||||
@import_admin_bp.route("/retry-failed", methods=["POST"])
|
||||
async def retry_failed():
|
||||
async with get_session() as session:
|
||||
failed_ids = (
|
||||
await session.execute(select(ImportTask.id).where(ImportTask.status == "failed"))
|
||||
).scalars().all()
|
||||
if not failed_ids:
|
||||
return jsonify({"retried": 0})
|
||||
await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(failed_ids))
|
||||
.values(status="queued", error=None, started_at=None, finished_at=None)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
from ..tasks.import_file import import_media_file
|
||||
for tid in failed_ids:
|
||||
import_media_file.delay(tid)
|
||||
|
||||
return jsonify({"retried": len(failed_ids)})
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-completed", methods=["POST"])
|
||||
async def clear_completed():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
age_days = body.get("age_days", 0)
|
||||
status_filter = body.get("status", ["complete", "skipped"])
|
||||
if not isinstance(status_filter, list):
|
||||
return jsonify({"error": "status must be a list"}), 400
|
||||
|
||||
cutoff = (
|
||||
datetime.now(UTC) - timedelta(days=int(age_days)) if age_days else None
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = delete(ImportTask).where(ImportTask.status.in_(status_filter))
|
||||
if cutoff is not None:
|
||||
stmt = stmt.where(ImportTask.finished_at < cutoff)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
return jsonify({"deleted": result.rowcount or 0})
|
||||
@@ -0,0 +1,141 @@
|
||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
||||
|
||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
||||
`export_file` field. All other kinds accept JSON. Apply-without-backup
|
||||
guard rejects non-dry-run ingests unless a pre_migration-tagged backup
|
||||
exists in the last 24h (override with body.force=true).
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import MigrationRun
|
||||
from ..tasks.migration import run_migration
|
||||
|
||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
||||
|
||||
_VALID_KINDS = frozenset({
|
||||
"backup", "gs_ingest", "ir_ingest", "tag_apply",
|
||||
"ml_queue", "verify", "rollback",
|
||||
})
|
||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback"})
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _has_recent_pre_migration_backup() -> bool:
|
||||
from ..services.migrators import backup as backup_mod
|
||||
images_root = Path("/images")
|
||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
||||
if manifest is None:
|
||||
return False
|
||||
created_at_str = manifest.get("created_at")
|
||||
if not created_at_str:
|
||||
return False
|
||||
created_at = datetime.fromisoformat(created_at_str)
|
||||
return (datetime.now(UTC) - created_at) < timedelta(hours=24)
|
||||
|
||||
|
||||
def _run_to_dict(run: MigrationRun) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"kind": run.kind,
|
||||
"status": run.status,
|
||||
"dry_run": run.dry_run,
|
||||
"started_at": run.started_at.isoformat(),
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"counts": run.counts or {},
|
||||
"error": run.error,
|
||||
"metadata": run.metadata_ or {},
|
||||
}
|
||||
|
||||
|
||||
@migrate_bp.route("/<kind>", methods=["POST"])
|
||||
async def create_run(kind: str):
|
||||
if kind not in _VALID_KINDS:
|
||||
return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}")
|
||||
|
||||
# Ingest kinds accept multipart/form-data; everything else takes JSON.
|
||||
if kind in _INGEST_KINDS:
|
||||
form = await request.form
|
||||
files = await request.files
|
||||
if "export_file" not in files:
|
||||
return _bad("missing_export_file", detail="multipart export_file required")
|
||||
export_file = files["export_file"]
|
||||
try:
|
||||
raw = export_file.read()
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
return _bad("invalid_export_file", detail=str(exc))
|
||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
||||
force = str(form.get("force", "false")).lower() in ("true", "1", "yes")
|
||||
params: dict = {"data": data, "dry_run": dry_run}
|
||||
else:
|
||||
body = await request.get_json()
|
||||
if body is None:
|
||||
body = {}
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body")
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
force = bool(body.get("force", False))
|
||||
params = dict(body)
|
||||
|
||||
is_apply = (kind in _APPLY_KINDS) and not dry_run
|
||||
if is_apply and not force and not _has_recent_pre_migration_backup():
|
||||
return _bad(
|
||||
"no_backup",
|
||||
detail="apply action requires a pre_migration-tagged backup "
|
||||
"in the last 24h (or force=true).",
|
||||
)
|
||||
|
||||
if kind == "backup":
|
||||
params.setdefault("tag", "pre_migration")
|
||||
|
||||
async with get_session() as session:
|
||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
||||
session.add(run)
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
run_id = run.id
|
||||
|
||||
run_migration.delay(run_id, kind, params)
|
||||
return jsonify({"run_id": run_id, "status": "pending"}), 202
|
||||
|
||||
|
||||
@migrate_bp.route("/runs/<int:run_id>", methods=["GET"])
|
||||
async def get_run(run_id: int):
|
||||
async with get_session() as session:
|
||||
run = (await session.execute(
|
||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
||||
)).scalar_one_or_none()
|
||||
if run is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_run_to_dict(run))
|
||||
|
||||
|
||||
@migrate_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
try:
|
||||
limit = int(request.args.get("limit", "10"))
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
if limit < 1 or limit > 100:
|
||||
return _bad("invalid_limit")
|
||||
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(MigrationRun)
|
||||
.order_by(MigrationRun.id.desc())
|
||||
.limit(limit)
|
||||
)).scalars().all()
|
||||
return jsonify([_run_to_dict(r) for r in rows])
|
||||
@@ -0,0 +1,74 @@
|
||||
"""ML admin API: settings, backfill trigger, centroid recompute trigger."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import MLSettings
|
||||
|
||||
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||
|
||||
|
||||
_EDITABLE = (
|
||||
"suggestion_threshold_artist",
|
||||
"suggestion_threshold_character",
|
||||
"suggestion_threshold_copyright",
|
||||
"suggestion_threshold_general",
|
||||
"centroid_similarity_threshold",
|
||||
"min_reference_images",
|
||||
)
|
||||
|
||||
|
||||
@ml_admin_bp.route("/settings", methods=["GET"])
|
||||
async def get_settings():
|
||||
from sqlalchemy import select
|
||||
|
||||
async with get_session() as session:
|
||||
s = (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
).scalar_one()
|
||||
return jsonify(
|
||||
{
|
||||
"suggestion_threshold_artist": s.suggestion_threshold_artist,
|
||||
"suggestion_threshold_character": s.suggestion_threshold_character,
|
||||
"suggestion_threshold_copyright": s.suggestion_threshold_copyright,
|
||||
"suggestion_threshold_general": s.suggestion_threshold_general,
|
||||
"centroid_similarity_threshold": s.centroid_similarity_threshold,
|
||||
"min_reference_images": s.min_reference_images,
|
||||
"tagger_model_version": s.tagger_model_version,
|
||||
"embedder_model_version": s.embedder_model_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@ml_admin_bp.route("/settings", methods=["PATCH"])
|
||||
async def patch_settings():
|
||||
from sqlalchemy import select
|
||||
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return jsonify({"error": "body must be an object"}), 400
|
||||
async with get_session() as session:
|
||||
s = (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
).scalar_one()
|
||||
for field in _EDITABLE:
|
||||
if field in body:
|
||||
setattr(s, field, body[field])
|
||||
await session.commit()
|
||||
return await get_settings()
|
||||
|
||||
|
||||
@ml_admin_bp.route("/backfill", methods=["POST"])
|
||||
async def trigger_backfill():
|
||||
from ..tasks.ml import backfill
|
||||
|
||||
r = backfill.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@ml_admin_bp.route("/recompute-centroids", methods=["POST"])
|
||||
async def trigger_recompute():
|
||||
from ..tasks.ml import recompute_centroids
|
||||
|
||||
r = recompute_centroids.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
@@ -0,0 +1,12 @@
|
||||
"""FC-3b: /api/platforms — informational, no auth, matches GS shape."""
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
|
||||
from ..services.platforms import PLATFORMS, to_dict
|
||||
|
||||
platforms_bp = Blueprint("platforms", __name__, url_prefix="/api/platforms")
|
||||
|
||||
|
||||
@platforms_bp.route("", methods=["GET"])
|
||||
async def list_platforms():
|
||||
return jsonify({"platforms": {k: to_dict(v) for k, v in PLATFORMS.items()}})
|
||||
@@ -0,0 +1,69 @@
|
||||
"""FC-3e: /api/posts — cursor-paginated unified posts feed."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.post_feed_service import PostFeedService
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
|
||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
@posts_bp.route("", methods=["GET"])
|
||||
async def list_posts():
|
||||
args = request.args
|
||||
|
||||
cursor = args.get("cursor") or None
|
||||
artist_id_raw = args.get("artist_id")
|
||||
platform = args.get("platform") or None
|
||||
limit_raw = args.get("limit", "24")
|
||||
|
||||
try:
|
||||
limit = int(limit_raw)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit", detail="limit must be an integer")
|
||||
if limit < 1 or limit > 100:
|
||||
return _bad("invalid_limit", detail="limit must be between 1 and 100")
|
||||
|
||||
artist_id = None
|
||||
if artist_id_raw is not None:
|
||||
try:
|
||||
artist_id = int(artist_id_raw)
|
||||
except ValueError:
|
||||
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
||||
|
||||
if platform is not None and platform not in KNOWN_PLATFORMS:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
detail=f"platform must be one of {sorted(KNOWN_PLATFORMS)}",
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
page = await PostFeedService(session).scroll(
|
||||
cursor=cursor, artist_id=artist_id,
|
||||
platform=platform, limit=limit,
|
||||
)
|
||||
except ValueError as exc:
|
||||
# Service raises ValueError for malformed cursors only;
|
||||
# limit bounds are validated above.
|
||||
return _bad("invalid_cursor", detail=str(exc))
|
||||
|
||||
return jsonify(page)
|
||||
|
||||
|
||||
@posts_bp.route("/<int:post_id>", methods=["GET"])
|
||||
async def get_post(post_id: int):
|
||||
async with get_session() as session:
|
||||
item = await PostFeedService(session).get_post(post_id)
|
||||
if item is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||
return jsonify(item)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Provenance API: image -> posts, post -> header payload.
|
||||
|
||||
Separate from the gallery/tag APIs by design (provenance is its own
|
||||
system). Read-only.
|
||||
"""
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.provenance_service import ProvenanceService
|
||||
|
||||
provenance_bp = Blueprint("provenance", __name__,
|
||||
url_prefix="/api/provenance")
|
||||
|
||||
|
||||
@provenance_bp.route("/image/<int:image_id>", methods=["GET"])
|
||||
async def image_provenance(image_id: int):
|
||||
async with get_session() as session:
|
||||
svc = ProvenanceService(session)
|
||||
payload = await svc.for_image(image_id)
|
||||
if payload is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(payload)
|
||||
|
||||
|
||||
@provenance_bp.route("/post/<int:post_id>", methods=["GET"])
|
||||
async def post_provenance(post_id: int):
|
||||
async with get_session() as session:
|
||||
svc = ProvenanceService(session)
|
||||
payload = await svc.for_post(post_id)
|
||||
if payload is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(payload)
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Settings API: import filters, system stats."""
|
||||
|
||||
import secrets
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||
|
||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
_EDITABLE_FIELDS = (
|
||||
"min_width",
|
||||
"min_height",
|
||||
"skip_transparent",
|
||||
"transparency_threshold",
|
||||
"skip_single_color",
|
||||
"single_color_threshold",
|
||||
"single_color_tolerance",
|
||||
"phash_threshold",
|
||||
"download_rate_limit_seconds",
|
||||
"download_validate_files",
|
||||
"download_schedule_default_seconds",
|
||||
"download_event_retention_days",
|
||||
"download_failure_warning_threshold",
|
||||
)
|
||||
|
||||
|
||||
@settings_bp.route("/settings/import", methods=["GET"])
|
||||
async def get_import_settings():
|
||||
async with get_session() as session:
|
||||
row = (
|
||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
||||
).scalar_one()
|
||||
return jsonify({
|
||||
"min_width": row.min_width,
|
||||
"min_height": row.min_height,
|
||||
"skip_transparent": row.skip_transparent,
|
||||
"transparency_threshold": row.transparency_threshold,
|
||||
"skip_single_color": row.skip_single_color,
|
||||
"single_color_threshold": row.single_color_threshold,
|
||||
"single_color_tolerance": row.single_color_tolerance,
|
||||
"phash_threshold": row.phash_threshold,
|
||||
"download_rate_limit_seconds": row.download_rate_limit_seconds,
|
||||
"download_validate_files": row.download_validate_files,
|
||||
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
||||
"download_event_retention_days": row.download_event_retention_days,
|
||||
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/import", methods=["PATCH"])
|
||||
async def update_import_settings():
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return jsonify({"error": "body must be a JSON object"}), 400
|
||||
|
||||
if "phash_threshold" in body and (
|
||||
not isinstance(body["phash_threshold"], int)
|
||||
or isinstance(body["phash_threshold"], bool)
|
||||
or body["phash_threshold"] < 0
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "phash_threshold must be a non-negative integer"}
|
||||
), 400
|
||||
|
||||
if "download_rate_limit_seconds" in body:
|
||||
val = body["download_rate_limit_seconds"]
|
||||
if not isinstance(val, (int, float)) or isinstance(val, bool) or val < 0:
|
||||
return jsonify(
|
||||
{"error": "download_rate_limit_seconds must be a non-negative number"}
|
||||
), 400
|
||||
if "download_validate_files" in body and not isinstance(
|
||||
body["download_validate_files"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "download_validate_files must be a boolean"}
|
||||
), 400
|
||||
|
||||
# FC-3d scheduling knobs — bounds validation.
|
||||
def _bad_int(name: str, lo: int, hi: int):
|
||||
return jsonify(
|
||||
{"error": f"{name} must be an integer in [{lo}, {hi}]"}
|
||||
), 400
|
||||
|
||||
if "download_schedule_default_seconds" in body:
|
||||
v = body["download_schedule_default_seconds"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 60 or v > 86400:
|
||||
return _bad_int("download_schedule_default_seconds", 60, 86400)
|
||||
if "download_event_retention_days" in body:
|
||||
v = body["download_event_retention_days"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 3650:
|
||||
return _bad_int("download_event_retention_days", 1, 3650)
|
||||
if "download_failure_warning_threshold" in body:
|
||||
v = body["download_failure_warning_threshold"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
|
||||
return _bad_int("download_failure_warning_threshold", 1, 100)
|
||||
|
||||
async with get_session() as session:
|
||||
row = (
|
||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
||||
).scalar_one()
|
||||
for field in _EDITABLE_FIELDS:
|
||||
if field in body:
|
||||
setattr(row, field, body[field])
|
||||
await session.commit()
|
||||
|
||||
return await get_import_settings()
|
||||
|
||||
|
||||
@settings_bp.route("/system/stats", methods=["GET"])
|
||||
async def system_stats():
|
||||
async with get_session() as session:
|
||||
total_images = (await session.execute(select(func.count(ImageRecord.id)))).scalar_one()
|
||||
total_tags = (await session.execute(select(func.count(Tag.id)))).scalar_one()
|
||||
storage_bytes = (
|
||||
(await session.execute(select(func.coalesce(func.sum(ImageRecord.size_bytes), 0)))).scalar_one()
|
||||
)
|
||||
|
||||
# Task counts grouped by status
|
||||
status_rows = (
|
||||
await session.execute(
|
||||
select(ImportTask.status, func.count(ImportTask.id)).group_by(ImportTask.status)
|
||||
)
|
||||
).all()
|
||||
status_counts = {row[0]: row[1] for row in status_rows}
|
||||
|
||||
# Integrity counts — FC-2e.
|
||||
integrity_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
ImageRecord.integrity_status,
|
||||
func.count(ImageRecord.id),
|
||||
).group_by(ImageRecord.integrity_status)
|
||||
)
|
||||
).all()
|
||||
integrity_counts = {row[0]: row[1] for row in integrity_rows}
|
||||
|
||||
# Active batch (most recent running)
|
||||
active_batch_row = (
|
||||
await session.execute(
|
||||
select(ImportBatch)
|
||||
.where(ImportBatch.status == "running")
|
||||
.order_by(ImportBatch.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
active_batch = None
|
||||
if active_batch_row:
|
||||
active_batch = {
|
||||
"id": active_batch_row.id,
|
||||
"source_path": active_batch_row.source_path,
|
||||
"started_at": active_batch_row.started_at.isoformat(),
|
||||
"total_files": active_batch_row.total_files,
|
||||
"imported": active_batch_row.imported,
|
||||
"skipped": active_batch_row.skipped,
|
||||
"failed": active_batch_row.failed,
|
||||
}
|
||||
|
||||
return jsonify({
|
||||
"total_images": total_images,
|
||||
"total_tags": total_tags,
|
||||
"storage_bytes": storage_bytes,
|
||||
"tasks": {
|
||||
"pending": status_counts.get("pending", 0),
|
||||
"queued": status_counts.get("queued", 0),
|
||||
"processing": status_counts.get("processing", 0),
|
||||
"complete": status_counts.get("complete", 0),
|
||||
"skipped": status_counts.get("skipped", 0),
|
||||
"failed": status_counts.get("failed", 0),
|
||||
},
|
||||
"integrity": {
|
||||
"unknown": int(integrity_counts.get("unknown", 0)),
|
||||
"ok": int(integrity_counts.get("ok", 0)),
|
||||
"corrupt": int(integrity_counts.get("corrupt", 0)),
|
||||
"failed_verification": int(
|
||||
integrity_counts.get("failed_verification", 0)
|
||||
),
|
||||
},
|
||||
"active_batch": active_batch,
|
||||
})
|
||||
|
||||
|
||||
# --- FC-3b: extension API key (lives in app_setting) -----------------------
|
||||
|
||||
|
||||
async def _get_or_seed_extension_api_key(session) -> str:
|
||||
row = (await session.execute(
|
||||
select(AppSetting).where(AppSetting.key == "extension_api_key")
|
||||
)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = AppSetting(key="extension_api_key", value=secrets.token_urlsafe(32))
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return row.value
|
||||
|
||||
|
||||
@settings_bp.route("/settings/extension_api_key", methods=["GET"])
|
||||
async def get_extension_api_key():
|
||||
async with get_session() as session:
|
||||
key = await _get_or_seed_extension_api_key(session)
|
||||
return jsonify({"key": key})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/extension_api_key/rotate", methods=["POST"])
|
||||
async def rotate_extension_api_key():
|
||||
async with get_session() as session:
|
||||
row = (await session.execute(
|
||||
select(AppSetting).where(AppSetting.key == "extension_api_key")
|
||||
)).scalar_one_or_none()
|
||||
new_value = secrets.token_urlsafe(32)
|
||||
if row is None:
|
||||
row = AppSetting(key="extension_api_key", value=new_value)
|
||||
session.add(row)
|
||||
else:
|
||||
row.value = new_value
|
||||
await session.commit()
|
||||
return jsonify({"key": new_value})
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Showcase API: scalable random sample of images."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.showcase_service import ShowcaseService
|
||||
|
||||
showcase_bp = Blueprint("showcase", __name__, url_prefix="/api/showcase")
|
||||
|
||||
|
||||
@showcase_bp.route("", methods=["GET"])
|
||||
async def random_showcase():
|
||||
try:
|
||||
limit = int(request.args.get("limit", "60"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
async with get_session() as session:
|
||||
svc = ShowcaseService(session)
|
||||
try:
|
||||
images = await svc.random_sample(limit=limit)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify({"images": images})
|
||||
@@ -0,0 +1,156 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, Source
|
||||
from ..services.source_service import (
|
||||
KNOWN_PLATFORMS,
|
||||
ArtistNotFoundError,
|
||||
DuplicateSourceError,
|
||||
EmptyUrlError,
|
||||
InvalidConfigError,
|
||||
SourceService,
|
||||
UnknownPlatformError,
|
||||
)
|
||||
|
||||
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
@sources_bp.route("", methods=["GET"])
|
||||
async def list_sources():
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = None
|
||||
if artist_id_raw is not None:
|
||||
try:
|
||||
artist_id = int(artist_id_raw)
|
||||
except ValueError:
|
||||
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
||||
async with get_session() as session:
|
||||
records = await SourceService(session).list(artist_id=artist_id)
|
||||
return jsonify([r.to_dict() for r in records])
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>", methods=["GET"])
|
||||
async def get_source(source_id: int):
|
||||
async with get_session() as session:
|
||||
record = await SourceService(session).get(source_id)
|
||||
if record is None:
|
||||
return _bad("not_found", status=404, detail=f"source id={source_id}")
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("", methods=["POST"])
|
||||
async def create_source():
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
try:
|
||||
artist_id = int(body["artist_id"])
|
||||
platform = body["platform"]
|
||||
url = body["url"]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return _bad("invalid_body", detail="artist_id, platform, url are required")
|
||||
|
||||
optional = {
|
||||
k: body[k]
|
||||
for k in ("enabled", "config_overrides", "check_interval_override")
|
||||
if k in body
|
||||
}
|
||||
|
||||
async with get_session() as session:
|
||||
svc = SourceService(session)
|
||||
try:
|
||||
record = await svc.create(
|
||||
artist_id=artist_id, platform=platform, url=url, **optional,
|
||||
)
|
||||
except ArtistNotFoundError:
|
||||
return _bad("artist_not_found", status=404, detail=f"artist id={artist_id}")
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
|
||||
except InvalidConfigError as exc:
|
||||
return _bad("invalid_config", detail=str(exc))
|
||||
except EmptyUrlError as exc:
|
||||
return _bad("empty_url", detail=str(exc))
|
||||
except DuplicateSourceError as exc:
|
||||
return _bad("duplicate", status=409, existing_id=exc.existing_id)
|
||||
return jsonify(record.to_dict()), 201
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>", methods=["PATCH"])
|
||||
async def patch_source(source_id: int):
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
async with get_session() as session:
|
||||
svc = SourceService(session)
|
||||
try:
|
||||
record = await svc.update(source_id, **body)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad("unknown_platform", detail=str(exc), known=sorted(KNOWN_PLATFORMS))
|
||||
except InvalidConfigError as exc:
|
||||
return _bad("invalid_config", detail=str(exc))
|
||||
except EmptyUrlError as exc:
|
||||
return _bad("empty_url", detail=str(exc))
|
||||
except DuplicateSourceError as exc:
|
||||
return _bad("duplicate", status=409, existing_id=exc.existing_id)
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>", methods=["DELETE"])
|
||||
async def delete_source(source_id: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SourceService(session).delete(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
return "", 204
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||
async def check_source(source_id: int):
|
||||
"""FC-3c: enqueue a download for this source.
|
||||
|
||||
Returns 202 with the new DownloadEvent id. If a pending/running
|
||||
event already exists for this source, returns 409 with that id."""
|
||||
async with get_session() as session:
|
||||
source = (await session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
return _bad("not_found", status=404, detail=f"source id={source_id}")
|
||||
if not source.enabled:
|
||||
return _bad("source_disabled", detail="enable the source first")
|
||||
|
||||
in_flight = (await session.execute(
|
||||
select(DownloadEvent.id).where(
|
||||
DownloadEvent.source_id == source_id,
|
||||
DownloadEvent.status.in_(["pending", "running"]),
|
||||
).order_by(DownloadEvent.id.desc()).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if in_flight is not None:
|
||||
return jsonify(
|
||||
{"download_event_id": in_flight, "status": "already_running"}
|
||||
), 409
|
||||
|
||||
event = DownloadEvent(source_id=source_id, status="pending")
|
||||
session.add(event)
|
||||
await session.commit()
|
||||
await session.refresh(event)
|
||||
event_id = event.id
|
||||
|
||||
from ..tasks.download import download_source
|
||||
download_source.delay(source_id)
|
||||
|
||||
return jsonify({"download_event_id": event_id, "status": "pending"}), 202
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Suggestions API: per-image ranked suggestions + accept/alias/dismiss."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.ml.allowlist import AllowlistService
|
||||
from ..services.ml.suggestions import SuggestionService
|
||||
|
||||
suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
|
||||
async def get_suggestions(image_id: int):
|
||||
async with get_session() as session:
|
||||
sl = await SuggestionService(session).for_image(image_id)
|
||||
return jsonify(
|
||||
{
|
||||
"by_category": {
|
||||
cat: [
|
||||
{
|
||||
"canonical_tag_id": s.canonical_tag_id,
|
||||
"display_name": s.display_name,
|
||||
"category": s.category,
|
||||
"score": round(s.score, 4),
|
||||
"source": s.source,
|
||||
"creates_new_tag": s.creates_new_tag,
|
||||
}
|
||||
for s in items
|
||||
]
|
||||
for cat, items in sl.by_category.items()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/accept", methods=["POST"]
|
||||
)
|
||||
async def accept_suggestion(image_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
tag_id = body["tag_id"]
|
||||
async with get_session() as session:
|
||||
newly_added = await AllowlistService(session).accept(image_id, tag_id)
|
||||
await session.commit()
|
||||
if newly_added:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=tag_id)
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/alias", methods=["POST"]
|
||||
)
|
||||
async def alias_suggestion(image_id: int):
|
||||
body = await request.get_json()
|
||||
required = {"alias_string", "alias_category", "canonical_tag_id"}
|
||||
if not body or not required.issubset(body):
|
||||
return jsonify({"error": f"required: {sorted(required)}"}), 400
|
||||
async with get_session() as session:
|
||||
newly_added = await AllowlistService(session).add_alias_and_accept(
|
||||
image_id,
|
||||
body["alias_string"],
|
||||
body["alias_category"],
|
||||
body["canonical_tag_id"],
|
||||
)
|
||||
await session.commit()
|
||||
if newly_added:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=body["canonical_tag_id"])
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/dismiss", methods=["POST"]
|
||||
)
|
||||
async def dismiss_suggestion(image_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).dismiss(image_id, body["tag_id"])
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route("/suggestions/bulk", methods=["POST"])
|
||||
async def bulk_suggestions():
|
||||
body = await request.get_json()
|
||||
if not body or "image_ids" not in body:
|
||||
return jsonify({"error": "image_ids required"}), 400
|
||||
raw = body["image_ids"]
|
||||
if not isinstance(raw, list) or not raw:
|
||||
return jsonify({"error": "image_ids must be a non-empty list"}), 400
|
||||
try:
|
||||
ids = [int(x) for x in raw]
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"error": "image_ids must be integers"}), 400
|
||||
if len(ids) > 200:
|
||||
return jsonify({"error": "selection too large (max 200)"}), 400
|
||||
try:
|
||||
threshold = float(body.get("threshold", 0.8))
|
||||
except (TypeError, ValueError):
|
||||
threshold = 0.8
|
||||
threshold = min(1.0, max(0.0, threshold))
|
||||
async with get_session() as session:
|
||||
suggestions = await SuggestionService(session).for_selection(
|
||||
ids, threshold=threshold
|
||||
)
|
||||
return jsonify(
|
||||
{
|
||||
"suggestions": suggestions,
|
||||
"evaluated": len(ids),
|
||||
"threshold": threshold,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Tags API: autocomplete, create, list/add/remove for an image."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import exists, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Tag, TagKind
|
||||
from ..models.tag_allowlist import TagAllowlist
|
||||
from ..services.bulk_tag_service import BulkTagService
|
||||
from ..services.series_service import SeriesError, SeriesService
|
||||
from ..services.tag_directory_service import TagDirectoryService
|
||||
from ..services.tag_service import (
|
||||
TagMergeConflict,
|
||||
TagService,
|
||||
TagValidationError,
|
||||
)
|
||||
|
||||
tags_bp = Blueprint("tags", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
def _coerce_kind(raw: str | None) -> TagKind | None:
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return TagKind(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_bulk_ids(
|
||||
body, *, max_ids: int = 200
|
||||
) -> tuple[list[int] | None, tuple | None]:
|
||||
"""Returns (image_ids, error). error is (json, status) or None."""
|
||||
if not body or "image_ids" not in body:
|
||||
return None, (jsonify({"error": "image_ids required"}), 400)
|
||||
raw = body["image_ids"]
|
||||
if not isinstance(raw, list) or not raw:
|
||||
return None, (
|
||||
jsonify({"error": "image_ids must be a non-empty list"}),
|
||||
400,
|
||||
)
|
||||
try:
|
||||
ids = [int(x) for x in raw]
|
||||
except (TypeError, ValueError):
|
||||
return None, (
|
||||
jsonify({"error": "image_ids must be integers"}),
|
||||
400,
|
||||
)
|
||||
if len(ids) > max_ids:
|
||||
return None, (
|
||||
jsonify({"error": f"selection too large (max {max_ids})"}),
|
||||
400,
|
||||
)
|
||||
return ids, None
|
||||
|
||||
|
||||
@tags_bp.route("/tags/autocomplete", methods=["GET"])
|
||||
async def autocomplete():
|
||||
q = request.args.get("q", "")
|
||||
kind = _coerce_kind(request.args.get("kind"))
|
||||
try:
|
||||
limit = int(request.args.get("limit", "20"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
hits = await svc.autocomplete(q, kind=kind, limit=limit)
|
||||
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"id": h.id,
|
||||
"name": h.name,
|
||||
"kind": h.kind,
|
||||
"fandom_id": h.fandom_id,
|
||||
"fandom_name": h.fandom_name,
|
||||
"image_count": h.image_count,
|
||||
}
|
||||
for h in hits
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/tags/directory", methods=["GET"])
|
||||
async def directory():
|
||||
kind = request.args.get("kind") or None
|
||||
if kind is not None and _coerce_kind(kind) is None:
|
||||
return jsonify({"error": f"invalid kind {kind!r}"}), 400
|
||||
q = request.args.get("q") or None
|
||||
cursor = request.args.get("cursor") or None
|
||||
try:
|
||||
limit = int(request.args.get("limit", "60"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
async with get_session() as session:
|
||||
svc = TagDirectoryService(session)
|
||||
try:
|
||||
page = await svc.list_tags(kind=kind, q=q, cursor=cursor, limit=limit)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify({"cards": page.cards, "next_cursor": page.next_cursor})
|
||||
|
||||
|
||||
@tags_bp.route("/tags", methods=["POST"])
|
||||
async def create_tag():
|
||||
body = await request.get_json()
|
||||
if not body or "name" not in body or "kind" not in body:
|
||||
return jsonify({"error": "name and kind required"}), 400
|
||||
name = body["name"]
|
||||
kind = _coerce_kind(body["kind"])
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400
|
||||
fandom_id = body.get("fandom_id")
|
||||
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
tag = await svc.find_or_create(name, kind, fandom_id=fandom_id)
|
||||
except TagValidationError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
await session.commit()
|
||||
return jsonify(
|
||||
{"id": tag.id, "name": tag.name, "kind": tag.kind.value, "fandom_id": tag.fandom_id}
|
||||
), 201
|
||||
|
||||
|
||||
@tags_bp.route("/images/<int:image_id>/tags", methods=["GET"])
|
||||
async def list_tags_for_image(image_id: int):
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
tags = await svc.list_for_image(image_id)
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"kind": t.kind.value,
|
||||
"fandom_id": t.fandom_id,
|
||||
}
|
||||
for t in tags
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/images/<int:image_id>/tags", methods=["POST"])
|
||||
async def add_tag_to_image(image_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
source = body.get("source", "manual")
|
||||
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
await svc.add_to_image(image_id, body["tag_id"], source=source)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route("/images/<int:image_id>/tags/<int:tag_id>", methods=["DELETE"])
|
||||
async def remove_tag_from_image(image_id: int, tag_id: int):
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
await svc.remove_from_image(image_id, tag_id)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>", methods=["PATCH"])
|
||||
async def rename_tag(tag_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "name" not in body:
|
||||
return jsonify({"error": "name required"}), 400
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
tag = await svc.rename(tag_id, body["name"])
|
||||
except TagMergeConflict as exc:
|
||||
return jsonify(
|
||||
{
|
||||
"error": str(exc),
|
||||
"target": {
|
||||
"id": exc.target_id,
|
||||
"name": exc.target_name,
|
||||
},
|
||||
"source_image_count": exc.source_image_count,
|
||||
"will_alias": exc.will_alias,
|
||||
}
|
||||
), 409
|
||||
except TagValidationError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
await session.commit()
|
||||
return jsonify(
|
||||
{"id": tag.id, "name": tag.name, "kind": tag.kind.value}
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:source_id>/merge", methods=["POST"])
|
||||
async def merge_tag(source_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "target_id" not in body:
|
||||
return jsonify({"error": "target_id required"}), 400
|
||||
target_id = body["target_id"]
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
result = await svc.merge(source_id, target_id)
|
||||
except TagValidationError as exc:
|
||||
msg = str(exc)
|
||||
status = 404 if "not found" in msg else 400
|
||||
return jsonify({"error": msg}), status
|
||||
await session.commit()
|
||||
target_allowlisted = await session.scalar(
|
||||
select(exists().where(TagAllowlist.tag_id == result.target_id))
|
||||
)
|
||||
if target_allowlisted:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=result.target_id)
|
||||
return jsonify(
|
||||
{
|
||||
"target": {
|
||||
"id": result.target_id,
|
||||
"name": result.target_name,
|
||||
"kind": result.target_kind,
|
||||
},
|
||||
"merged_count": result.merged_count,
|
||||
"alias_created": result.alias_created,
|
||||
"source_deleted": result.source_deleted,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/images/common-tags", methods=["POST"])
|
||||
async def common_tags():
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body)
|
||||
if err:
|
||||
return err
|
||||
async with get_session() as session:
|
||||
tags = await BulkTagService(session).common_tags(ids)
|
||||
return jsonify({"tags": tags})
|
||||
|
||||
|
||||
@tags_bp.route("/images/bulk/tags", methods=["POST"])
|
||||
async def bulk_add_tag():
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body)
|
||||
if err:
|
||||
return err
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
source = body.get("source", "manual")
|
||||
if source not in ("manual", "ml_accepted"):
|
||||
return jsonify({"error": f"invalid source {source!r}"}), 400
|
||||
async with get_session() as session:
|
||||
if await session.get(Tag, body["tag_id"]) is None:
|
||||
return jsonify({"error": "tag not found"}), 404
|
||||
try:
|
||||
added = await BulkTagService(session).bulk_add(
|
||||
ids, body["tag_id"], source=source
|
||||
)
|
||||
except IntegrityError:
|
||||
return jsonify(
|
||||
{"error": "one or more image_ids do not exist"}
|
||||
), 400
|
||||
await session.commit()
|
||||
return jsonify({"added_count": added, "total": len(ids)})
|
||||
|
||||
|
||||
@tags_bp.route("/images/bulk/tags/remove", methods=["POST"])
|
||||
async def bulk_remove_tag():
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body)
|
||||
if err:
|
||||
return err
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
async with get_session() as session:
|
||||
if await session.get(Tag, body["tag_id"]) is None:
|
||||
return jsonify({"error": "tag not found"}), 404
|
||||
removed = await BulkTagService(session).bulk_remove(
|
||||
ids, body["tag_id"]
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"removed_count": removed})
|
||||
|
||||
|
||||
def _series_err(exc: SeriesError):
|
||||
msg = str(exc)
|
||||
status = 404 if ("not found" in msg or "not a series" in msg) else 400
|
||||
return jsonify({"error": msg}), status
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pages", methods=["GET"])
|
||||
async def series_pages(tag_id: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
data = await SeriesService(session).list_pages(tag_id)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pages", methods=["POST"])
|
||||
async def series_add(tag_id: int):
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body, max_ids=500)
|
||||
if err:
|
||||
return err
|
||||
async with get_session() as session:
|
||||
try:
|
||||
n = await SeriesService(session).add_images(tag_id, ids)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"added_count": n})
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pages/remove", methods=["POST"])
|
||||
async def series_remove(tag_id: int):
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body, max_ids=500)
|
||||
if err:
|
||||
return err
|
||||
async with get_session() as session:
|
||||
try:
|
||||
n = await SeriesService(session).remove_images(tag_id, ids)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"removed_count": n})
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/reorder", methods=["POST"])
|
||||
async def series_reorder(tag_id: int):
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body, max_ids=500)
|
||||
if err:
|
||||
return err
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesService(session).reorder(tag_id, ids)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/cover", methods=["POST"])
|
||||
async def series_cover(tag_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "image_id" not in body:
|
||||
return jsonify({"error": "image_id required"}), 400
|
||||
try:
|
||||
image_id = int(body["image_id"])
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"error": "image_id must be an integer"}), 400
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesService(session).set_cover(tag_id, image_id)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Celery configuration with separate queue lanes.
|
||||
|
||||
Queues:
|
||||
import — filesystem import path (FC-2)
|
||||
ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
|
||||
thumbnail — image and video thumbnail generation (FC-2)
|
||||
download — gallery-dl tasks (FC-3)
|
||||
scan — periodic source checks (FC-3) — kept separate so long imports
|
||||
don't starve the scheduler
|
||||
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
||||
default — anything not explicitly routed
|
||||
"""
|
||||
|
||||
from celery import Celery
|
||||
|
||||
from .config import get_config
|
||||
|
||||
|
||||
def make_celery() -> Celery:
|
||||
cfg = get_config()
|
||||
app = Celery(
|
||||
"fabledcurator",
|
||||
broker=cfg.celery_broker_url,
|
||||
backend=cfg.celery_result_backend,
|
||||
include=[
|
||||
"backend.app.tasks.smoke",
|
||||
"backend.app.tasks.scan",
|
||||
"backend.app.tasks.import_file",
|
||||
"backend.app.tasks.thumbnail",
|
||||
"backend.app.tasks.maintenance",
|
||||
"backend.app.tasks.migration",
|
||||
"backend.app.tasks.ml",
|
||||
"backend.app.tasks.download",
|
||||
],
|
||||
)
|
||||
app.conf.update(
|
||||
task_default_queue="default",
|
||||
task_routes={
|
||||
"backend.app.tasks.import_file.*": {"queue": "import"},
|
||||
"backend.app.tasks.ml.*": {"queue": "ml"},
|
||||
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
||||
"backend.app.tasks.download.*": {"queue": "download"},
|
||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
||||
},
|
||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||
task_acks_late=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
beat_schedule={
|
||||
"recover-interrupted-tasks": {
|
||||
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
||||
"schedule": 300.0, # every 5 minutes
|
||||
},
|
||||
"cleanup-old-tasks": {
|
||||
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
"ml-backfill-daily": {
|
||||
"task": "backend.app.tasks.ml.backfill",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
"recompute-centroids-daily": {
|
||||
"task": "backend.app.tasks.ml.recompute_centroids",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
"apply-allowlist-sweep-daily": {
|
||||
"task": "backend.app.tasks.ml.apply_allowlist_tags",
|
||||
"schedule": 86400.0,
|
||||
},
|
||||
"integrity-verify-weekly": {
|
||||
"task": "backend.app.tasks.maintenance.verify_integrity",
|
||||
"schedule": 604800.0, # weekly
|
||||
},
|
||||
"fc3d-tick-due-sources": {
|
||||
"task": "backend.app.tasks.scan.tick_due_sources",
|
||||
"schedule": 60.0, # every minute
|
||||
},
|
||||
"fc3d-cleanup-download-events": {
|
||||
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
},
|
||||
timezone="UTC",
|
||||
)
|
||||
return app
|
||||
|
||||
|
||||
celery = make_celery()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Application configuration loaded from environment variables."""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
db_user: str
|
||||
db_password: str
|
||||
db_host: str
|
||||
db_port: int
|
||||
db_name: str
|
||||
|
||||
celery_broker_url: str
|
||||
celery_result_backend: str
|
||||
|
||||
secret_key: str
|
||||
extension_api_key: str # used by the Firefox extension; lands in FC-3 but read here
|
||||
log_level: str
|
||||
|
||||
@property
|
||||
def database_url(self) -> str:
|
||||
return (
|
||||
f"postgresql+asyncpg://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
)
|
||||
|
||||
@property
|
||||
def database_url_sync(self) -> str:
|
||||
# Alembic uses sync driver
|
||||
return (
|
||||
f"postgresql+psycopg://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_config() -> Config:
|
||||
return Config(
|
||||
db_user=os.environ.get("DB_USER", "fabledcurator"),
|
||||
db_password=os.environ["DB_PASSWORD"],
|
||||
db_host=os.environ.get("DB_HOST", "postgres"),
|
||||
db_port=int(os.environ.get("DB_PORT", "5432")),
|
||||
db_name=os.environ.get("DB_NAME", "fabledcurator"),
|
||||
celery_broker_url=os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"),
|
||||
celery_result_backend=os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"),
|
||||
secret_key=os.environ["SECRET_KEY"],
|
||||
extension_api_key=os.environ.get("EXTENSION_API_KEY", ""),
|
||||
log_level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Process-wide singleton async engine + session helper.
|
||||
|
||||
Previously every API blueprint lazily built its own engine via make_engine(),
|
||||
so the process ran N connection pools. This module owns ONE engine for the
|
||||
whole app; create_app() disposes it on shutdown via after_serving.
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from .config import get_config
|
||||
|
||||
_engine: AsyncEngine | None = None
|
||||
_sessionmaker: async_sessionmaker | None = None
|
||||
|
||||
|
||||
def get_engine() -> AsyncEngine:
|
||||
global _engine, _sessionmaker
|
||||
if _engine is None:
|
||||
_engine = create_async_engine(
|
||||
get_config().database_url, pool_pre_ping=True, future=True
|
||||
)
|
||||
_sessionmaker = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
return _engine
|
||||
|
||||
|
||||
def _get_sessionmaker() -> async_sessionmaker:
|
||||
if _sessionmaker is None:
|
||||
get_engine()
|
||||
assert _sessionmaker is not None
|
||||
return _sessionmaker
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_session():
|
||||
"""`async with get_session() as session:` — one shared pool."""
|
||||
Session = _get_sessionmaker()
|
||||
async with Session() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def dispose_engine() -> None:
|
||||
global _engine, _sessionmaker
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
_engine = None
|
||||
_sessionmaker = None
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Serves the built Vue SPA from frontend/dist/ with history-mode fallback."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, send_from_directory
|
||||
|
||||
FRONTEND_DIST = Path(__file__).resolve().parent.parent.parent / "frontend" / "dist"
|
||||
|
||||
frontend_bp = Blueprint("frontend", __name__)
|
||||
|
||||
|
||||
@frontend_bp.route("/")
|
||||
@frontend_bp.route("/<path:subpath>")
|
||||
async def serve_spa(subpath: str = ""):
|
||||
if subpath and (FRONTEND_DIST / subpath).is_file():
|
||||
return await send_from_directory(FRONTEND_DIST, subpath)
|
||||
return await send_from_directory(FRONTEND_DIST, "index.html")
|
||||
@@ -0,0 +1,49 @@
|
||||
"""All ORM models. Import this module to make every model visible to Alembic."""
|
||||
|
||||
from .app_setting import AppSetting
|
||||
from .artist import Artist
|
||||
from .base import Base
|
||||
from .credential import Credential
|
||||
from .download_event import DownloadEvent
|
||||
from .image_provenance import ImageProvenance
|
||||
from .image_record import ImageRecord
|
||||
from .import_batch import ImportBatch
|
||||
from .import_settings import ImportSettings
|
||||
from .import_task import ImportTask
|
||||
from .migration_run import MigrationRun
|
||||
from .ml_settings import MLSettings
|
||||
from .post import Post
|
||||
from .post_attachment import PostAttachment
|
||||
from .series_page import SeriesPage
|
||||
from .source import Source
|
||||
from .tag import Tag, TagKind, image_tag
|
||||
from .tag_alias import TagAlias
|
||||
from .tag_allowlist import TagAllowlist
|
||||
from .tag_reference_embedding import TagReferenceEmbedding
|
||||
from .tag_suggestion_rejection import TagSuggestionRejection
|
||||
|
||||
__all__ = [
|
||||
"Base",
|
||||
"AppSetting",
|
||||
"Artist",
|
||||
"Source",
|
||||
"Credential",
|
||||
"Post",
|
||||
"PostAttachment",
|
||||
"SeriesPage",
|
||||
"ImageRecord",
|
||||
"ImageProvenance",
|
||||
"Tag",
|
||||
"TagKind",
|
||||
"image_tag",
|
||||
"DownloadEvent",
|
||||
"ImportBatch",
|
||||
"ImportTask",
|
||||
"ImportSettings",
|
||||
"MLSettings",
|
||||
"MigrationRun",
|
||||
"TagAlias",
|
||||
"TagAllowlist",
|
||||
"TagReferenceEmbedding",
|
||||
"TagSuggestionRejection",
|
||||
]
|
||||
@@ -0,0 +1,24 @@
|
||||
"""AppSetting — small key/value app config (extension API key, etc.).
|
||||
|
||||
For settings that need DB persistence + runtime mutation. Use
|
||||
`ImportSettings` for the structured import-tuning settings; this
|
||||
table is for one-off scalar values keyed by string.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class AppSetting(Base):
|
||||
__tablename__ = "app_setting"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
value: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False,
|
||||
server_default=func.now(), onupdate=func.now(),
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Artist — unified entity that is both the gallery's ``artist:`` tag concept
|
||||
and GallerySubscriber's Subscription. ``is_subscription`` is True if any
|
||||
Sources are attached.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Artist(Base):
|
||||
__tablename__ = "artist"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# True once a Source is attached; flips false if all sources removed.
|
||||
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
|
||||
# Per-artist scheduling overrides; null means "use global default".
|
||||
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
sources = relationship("Source", back_populates="artist", cascade="all, delete-orphan")
|
||||
@@ -0,0 +1,17 @@
|
||||
"""SQLAlchemy declarative base with standardized constraint naming."""
|
||||
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
# Stable constraint names so Alembic autogeneration produces clean diffs.
|
||||
NAMING_CONVENTION = {
|
||||
"ix": "ix_%(column_0_label)s",
|
||||
"uq": "uq_%(table_name)s_%(column_0_name)s",
|
||||
"ck": "ck_%(table_name)s_%(constraint_name)s",
|
||||
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
|
||||
"pk": "pk_%(table_name)s",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Credential — encrypted blob (cookies/token) scoped per-platform,
|
||||
not per-source. One Patreon credential serves every Patreon source.
|
||||
|
||||
Schema aligned with GallerySubscriber's wire format (FC-3b) so the
|
||||
existing browser extension hits FC unmodified.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, LargeBinary, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Credential(Base):
|
||||
__tablename__ = "credential"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
platform: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
credential_type: Mapped[str] = mapped_column(String(32), nullable=False) # cookies|token
|
||||
encrypted_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
|
||||
captured_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_verified: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""DownloadEvent — log of every gallery-dl run, populated by FC-3."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DownloadEvent(Base):
|
||||
__tablename__ = "download_event"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
post_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False) # pending|running|ok|error|skipped
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
metadata_: Mapped[dict] = mapped_column(
|
||||
"metadata", JSONB, nullable=False, default=dict,
|
||||
server_default=sa.text("'{}'::jsonb"),
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""ImageProvenance — links an ImageRecord to a Post.
|
||||
|
||||
Many-to-one (one image, many provenance rows) enables the enrich-on-duplicate
|
||||
rule (spec §3): when a downloaded image is a pHash dupe of an existing
|
||||
record, we append a new provenance row to the existing record rather than
|
||||
dropping the metadata.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class ImageProvenance(Base):
|
||||
__tablename__ = "image_provenance"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
post_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
captured_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""ImageRecord — the gallery's primary entity, ported from ImageRepo.
|
||||
|
||||
ML fields and thumbnails are declared now (in FC-1) so FC-2 can populate them
|
||||
without a schema migration. The SigLIP embedding column uses pgvector's Vector
|
||||
type — pgvector extension is enabled in the initial migration.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
BigInteger,
|
||||
DateTime,
|
||||
Enum,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
|
||||
|
||||
|
||||
class ImageRecord(Base):
|
||||
__tablename__ = "image_record"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
|
||||
# On-disk identity
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
||||
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
width: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
height: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
# Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
|
||||
# Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
|
||||
integrity_status: Mapped[str] = mapped_column(
|
||||
String(24), nullable=False, default="unknown", index=True
|
||||
)
|
||||
|
||||
# Thumbnail (populated by FC-2)
|
||||
thumbnail_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# Origin / provenance pointers
|
||||
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_CHOICES, name="origin_enum"), nullable=False)
|
||||
primary_post_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
# FC-2d-vii-c: canonical per-image artist (the single source of truth
|
||||
# for attribution; provenance posts remain lineage detail).
|
||||
artist_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
# ML fields (populated by FC-2's ml-worker)
|
||||
tagger_predictions: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
tagger_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# 1152 = SigLIP-so400m embedding dim. Swapping models in FC-2 may require
|
||||
# a column-width migration.
|
||||
siglip_embedding: Mapped[list[float] | None] = mapped_column(Vector(1152), nullable=True)
|
||||
siglip_model_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
|
||||
# Centroid score cache (populated post-tagging)
|
||||
centroid_scores: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""ImportBatch — one scan run, aggregating many ImportTasks."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class ImportBatch(Base):
|
||||
__tablename__ = "import_batch"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
scan_mode: Mapped[str] = mapped_column(String(16), nullable=False) # quick|deep
|
||||
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||
# running | complete | cancelled
|
||||
|
||||
tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""ImportSettings — single-row table holding the importer's tunable knobs.
|
||||
|
||||
Enforced as a single row via a CHECK (id = 1) constraint. The application
|
||||
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
||||
"""
|
||||
|
||||
from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class ImportSettings(Base):
|
||||
__tablename__ = "import_settings"
|
||||
# Bare constraint name — Base.metadata's naming convention applies the
|
||||
# ck_<table>_<name> prefix, producing the final ck_import_settings_singleton.
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import")
|
||||
|
||||
min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9)
|
||||
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95)
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
||||
|
||||
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
|
||||
|
||||
# FC-3c downloader knobs
|
||||
download_rate_limit_seconds: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=3.0
|
||||
)
|
||||
download_validate_files: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
|
||||
# FC-3d scheduling knobs
|
||||
download_schedule_default_seconds: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=28800
|
||||
)
|
||||
download_event_retention_days: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=90
|
||||
)
|
||||
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=5
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""ImportTask — a single source file to import as part of a batch.
|
||||
|
||||
State machine: pending -> queued -> processing -> complete | skipped | failed.
|
||||
Workers crashing mid-task leave rows in 'processing'; the recovery sweep
|
||||
(tasks/maintenance.py::recover_interrupted_tasks) re-queues rows that have
|
||||
been processing longer than the stuck-task threshold.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class ImportTask(Base):
|
||||
__tablename__ = "import_task"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
batch_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
|
||||
source_path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||
|
||||
result_image_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
size_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
batch = relationship("ImportBatch", back_populates="tasks")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc).
|
||||
|
||||
kind/status are String(32) not Postgres ENUM so adding kinds later
|
||||
doesn't need a schema migration. The API layer validates values.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class MigrationRun(Base):
|
||||
__tablename__ = "migration_run"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
counts: Mapped[dict] = mapped_column(
|
||||
JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"),
|
||||
)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
metadata_: Mapped[dict] = mapped_column(
|
||||
"metadata", JSONB, nullable=False, default=dict,
|
||||
server_default=sa.text("'{}'::jsonb"),
|
||||
)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""MLSettings — single-row table holding ML pipeline tunables."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, Float, Integer, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class MLSettings(Base):
|
||||
__tablename__ = "ml_settings"
|
||||
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||
# producing the final ck_ml_settings_singleton (matches migration 0003).
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
suggestion_threshold_artist: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
)
|
||||
suggestion_threshold_character: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
suggestion_threshold_copyright: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
suggestion_threshold_general: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.95
|
||||
)
|
||||
centroid_similarity_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.55
|
||||
)
|
||||
min_reference_images: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=5
|
||||
)
|
||||
tagger_model_version: Mapped[str] = mapped_column(
|
||||
String(128), nullable=False, default="camie-tagger-v2"
|
||||
)
|
||||
embedder_model_version: Mapped[str] = mapped_column(
|
||||
String(128), nullable=False, default="siglip-so400m-patch14-384"
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Post — provenance anchor for content downloaded from a Source.
|
||||
|
||||
A Post is one creator post; it may contain many images/videos.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Post(Base):
|
||||
__tablename__ = "post"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
post_title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
post_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
raw_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
downloaded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
"""PostAttachment — a non-art file preserved from a post.
|
||||
|
||||
Art images become ImageRecords; everything else a post contained
|
||||
(archives, .exe, .pdf, ...) is captured here so nothing is lost.
|
||||
post_id is nullable (set only when an adjacent sidecar yields a Post);
|
||||
artist_id mirrors the canonical attribution model (FC-2d-vii-c). Both
|
||||
FKs are SET NULL so deleting a Post/Artist never deletes the preserved
|
||||
binary row.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PostAttachment(Base):
|
||||
__tablename__ = "post_attachment"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
post_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
artist_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
sha256: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, unique=True, index=True
|
||||
)
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
original_filename: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
ext: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
mime: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
captured_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
"""SeriesPage — ordered image membership for a series-kind Tag.
|
||||
|
||||
A series IS a Tag with kind='series'; series_page gives it ordered pages.
|
||||
An image belongs to at most one series (UNIQUE image_id). Cover = the
|
||||
lowest page_number. page_number is an ordering key only (not unique) —
|
||||
reorder rewrites 1..N wholesale.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class SeriesPage(Base):
|
||||
__tablename__ = "series_page"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
series_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
image_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
page_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Source — a platform-specific URL owned by an Artist (e.g., a Patreon URL).
|
||||
|
||||
Multiple sources per artist support creators with cross-platform presence.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class Source(Base):
|
||||
__tablename__ = "source"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
artist_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
|
||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
artist = relationship("Artist", back_populates="sources")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Tag + ImageTag association.
|
||||
|
||||
Tags carry a ``kind`` enum and (for kind='character') an optional
|
||||
``fandom_id`` FK to another tag of kind='fandom'. The uniqueness key is
|
||||
(name, kind, COALESCE(fandom_id, 0)) so the same name can exist across
|
||||
different kinds and the same character name can exist in different fandoms.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
Column,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy import (
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagKind(StrEnum):
|
||||
artist = "artist"
|
||||
character = "character"
|
||||
fandom = "fandom"
|
||||
general = "general"
|
||||
series = "series"
|
||||
archive = "archive"
|
||||
post = "post"
|
||||
meta = "meta"
|
||||
rating = "rating"
|
||||
|
||||
|
||||
image_tag = Table(
|
||||
"image_tag",
|
||||
Base.metadata,
|
||||
Column(
|
||||
"image_record_id",
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
|
||||
Column("source", String(32), nullable=False, default="manual"),
|
||||
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
|
||||
)
|
||||
|
||||
|
||||
class Tag(Base):
|
||||
__tablename__ = "tag"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
name="ck_tag_fandom_requires_character",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
kind: Mapped[TagKind] = mapped_column(
|
||||
SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]),
|
||||
nullable=False,
|
||||
default=TagKind.general,
|
||||
)
|
||||
fandom_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
images = relationship("ImageRecord", secondary=image_tag, backref="tags")
|
||||
fandom = relationship("Tag", remote_side="Tag.id", foreign_keys=[fandom_id])
|
||||
@@ -0,0 +1,24 @@
|
||||
"""TagAlias — maps a model's (name, category) prediction to the operator's
|
||||
canonical tag. Resolved at suggestion-read time so raw predictions stay
|
||||
unmolested in image_record.tagger_predictions.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagAlias(Base):
|
||||
__tablename__ = "tag_alias"
|
||||
|
||||
alias_string: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
alias_category: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
canonical_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""TagAllowlist — tags the operator opted-in to auto-apply via maintenance."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, Float, ForeignKey, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagAllowlist(Base):
|
||||
__tablename__ = "tag_allowlist"
|
||||
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||
# producing the final ck_tag_allowlist_confidence_range (matches migration 0003).
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"min_confidence > 0 AND min_confidence <= 1",
|
||||
name="confidence_range",
|
||||
),
|
||||
)
|
||||
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
min_confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.95)
|
||||
added_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""TagReferenceEmbedding — per-tag centroid (mean SigLIP embedding of members)."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagReferenceEmbedding(Base):
|
||||
__tablename__ = "tag_reference_embedding"
|
||||
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
embedding: Mapped[list[float]] = mapped_column(Vector(1152), nullable=False)
|
||||
reference_count: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
model_version: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
"""TagSuggestionRejection — per-image dismissed suggestions.
|
||||
|
||||
Prevents re-suggestion AND prevents allowlist auto-apply on that image.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class TagSuggestionRejection(Base):
|
||||
__tablename__ = "tag_suggestion_rejection"
|
||||
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True
|
||||
)
|
||||
rejected_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""Operational scripts run from the container entrypoint or CLI."""
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Self-heal model weights into /models. Idempotent — only missing files
|
||||
are fetched. Called by the ml-worker entrypoint before Celery starts.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
MODEL_ROOT = Path(os.environ.get("ML_MODEL_DIR", "/models"))
|
||||
CAMIE_REPO = os.environ.get("CAMIE_HF_REPO", "Camais03/camie-tagger-v2")
|
||||
SIGLIP_REPO = os.environ.get(
|
||||
"SIGLIP_HF_REPO", "google/siglip-so400m-patch14-384"
|
||||
)
|
||||
|
||||
|
||||
def _snapshot(repo_id: str, dest: Path, allow_patterns: list[str] | None) -> None:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_download(
|
||||
repo_id=repo_id,
|
||||
local_dir=str(dest),
|
||||
allow_patterns=allow_patterns,
|
||||
)
|
||||
|
||||
|
||||
def ensure_camie() -> None:
|
||||
dest = MODEL_ROOT / "camie"
|
||||
if (dest / "model.onnx").is_file() and (dest / "selected_tags.csv").is_file():
|
||||
print(f"[download_models] Camie present at {dest}")
|
||||
return
|
||||
print(f"[download_models] Fetching {CAMIE_REPO} -> {dest}")
|
||||
_snapshot(CAMIE_REPO, dest, ["model.onnx", "selected_tags.csv", "*.json"])
|
||||
|
||||
|
||||
def ensure_siglip() -> None:
|
||||
dest = MODEL_ROOT / "siglip"
|
||||
if (dest / "config.json").is_file() and any(dest.glob("*.safetensors")):
|
||||
print(f"[download_models] SigLIP present at {dest}")
|
||||
return
|
||||
print(f"[download_models] Fetching {SIGLIP_REPO} -> {dest}")
|
||||
_snapshot(SIGLIP_REPO, dest, None)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ensure_camie()
|
||||
ensure_siglip()
|
||||
print("[download_models] Done.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1 @@
|
||||
"""Business-logic services consumed by API blueprints and Celery tasks."""
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Best-effort archive member extraction (filesystem-import aid).
|
||||
|
||||
zip/cbz via stdlib zipfile; rar via rarfile (needs an unrar/bsdtar
|
||||
binary); 7z via py7zr (pure-Python). NEVER raises: on any failure the
|
||||
context yields an empty list, and the caller still preserves the archive
|
||||
file itself as a PostAttachment, so nothing is lost.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import zipfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ARCHIVE_EXTS = {".zip", ".cbz", ".rar", ".7z"}
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
return Path(path).suffix.lower() in ARCHIVE_EXTS
|
||||
|
||||
|
||||
@contextmanager
|
||||
def extract_archive(path: Path):
|
||||
"""Context manager yielding a list of (member_name, extracted_path)
|
||||
for every regular-file member. Temp dir is removed on exit. Members
|
||||
are only ever under the temp dir (libs' safe extractall + rglob from
|
||||
base). Fail-soft: any error → yields []."""
|
||||
tmp = TemporaryDirectory(prefix="fc_arch_")
|
||||
base = Path(tmp.name)
|
||||
members: list[tuple[str, Path]] = []
|
||||
try:
|
||||
try:
|
||||
ext = Path(path).suffix.lower()
|
||||
if ext in (".zip", ".cbz"):
|
||||
with zipfile.ZipFile(path) as zf:
|
||||
zf.extractall(base)
|
||||
elif ext == ".rar":
|
||||
import rarfile
|
||||
|
||||
with rarfile.RarFile(path) as rf:
|
||||
rf.extractall(base)
|
||||
elif ext == ".7z":
|
||||
import py7zr
|
||||
|
||||
with py7zr.SevenZipFile(path, "r") as zf:
|
||||
zf.extractall(base)
|
||||
for p in sorted(base.rglob("*")):
|
||||
if p.is_file():
|
||||
members.append((str(p.relative_to(base)), p))
|
||||
except Exception as exc:
|
||||
log.warning("archive extract failed for %s: %s", path, exc)
|
||||
members = []
|
||||
yield members
|
||||
finally:
|
||||
tmp.cleanup()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""FC-3f: cursor-paginated artists directory.
|
||||
|
||||
Mirrors TagDirectoryService surface-for-surface, but backed by
|
||||
Artist + Source + ImageRecord.artist_id (FC-2d-vii-c authoritative
|
||||
attribution). Pure read-surface; no writes.
|
||||
|
||||
Cursor wire format is identical to tag_directory_service: base64 of
|
||||
"<name>|<id>". The _encode/_decode helpers are copied (not imported)
|
||||
to keep the two services decoupled.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import and_, exists, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, ImageRecord, Source
|
||||
from .gallery_service import thumbnail_url
|
||||
|
||||
_SEP = "|"
|
||||
_PREVIEW_COUNT = 3
|
||||
|
||||
|
||||
def _encode(name: str, artist_id: int) -> str:
|
||||
return base64.urlsafe_b64encode(f"{name}{_SEP}{artist_id}".encode()).decode()
|
||||
|
||||
|
||||
def _decode(cursor: str) -> tuple[str, int]:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||
name, aid = raw.rsplit(_SEP, 1)
|
||||
return name, int(aid)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DirectoryPage:
|
||||
cards: list[dict]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class ArtistDirectoryService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def list_artists(
|
||||
self,
|
||||
*,
|
||||
q: str | None,
|
||||
platform: str | None,
|
||||
cursor: str | None,
|
||||
limit: int = 60,
|
||||
) -> DirectoryPage:
|
||||
if limit < 1 or limit > 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
|
||||
count_col = func.count(ImageRecord.id).label("image_count")
|
||||
stmt = (
|
||||
select(Artist, count_col)
|
||||
.outerjoin(ImageRecord, ImageRecord.artist_id == Artist.id)
|
||||
.group_by(Artist.id)
|
||||
)
|
||||
if q:
|
||||
stmt = stmt.where(Artist.name.ilike(f"%{q}%"))
|
||||
if platform is not None:
|
||||
# Correlated EXISTS — NOT a JOIN. A JOIN to Source duplicates
|
||||
# the artist row when the artist has multiple sources on this
|
||||
# platform, breaking the keyset cursor and image_count.
|
||||
stmt = stmt.where(
|
||||
exists().where(
|
||||
and_(
|
||||
Source.artist_id == Artist.id,
|
||||
Source.platform == platform,
|
||||
)
|
||||
)
|
||||
)
|
||||
if cursor:
|
||||
c_name, c_id = _decode(cursor)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
Artist.name > c_name,
|
||||
and_(Artist.name == c_name, Artist.id > c_id),
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(Artist.name.asc(), Artist.id.asc()).limit(limit + 1)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
|
||||
next_cursor: str | None = None
|
||||
if len(rows) > limit:
|
||||
last_artist = rows[limit - 1][0]
|
||||
next_cursor = _encode(last_artist.name, last_artist.id)
|
||||
rows = rows[:limit]
|
||||
|
||||
artist_ids = [a.id for a, _ in rows]
|
||||
previews = await self._previews(artist_ids)
|
||||
|
||||
cards = [
|
||||
{
|
||||
"id": artist.id,
|
||||
"name": artist.name,
|
||||
"slug": artist.slug,
|
||||
"is_subscription": bool(artist.is_subscription),
|
||||
"image_count": int(image_count),
|
||||
"preview_thumbnails": previews.get(artist.id, []),
|
||||
}
|
||||
for artist, image_count in rows
|
||||
]
|
||||
return DirectoryPage(cards=cards, next_cursor=next_cursor)
|
||||
|
||||
async def _previews(self, artist_ids: list[int]) -> dict[int, list[str]]:
|
||||
"""artist_id -> [thumbnail_url, ...up to _PREVIEW_COUNT].
|
||||
|
||||
Window function ranks ImageRecord rows per artist by id DESC
|
||||
(newest first) and slices to the top _PREVIEW_COUNT in one query.
|
||||
"""
|
||||
if not artist_ids:
|
||||
return {}
|
||||
rn = func.row_number().over(
|
||||
partition_by=ImageRecord.artist_id,
|
||||
order_by=ImageRecord.id.desc(),
|
||||
).label("rn")
|
||||
sub = (
|
||||
select(
|
||||
ImageRecord.artist_id.label("artist_id"),
|
||||
ImageRecord.sha256.label("sha256"),
|
||||
ImageRecord.mime.label("mime"),
|
||||
rn,
|
||||
)
|
||||
.where(ImageRecord.artist_id.in_(artist_ids))
|
||||
.subquery()
|
||||
)
|
||||
stmt = (
|
||||
select(sub.c.artist_id, sub.c.sha256, sub.c.mime)
|
||||
.where(sub.c.rn <= _PREVIEW_COUNT)
|
||||
.order_by(sub.c.artist_id, sub.c.rn)
|
||||
)
|
||||
out: dict[int, list[str]] = {}
|
||||
for aid, sha, mime in (await self.session.execute(stmt)).all():
|
||||
out.setdefault(aid, []).append(thumbnail_url(sha, mime))
|
||||
return out
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Artist overview + paged images.
|
||||
|
||||
Images are linked to an artist through the provenance chain
|
||||
Source(artist_id) -> ImageProvenance(source_id) -> ImageRecord. DISTINCT
|
||||
guards against an image having several provenance rows for one artist.
|
||||
Dates come from Post.post_date via ImageProvenance.post_id.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
Source,
|
||||
Tag,
|
||||
)
|
||||
from ..models.tag import image_tag
|
||||
from ..utils.slug import slugify
|
||||
from .gallery_service import decode_cursor, encode_cursor, thumbnail_url
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArtistImagesPage:
|
||||
images: list[dict]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class ArtistService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def _artist_by_slug(self, slug: str) -> Artist | None:
|
||||
return (
|
||||
await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
async def overview(self, slug: str) -> dict | None:
|
||||
artist = await self._artist_by_slug(slug)
|
||||
if artist is None:
|
||||
return None
|
||||
aid = artist.id
|
||||
|
||||
img_join = (
|
||||
select(ImageRecord.id).where(ImageRecord.artist_id == aid)
|
||||
)
|
||||
image_count = (
|
||||
await self.session.execute(
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.artist_id == aid)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
date_row = (
|
||||
await self.session.execute(
|
||||
select(func.min(Post.post_date), func.max(Post.post_date))
|
||||
.select_from(Post)
|
||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(Source.artist_id == aid)
|
||||
)
|
||||
).first()
|
||||
dmin, dmax = date_row if date_row else (None, None)
|
||||
|
||||
cooccurring = (
|
||||
await self.session.execute(
|
||||
select(
|
||||
Tag.id, Tag.name, Tag.kind,
|
||||
func.count(image_tag.c.image_record_id).label("cnt"),
|
||||
)
|
||||
.select_from(Tag)
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
.where(image_tag.c.image_record_id.in_(img_join))
|
||||
.group_by(Tag.id, Tag.name, Tag.kind)
|
||||
.order_by(func.count(image_tag.c.image_record_id).desc())
|
||||
.limit(20)
|
||||
)
|
||||
).all()
|
||||
|
||||
sources = (
|
||||
await self.session.execute(
|
||||
select(
|
||||
Source.id, Source.platform, Source.url,
|
||||
func.count(func.distinct(ImageProvenance.image_record_id)).label("cnt"),
|
||||
)
|
||||
.select_from(Source)
|
||||
.outerjoin(ImageProvenance, ImageProvenance.source_id == Source.id)
|
||||
.where(Source.artist_id == aid)
|
||||
.group_by(Source.id, Source.platform, Source.url)
|
||||
.order_by(Source.id)
|
||||
)
|
||||
).all()
|
||||
|
||||
month = func.date_trunc("month", Post.post_date).label("m")
|
||||
activity = (
|
||||
await self.session.execute(
|
||||
select(month, func.count(func.distinct(ImageProvenance.image_record_id)))
|
||||
.select_from(Post)
|
||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(and_(Source.artist_id == aid, Post.post_date.isnot(None)))
|
||||
.group_by(month)
|
||||
.order_by(month)
|
||||
)
|
||||
).all()
|
||||
|
||||
return {
|
||||
"id": artist.id,
|
||||
"name": artist.name,
|
||||
"slug": artist.slug,
|
||||
"is_subscription": bool(artist.is_subscription),
|
||||
"image_count": int(image_count),
|
||||
"date_range": {
|
||||
"min": dmin.isoformat() if dmin else None,
|
||||
"max": dmax.isoformat() if dmax else None,
|
||||
},
|
||||
"cooccurring_tags": [
|
||||
{
|
||||
"id": tid,
|
||||
"name": name,
|
||||
"kind": kind.value if hasattr(kind, "value") else kind,
|
||||
"count": int(cnt),
|
||||
}
|
||||
for tid, name, kind, cnt in cooccurring
|
||||
],
|
||||
"sources": [
|
||||
{
|
||||
"id": sid,
|
||||
"platform": platform,
|
||||
"url": url,
|
||||
"image_count": int(cnt),
|
||||
}
|
||||
for sid, platform, url, cnt in sources
|
||||
],
|
||||
"activity": [
|
||||
{"month": m.isoformat(), "count": int(cnt)}
|
||||
for m, cnt in activity
|
||||
],
|
||||
}
|
||||
|
||||
async def images(
|
||||
self, slug: str, cursor: str | None, limit: int = 60
|
||||
) -> ArtistImagesPage | None:
|
||||
if limit < 1 or limit > 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
artist = await self._artist_by_slug(slug)
|
||||
if artist is None:
|
||||
return None
|
||||
|
||||
# Dedupe via IN-subquery rather than JOIN + DISTINCT: an image can
|
||||
# have several provenance rows for one artist, and SELECT DISTINCT
|
||||
# over ImageRecord fails in Postgres because its json columns have
|
||||
# no equality operator.
|
||||
stmt = select(ImageRecord).where(
|
||||
ImageRecord.artist_id == artist.id
|
||||
)
|
||||
if cursor:
|
||||
cur_ts, cur_id = decode_cursor(cursor)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
ImageRecord.created_at < cur_ts,
|
||||
and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id),
|
||||
)
|
||||
)
|
||||
stmt = stmt.order_by(
|
||||
ImageRecord.created_at.desc(), ImageRecord.id.desc()
|
||||
).limit(limit + 1)
|
||||
rows = (await self.session.execute(stmt)).scalars().all()
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = rows[limit - 1]
|
||||
next_cursor = encode_cursor(last.created_at, last.id)
|
||||
rows = rows[:limit]
|
||||
|
||||
return ArtistImagesPage(
|
||||
images=[
|
||||
{
|
||||
"id": r.id,
|
||||
"sha256": r.sha256,
|
||||
"mime": r.mime,
|
||||
"width": r.width,
|
||||
"height": r.height,
|
||||
"thumbnail_url": thumbnail_url(r.sha256, r.mime),
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
next_cursor=next_cursor,
|
||||
)
|
||||
|
||||
async def find_or_create(self, name: str) -> tuple[Artist, bool]:
|
||||
"""Return (artist, created). Slug-keyed; idempotent under races."""
|
||||
cleaned = (name or "").strip()
|
||||
if not cleaned:
|
||||
raise ValueError("artist name must not be empty")
|
||||
slug = slugify(cleaned)
|
||||
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
|
||||
artist = Artist(name=cleaned, slug=slug)
|
||||
self.session.add(artist)
|
||||
try:
|
||||
await self.session.flush()
|
||||
except IntegrityError:
|
||||
await self.session.rollback()
|
||||
existing = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one()
|
||||
return existing, False
|
||||
await self.session.commit()
|
||||
return artist, True
|
||||
|
||||
async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]:
|
||||
cleaned = (prefix or "").strip()
|
||||
if not cleaned:
|
||||
return []
|
||||
like = f"%{cleaned.lower()}%"
|
||||
prefix_like = f"{cleaned.lower()}%"
|
||||
# Rank: exact (0) < prefix (1) < substring (2).
|
||||
rank = case(
|
||||
(func.lower(Artist.name) == cleaned.lower(), 0),
|
||||
(func.lower(Artist.name).like(prefix_like), 1),
|
||||
else_=2,
|
||||
).label("rank")
|
||||
rows = (await self.session.execute(
|
||||
select(Artist, rank)
|
||||
.where(func.lower(Artist.name).like(like))
|
||||
.order_by(rank, Artist.name.asc())
|
||||
.limit(limit)
|
||||
)).all()
|
||||
return [a for a, _ in rows]
|
||||
@@ -0,0 +1,27 @@
|
||||
"""sha-addressed store for preserved non-art post files.
|
||||
|
||||
Arbitrary binaries — never opened or validated as images. Parallels the
|
||||
thumbnail store layout (/images/attachments/<sha[:3]>/<sha><ext>).
|
||||
"""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class AttachmentStore:
|
||||
def __init__(self, images_root: Path):
|
||||
self.root = Path(images_root) / "attachments"
|
||||
|
||||
def store(self, src: Path, sha256: str) -> str:
|
||||
"""Copy src into the sha-addressed store; idempotent on sha.
|
||||
Returns the stored absolute path as a string."""
|
||||
ext = Path(src).suffix.lower()
|
||||
dest_dir = self.root / sha256[:3]
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = dest_dir / f"{sha256}{ext}"
|
||||
if dest.exists():
|
||||
return str(dest)
|
||||
partial = dest.with_suffix(dest.suffix + ".partial")
|
||||
shutil.copy2(src, partial)
|
||||
partial.rename(dest)
|
||||
return str(dest)
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Bulk tag operations over a set of images (Core, set-based, atomic)."""
|
||||
|
||||
from sqlalchemy import and_, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Tag
|
||||
from ..models.tag import image_tag
|
||||
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
||||
|
||||
|
||||
class BulkTagService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def common_tags(self, image_ids: list[int]) -> list[dict]:
|
||||
"""Tags present on EVERY image in image_ids."""
|
||||
if not image_ids:
|
||||
return []
|
||||
n = len(set(image_ids))
|
||||
stmt = (
|
||||
select(Tag.id, Tag.name, Tag.kind)
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
.where(image_tag.c.image_record_id.in_(image_ids))
|
||||
.group_by(Tag.id, Tag.name, Tag.kind)
|
||||
.having(
|
||||
func.count(func.distinct(image_tag.c.image_record_id)) == n
|
||||
)
|
||||
.order_by(Tag.name.asc())
|
||||
)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
return [
|
||||
{
|
||||
"id": r.id,
|
||||
"name": r.name,
|
||||
"kind": r.kind.value if hasattr(r.kind, "value") else r.kind,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def bulk_add(
|
||||
self, image_ids: list[int], tag_id: int, source: str = "manual"
|
||||
) -> int:
|
||||
"""Apply tag_id to every image. Idempotent per (image, tag).
|
||||
Returns the number of links actually created."""
|
||||
if not image_ids:
|
||||
return 0
|
||||
stmt = (
|
||||
pg_insert(image_tag)
|
||||
.values(
|
||||
[
|
||||
{
|
||||
"image_record_id": iid,
|
||||
"tag_id": tag_id,
|
||||
"source": source,
|
||||
}
|
||||
for iid in set(image_ids)
|
||||
]
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=["image_record_id", "tag_id"]
|
||||
)
|
||||
)
|
||||
res = await self.session.execute(stmt)
|
||||
return res.rowcount or 0
|
||||
|
||||
async def bulk_remove(self, image_ids: list[int], tag_id: int) -> int:
|
||||
"""Remove tag_id from every image, and record a per-image rejection
|
||||
so the proactive allowlist sweep won't immediately re-add it
|
||||
(mirrors single-image removal semantics). Returns links removed."""
|
||||
if not image_ids:
|
||||
return 0
|
||||
res = await self.session.execute(
|
||||
image_tag.delete().where(
|
||||
and_(
|
||||
image_tag.c.tag_id == tag_id,
|
||||
image_tag.c.image_record_id.in_(image_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
await self.session.execute(
|
||||
pg_insert(TagSuggestionRejection)
|
||||
.values(
|
||||
[
|
||||
{"image_record_id": iid, "tag_id": tag_id}
|
||||
for iid in set(image_ids)
|
||||
]
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=["image_record_id", "tag_id"]
|
||||
)
|
||||
)
|
||||
return res.rowcount or 0
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Fernet-based encryption for credential blobs.
|
||||
|
||||
The key is a single 32-byte value (urlsafe-base64-encoded; what
|
||||
Fernet.generate_key produces) stored at a fixed path inside the
|
||||
images/data root. Created on first boot if absent; mode 0600. No KDF
|
||||
needed — the file contents are already maximum-entropy random bytes.
|
||||
|
||||
Operator backup procedure must include this file alongside the rest
|
||||
of /images/ — losing it makes existing encrypted_blob rows
|
||||
undecryptable (recovery = delete the rows and re-upload).
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
|
||||
class InvalidCredentialBlob(Exception):
|
||||
"""Raised when decryption fails (wrong key, tampered blob, …)."""
|
||||
|
||||
|
||||
class CredentialCrypto:
|
||||
"""Fernet encrypt/decrypt with an on-disk key file.
|
||||
|
||||
Instantiate with a path; the file is created on first access and
|
||||
reused thereafter. Tests pass a tmp_path; production calls with
|
||||
`IMAGES_ROOT / "secrets" / "credential_key.b64"`.
|
||||
"""
|
||||
|
||||
def __init__(self, key_path: Path):
|
||||
self._key_path = Path(key_path)
|
||||
self._fernet = Fernet(self._load_or_create_key())
|
||||
|
||||
def _load_or_create_key(self) -> bytes:
|
||||
if self._key_path.exists():
|
||||
return self._key_path.read_bytes()
|
||||
parent = self._key_path.parent
|
||||
parent.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(parent, 0o700)
|
||||
key = Fernet.generate_key()
|
||||
self._key_path.write_bytes(key)
|
||||
os.chmod(self._key_path, 0o600)
|
||||
return key
|
||||
|
||||
def encrypt(self, plaintext: str) -> bytes:
|
||||
return self._fernet.encrypt(plaintext.encode("utf-8"))
|
||||
|
||||
def decrypt(self, ciphertext: bytes) -> str:
|
||||
try:
|
||||
return self._fernet.decrypt(ciphertext).decode("utf-8")
|
||||
except InvalidToken as exc:
|
||||
raise InvalidCredentialBlob(
|
||||
"credential blob is corrupt or encrypted with a different key"
|
||||
) from exc
|
||||
@@ -0,0 +1,199 @@
|
||||
"""FC-3b: CRUD over Credential rows + on-demand cookies-file
|
||||
materialisation for gallery-dl (consumed by FC-3c).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Credential
|
||||
from .credential_crypto import CredentialCrypto
|
||||
from .platforms import PLATFORMS, auth_type_for
|
||||
|
||||
|
||||
class CredentialServiceError(Exception):
|
||||
"""Base."""
|
||||
|
||||
|
||||
class UnknownPlatformError(CredentialServiceError):
|
||||
pass
|
||||
|
||||
|
||||
class WrongAuthTypeError(CredentialServiceError):
|
||||
def __init__(self, expected: str):
|
||||
super().__init__(f"wrong credential_type for platform; expected {expected!r}")
|
||||
self.expected = expected
|
||||
|
||||
|
||||
class EmptyDataError(CredentialServiceError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CredentialRecord:
|
||||
platform: str
|
||||
credential_type: str
|
||||
captured_at: str
|
||||
expires_at: str | None
|
||||
last_verified: str | None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"platform": self.platform,
|
||||
"credential_type": self.credential_type,
|
||||
"captured_at": self.captured_at,
|
||||
"expires_at": self.expires_at,
|
||||
"last_verified": self.last_verified,
|
||||
}
|
||||
|
||||
|
||||
def _to_record(row: Credential) -> CredentialRecord:
|
||||
return CredentialRecord(
|
||||
platform=row.platform,
|
||||
credential_type=row.credential_type,
|
||||
captured_at=row.captured_at.isoformat() if row.captured_at else "",
|
||||
expires_at=row.expires_at.isoformat() if row.expires_at else None,
|
||||
last_verified=row.last_verified.isoformat() if row.last_verified else None,
|
||||
)
|
||||
|
||||
|
||||
# Default cookies dir under the image root; tests override.
|
||||
_DEFAULT_COOKIES_DIR = Path("/images/cookies")
|
||||
|
||||
|
||||
class CredentialService:
|
||||
def __init__(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
crypto: CredentialCrypto,
|
||||
cookies_dir: Path | None = None,
|
||||
):
|
||||
self.session = session
|
||||
self.crypto = crypto
|
||||
self.cookies_dir = Path(cookies_dir) if cookies_dir else _DEFAULT_COOKIES_DIR
|
||||
|
||||
async def list(self) -> list[CredentialRecord]:
|
||||
rows = (await self.session.execute(
|
||||
select(Credential).order_by(Credential.platform.asc())
|
||||
)).scalars().all()
|
||||
return [_to_record(r) for r in rows]
|
||||
|
||||
async def get(self, platform: str) -> CredentialRecord | None:
|
||||
row = (await self.session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none()
|
||||
return _to_record(row) if row else None
|
||||
|
||||
async def upsert(
|
||||
self,
|
||||
*,
|
||||
platform: str,
|
||||
credential_type: str,
|
||||
data: str,
|
||||
expires_at: datetime | None = None,
|
||||
) -> CredentialRecord:
|
||||
if platform not in PLATFORMS:
|
||||
raise UnknownPlatformError(f"unknown platform: {platform!r}")
|
||||
expected = auth_type_for(platform)
|
||||
if credential_type != expected:
|
||||
raise WrongAuthTypeError(expected=expected or "")
|
||||
cleaned = (data or "").strip()
|
||||
if not cleaned:
|
||||
raise EmptyDataError("data must be a non-empty string")
|
||||
blob = self.crypto.encrypt(cleaned)
|
||||
|
||||
row = (await self.session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = Credential(
|
||||
platform=platform,
|
||||
credential_type=credential_type,
|
||||
encrypted_blob=blob,
|
||||
expires_at=expires_at,
|
||||
)
|
||||
self.session.add(row)
|
||||
else:
|
||||
row.credential_type = credential_type
|
||||
row.encrypted_blob = blob
|
||||
row.expires_at = expires_at
|
||||
row.last_verified = None # invalidate prior verification
|
||||
await self.session.commit()
|
||||
await self.session.refresh(row)
|
||||
return _to_record(row)
|
||||
|
||||
async def delete(self, platform: str) -> None:
|
||||
row = (await self.session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none()
|
||||
if row is None:
|
||||
raise LookupError(f"no credential for platform {platform!r}")
|
||||
await self.session.delete(row)
|
||||
await self.session.commit()
|
||||
|
||||
async def get_cookies_path(self, platform: str) -> Path | None:
|
||||
"""Decrypt the credential and write a Netscape cookies.txt for
|
||||
gallery-dl. Returns None if no credential or wrong kind."""
|
||||
row = (await self.session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none()
|
||||
if row is None or row.credential_type != "cookies":
|
||||
return None
|
||||
plaintext = self.crypto.decrypt(row.encrypted_blob)
|
||||
netscape = _to_netscape(plaintext)
|
||||
self.cookies_dir.mkdir(parents=True, exist_ok=True)
|
||||
out = self.cookies_dir / f"{platform}_cookies.txt"
|
||||
out.write_text(netscape)
|
||||
os.chmod(out, 0o600)
|
||||
return out
|
||||
|
||||
async def get_token(self, platform: str) -> str | None:
|
||||
row = (await self.session.execute(
|
||||
select(Credential).where(Credential.platform == platform)
|
||||
)).scalar_one_or_none()
|
||||
if row is None or row.credential_type != "token":
|
||||
return None
|
||||
return self.crypto.decrypt(row.encrypted_blob)
|
||||
|
||||
|
||||
def _to_netscape(plaintext: str) -> str:
|
||||
"""Accept either Netscape-format text (the extension's output) or a
|
||||
JSON array of cookie dicts (a manual-paste edge case); produce
|
||||
Netscape-format text suitable for gallery-dl --cookies."""
|
||||
stripped = plaintext.strip()
|
||||
if not stripped:
|
||||
return ""
|
||||
if stripped.startswith("#") or "\t" in stripped:
|
||||
return plaintext # already Netscape
|
||||
try:
|
||||
loaded = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
return plaintext # write as-is and hope; gallery-dl will complain if invalid
|
||||
if isinstance(loaded, dict):
|
||||
loaded = [loaded]
|
||||
if not isinstance(loaded, list):
|
||||
return plaintext
|
||||
lines = ["# Netscape HTTP Cookie File"]
|
||||
for c in loaded:
|
||||
domain = str(c.get("domain", ""))
|
||||
if domain and not domain.startswith("."):
|
||||
domain = "." + domain
|
||||
flag = "TRUE"
|
||||
path = str(c.get("path", "/"))
|
||||
secure = "TRUE" if c.get("secure", False) else "FALSE"
|
||||
expiration = c.get("expiration") or c.get("expirationDate") or 0
|
||||
try:
|
||||
expiration_str = str(int(float(expiration)))
|
||||
except (TypeError, ValueError):
|
||||
expiration_str = "0"
|
||||
name = str(c.get("name", ""))
|
||||
value = str(c.get("value", ""))
|
||||
lines.append("\t".join([domain, flag, path, secure, expiration_str, name, value]))
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,323 @@
|
||||
"""FC-3c download orchestrator.
|
||||
|
||||
Three-phase pipeline:
|
||||
Phase 1 — DB setup (brief session): load source, mark event running,
|
||||
fetch credentials + settings.
|
||||
Phase 2 — Execute (no DB connection): call GalleryDLService; on
|
||||
Patreon campaign-ID failure, resolve vanity and retry once.
|
||||
Phase 3 — Import + persist (fresh session): attach_in_place each
|
||||
written file; clean up duplicates; persist rich metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session as SyncSession
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
from .credential_service import CredentialService
|
||||
from .gallery_dl import GalleryDLService, SourceConfig
|
||||
from .importer import Importer
|
||||
from .patreon_resolver import resolve_campaign_id
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_PATREON_VANITY_RE = re.compile(
|
||||
r"^https?://(?:www\.)?patreon\.com/(?:c/)?(?!id:)([^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CAMPAIGN_ID_FAILURE_PATTERN = "failed to extract campaign id"
|
||||
|
||||
|
||||
def _extract_patreon_vanity(url: str) -> str | None:
|
||||
m = _PATREON_VANITY_RE.match(url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _looks_like_campaign_id_failure(stdout: str, stderr: str) -> bool:
|
||||
return _CAMPAIGN_ID_FAILURE_PATTERN in f"{stdout}\n{stderr}".lower()
|
||||
|
||||
|
||||
def _effective_url(platform: str, source_url: str, overrides: dict) -> str:
|
||||
if platform == "patreon" and overrides.get("patreon_campaign_id"):
|
||||
return f"https://www.patreon.com/id:{overrides['patreon_campaign_id']}"
|
||||
return source_url
|
||||
|
||||
|
||||
class DownloadService:
|
||||
"""Async orchestrator. The Celery task runs `asyncio.run(svc.download_source(N))`.
|
||||
|
||||
The Importer is sync (existing FC convention); calls to attach_in_place
|
||||
happen via run_in_executor.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
async_session: AsyncSession,
|
||||
sync_session: SyncSession,
|
||||
gdl: GalleryDLService,
|
||||
importer: Importer,
|
||||
cred_service: CredentialService,
|
||||
):
|
||||
self.async_session = async_session
|
||||
self.sync_session = sync_session
|
||||
self.gdl = gdl
|
||||
self.importer = importer
|
||||
self.cred_service = cred_service
|
||||
|
||||
async def download_source(self, source_id: int) -> int:
|
||||
"""Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
|
||||
setup = await self._phase1_setup(source_id)
|
||||
if setup["status"] in ("skipped", "in_flight"):
|
||||
return setup["event_id"]
|
||||
ctx = setup
|
||||
|
||||
source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
|
||||
effective_url = _effective_url(
|
||||
ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
|
||||
)
|
||||
|
||||
dl_result = await self.gdl.download(
|
||||
url=effective_url,
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform=ctx["platform"],
|
||||
source_config=source_config,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
auth_token=ctx["auth_token"],
|
||||
)
|
||||
|
||||
resolved_campaign_id: str | None = None
|
||||
if (
|
||||
ctx["platform"] == "patreon"
|
||||
and not dl_result.success
|
||||
and "patreon_campaign_id" not in (ctx["config_overrides"] or {})
|
||||
and _looks_like_campaign_id_failure(dl_result.stdout, dl_result.stderr)
|
||||
):
|
||||
vanity = _extract_patreon_vanity(ctx["url"])
|
||||
if vanity:
|
||||
log.info(
|
||||
"Attempting campaign-ID resolution for %s (%s)",
|
||||
ctx["artist_slug"], vanity,
|
||||
)
|
||||
resolved_campaign_id = await resolve_campaign_id(
|
||||
vanity, ctx["cookies_path"]
|
||||
)
|
||||
if resolved_campaign_id:
|
||||
dl_result = await self.gdl.download(
|
||||
url=f"https://www.patreon.com/id:{resolved_campaign_id}",
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform=ctx["platform"],
|
||||
source_config=source_config,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
auth_token=ctx["auth_token"],
|
||||
)
|
||||
|
||||
return await self._phase3_persist(
|
||||
ctx["event_id"], ctx, dl_result, resolved_campaign_id,
|
||||
)
|
||||
|
||||
async def _phase1_setup(self, source_id: int) -> dict[str, Any]:
|
||||
source = (await self.async_session.execute(
|
||||
select(Source).options(joinedload(Source.artist)).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
raise LookupError(f"source {source_id} not found")
|
||||
|
||||
if not source.enabled:
|
||||
ev = DownloadEvent(source_id=source_id, status="pending")
|
||||
self.async_session.add(ev)
|
||||
await self.async_session.commit()
|
||||
await self.async_session.refresh(ev)
|
||||
await self._finalize_event_and_source(
|
||||
event_id=ev.id, source_id=source_id, status="skipped",
|
||||
error_message="source disabled",
|
||||
)
|
||||
return {"status": "skipped", "event_id": ev.id}
|
||||
|
||||
existing = (await self.async_session.execute(
|
||||
select(DownloadEvent).where(
|
||||
DownloadEvent.source_id == source_id,
|
||||
DownloadEvent.status.in_(["pending", "running"]),
|
||||
).order_by(DownloadEvent.id.desc()).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
if existing and existing.status == "running":
|
||||
return {"status": "in_flight", "event_id": existing.id}
|
||||
if existing and existing.status == "pending":
|
||||
existing.status = "running"
|
||||
await self.async_session.commit()
|
||||
event_id = existing.id
|
||||
else:
|
||||
ev = DownloadEvent(source_id=source_id, status="running")
|
||||
self.async_session.add(ev)
|
||||
await self.async_session.commit()
|
||||
await self.async_session.refresh(ev)
|
||||
event_id = ev.id
|
||||
|
||||
artist = source.artist
|
||||
if source.platform in ("discord", "pixiv"):
|
||||
cookies_path = None
|
||||
auth_token = await self.cred_service.get_token(source.platform)
|
||||
else:
|
||||
cookies_path_obj = await self.cred_service.get_cookies_path(source.platform)
|
||||
cookies_path = str(cookies_path_obj) if cookies_path_obj else None
|
||||
auth_token = None
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"event_id": event_id,
|
||||
"source_id": source_id,
|
||||
"url": source.url,
|
||||
"platform": source.platform,
|
||||
"artist_slug": artist.slug if artist else "_unknown",
|
||||
"artist_id": artist.id if artist else None,
|
||||
"config_overrides": dict(source.config_overrides or {}),
|
||||
"cookies_path": cookies_path,
|
||||
"auth_token": auth_token,
|
||||
}
|
||||
|
||||
async def _phase3_persist(
|
||||
self,
|
||||
event_id: int,
|
||||
ctx: dict,
|
||||
dl_result,
|
||||
resolved_campaign_id: str | None,
|
||||
) -> int:
|
||||
# Cache Patreon campaign id on the source.
|
||||
if resolved_campaign_id:
|
||||
src = self.sync_session.get(Source, ctx["source_id"])
|
||||
if src is not None:
|
||||
src.config_overrides = {
|
||||
**(src.config_overrides or {}),
|
||||
"patreon_campaign_id": resolved_campaign_id,
|
||||
}
|
||||
self.sync_session.commit()
|
||||
|
||||
artist = None
|
||||
source_row = None
|
||||
if ctx.get("artist_id"):
|
||||
artist = self.sync_session.get(Artist, ctx["artist_id"])
|
||||
source_row = self.sync_session.get(Source, ctx["source_id"])
|
||||
|
||||
import_summary = {"attached": 0, "skipped": 0, "errors": 0}
|
||||
bytes_downloaded = 0
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
for path_str in dl_result.written_paths or []:
|
||||
if path_str in (dl_result.quarantined_paths or []):
|
||||
continue
|
||||
path = Path(path_str)
|
||||
# Sync stdlib filesystem ops are intentional here: this orchestrator
|
||||
# runs under `asyncio.run` inside a Celery task — no other awaitables
|
||||
# compete for the loop. ASYNC240 suppressed below.
|
||||
if not path.exists(): # noqa: ASYNC240
|
||||
continue
|
||||
|
||||
def _attach(p=path):
|
||||
return self.importer.attach_in_place(
|
||||
p, artist=artist, source=source_row,
|
||||
)
|
||||
|
||||
result = await loop.run_in_executor(None, _attach)
|
||||
|
||||
if result.status in ("imported", "superseded"):
|
||||
import_summary["attached"] += 1
|
||||
try:
|
||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
||||
"duplicate_hash", "duplicate_phash",
|
||||
):
|
||||
import_summary["skipped"] += 1
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
import_summary["errors"] += 1
|
||||
|
||||
ev = (await self.async_session.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
|
||||
run_stats = self.gdl._compute_run_stats(
|
||||
dl_result.return_code, dl_result.stdout, dl_result.stderr
|
||||
)
|
||||
run_stats["quarantined_count"] = dl_result.files_quarantined
|
||||
stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
|
||||
|
||||
status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error"
|
||||
ev.status = status
|
||||
ev.finished_at = datetime.now(UTC)
|
||||
ev.files_count = import_summary["attached"]
|
||||
ev.bytes_downloaded = bytes_downloaded
|
||||
ev.error = dl_result.error_message if not dl_result.success else None
|
||||
ev.metadata_ = {
|
||||
"run_stats": run_stats,
|
||||
"error_type": dl_result.error_type.value if dl_result.error_type else None,
|
||||
"stdout": self.gdl._truncate_log(dl_result.stdout) or None,
|
||||
"stderr": self.gdl._truncate_log(dl_result.stderr) or None,
|
||||
"stderr_errors_warnings": stderr_summary or None,
|
||||
"duration_seconds": dl_result.duration_seconds,
|
||||
"quarantined_paths": dl_result.quarantined_paths or None,
|
||||
"import_summary": import_summary,
|
||||
}
|
||||
await self._update_source_health(
|
||||
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
||||
)
|
||||
await self.async_session.commit()
|
||||
return event_id
|
||||
|
||||
async def _update_source_health(
|
||||
self, *, source_id: int, status: str, error_message: str | None,
|
||||
) -> None:
|
||||
"""FC-3d: update Source.{consecutive_failures, last_error, last_checked_at}.
|
||||
|
||||
ok -> failures = 0, error = None, checked_at = now
|
||||
error -> failures += 1, error = error_message, checked_at = now
|
||||
skipped -> failures unchanged, error = None, checked_at = now
|
||||
"""
|
||||
source = (await self.async_session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
now = datetime.now(UTC)
|
||||
if status == "ok":
|
||||
source.consecutive_failures = 0
|
||||
source.last_error = None
|
||||
elif status == "error":
|
||||
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
||||
source.last_error = error_message
|
||||
elif status == "skipped":
|
||||
source.last_error = None
|
||||
source.last_checked_at = now
|
||||
|
||||
async def _finalize_event_and_source(
|
||||
self, *, event_id: int, source_id: int, status: str,
|
||||
error_message: str | None,
|
||||
) -> None:
|
||||
"""Finalize the event AND the source-health columns in one
|
||||
transactional step. Used by the skipped early-out path and by
|
||||
unit tests that exercise the source-health writes directly.
|
||||
"""
|
||||
ev = (await self.async_session.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
ev.status = status
|
||||
ev.finished_at = datetime.now(UTC)
|
||||
ev.error = error_message
|
||||
await self._update_source_health(
|
||||
source_id=source_id, status=status, error_message=error_message,
|
||||
)
|
||||
await self.async_session.commit()
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Magic-byte head/tail validator for fresh gallery-dl writes.
|
||||
|
||||
Catches truncated downloads (servers lying about Content-Length, TCP
|
||||
RST mid-transfer, etc.) before they enter the library. O(1) per file
|
||||
— reads only head and tail bytes; never decodes pixels.
|
||||
|
||||
Fail-open: unknown formats and unreadable files pass through. Format
|
||||
detection is by extension; we only validate what the downloader is
|
||||
expected to write.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
JPEG_HEAD = b"\xff\xd8\xff"
|
||||
JPEG_TAIL = b"\xff\xd9"
|
||||
|
||||
PNG_HEAD = b"\x89PNG\r\n\x1a\n"
|
||||
PNG_TAIL = b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
|
||||
GIF_HEAD_87A = b"GIF87a"
|
||||
GIF_HEAD_89A = b"GIF89a"
|
||||
GIF_TAIL = b"\x3b"
|
||||
|
||||
WEBP_RIFF = b"RIFF"
|
||||
WEBP_FORMAT = b"WEBP"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidationResult:
|
||||
ok: bool
|
||||
format: str | None = None
|
||||
reason: str | None = None
|
||||
size: int = 0
|
||||
|
||||
|
||||
VALIDATED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
|
||||
|
||||
|
||||
def is_validatable(path: Path) -> bool:
|
||||
return path.suffix.lower() in VALIDATED_EXTENSIONS
|
||||
|
||||
|
||||
def validate_file(path: Path) -> ValidationResult:
|
||||
"""Returns ok=False if the file is structurally incomplete.
|
||||
|
||||
Never raises. Unknown formats return ok=True with format=None.
|
||||
Unreadable files surface as ok=False so callers can quarantine.
|
||||
"""
|
||||
ext = path.suffix.lower()
|
||||
if ext not in VALIDATED_EXTENSIONS:
|
||||
return ValidationResult(ok=True, format=None)
|
||||
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except FileNotFoundError:
|
||||
return ValidationResult(ok=False, reason="file not found")
|
||||
except OSError as exc:
|
||||
return ValidationResult(ok=False, reason=f"stat failed: {exc}")
|
||||
|
||||
if size < 16:
|
||||
return ValidationResult(ok=False, format=ext.lstrip("."), reason="too small", size=size)
|
||||
|
||||
try:
|
||||
with path.open("rb") as fh:
|
||||
head = fh.read(32)
|
||||
fh.seek(max(0, size - 32))
|
||||
tail = fh.read(32)
|
||||
except OSError as exc:
|
||||
return ValidationResult(ok=False, reason=f"read failed: {exc}", size=size)
|
||||
|
||||
if ext in (".jpg", ".jpeg"):
|
||||
if not head.startswith(JPEG_HEAD):
|
||||
return ValidationResult(ok=False, format="jpeg", reason="missing JPEG SOI marker", size=size)
|
||||
if not tail.endswith(JPEG_TAIL):
|
||||
return ValidationResult(ok=False, format="jpeg", reason="missing JPEG EOI marker", size=size)
|
||||
return ValidationResult(ok=True, format="jpeg", size=size)
|
||||
|
||||
if ext == ".png":
|
||||
if not head.startswith(PNG_HEAD):
|
||||
return ValidationResult(ok=False, format="png", reason="missing PNG signature", size=size)
|
||||
if not tail.endswith(PNG_TAIL):
|
||||
return ValidationResult(ok=False, format="png", reason="missing PNG IEND chunk", size=size)
|
||||
return ValidationResult(ok=True, format="png", size=size)
|
||||
|
||||
if ext == ".gif":
|
||||
if not (head.startswith(GIF_HEAD_87A) or head.startswith(GIF_HEAD_89A)):
|
||||
return ValidationResult(ok=False, format="gif", reason="missing GIF signature", size=size)
|
||||
if not tail.endswith(GIF_TAIL):
|
||||
return ValidationResult(ok=False, format="gif", reason="missing GIF trailer", size=size)
|
||||
return ValidationResult(ok=True, format="gif", size=size)
|
||||
|
||||
if ext == ".webp":
|
||||
if not (head.startswith(WEBP_RIFF) and WEBP_FORMAT in head[:16]):
|
||||
return ValidationResult(ok=False, format="webp", reason="missing WebP RIFF/WEBP marker", size=size)
|
||||
try:
|
||||
chunk_size = int.from_bytes(head[4:8], "little")
|
||||
except Exception:
|
||||
return ValidationResult(ok=False, format="webp", reason="malformed WebP chunk size", size=size)
|
||||
if chunk_size + 8 != size:
|
||||
return ValidationResult(
|
||||
ok=False, format="webp",
|
||||
reason=f"WebP RIFF size mismatch: header says {chunk_size + 8}, file is {size}",
|
||||
size=size,
|
||||
)
|
||||
return ValidationResult(ok=True, format="webp", size=size)
|
||||
|
||||
return ValidationResult(ok=True, format=None, size=size)
|
||||
@@ -0,0 +1,660 @@
|
||||
"""Gallery-dl subprocess wrapper.
|
||||
|
||||
Ported from GallerySubscriber (~/Nextcloud/Projects/GallerySubscriber/
|
||||
backend/app/services/gallery_dl.py). FC adaptations:
|
||||
- `artist_slug` replaces GS's `subscription_name` throughout
|
||||
- Validation uses backend/app/services/file_validator.py
|
||||
- Config dir under `<images_root>/.gallery-dl/`, not a separate config_path
|
||||
- PLATFORM_DEFAULTS scoped to the GS-6 platforms registered in FC-3b
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from .file_validator import is_validatable, validate_file
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ErrorType(StrEnum):
|
||||
AUTH_ERROR = "auth_error"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
NOT_FOUND = "not_found"
|
||||
ACCESS_DENIED = "access_denied"
|
||||
NETWORK_ERROR = "network_error"
|
||||
NO_NEW_CONTENT = "no_new_content"
|
||||
TIER_LIMITED = "tier_limited"
|
||||
TIMEOUT = "timeout"
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
VALIDATION_FAILED = "validation_failed"
|
||||
UNKNOWN_ERROR = "unknown_error"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||||
sleep: float | None = None
|
||||
sleep_request: float | None = None
|
||||
directory_pattern: str | None = None
|
||||
filename_pattern: str | None = None
|
||||
skip_existing: bool = True
|
||||
save_metadata: bool = True
|
||||
timeout: int = 3600
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> SourceConfig:
|
||||
return cls(
|
||||
content_types=data.get("content_types", ["all"]),
|
||||
sleep=data.get("sleep"),
|
||||
sleep_request=data.get("sleep_request"),
|
||||
directory_pattern=data.get("directory_pattern"),
|
||||
filename_pattern=data.get("filename_pattern"),
|
||||
skip_existing=data.get("skip_existing", True),
|
||||
save_metadata=data.get("save_metadata", True),
|
||||
timeout=data.get("timeout", 3600),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadResult:
|
||||
success: bool
|
||||
url: str
|
||||
artist_slug: str
|
||||
platform: str
|
||||
files_downloaded: int = 0
|
||||
files_quarantined: int = 0
|
||||
quarantined_paths: list[str] = field(default_factory=list)
|
||||
written_paths: list[str] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
error_type: ErrorType | None = None
|
||||
error_message: str | None = None
|
||||
duration_seconds: float = 0.0
|
||||
started_at: str | None = None
|
||||
completed_at: str | None = None
|
||||
|
||||
|
||||
def _summarize_validation_failures(failures: list[dict]) -> str:
|
||||
if not failures:
|
||||
return "Validation failed"
|
||||
reasons: dict[str, int] = {}
|
||||
for f in failures:
|
||||
key = f.get("reason") or "unknown"
|
||||
reasons[key] = reasons.get(key, 0) + 1
|
||||
top_reason, top_count = max(reasons.items(), key=lambda kv: kv[1])
|
||||
n = len(failures)
|
||||
if top_count == n:
|
||||
return f"{n} file{'s' if n != 1 else ''} quarantined: {top_reason}"
|
||||
return f"{n} files quarantined ({top_count}× {top_reason}, mixed)"
|
||||
|
||||
|
||||
class GalleryDLService:
|
||||
"""Service for executing gallery-dl downloads."""
|
||||
|
||||
AUTH_ERROR_PATTERNS = [
|
||||
"unauthorized", "login required", "please login", "must be logged in",
|
||||
"authentication required", "authenticationerror", "refresh-token",
|
||||
"session expired", "invalid cookie", "cookies are expired",
|
||||
"cookies have expired", "not logged in",
|
||||
]
|
||||
RATE_LIMIT_PATTERNS = [
|
||||
'" 429 ', "rate limit", "too many requests", "ratelimit", "throttl",
|
||||
]
|
||||
NOT_FOUND_PATTERNS = [
|
||||
'" 404 ', "error 404", "status: 404", "status code: 404",
|
||||
"404 not found", "page not found", "user not found",
|
||||
"creator not found", "artist not found",
|
||||
"content no longer available", "has been deleted",
|
||||
"account deleted", "profile does not exist",
|
||||
]
|
||||
GENERIC_NOT_FOUND_PATTERNS = ["does not exist", "no longer available"]
|
||||
NETWORK_ERROR_PATTERNS = [
|
||||
"timed out", "read timed out", "connect timed out",
|
||||
"connection timed out", "connection refused", "connection reset",
|
||||
"network unreachable", "name resolution failed",
|
||||
"name or service not known", "nodename nor servname provided",
|
||||
"ssl: certificate_verify_failed", "ssl: wrong_version_number",
|
||||
"certificate verify failed", "[errno",
|
||||
]
|
||||
ACCESS_DENIED_PATTERNS = [
|
||||
'" 403 ', "error 403", "forbidden", "access denied",
|
||||
"permission denied", "tier required", "pledge required",
|
||||
]
|
||||
|
||||
# Per-platform defaults. Lifted from GS — same six platforms FC supports.
|
||||
PLATFORM_DEFAULTS = {
|
||||
"patreon": {
|
||||
"content_types": ["images", "image_large", "attachments", "postfile", "content"],
|
||||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||||
"filename": "{num:>02}_{filename}.{extension}",
|
||||
"videos": True,
|
||||
"embeds": True,
|
||||
"cursor": True,
|
||||
},
|
||||
"subscribestar": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
|
||||
"filename": "{num:>02}_{filename}.{extension}",
|
||||
},
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||
"include": "all",
|
||||
},
|
||||
"discord": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{channel[name]}"],
|
||||
"filename": "{date:%Y%m%d}_{id}_{filename}.{extension}",
|
||||
"embeds": "all",
|
||||
"stickers": True,
|
||||
"reactions": False,
|
||||
"threads": True,
|
||||
},
|
||||
"pixiv": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{category}"],
|
||||
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
|
||||
"ugoira": True,
|
||||
},
|
||||
"deviantart": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{index:>03}_{title[:50]}.{extension}",
|
||||
"flat": True,
|
||||
"original": True,
|
||||
"mature": True,
|
||||
"metadata": True,
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
rate_limit: float = 3.0,
|
||||
validate_files: bool = True,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self._rate_limit = rate_limit
|
||||
self._validate_files = validate_files
|
||||
self._config_dir = self.images_root / ".gallery-dl"
|
||||
self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
(self._config_dir / "temp").mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
def _get_default_config(self) -> dict:
|
||||
config = {
|
||||
"extractor": {
|
||||
"base-directory": str(self.images_root),
|
||||
"archive": str(self._config_dir / "archive.sqlite3"),
|
||||
"skip": True,
|
||||
"sleep": self._rate_limit,
|
||||
"sleep-request": max(0.5, self._rate_limit / 4),
|
||||
"retries": 3,
|
||||
"timeout": 30.0,
|
||||
"verify": True,
|
||||
"postprocessors": [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "json",
|
||||
"directory": ".",
|
||||
"filename": "{filename}.json",
|
||||
}
|
||||
],
|
||||
},
|
||||
"downloader": {
|
||||
"part": True,
|
||||
"part-directory": str(self._config_dir / "temp"),
|
||||
"retries": 3,
|
||||
"timeout": 120.0,
|
||||
},
|
||||
"output": {"progress": True},
|
||||
}
|
||||
for platform_name, defaults in self.PLATFORM_DEFAULTS.items():
|
||||
config["extractor"][platform_name] = {
|
||||
key: value for key, value in defaults.items() if key != "content_types"
|
||||
}
|
||||
return config
|
||||
|
||||
def _build_config_for_source(
|
||||
self,
|
||||
platform: str,
|
||||
source_config: SourceConfig,
|
||||
artist_slug: str,
|
||||
) -> dict:
|
||||
config = json.loads(json.dumps(self._get_default_config())) # deep copy
|
||||
|
||||
destination = str(self.images_root / artist_slug / platform)
|
||||
config["extractor"]["base-directory"] = destination
|
||||
|
||||
if source_config.sleep is not None:
|
||||
config["extractor"]["sleep"] = source_config.sleep
|
||||
if source_config.sleep_request is not None:
|
||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||
config["extractor"]["skip"] = source_config.skip_existing
|
||||
|
||||
if source_config.save_metadata:
|
||||
config["extractor"]["postprocessors"] = [
|
||||
{
|
||||
"name": "metadata",
|
||||
"mode": "json",
|
||||
"directory": ".",
|
||||
"filename": "{filename}.json",
|
||||
}
|
||||
]
|
||||
else:
|
||||
config["extractor"].pop("postprocessors", None)
|
||||
|
||||
platform_section = config["extractor"].setdefault(platform, {})
|
||||
|
||||
if platform == "patreon":
|
||||
if "all" in source_config.content_types:
|
||||
platform_section["files"] = [
|
||||
"images", "image_large", "attachments", "postfile", "content",
|
||||
]
|
||||
else:
|
||||
platform_section["files"] = source_config.content_types
|
||||
elif platform == "hentaifoundry":
|
||||
if "pictures" in source_config.content_types or "all" in source_config.content_types:
|
||||
platform_section["include"] = "all"
|
||||
|
||||
if source_config.directory_pattern:
|
||||
platform_section["directory"] = source_config.directory_pattern
|
||||
if source_config.filename_pattern:
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
|
||||
def _categorize_error(
|
||||
self, return_code: int, stdout: str, stderr: str,
|
||||
) -> tuple[ErrorType, str]:
|
||||
combined = f"{stdout} {stderr}".lower()
|
||||
|
||||
skip_line_count = len([
|
||||
line for line in stdout.split("\n") if line.strip().startswith("#")
|
||||
])
|
||||
skip_text_indicators = ["skipping", "already exists", "archive"]
|
||||
has_skip_text = any(ind in combined for ind in skip_text_indicators)
|
||||
|
||||
actual_error_indicators = [
|
||||
"][error]", "exception", "traceback",
|
||||
"download failed", "extraction failed",
|
||||
]
|
||||
per_item_patterns = [
|
||||
"not allowed to view", "unable to get post",
|
||||
"failed to extract campaign id", "failed to download",
|
||||
"cannot import yt-dlp", "cannot import youtube-dl",
|
||||
]
|
||||
all_lines = combined.split("\n")
|
||||
error_lines = [
|
||||
line for line in all_lines
|
||||
if any(ind in line for ind in actual_error_indicators)
|
||||
]
|
||||
per_item_error_lines = [
|
||||
line for line in error_lines
|
||||
if any(p in line for p in per_item_patterns)
|
||||
]
|
||||
source_level_error_lines = [
|
||||
line for line in error_lines
|
||||
if line not in set(per_item_error_lines)
|
||||
]
|
||||
has_actual_error = bool(source_level_error_lines)
|
||||
|
||||
if per_item_error_lines:
|
||||
per_item_set = set(per_item_error_lines)
|
||||
combined = "\n".join(line for line in all_lines if line not in per_item_set)
|
||||
|
||||
if (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
if return_code == 0 and not stdout.strip():
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
has_401_error = (
|
||||
'" 401 ' in combined
|
||||
or "401 unauthorized" in combined
|
||||
or "error 401" in combined
|
||||
or "status: 401" in combined
|
||||
or "status code: 401" in combined
|
||||
)
|
||||
if has_401_error or any(p in combined for p in self.AUTH_ERROR_PATTERNS):
|
||||
return ErrorType.AUTH_ERROR, "Authentication failed — cookies may be expired or invalid"
|
||||
|
||||
if any(p in combined for p in self.RATE_LIMIT_PATTERNS):
|
||||
return ErrorType.RATE_LIMITED, "Rate limited by server — try increasing sleep time"
|
||||
|
||||
has_specific_404 = any(p in combined for p in self.NOT_FOUND_PATTERNS)
|
||||
has_generic_with_error = any(
|
||||
p in combined and "][error]" in combined
|
||||
for p in self.GENERIC_NOT_FOUND_PATTERNS
|
||||
)
|
||||
if has_specific_404 or has_generic_with_error:
|
||||
return ErrorType.NOT_FOUND, "URL not found — artist may have changed username or deleted content"
|
||||
|
||||
if any(p in combined for p in self.NETWORK_ERROR_PATTERNS):
|
||||
return ErrorType.NETWORK_ERROR, "Network error — check internet connection"
|
||||
|
||||
if any(p in combined for p in self.ACCESS_DENIED_PATTERNS):
|
||||
return ErrorType.ACCESS_DENIED, "Access denied — may need higher subscription tier"
|
||||
|
||||
if "http error" in combined or "httperror" in combined:
|
||||
return ErrorType.HTTP_ERROR, "HTTP error occurred"
|
||||
|
||||
if "no suitable" in combined or "unsupported" in combined or "no extractor" in combined:
|
||||
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
|
||||
|
||||
if return_code in (1, 4) and (skip_line_count > 0 or has_skip_text) and not has_actual_error:
|
||||
return ErrorType.NO_NEW_CONTENT, "No new content to download"
|
||||
|
||||
if return_code in (1, 4) and not has_actual_error:
|
||||
tier_gated_lines = [
|
||||
line for line in combined.split("\n")
|
||||
if "][warning]" in line and "not allowed to view post" in line
|
||||
]
|
||||
if tier_gated_lines:
|
||||
count = len(tier_gated_lines)
|
||||
return (
|
||||
ErrorType.TIER_LIMITED,
|
||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||
)
|
||||
|
||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||
|
||||
def _count_downloaded_files(self, stdout: str) -> int:
|
||||
if not stdout:
|
||||
return 0
|
||||
return sum(1 for line in stdout.splitlines() if line.strip().startswith("/"))
|
||||
|
||||
def _written_paths(self, stdout: str) -> list[Path]:
|
||||
if not stdout:
|
||||
return []
|
||||
return [
|
||||
Path(stripped)
|
||||
for line in stdout.splitlines()
|
||||
if (stripped := line.strip()).startswith("/")
|
||||
]
|
||||
|
||||
def _validate_and_quarantine(
|
||||
self,
|
||||
written_paths: list[Path],
|
||||
artist_slug: str,
|
||||
platform: str,
|
||||
url: str,
|
||||
) -> tuple[list[str], list[dict]]:
|
||||
quarantined_relpaths: list[str] = []
|
||||
failures: list[dict] = []
|
||||
if not written_paths:
|
||||
return quarantined_relpaths, failures
|
||||
|
||||
quarantine_root = (
|
||||
self.images_root / "_quarantine" / artist_slug / platform
|
||||
)
|
||||
|
||||
for path in written_paths:
|
||||
if not is_validatable(path):
|
||||
continue
|
||||
try:
|
||||
result = validate_file(path)
|
||||
except Exception as exc:
|
||||
log.warning("Validator raised on %s: %s", path, exc)
|
||||
continue
|
||||
if result.ok:
|
||||
continue
|
||||
try:
|
||||
quarantine_root.mkdir(parents=True, exist_ok=True)
|
||||
dest = quarantine_root / path.name
|
||||
counter = 1
|
||||
while dest.exists():
|
||||
dest = quarantine_root / f"{path.stem}.{counter}{path.suffix}"
|
||||
counter += 1
|
||||
path.rename(dest)
|
||||
sidecar = dest.with_suffix(dest.suffix + ".quarantine.json")
|
||||
sidecar.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"original_path": str(path),
|
||||
"source_url": url,
|
||||
"artist_slug": artist_slug,
|
||||
"platform": platform,
|
||||
"format": result.format,
|
||||
"reason": result.reason,
|
||||
"size": result.size,
|
||||
"quarantined_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
except OSError as exc:
|
||||
log.error("Failed to quarantine %s: %s. File left in place.", path, exc)
|
||||
continue
|
||||
|
||||
log.warning(
|
||||
"Quarantined corrupt file: %s → %s (%s: %s)",
|
||||
path, dest, result.format, result.reason,
|
||||
)
|
||||
quarantined_relpaths.append(str(dest))
|
||||
failures.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"format": result.format,
|
||||
"reason": result.reason,
|
||||
"size": result.size,
|
||||
}
|
||||
)
|
||||
return quarantined_relpaths, failures
|
||||
|
||||
def _compute_run_stats(self, return_code: int, stdout: str, stderr: str) -> dict:
|
||||
stdout = stdout or ""
|
||||
stderr = stderr or ""
|
||||
|
||||
skipped_stdout = sum(
|
||||
1 for line in stdout.splitlines() if line.strip().startswith("#")
|
||||
)
|
||||
skipped_stderr = sum(
|
||||
1 for line in stderr.splitlines() if "] skipping " in line.lower()
|
||||
)
|
||||
per_item_failures = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "[download][error]" in line.lower() and "failed to download" in line.lower()
|
||||
)
|
||||
warning_count = sum(
|
||||
1 for line in stderr.splitlines() if "][warning]" in line.lower()
|
||||
)
|
||||
tier_gated_count = sum(
|
||||
1 for line in stderr.splitlines()
|
||||
if "][warning]" in line.lower() and "not allowed to view post" in line.lower()
|
||||
)
|
||||
|
||||
return {
|
||||
"exit_code": return_code,
|
||||
"downloaded_count": self._count_downloaded_files(stdout),
|
||||
"skipped_count": skipped_stdout + skipped_stderr,
|
||||
"per_item_failures": per_item_failures,
|
||||
"warning_count": warning_count,
|
||||
"tier_gated_count": tier_gated_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_errors_warnings(stderr: str) -> str:
|
||||
if not stderr:
|
||||
return ""
|
||||
kept = [
|
||||
line for line in stderr.splitlines()
|
||||
if "][error]" in line.lower() or "][warning]" in line.lower()
|
||||
]
|
||||
return "\n".join(kept)
|
||||
|
||||
@staticmethod
|
||||
def _truncate_log(text: str, max_bytes: int = 500_000) -> str:
|
||||
if not text:
|
||||
return text
|
||||
encoded = text.encode("utf-8")
|
||||
if len(encoded) <= max_bytes:
|
||||
return text
|
||||
half = max_bytes // 2
|
||||
head = encoded[:half].decode("utf-8", errors="ignore")
|
||||
tail = encoded[-half:].decode("utf-8", errors="ignore")
|
||||
head_lines = head.count("\n")
|
||||
tail_lines = tail.count("\n")
|
||||
total_lines = text.count("\n")
|
||||
elided = max(0, total_lines - head_lines - tail_lines)
|
||||
marker = f"\n\n... [{elided} lines elided, {len(encoded) - max_bytes} bytes] ...\n\n"
|
||||
return head + marker + tail
|
||||
|
||||
async def download(
|
||||
self,
|
||||
url: str,
|
||||
artist_slug: str,
|
||||
platform: str,
|
||||
source_config: SourceConfig | None = None,
|
||||
cookies_path: str | None = None,
|
||||
auth_token: str | None = None,
|
||||
) -> DownloadResult:
|
||||
start_time = time.time()
|
||||
started_at = datetime.now(UTC).isoformat()
|
||||
|
||||
if source_config is None:
|
||||
source_config = SourceConfig()
|
||||
|
||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})
|
||||
config["extractor"]["discord"]["token"] = auth_token
|
||||
if auth_token and platform == "pixiv":
|
||||
config["extractor"].setdefault("pixiv", {})
|
||||
config["extractor"]["pixiv"]["refresh-token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
) as fh:
|
||||
json.dump(config, fh, indent=2)
|
||||
temp_config_path = fh.name
|
||||
|
||||
try:
|
||||
cmd = [
|
||||
sys.executable, "-m", "gallery_dl",
|
||||
"--config", temp_config_path, "--verbose", url,
|
||||
]
|
||||
log.info("Executing gallery-dl for %s/%s: %s", artist_slug, platform, url)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
proc = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: subprocess.run(
|
||||
cmd, capture_output=True, text=True,
|
||||
timeout=source_config.timeout,
|
||||
),
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
completed_at = datetime.now(UTC).isoformat()
|
||||
|
||||
quarantined_paths: list[str] = []
|
||||
quarantine_failures: list[dict] = []
|
||||
if self._validate_files:
|
||||
quarantined_paths, quarantine_failures = self._validate_and_quarantine(
|
||||
written_paths=self._written_paths(proc.stdout),
|
||||
artist_slug=artist_slug,
|
||||
platform=platform,
|
||||
url=url,
|
||||
)
|
||||
|
||||
written_paths = [
|
||||
str(p) for p in self._written_paths(proc.stdout)
|
||||
if str(p) not in set(quarantined_paths)
|
||||
]
|
||||
files_written = self._count_downloaded_files(proc.stdout)
|
||||
files_quarantined = len(quarantined_paths)
|
||||
files_kept = max(0, files_written - files_quarantined)
|
||||
|
||||
if proc.returncode == 0:
|
||||
if files_quarantined > 0:
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept,
|
||||
files_quarantined=files_quarantined,
|
||||
quarantined_paths=quarantined_paths,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
error_type=ErrorType.VALIDATION_FAILED,
|
||||
error_message=_summarize_validation_failures(quarantine_failures),
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
return DownloadResult(
|
||||
success=True, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
|
||||
error_type, error_message = self._categorize_error(
|
||||
proc.returncode, proc.stdout, proc.stderr
|
||||
)
|
||||
success = error_type in (ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED)
|
||||
|
||||
if files_quarantined > 0:
|
||||
error_type = ErrorType.VALIDATION_FAILED
|
||||
error_message = _summarize_validation_failures(quarantine_failures)
|
||||
success = False
|
||||
|
||||
return DownloadResult(
|
||||
success=success, url=url, artist_slug=artist_slug, platform=platform,
|
||||
files_downloaded=files_kept if success else 0,
|
||||
files_quarantined=files_quarantined,
|
||||
quarantined_paths=quarantined_paths,
|
||||
written_paths=written_paths,
|
||||
stdout=proc.stdout, stderr=proc.stderr,
|
||||
return_code=proc.returncode,
|
||||
error_type=error_type, error_message=error_message,
|
||||
duration_seconds=duration,
|
||||
started_at=started_at, completed_at=completed_at,
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
duration = time.time() - start_time
|
||||
log.error("Download timeout for %s/%s after %.1fs", artist_slug, platform, duration)
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
error_type=ErrorType.TIMEOUT,
|
||||
error_message=f"Download timed out after {source_config.timeout} seconds",
|
||||
duration_seconds=duration,
|
||||
started_at=started_at,
|
||||
completed_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
except Exception as exc:
|
||||
duration = time.time() - start_time
|
||||
log.exception("Unexpected error downloading %s/%s: %s", artist_slug, platform, exc)
|
||||
return DownloadResult(
|
||||
success=False, url=url, artist_slug=artist_slug, platform=platform,
|
||||
error_type=ErrorType.UNKNOWN_ERROR, error_message=str(exc),
|
||||
duration_seconds=duration,
|
||||
started_at=started_at,
|
||||
completed_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
Path(temp_config_path).unlink() # noqa: ASYNC240
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,308 @@
|
||||
"""Cursor-paginated gallery queries.
|
||||
|
||||
Cursor format: opaque base64-encoded "<iso8601_created_at>:<image_id>".
|
||||
Pagination key is (created_at DESC, id DESC) so we don't drift when new
|
||||
imports arrive between page loads. Decoding rejects malformed cursors with
|
||||
a ValueError; the API layer translates that to HTTP 400.
|
||||
"""
|
||||
|
||||
import base64
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import and_, exists, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, ImageProvenance, ImageRecord, Source, Tag
|
||||
from ..models.tag import image_tag
|
||||
|
||||
CURSOR_SEPARATOR = "|"
|
||||
|
||||
|
||||
def encode_cursor(created_at: datetime, image_id: int) -> str:
|
||||
raw = f"{created_at.isoformat()}{CURSOR_SEPARATOR}{image_id}"
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode()).decode()
|
||||
ts_part, id_part = raw.split(CURSOR_SEPARATOR, 1)
|
||||
return datetime.fromisoformat(ts_part), int(id_part)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid cursor: {cursor!r}") from exc
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GalleryImage:
|
||||
id: int
|
||||
path: str
|
||||
sha256: str
|
||||
mime: str
|
||||
width: int | None
|
||||
height: int | None
|
||||
created_at: datetime
|
||||
thumbnail_url: str
|
||||
artist: dict | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GalleryPage:
|
||||
images: list[GalleryImage]
|
||||
next_cursor: str | None
|
||||
date_groups: list[tuple[int, int, list[int]]] # (year, month, [image_id...])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TimelineBucket:
|
||||
year: int
|
||||
month: int
|
||||
count: int
|
||||
|
||||
|
||||
def thumbnail_url(sha256_hex: str, mime: str) -> str:
|
||||
# Quart serves /images/* via the frontend blueprint (FC-1); thumbnails go
|
||||
# under /images/thumbs/. The MIME determines the extension.
|
||||
ext = ".png" if mime in ("image/png", "image/gif") else ".jpg"
|
||||
bucket = sha256_hex[:3]
|
||||
return f"/images/thumbs/{bucket}/{sha256_hex}{ext}"
|
||||
|
||||
|
||||
def _require_single_filter(tag_id, post_id, artist_id) -> None:
|
||||
if sum(x is not None for x in (tag_id, post_id, artist_id)) > 1:
|
||||
raise ValueError(
|
||||
"tag_id, post_id, artist_id are mutually exclusive"
|
||||
)
|
||||
|
||||
|
||||
def _provenance_clause(post_id, artist_id):
|
||||
"""Correlated EXISTS clause (NOT a join) so an image with multiple
|
||||
matching provenance rows is returned exactly once and the
|
||||
(created_at DESC, id DESC) cursor ordering is unaffected."""
|
||||
if post_id is not None:
|
||||
return exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id == post_id,
|
||||
)
|
||||
if artist_id is not None:
|
||||
return exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.source_id == Source.id,
|
||||
Source.artist_id == artist_id,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
||||
"""Map image_id -> {"name","slug"} via the canonical
|
||||
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
||||
if not image_ids:
|
||||
return {}
|
||||
stmt = (
|
||||
select(ImageRecord.id, Artist.name, Artist.slug)
|
||||
.join(Artist, Artist.id == ImageRecord.artist_id)
|
||||
.where(ImageRecord.id.in_(image_ids))
|
||||
)
|
||||
return {
|
||||
img_id: {"name": name, "slug": slug}
|
||||
for img_id, name, slug in (await session.execute(stmt)).all()
|
||||
}
|
||||
|
||||
|
||||
class GalleryService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def scroll(
|
||||
self,
|
||||
cursor: str | None,
|
||||
limit: int = 50,
|
||||
tag_id: int | None = None,
|
||||
post_id: int | None = None,
|
||||
artist_id: int | None = None,
|
||||
) -> GalleryPage:
|
||||
if limit < 1 or limit > 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
_require_single_filter(tag_id, post_id, artist_id)
|
||||
|
||||
stmt = select(ImageRecord)
|
||||
if tag_id is not None:
|
||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||
image_tag.c.tag_id == tag_id
|
||||
)
|
||||
prov = _provenance_clause(post_id, artist_id)
|
||||
if prov is not None:
|
||||
stmt = stmt.where(prov)
|
||||
|
||||
if cursor:
|
||||
cur_ts, cur_id = decode_cursor(cursor)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
ImageRecord.created_at < cur_ts,
|
||||
and_(ImageRecord.created_at == cur_ts, ImageRecord.id < cur_id),
|
||||
)
|
||||
)
|
||||
|
||||
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(limit + 1)
|
||||
rows = (await self.session.execute(stmt)).scalars().all()
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = rows[limit - 1]
|
||||
next_cursor = encode_cursor(last.created_at, last.id)
|
||||
rows = rows[:limit]
|
||||
|
||||
artists = await _artists_for(self.session, [r.id for r in rows])
|
||||
images = [
|
||||
GalleryImage(
|
||||
id=r.id,
|
||||
path=r.path,
|
||||
sha256=r.sha256,
|
||||
mime=r.mime,
|
||||
width=r.width,
|
||||
height=r.height,
|
||||
created_at=r.created_at,
|
||||
thumbnail_url=thumbnail_url(r.sha256, r.mime),
|
||||
artist=artists.get(r.id),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
return GalleryPage(
|
||||
images=images,
|
||||
next_cursor=next_cursor,
|
||||
date_groups=_group_by_year_month(images),
|
||||
)
|
||||
|
||||
async def timeline(
|
||||
self,
|
||||
tag_id: int | None = None,
|
||||
post_id: int | None = None,
|
||||
artist_id: int | None = None,
|
||||
) -> list[TimelineBucket]:
|
||||
year_col = func.date_part("year", ImageRecord.created_at).label("yr")
|
||||
month_col = func.date_part("month", ImageRecord.created_at).label("mo")
|
||||
stmt = select(
|
||||
year_col, month_col, func.count(ImageRecord.id).label("cnt")
|
||||
)
|
||||
_require_single_filter(tag_id, post_id, artist_id)
|
||||
if tag_id is not None:
|
||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||
image_tag.c.tag_id == tag_id
|
||||
)
|
||||
prov = _provenance_clause(post_id, artist_id)
|
||||
if prov is not None:
|
||||
stmt = stmt.where(prov)
|
||||
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
return [TimelineBucket(year=int(r.yr), month=int(r.mo), count=int(r.cnt)) for r in rows]
|
||||
|
||||
async def jump_cursor(
|
||||
self, year: int, month: int, tag_id: int | None = None,
|
||||
post_id: int | None = None, artist_id: int | None = None,
|
||||
) -> str | None:
|
||||
"""Returns a cursor that, when passed to scroll(), positions at the
|
||||
first image of the given year-month. None if the bucket is empty.
|
||||
"""
|
||||
from sqlalchemy import extract
|
||||
|
||||
stmt = select(ImageRecord).where(
|
||||
extract("year", ImageRecord.created_at) == year,
|
||||
extract("month", ImageRecord.created_at) == month,
|
||||
)
|
||||
_require_single_filter(tag_id, post_id, artist_id)
|
||||
if tag_id is not None:
|
||||
stmt = stmt.join(image_tag, image_tag.c.image_record_id == ImageRecord.id).where(
|
||||
image_tag.c.tag_id == tag_id
|
||||
)
|
||||
prov = _provenance_clause(post_id, artist_id)
|
||||
if prov is not None:
|
||||
stmt = stmt.where(prov)
|
||||
stmt = stmt.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc()).limit(1)
|
||||
first = (await self.session.execute(stmt)).scalar_one_or_none()
|
||||
if first is None:
|
||||
return None
|
||||
# Cursor is exclusive; we encode a cursor with id+1 so the row itself
|
||||
# is the first result in the next scroll().
|
||||
return encode_cursor(first.created_at, first.id + 1)
|
||||
|
||||
async def get_image_with_tags(self, image_id: int) -> dict | None:
|
||||
record = await self.session.get(ImageRecord, image_id)
|
||||
if record is None:
|
||||
return None
|
||||
tag_stmt = (
|
||||
select(Tag)
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
.where(image_tag.c.image_record_id == image_id)
|
||||
.order_by(Tag.kind.asc(), Tag.name.asc())
|
||||
)
|
||||
tags = (await self.session.execute(tag_stmt)).scalars().all()
|
||||
neighbors = await self._neighbors(record)
|
||||
return {
|
||||
"id": record.id,
|
||||
"path": record.path,
|
||||
"sha256": record.sha256,
|
||||
"mime": record.mime,
|
||||
"width": record.width,
|
||||
"height": record.height,
|
||||
"size_bytes": record.size_bytes,
|
||||
"integrity_status": record.integrity_status,
|
||||
"created_at": record.created_at.isoformat(),
|
||||
"thumbnail_url": thumbnail_url(record.sha256, record.mime),
|
||||
"image_url": f"/images/{record.path.split('/images/', 1)[-1]}",
|
||||
"tags": [
|
||||
{
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"kind": t.kind.value if hasattr(t.kind, "value") else t.kind,
|
||||
"fandom_id": t.fandom_id,
|
||||
}
|
||||
for t in tags
|
||||
],
|
||||
"neighbors": neighbors,
|
||||
}
|
||||
|
||||
async def _neighbors(self, record: ImageRecord) -> dict:
|
||||
prev_stmt = (
|
||||
select(ImageRecord.id)
|
||||
.where(
|
||||
or_(
|
||||
ImageRecord.created_at > record.created_at,
|
||||
and_(
|
||||
ImageRecord.created_at == record.created_at,
|
||||
ImageRecord.id > record.id,
|
||||
),
|
||||
)
|
||||
)
|
||||
.order_by(ImageRecord.created_at.asc(), ImageRecord.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
next_stmt = (
|
||||
select(ImageRecord.id)
|
||||
.where(
|
||||
or_(
|
||||
ImageRecord.created_at < record.created_at,
|
||||
and_(
|
||||
ImageRecord.created_at == record.created_at,
|
||||
ImageRecord.id < record.id,
|
||||
),
|
||||
)
|
||||
)
|
||||
.order_by(ImageRecord.created_at.desc(), ImageRecord.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
prev_id = (await self.session.execute(prev_stmt)).scalar_one_or_none()
|
||||
next_id = (await self.session.execute(next_stmt)).scalar_one_or_none()
|
||||
return {"prev_id": prev_id, "next_id": next_id}
|
||||
|
||||
|
||||
def _group_by_year_month(
|
||||
images: list[GalleryImage],
|
||||
) -> list[tuple[int, int, list[int]]]:
|
||||
groups: list[tuple[int, int, list[int]]] = []
|
||||
for img in images:
|
||||
y, m = img.created_at.year, img.created_at.month
|
||||
if groups and groups[-1][0] == y and groups[-1][1] == m:
|
||||
groups[-1][2].append(img.id)
|
||||
else:
|
||||
groups.append((y, m, [img.id]))
|
||||
return groups
|
||||
@@ -0,0 +1,761 @@
|
||||
"""Single-file import pipeline.
|
||||
|
||||
The Importer is the synchronous core of FC-2a's import path. A Celery task
|
||||
(tasks/import_file.py) instantiates one per job and calls import_one().
|
||||
|
||||
The service is intentionally not async — Celery tasks are run in a
|
||||
synchronous worker process, and the DB session is a sync session passed in
|
||||
by the task. The Quart side uses an async session via the same models.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Post,
|
||||
PostAttachment,
|
||||
Source,
|
||||
)
|
||||
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
|
||||
from ..utils.phash import compute_phash, find_similar
|
||||
from ..utils.sidecar import find_sidecar, parse_sidecar
|
||||
from ..utils.slug import slugify
|
||||
from .archive_extractor import extract_archive, is_archive
|
||||
from .attachment_store import AttachmentStore
|
||||
from .thumbnailer import Thumbnailer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkipReason(StrEnum):
|
||||
too_small = "too_small"
|
||||
too_transparent = "too_transparent"
|
||||
single_color = "single_color"
|
||||
duplicate_hash = "duplicate_hash"
|
||||
duplicate_phash = "duplicate_phash"
|
||||
invalid_image = "invalid_image"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached'
|
||||
image_id: int | None = None
|
||||
skip_reason: SkipReason | None = None
|
||||
error: str | None = None
|
||||
member_image_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
|
||||
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v", ".wmv", ".flv"}
|
||||
ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS
|
||||
|
||||
|
||||
def is_supported(path: Path) -> bool:
|
||||
return path.suffix.lower() in ALL_EXTS
|
||||
|
||||
|
||||
def is_video(path: Path) -> bool:
|
||||
return path.suffix.lower() in VIDEO_EXTS
|
||||
|
||||
|
||||
def _mime_for(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
image_mimes = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".bmp": "image/bmp",
|
||||
".tif": "image/tiff",
|
||||
".tiff": "image/tiff",
|
||||
}
|
||||
video_mimes = {
|
||||
".mp4": "video/mp4",
|
||||
".mov": "video/quicktime",
|
||||
".avi": "video/x-msvideo",
|
||||
".mkv": "video/x-matroska",
|
||||
".webm": "video/webm",
|
||||
".m4v": "video/x-m4v",
|
||||
".wmv": "video/x-ms-wmv",
|
||||
".flv": "video/x-flv",
|
||||
}
|
||||
return image_mimes.get(suffix) or video_mimes.get(suffix) or "application/octet-stream"
|
||||
|
||||
|
||||
def _sha256_of(path: Path, chunk_size: int = 1 << 20) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fp:
|
||||
while True:
|
||||
block = fp.read(chunk_size)
|
||||
if not block:
|
||||
break
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class Importer:
|
||||
"""Imports one file at a time. Constructed per-task by the Celery task."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: Session,
|
||||
images_root: Path,
|
||||
import_root: Path,
|
||||
thumbnailer: Thumbnailer,
|
||||
settings: ImportSettings,
|
||||
deep: bool = False,
|
||||
):
|
||||
self.session = session
|
||||
self.images_root = images_root
|
||||
self.import_root = import_root
|
||||
self.thumbnailer = thumbnailer
|
||||
self.settings = settings
|
||||
self.deep = deep
|
||||
self.attachments = AttachmentStore(images_root)
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||
media members (one Post via the archive-adjacent sidecar) and
|
||||
preserve the archive itself. Other non-media → PostAttachment.
|
||||
Nothing is skipped for being non-media (FC-2d-iii)."""
|
||||
if source.suffix.lower() == ".json":
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error="sidecar json is metadata, not content",
|
||||
)
|
||||
if is_archive(source):
|
||||
return self._import_archive(source)
|
||||
if not is_supported(source):
|
||||
return self._capture_attachment(source)
|
||||
return self._import_media(source, source)
|
||||
|
||||
def _resolve_artist(self, attribution_path: Path) -> Artist | None:
|
||||
name = derive_top_level_artist(attribution_path, self.import_root)
|
||||
return self._upsert_artist(name) if name else None
|
||||
|
||||
def _post_for_sidecar(
|
||||
self, source: Path, artist: Artist | None
|
||||
) -> Post | None:
|
||||
"""If a sidecar sits next to `source`, ensure its Source+Post
|
||||
exist (idempotent) and return the Post — so attachments can link
|
||||
to the same Post the per-member _apply_sidecar will reuse."""
|
||||
sc = find_sidecar(source)
|
||||
if sc is None or artist is None:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(sc.read_text("utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("sidecar JSON is not an object")
|
||||
except Exception as exc:
|
||||
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
return post
|
||||
|
||||
def _capture_attachment(
|
||||
self, source: Path, *, post: Post | None = None,
|
||||
artist: Artist | None = None, resolved: bool = False,
|
||||
) -> ImportResult:
|
||||
if not resolved:
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
sha = _sha256_of(source)
|
||||
existing = self.session.execute(
|
||||
select(PostAttachment).where(PostAttachment.sha256 == sha)
|
||||
).scalar_one_or_none()
|
||||
if existing is None:
|
||||
stored = self.attachments.store(source, sha)
|
||||
self.session.add(PostAttachment(
|
||||
post_id=post.id if post else None,
|
||||
artist_id=artist.id if artist else None,
|
||||
sha256=sha,
|
||||
path=stored,
|
||||
original_filename=source.name,
|
||||
ext=source.suffix.lower(),
|
||||
mime=_mime_for(source),
|
||||
size_bytes=source.stat().st_size,
|
||||
))
|
||||
self.session.flush()
|
||||
self.session.commit()
|
||||
return ImportResult(status="attached")
|
||||
|
||||
def _import_archive(self, source: Path) -> ImportResult:
|
||||
artist = self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist)
|
||||
member_ids: list[int] = []
|
||||
with extract_archive(source) as members:
|
||||
for _name, member_path in members:
|
||||
if not is_supported(member_path):
|
||||
continue # non-media preserved via the stored archive
|
||||
res = self._import_media(member_path, source)
|
||||
if res.status in ("imported", "superseded") and res.image_id:
|
||||
member_ids.append(res.image_id)
|
||||
# Preserve the archive itself (links to the same Post/Artist).
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist, resolved=True
|
||||
)
|
||||
if member_ids:
|
||||
return ImportResult(
|
||||
status="imported", image_id=member_ids[0],
|
||||
member_image_ids=member_ids,
|
||||
)
|
||||
return ImportResult(status="attached")
|
||||
|
||||
def _import_media(
|
||||
self, source: Path, attribution_path: Path
|
||||
) -> ImportResult:
|
||||
"""The media import pipeline (filters, dedup, copy, provenance).
|
||||
|
||||
`attribution_path` anchors artist/subdir/sidecar derivation: for
|
||||
a normal file it equals `source`; for an archive member it is the
|
||||
archive's real path (the member lives in a tempdir).
|
||||
"""
|
||||
if not is_supported(source):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"unsupported extension {source.suffix}",
|
||||
)
|
||||
|
||||
# Compute file dimensions (images only) and apply filters.
|
||||
width = height = None
|
||||
has_alpha = False
|
||||
if not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
im.verify()
|
||||
with Image.open(source) as im:
|
||||
width, height = im.size
|
||||
has_alpha = im.mode in ("RGBA", "LA") or (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
)
|
||||
except Exception as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image, error=str(exc),
|
||||
)
|
||||
|
||||
if (self.settings.min_width and width < self.settings.min_width) or (
|
||||
self.settings.min_height and height < self.settings.min_height
|
||||
):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_small,
|
||||
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
||||
)
|
||||
|
||||
if self.settings.skip_transparent and has_alpha:
|
||||
pct = self._transparency_pct(source)
|
||||
if pct >= self.settings.transparency_threshold:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
# Hash dedup (exact).
|
||||
sha = _sha256_of(source)
|
||||
existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
|
||||
existing = self.session.execute(existing_stmt).scalar_one_or_none()
|
||||
if existing:
|
||||
if not self.deep:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="sha256 already present",
|
||||
)
|
||||
return self._deep_rederive(existing, source, attribution_path)
|
||||
|
||||
# Perceptual near-dup (images only; videos keep phash NULL).
|
||||
phash = None
|
||||
if not is_video(source):
|
||||
with Image.open(source) as im:
|
||||
phash = compute_phash(im)
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
)
|
||||
if rel == "larger_exists":
|
||||
return ImportResult(
|
||||
status="skipped",
|
||||
skip_reason=SkipReason.duplicate_phash,
|
||||
image_id=match_id,
|
||||
error="perceptual near-duplicate of larger existing image",
|
||||
)
|
||||
if rel == "smaller_exists":
|
||||
target = self.session.get(ImageRecord, match_id)
|
||||
self._supersede(target, source, sha, phash, width, height)
|
||||
return ImportResult(
|
||||
status="superseded", image_id=match_id
|
||||
)
|
||||
|
||||
dest = self._copy_to_library(source, sha, attribution_path)
|
||||
|
||||
record = ImageRecord(
|
||||
path=str(dest),
|
||||
sha256=sha,
|
||||
phash=phash,
|
||||
size_bytes=dest.stat().st_size,
|
||||
mime=_mime_for(source),
|
||||
width=width,
|
||||
height=height,
|
||||
origin="imported_filesystem",
|
||||
integrity_status="unknown",
|
||||
)
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
|
||||
# Folder→artist (anchored to attribution_path).
|
||||
artist = None
|
||||
artist_name = derive_top_level_artist(
|
||||
attribution_path, self.import_root
|
||||
)
|
||||
if artist_name:
|
||||
artist = self._attach_artist(record, artist_name)
|
||||
|
||||
# Sidecar provenance (best-effort; never fails the import).
|
||||
self._apply_sidecar(record, attribution_path, artist)
|
||||
|
||||
# Thumbnail is queued separately by the calling task; the importer
|
||||
# does not generate thumbnails inline so the import queue stays moving.
|
||||
|
||||
self.session.commit()
|
||||
return ImportResult(status="imported", image_id=record.id)
|
||||
|
||||
def _deep_rederive(
|
||||
self, existing: ImageRecord, source: Path, attribution_path: Path
|
||||
) -> ImportResult:
|
||||
"""Deep scan: backfill phash/provenance/artist on an
|
||||
already-imported record. METADATA ONLY — never re-runs the pHash
|
||||
near-dup / supersede path. NULL-only, idempotent."""
|
||||
if existing.phash is None and not is_video(source):
|
||||
try:
|
||||
with Image.open(source) as im:
|
||||
ph = compute_phash(im)
|
||||
if ph is not None:
|
||||
existing.phash = ph
|
||||
except Exception as exc:
|
||||
log.warning("deep rephash failed for %s: %s", source, exc)
|
||||
|
||||
artist = None
|
||||
name = derive_top_level_artist(attribution_path, self.import_root)
|
||||
if name:
|
||||
artist = self._upsert_artist(name)
|
||||
if existing.artist_id is None:
|
||||
existing.artist_id = artist.id
|
||||
|
||||
self._apply_sidecar(existing, attribution_path, artist)
|
||||
self.session.commit()
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="deep: re-derived",
|
||||
)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
path: Path,
|
||||
*,
|
||||
sidecar_path: Path | None = None,
|
||||
artist: Artist | None = None,
|
||||
source: Source | None = None,
|
||||
) -> ImportResult:
|
||||
"""Attach a file ALREADY at its final library location (FC-3c).
|
||||
|
||||
Mirrors _import_media but skips the copy step — the file is
|
||||
where it'll permanently live. Caller (DownloadService) already
|
||||
has artist/source context from the subscription Source row; pass
|
||||
them through. The sidecar JSON gallery-dl emits next to each
|
||||
downloaded file is read by `_apply_sidecar` via `find_sidecar`.
|
||||
|
||||
Caller's responsibilities after this returns:
|
||||
- duplicate_hash / duplicate_phash skip → delete the on-disk file
|
||||
- superseded → file stays where it is (now canonical)
|
||||
- imported → file stays where it is
|
||||
- failed → file untouched; caller decides
|
||||
"""
|
||||
if not is_supported(path):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"unsupported extension {path.suffix}",
|
||||
)
|
||||
|
||||
# Format / dimension / transparency filters (mirror _import_media).
|
||||
width = height = None
|
||||
has_alpha = False
|
||||
if not is_video(path):
|
||||
try:
|
||||
with Image.open(path) as im:
|
||||
im.verify()
|
||||
with Image.open(path) as im:
|
||||
width, height = im.size
|
||||
has_alpha = im.mode in ("RGBA", "LA") or (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
)
|
||||
except Exception as exc:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
if (self.settings.min_width and width < self.settings.min_width) or (
|
||||
self.settings.min_height and height < self.settings.min_height
|
||||
):
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_small,
|
||||
error=f"{width}x{height} below min {self.settings.min_width}x{self.settings.min_height}",
|
||||
)
|
||||
|
||||
if self.settings.skip_transparent and has_alpha:
|
||||
pct = self._transparency_pct(path)
|
||||
if pct >= self.settings.transparency_threshold:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.too_transparent,
|
||||
error=f"{pct:.2%} transparent",
|
||||
)
|
||||
|
||||
# Hash dedup
|
||||
sha = _sha256_of(path)
|
||||
existing = self.session.execute(
|
||||
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.duplicate_hash,
|
||||
image_id=existing.id, error="sha256 already present",
|
||||
)
|
||||
|
||||
# phash dedup + supersede
|
||||
phash = None
|
||||
if not is_video(path):
|
||||
try:
|
||||
with Image.open(path) as im:
|
||||
phash = compute_phash(im)
|
||||
except Exception:
|
||||
phash = None
|
||||
if phash is not None:
|
||||
cand_rows = self.session.execute(
|
||||
select(
|
||||
ImageRecord.phash,
|
||||
ImageRecord.width,
|
||||
ImageRecord.height,
|
||||
ImageRecord.id,
|
||||
).where(ImageRecord.phash.is_not(None))
|
||||
).all()
|
||||
candidates = [
|
||||
(c.phash, c.width or 0, c.height or 0, c.id)
|
||||
for c in cand_rows
|
||||
]
|
||||
rel, match_id = find_similar(
|
||||
phash, width or 0, height or 0,
|
||||
candidates, self.settings.phash_threshold,
|
||||
)
|
||||
if rel == "larger_exists":
|
||||
return ImportResult(
|
||||
status="skipped",
|
||||
skip_reason=SkipReason.duplicate_phash,
|
||||
image_id=match_id,
|
||||
error="perceptual near-duplicate of larger existing image",
|
||||
)
|
||||
if rel == "smaller_exists":
|
||||
target = self.session.get(ImageRecord, match_id)
|
||||
self._supersede(
|
||||
target, path, sha, phash, width, height, new_path=path
|
||||
)
|
||||
return ImportResult(status="superseded", image_id=match_id)
|
||||
|
||||
# Create record — the path IS where the file lives.
|
||||
record = ImageRecord(
|
||||
path=str(path),
|
||||
sha256=sha,
|
||||
phash=phash,
|
||||
size_bytes=path.stat().st_size,
|
||||
mime=_mime_for(path),
|
||||
width=width,
|
||||
height=height,
|
||||
origin="downloaded",
|
||||
integrity_status="unknown",
|
||||
)
|
||||
if artist is not None:
|
||||
record.artist_id = artist.id
|
||||
self.session.add(record)
|
||||
self.session.flush()
|
||||
|
||||
# Sidecar provenance (best-effort). When `source` is passed, link
|
||||
# the post to that subscription Source instead of creating a new
|
||||
# per-post Source row.
|
||||
self._apply_sidecar(record, path, artist, explicit_source=source)
|
||||
|
||||
self.session.commit()
|
||||
return ImportResult(status="imported", image_id=record.id)
|
||||
|
||||
def _upsert_artist(self, name: str) -> Artist:
|
||||
slug = slugify(name)
|
||||
artist = self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
).scalar_one_or_none()
|
||||
if artist is None:
|
||||
artist = Artist(name=name, slug=slug, is_subscription=False)
|
||||
self.session.add(artist)
|
||||
self.session.flush()
|
||||
return artist
|
||||
|
||||
def _attach_artist(self, record: ImageRecord, artist_name: str) -> Artist:
|
||||
"""Upsert the Artist and set it as the image's canonical artist.
|
||||
Artist-kind tags were retired in FC-2d-vii-c — the Artist row is
|
||||
the single source of truth. Returns the Artist (sidecar
|
||||
provenance attaches its Source to it)."""
|
||||
artist = self._upsert_artist(artist_name)
|
||||
if record.artist_id is None:
|
||||
record.artist_id = artist.id
|
||||
self.session.flush()
|
||||
return artist
|
||||
|
||||
@staticmethod
|
||||
def _sidecar_artist_name(data: dict) -> str | None:
|
||||
for k in ("artist", "author_name"):
|
||||
v = data.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
for k in ("user", "creator"):
|
||||
v = data.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()
|
||||
if isinstance(v, dict):
|
||||
for sub in ("name", "full_name", "vanity", "account"):
|
||||
sv = v.get(sub)
|
||||
if isinstance(sv, str) and sv.strip():
|
||||
return sv.strip()
|
||||
return None
|
||||
|
||||
def _apply_sidecar(
|
||||
self,
|
||||
record: ImageRecord,
|
||||
source: Path,
|
||||
artist: Artist | None,
|
||||
*,
|
||||
explicit_source: Source | None = None,
|
||||
) -> None:
|
||||
"""Read the sidecar adjacent to `source` and wire up provenance.
|
||||
|
||||
If `explicit_source` is passed (FC-3c attach_in_place case), the
|
||||
sidecar's post is linked to that Source instead of looking up /
|
||||
creating one keyed by the sidecar's post_url. Without it, the
|
||||
filesystem importer's per-post-Source creation is preserved.
|
||||
"""
|
||||
sc = find_sidecar(source)
|
||||
if sc is None:
|
||||
return
|
||||
try:
|
||||
data = json.loads(sc.read_text("utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("sidecar JSON is not an object")
|
||||
except Exception as exc:
|
||||
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
||||
return
|
||||
|
||||
sd = parse_sidecar(data)
|
||||
|
||||
if artist is None:
|
||||
name = self._sidecar_artist_name(data)
|
||||
if name:
|
||||
artist = self._upsert_artist(name)
|
||||
if artist is None:
|
||||
log.warning(
|
||||
"sidecar present for %s but no artist; skipping provenance",
|
||||
source,
|
||||
)
|
||||
return
|
||||
|
||||
if record.artist_id is None:
|
||||
record.artist_id = artist.id
|
||||
|
||||
if explicit_source is not None:
|
||||
src = explicit_source
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
if sd.post_url is not None:
|
||||
post.post_url = sd.post_url
|
||||
if sd.post_title is not None:
|
||||
post.post_title = sd.post_title
|
||||
if sd.post_date is not None:
|
||||
post.post_date = sd.post_date
|
||||
if sd.description is not None:
|
||||
post.description = sd.description
|
||||
if sd.attachment_count is not None:
|
||||
post.attachment_count = sd.attachment_count
|
||||
post.raw_metadata = sd.raw
|
||||
|
||||
exists = self.session.execute(
|
||||
select(ImageProvenance.id).where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
ImageProvenance.post_id == post.id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists is None:
|
||||
self.session.add(
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=src.id,
|
||||
captured_metadata=sd.raw,
|
||||
)
|
||||
)
|
||||
if record.primary_post_id is None:
|
||||
record.primary_post_id = post.id
|
||||
self.session.flush()
|
||||
|
||||
def _copy_to_library(
|
||||
self, source: Path, sha: str, attribution_path: Path
|
||||
) -> Path:
|
||||
"""Copy `source` to its final library path. Returns the destination.
|
||||
|
||||
Atomic write: copies to .partial then renames. Shared by
|
||||
_import_media (filesystem scan) and _supersede (when new_path is
|
||||
not passed). FC-3c's attach_in_place skips this helper entirely
|
||||
— the file is already at its final home.
|
||||
"""
|
||||
subdir = derive_subdir(attribution_path, self.import_root)
|
||||
dest_dir = self.images_root / subdir if subdir else self.images_root
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_name = hash_suffixed_name(source.stem, sha, source.suffix)
|
||||
dest = dest_dir / dest_name
|
||||
partial = dest.with_suffix(dest.suffix + ".partial")
|
||||
shutil.copy2(source, partial)
|
||||
partial.rename(dest)
|
||||
return dest
|
||||
|
||||
def _supersede(
|
||||
self, existing: ImageRecord, source: Path, sha: str,
|
||||
phash: str, width: int | None, height: int | None,
|
||||
*, new_path: Path | None = None,
|
||||
) -> None:
|
||||
"""Replace `existing`'s file with the larger `source`, keeping the
|
||||
row id (so tags/series/curation stay attached). ML is cleared so
|
||||
the import task re-derives it on the new pixels.
|
||||
|
||||
If `new_path` is provided, `source` is assumed to ALREADY be at
|
||||
that path (FC-3c attach_in_place case) — skip the copy step.
|
||||
Otherwise the file is copied via _copy_to_library."""
|
||||
if new_path is None:
|
||||
dest = self._copy_to_library(source, sha, source)
|
||||
else:
|
||||
dest = new_path
|
||||
|
||||
old_path = existing.path
|
||||
old_thumb = existing.thumbnail_path
|
||||
|
||||
existing.path = str(dest)
|
||||
existing.sha256 = sha
|
||||
existing.phash = phash
|
||||
existing.size_bytes = dest.stat().st_size
|
||||
existing.mime = _mime_for(source)
|
||||
existing.width = width
|
||||
existing.height = height
|
||||
existing.thumbnail_path = None
|
||||
existing.integrity_status = "unknown"
|
||||
existing.tagger_predictions = None
|
||||
existing.tagger_model_version = None
|
||||
existing.siglip_embedding = None
|
||||
existing.siglip_model_version = None
|
||||
existing.centroid_scores = None
|
||||
# created_at intentionally preserved; updated_at auto-bumps.
|
||||
self.session.flush()
|
||||
self.session.commit()
|
||||
|
||||
for stale in (old_path, old_thumb):
|
||||
if not stale or stale == str(dest):
|
||||
# If the supersede kept the file in place (new_path == old
|
||||
# location), don't delete it.
|
||||
continue
|
||||
try:
|
||||
p = Path(stale)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
except Exception:
|
||||
# Benign orphan; the DB swap already committed. Don't undo it.
|
||||
pass
|
||||
|
||||
def _transparency_pct(self, source: Path) -> float:
|
||||
"""Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
|
||||
with Image.open(source) as im:
|
||||
if im.mode not in ("RGBA", "LA") and not (
|
||||
im.mode == "P" and "transparency" in im.info
|
||||
):
|
||||
return 0.0
|
||||
if im.mode != "RGBA":
|
||||
im = im.convert("RGBA")
|
||||
alpha = im.getchannel("A")
|
||||
histogram = alpha.histogram()
|
||||
transparent = histogram[0]
|
||||
total = sum(histogram)
|
||||
return transparent / total if total else 0.0
|
||||
@@ -0,0 +1,6 @@
|
||||
"""FC-5 migration tooling.
|
||||
|
||||
One module per concern (backup/rollback/gs/ir/overlap/ml_queue/verify).
|
||||
Each migrator returns a counts dict; the run_migration task wires
|
||||
that dict into MigrationRun.counts so the UI polling shows progress.
|
||||
"""
|
||||
@@ -0,0 +1,124 @@
|
||||
"""pg_dump + tar.zst-based backup, restorable via pair of subprocess calls.
|
||||
|
||||
Backups live under <images_root>/_backups/. Each backup is two files
|
||||
(SQL + tarball) plus a manifest JSON. Tagged backups (e.g. tag='pre_migration')
|
||||
are how rollback.py finds the most recent restorable snapshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_BACKUPS_DIRNAME = "_backups"
|
||||
|
||||
|
||||
def _backups_dir(images_root: Path | None = None) -> Path:
|
||||
# Overridable for tests via monkeypatch.
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
p = root / _BACKUPS_DIRNAME
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _run_subprocess(cmd: list[str], **kwargs: Any):
|
||||
# Overridable for tests via monkeypatch.
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
**{k: v for k, v in kwargs.items() if not k.startswith("_")},
|
||||
)
|
||||
|
||||
|
||||
def create_backup(
|
||||
*, db_url: str, images_root: Path, tag: str = "manual",
|
||||
) -> dict:
|
||||
"""Create a backup: pg_dump SQL + tar.zst of images.
|
||||
|
||||
Returns a manifest dict. Writes <ts>.sql, <ts>.tar.zst, <ts>.json into
|
||||
<images_root>/_backups/.
|
||||
"""
|
||||
ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
out_dir = _backups_dir(images_root)
|
||||
sql_path = out_dir / f"fc_{ts}.sql"
|
||||
tar_path = out_dir / f"fc_{ts}.tar.zst"
|
||||
manifest_path = out_dir / f"fc_{ts}.json"
|
||||
|
||||
_run_subprocess(
|
||||
["pg_dump", "--no-owner", "--no-acl", "-f", str(sql_path), db_url],
|
||||
_test_ts=ts,
|
||||
)
|
||||
_run_subprocess(
|
||||
[
|
||||
"tar", "--zstd", "-cf", str(tar_path),
|
||||
"-C", str(images_root.parent), images_root.name,
|
||||
f"--exclude={images_root.name}/_backups",
|
||||
f"--exclude={images_root.name}/_quarantine",
|
||||
],
|
||||
_test_ts=ts,
|
||||
)
|
||||
|
||||
manifest = {
|
||||
"backup_id": ts,
|
||||
"tag": tag,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"sql_path": str(sql_path),
|
||||
"tar_path": str(tar_path),
|
||||
}
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2))
|
||||
return manifest
|
||||
|
||||
|
||||
def list_backups(images_root: Path) -> list[dict]:
|
||||
out_dir = _backups_dir(images_root)
|
||||
items = []
|
||||
for mf in sorted(out_dir.glob("fc_*.json"), reverse=True):
|
||||
try:
|
||||
items.append(json.loads(mf.read_text()))
|
||||
except Exception:
|
||||
continue
|
||||
return items
|
||||
|
||||
|
||||
def find_latest_backup(images_root: Path, *, tag: str) -> dict | None:
|
||||
for mf in list_backups(images_root):
|
||||
if mf.get("tag") == tag:
|
||||
return mf
|
||||
return None
|
||||
|
||||
|
||||
def restore_backup(
|
||||
*, manifest: dict, db_url: str, images_root: Path,
|
||||
) -> dict:
|
||||
"""Restore from a backup manifest.
|
||||
|
||||
1. Replay the .sql via psql.
|
||||
2. Wipe /images/ contents (except _backups/, which holds the file we're using).
|
||||
3. Untar the .tar.zst into /images/.
|
||||
"""
|
||||
sql_path = Path(manifest["sql_path"])
|
||||
tar_path = Path(manifest["tar_path"])
|
||||
|
||||
_run_subprocess(
|
||||
["psql", "-d", db_url, "-f", str(sql_path)],
|
||||
)
|
||||
|
||||
# Wipe everything in images_root EXCEPT _backups/ (we'd delete the backup
|
||||
# we're restoring from!).
|
||||
for entry in images_root.iterdir():
|
||||
if entry.name == _BACKUPS_DIRNAME:
|
||||
continue
|
||||
if entry.is_dir():
|
||||
shutil.rmtree(entry)
|
||||
else:
|
||||
entry.unlink()
|
||||
|
||||
_run_subprocess(
|
||||
["tar", "--zstd", "-xf", str(tar_path), "-C", str(images_root.parent)],
|
||||
)
|
||||
|
||||
return {"restored_from": manifest["backup_id"]}
|
||||
@@ -0,0 +1,126 @@
|
||||
"""GallerySubscriber export → FabledCurator ingest.
|
||||
|
||||
Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection
|
||||
to GS). Creates Artist (from subscriptions) + Source (nested under each
|
||||
subscription) + Credential (re-encrypted with FC's key). Idempotent on
|
||||
natural keys: Artist.slug, (artist_id, platform, url), Credential.platform.
|
||||
|
||||
Credentials arrive plaintext in the export — GS's export script
|
||||
decrypts using GS's Fernet key in GS's own process. FC re-encrypts
|
||||
with FC's CredentialCrypto.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Artist, Credential, Source
|
||||
from ...utils.slug import slugify
|
||||
from ..credential_crypto import CredentialCrypto
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
||||
}
|
||||
|
||||
|
||||
async def migrate_async(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
data: dict,
|
||||
fc_crypto: CredentialCrypto | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Ingest a parsed gallerysubscriber-export-v1.json dict."""
|
||||
if data.get("source_app") != "gallerysubscriber":
|
||||
raise ValueError("export source_app must be 'gallerysubscriber'")
|
||||
if data.get("schema_version") != 1:
|
||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
||||
|
||||
counts = _zero_counts()
|
||||
|
||||
# Phase 1: subscriptions → Artist; nested sources within each.
|
||||
for sub in data.get("subscriptions", []):
|
||||
counts["rows_processed"] += 1
|
||||
slug = slugify(sub["name"])
|
||||
artist = (await db.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
# Continue to nested sources, but they can't link without an artist row.
|
||||
continue
|
||||
notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None
|
||||
artist = Artist(
|
||||
name=sub["name"], slug=slug,
|
||||
is_subscription=True,
|
||||
auto_check=bool(sub.get("enabled", True)),
|
||||
notes=notes,
|
||||
)
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
counts["rows_inserted"] += 1
|
||||
else:
|
||||
counts["rows_skipped"] += 1
|
||||
|
||||
# Nested sources under this subscription.
|
||||
for src in sub.get("sources", []):
|
||||
counts["rows_processed"] += 1
|
||||
existing = (await db.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == src["platform"],
|
||||
Source.url == src["url"],
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(Source(
|
||||
artist_id=artist.id,
|
||||
platform=src["platform"],
|
||||
url=src["url"],
|
||||
enabled=bool(src.get("enabled", True)),
|
||||
check_interval_override=src.get("check_interval"),
|
||||
config_overrides=src.get("metadata") or {},
|
||||
))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
# Phase 2: credentials.
|
||||
for cred in data.get("credentials", []):
|
||||
counts["rows_processed"] += 1
|
||||
existing = (await db.execute(
|
||||
select(Credential).where(Credential.platform == cred["platform"])
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
if fc_crypto is None:
|
||||
# Without a crypto helper we can't encrypt — skip rather than
|
||||
# store plaintext.
|
||||
counts["rows_skipped"] += 1
|
||||
counts["conflicts"] += 1
|
||||
continue
|
||||
encrypted = fc_crypto.encrypt(cred["plaintext"])
|
||||
db.add(Credential(
|
||||
platform=cred["platform"],
|
||||
credential_type=cred.get("credential_type") or "cookies",
|
||||
encrypted_blob=encrypted,
|
||||
expires_at=cred.get("expires_at"),
|
||||
))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
return counts
|
||||
@@ -0,0 +1,141 @@
|
||||
"""ImageRepo export → FabledCurator ingest.
|
||||
|
||||
Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR).
|
||||
Creates Tag rows (skipping artist/post kinds, resolving fandom_name to
|
||||
FK). Writes the per-image-sha256 artist assignments + tag associations
|
||||
+ series page assignments to /images/_migration_state/ir_tag_manifest.json
|
||||
so tag_apply.py can join them to ImageRecord rows AFTER the operator
|
||||
runs FC's filesystem scan.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Tag, TagKind
|
||||
|
||||
_SKIP_KINDS = frozenset({"artist", "post"})
|
||||
_MIGRATION_STATE_DIRNAME = "_migration_state"
|
||||
_IR_MANIFEST_FILENAME = "ir_tag_manifest.json"
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
||||
}
|
||||
|
||||
|
||||
def manifest_path(images_root: Path | None = None) -> Path:
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
p = root / _MIGRATION_STATE_DIRNAME
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p / _IR_MANIFEST_FILENAME
|
||||
|
||||
|
||||
async def _resolve_fandom_id(
|
||||
db: AsyncSession, fandom_name: str | None, dry_run: bool,
|
||||
) -> int | None:
|
||||
"""Find-or-create a fandom-kind Tag by name."""
|
||||
if not fandom_name:
|
||||
return None
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom")
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing.id
|
||||
if dry_run:
|
||||
return None
|
||||
t = Tag(name=fandom_name, kind=TagKind.fandom)
|
||||
db.add(t)
|
||||
await db.flush()
|
||||
return t.id
|
||||
|
||||
|
||||
async def migrate_async(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
data: dict,
|
||||
images_root: Path | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Ingest a parsed imagerepo-export-v1.json dict.
|
||||
|
||||
Creates Tag rows + writes the IR tag manifest file. Tag-to-image
|
||||
binding happens later in tag_apply.py (after FC's filesystem scan
|
||||
populates image_record.sha256 → id).
|
||||
"""
|
||||
if data.get("source_app") != "imagerepo":
|
||||
raise ValueError("export source_app must be 'imagerepo'")
|
||||
if data.get("schema_version") != 1:
|
||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
||||
|
||||
counts = _zero_counts()
|
||||
|
||||
# Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id).
|
||||
# First pass: create all fandom-kind tags so they're available for FK resolution.
|
||||
for tag in data.get("tags", []):
|
||||
kind = tag.get("kind") or "general"
|
||||
if kind != "fandom":
|
||||
continue
|
||||
counts["rows_processed"] += 1
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom")
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(Tag(name=tag["name"], kind=TagKind.fandom))
|
||||
counts["rows_inserted"] += 1
|
||||
if not dry_run:
|
||||
await db.flush()
|
||||
|
||||
# Second pass: every other kind.
|
||||
for tag in data.get("tags", []):
|
||||
kind_str = tag.get("kind") or "general"
|
||||
if kind_str in _SKIP_KINDS:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if kind_str == "fandom":
|
||||
continue # handled above
|
||||
counts["rows_processed"] += 1
|
||||
try:
|
||||
kind = TagKind(kind_str)
|
||||
except ValueError:
|
||||
kind = TagKind.general
|
||||
fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run)
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == kind)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
|
||||
# Phase 2: write the per-image manifest for tag_apply.py to consume later.
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"image_artist_assignments": data.get("image_artist_assignments", []),
|
||||
"image_tag_associations": data.get("image_tag_associations", []),
|
||||
"series_pages": data.get("series_pages", []),
|
||||
}
|
||||
counts["rows_processed"] += len(manifest["image_artist_assignments"])
|
||||
counts["rows_processed"] += len(manifest["image_tag_associations"])
|
||||
counts["rows_processed"] += len(manifest["series_pages"])
|
||||
if not dry_run:
|
||||
manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
|
||||
|
||||
return counts
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Queue every migrated image_record with no embedding for ML re-processing."""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import ImageRecord
|
||||
|
||||
|
||||
async def queue_all_unprocessed_async(db: AsyncSession) -> int:
|
||||
"""Find every ImageRecord with siglip_embedding IS NULL, fire
|
||||
tag_and_embed.delay(id) for each. Returns count queued.
|
||||
"""
|
||||
from ...tasks.ml import tag_and_embed
|
||||
|
||||
rows = (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None))
|
||||
)).scalars().all()
|
||||
for image_id in rows:
|
||||
tag_and_embed.delay(image_id)
|
||||
return len(rows)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Restore from the most recent 'pre_migration'-tagged backup."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from . import backup as backup_mod
|
||||
|
||||
|
||||
class NoBackupFoundError(Exception):
|
||||
"""Raised when rollback() is called with no pre_migration backup on disk."""
|
||||
|
||||
|
||||
def rollback_to_pre_migration(*, db_url: str, images_root: Path) -> dict:
|
||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
||||
if manifest is None:
|
||||
raise NoBackupFoundError(
|
||||
"no pre_migration-tagged backup found under <images_root>/_backups/"
|
||||
)
|
||||
return backup_mod.restore_backup(
|
||||
manifest=manifest, db_url=db_url, images_root=images_root,
|
||||
)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Apply the IR tag manifest after FC's filesystem scan.
|
||||
|
||||
Reads /images/_migration_state/ir_tag_manifest.json and joins each entry
|
||||
to an ImageRecord row by sha256 (which exists after the operator runs
|
||||
FC's filesystem scan over the mounted IR images dir).
|
||||
|
||||
- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug).
|
||||
- image_tag_associations → image_tag insert (idempotent).
|
||||
- series_pages → series_page insert (idempotent on image_id unique).
|
||||
|
||||
Unmatched sha256s are logged into the result's `unmatched` list so the
|
||||
Celery task can drop them into MigrationRun.metadata for the operator
|
||||
to inspect.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag
|
||||
from ...utils.slug import slugify
|
||||
from .ir_ingest import manifest_path
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
||||
}
|
||||
|
||||
|
||||
async def _ensure_artist_id(
|
||||
db: AsyncSession, artist_name: str, dry_run: bool,
|
||||
) -> int | None:
|
||||
if not artist_name or not artist_name.strip():
|
||||
return None
|
||||
slug = slugify(artist_name)
|
||||
existing = (await db.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing.id
|
||||
if dry_run:
|
||||
return None
|
||||
a = Artist(name=artist_name, slug=slug, is_subscription=False)
|
||||
db.add(a)
|
||||
await db.flush()
|
||||
return a.id
|
||||
|
||||
|
||||
async def _resolve_tag_id(
|
||||
db: AsyncSession, tag_name: str, tag_kind: str,
|
||||
) -> int | None:
|
||||
try:
|
||||
kind = TagKind(tag_kind)
|
||||
except ValueError:
|
||||
kind = TagKind.general
|
||||
row = (await db.execute(
|
||||
select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind)
|
||||
)).scalar_one_or_none()
|
||||
return row
|
||||
|
||||
|
||||
async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None:
|
||||
return (await db.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def apply_async(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
images_root: Path | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Apply the manifest. Returns counts + an `unmatched` list of sha256s."""
|
||||
mf_path = manifest_path(images_root)
|
||||
if not mf_path.exists():
|
||||
raise FileNotFoundError(f"no IR tag manifest at {mf_path}")
|
||||
|
||||
manifest = json.loads(mf_path.read_text())
|
||||
counts = _zero_counts()
|
||||
unmatched: list[dict] = []
|
||||
|
||||
# 1. Artist assignments.
|
||||
for entry in manifest.get("image_artist_assignments", []):
|
||||
counts["rows_processed"] += 1
|
||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
||||
if img_id is None:
|
||||
unmatched.append({"kind": "artist", **entry})
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
aid = await _ensure_artist_id(db, entry["artist_name"], dry_run)
|
||||
if aid is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
img = await db.get(ImageRecord, img_id)
|
||||
if img is not None and img.artist_id != aid:
|
||||
img.artist_id = aid
|
||||
counts["rows_inserted"] += 1
|
||||
else:
|
||||
counts["rows_skipped"] += 1
|
||||
|
||||
# 2. Tag associations.
|
||||
for entry in manifest.get("image_tag_associations", []):
|
||||
counts["rows_processed"] += 1
|
||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
||||
if img_id is None:
|
||||
unmatched.append({"kind": "tag", **entry})
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
tag_id = await _resolve_tag_id(
|
||||
db, entry["tag_name"], entry.get("tag_kind") or "general",
|
||||
)
|
||||
if tag_id is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
# Skip if association already exists.
|
||||
already = (await db.execute(
|
||||
select(image_tag.c.image_record_id).where(
|
||||
image_tag.c.image_record_id == img_id,
|
||||
image_tag.c.tag_id == tag_id,
|
||||
)
|
||||
)).first()
|
||||
if already is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
await db.execute(image_tag.insert().values(
|
||||
image_record_id=img_id, tag_id=tag_id, source="manual",
|
||||
))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
# 3. Series pages.
|
||||
for entry in manifest.get("series_pages", []):
|
||||
counts["rows_processed"] += 1
|
||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
||||
if img_id is None:
|
||||
unmatched.append({"kind": "series", **entry})
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series")
|
||||
if series_tag_id is None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
existing = (await db.execute(
|
||||
select(SeriesPage).where(SeriesPage.image_id == img_id)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(SeriesPage(
|
||||
series_tag_id=series_tag_id,
|
||||
image_id=img_id,
|
||||
page_number=entry["page_number"],
|
||||
))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
return {"counts": counts, "unmatched": unmatched}
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Post-migration verification: row counts + sha256 sampling."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Artist, Credential, ImageRecord, Source, Tag
|
||||
|
||||
|
||||
async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict:
|
||||
"""Return per-check status dicts. `expected` is optional row-count
|
||||
assertions; checks default to status='ok' when no expected provided."""
|
||||
expected = expected or {}
|
||||
results: dict[str, dict] = {}
|
||||
|
||||
checks = {
|
||||
"artist_subscriptions": (
|
||||
select(func.count(Artist.id)).where(Artist.is_subscription.is_(True))
|
||||
),
|
||||
"source_count": select(func.count(Source.id)),
|
||||
"credential_count": select(func.count(Credential.id)),
|
||||
"tag_count": select(func.count(Tag.id)),
|
||||
"image_record_imported_or_downloaded": (
|
||||
select(func.count(ImageRecord.id))
|
||||
.where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"]))
|
||||
),
|
||||
}
|
||||
for name, stmt in checks.items():
|
||||
actual = (await db.execute(stmt)).scalar_one()
|
||||
exp = expected.get(name)
|
||||
status = "ok" if exp is None or exp == actual else "mismatch"
|
||||
results[name] = {"status": status, "actual": int(actual), "expected": exp}
|
||||
return results
|
||||
|
||||
|
||||
async def verify_sha256_sample(
|
||||
db: AsyncSession, *, sample_size: int = 20,
|
||||
) -> dict:
|
||||
"""Sample N image_records; verify file exists + sha256 matches."""
|
||||
rows = (await db.execute(
|
||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
||||
.order_by(func.random()).limit(sample_size)
|
||||
)).all()
|
||||
|
||||
matched = 0
|
||||
mismatched = 0
|
||||
missing = 0
|
||||
samples: list[dict] = []
|
||||
for img_id, path, expected_sha in rows:
|
||||
p = Path(path)
|
||||
# Sync stdlib filesystem ops are intentional: this verify pass runs
|
||||
# inside a Celery task under asyncio.run; no other awaitables compete
|
||||
# for the loop. Same pattern as download_service.py.
|
||||
if not p.exists(): # noqa: ASYNC240
|
||||
missing += 1
|
||||
samples.append({"id": img_id, "path": path, "result": "missing"})
|
||||
continue
|
||||
h = hashlib.sha256()
|
||||
with p.open("rb") as f: # noqa: ASYNC230
|
||||
for chunk in iter(lambda: f.read(65536), b""):
|
||||
h.update(chunk)
|
||||
if h.hexdigest() == expected_sha:
|
||||
matched += 1
|
||||
samples.append({"id": img_id, "result": "ok"})
|
||||
else:
|
||||
mismatched += 1
|
||||
samples.append({
|
||||
"id": img_id, "path": path, "result": "mismatch",
|
||||
"expected_sha": expected_sha, "actual_sha": h.hexdigest(),
|
||||
})
|
||||
return {
|
||||
"sample_size": len(rows),
|
||||
"matched": matched,
|
||||
"mismatched": mismatched,
|
||||
"missing": missing,
|
||||
"samples": samples,
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
"""ML pipeline services: tagger, embedder, suggestions, centroids, allowlist, aliases."""
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Alias resolution + CRUD.
|
||||
|
||||
A tag_alias maps (model_name, model_category) -> canonical Tag. Resolution
|
||||
happens at suggestion-read time so raw tagger_predictions stay unmolested.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Tag, TagAlias
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AliasRow:
|
||||
alias_string: str
|
||||
alias_category: str
|
||||
canonical_tag_id: int
|
||||
canonical_tag_name: str
|
||||
|
||||
|
||||
class AliasService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def resolve(self, name: str, category: str) -> Tag | None:
|
||||
"""Return the canonical Tag for (name, category), or None if no alias."""
|
||||
stmt = (
|
||||
select(Tag)
|
||||
.join(TagAlias, TagAlias.canonical_tag_id == Tag.id)
|
||||
.where(TagAlias.alias_string == name)
|
||||
.where(TagAlias.alias_category == category)
|
||||
)
|
||||
return (await self.session.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
async def resolve_many(
|
||||
self, pairs: list[tuple[str, str]]
|
||||
) -> dict[tuple[str, str], Tag]:
|
||||
"""Batch-resolve. Returns only the pairs that have an alias.
|
||||
|
||||
Used by SuggestionService so it does one query instead of N.
|
||||
"""
|
||||
if not pairs:
|
||||
return {}
|
||||
strings = {p[0] for p in pairs}
|
||||
stmt = (
|
||||
select(TagAlias, Tag)
|
||||
.join(Tag, Tag.id == TagAlias.canonical_tag_id)
|
||||
.where(TagAlias.alias_string.in_(strings))
|
||||
)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
wanted = set(pairs)
|
||||
out: dict[tuple[str, str], Tag] = {}
|
||||
for alias, tag in rows:
|
||||
key = (alias.alias_string, alias.alias_category)
|
||||
if key in wanted:
|
||||
out[key] = tag
|
||||
return out
|
||||
|
||||
async def create(
|
||||
self, alias_string: str, alias_category: str, canonical_tag_id: int
|
||||
) -> None:
|
||||
"""Idempotent create (ON CONFLICT DO NOTHING)."""
|
||||
stmt = insert(TagAlias).values(
|
||||
alias_string=alias_string,
|
||||
alias_category=alias_category,
|
||||
canonical_tag_id=canonical_tag_id,
|
||||
)
|
||||
stmt = stmt.on_conflict_do_nothing(
|
||||
index_elements=["alias_string", "alias_category"]
|
||||
)
|
||||
await self.session.execute(stmt)
|
||||
|
||||
async def remove(self, alias_string: str, alias_category: str) -> None:
|
||||
await self.session.execute(
|
||||
delete(TagAlias)
|
||||
.where(TagAlias.alias_string == alias_string)
|
||||
.where(TagAlias.alias_category == alias_category)
|
||||
)
|
||||
|
||||
async def list_all(self) -> Sequence[AliasRow]:
|
||||
stmt = (
|
||||
select(
|
||||
TagAlias.alias_string,
|
||||
TagAlias.alias_category,
|
||||
TagAlias.canonical_tag_id,
|
||||
Tag.name,
|
||||
)
|
||||
.join(Tag, Tag.id == TagAlias.canonical_tag_id)
|
||||
.order_by(TagAlias.alias_string.asc())
|
||||
)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
return [
|
||||
AliasRow(
|
||||
alias_string=r[0],
|
||||
alias_category=r[1],
|
||||
canonical_tag_id=r[2],
|
||||
canonical_tag_name=r[3],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user