refactor(I5): remove one-and-done GS/IR migration tooling
The GS/IR migration cutover is complete, so the runbook tooling is dead weight. Removed: - services/migrators/ (gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup), tasks/migration.py, api/migrate.py (+ blueprint registration) - MigrationRun model; alembic 0027 drops the migration_run table - frontend LegacyMigrationCard + migration store (+ MaintenancePanel ref) - celery include + task route + celery_signals queue mapping for migration.* - the 1 GB MAX_CONTENT_LENGTH / MAX_FORM_MEMORY override (added solely for the ir_ingest upload) - migration-surface tests (test_api_migrate, test_migration_verify, test_ir_ingest, test_gs_ingest, test_tag_apply) Kept: the alembic schema-migration tests (test_migration_00XX — unrelated) and cleanup_service.py (the permanent artist-cascade/unlink home). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,50 @@
|
|||||||
|
"""drop migration_run — one-and-done GS/IR migration tooling removed
|
||||||
|
|
||||||
|
Revision ID: 0027
|
||||||
|
Revises: 0026
|
||||||
|
Create Date: 2026-05-29
|
||||||
|
|
||||||
|
The GS/IR migration tooling (services/migrators, /api/migrate, the
|
||||||
|
run_migration task, LegacyMigrationCard, and the MigrationRun model) was
|
||||||
|
removed after the migration cutover completed. This drops its now-orphaned
|
||||||
|
run-log table. Downgrade recreates the table (mirrors the old model) so the
|
||||||
|
migration is reversible.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
revision: str = "0027"
|
||||||
|
down_revision: Union[str, None] = "0026"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.drop_table("migration_run")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"migration_run",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("status", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||||
|
sa.Column(
|
||||||
|
"started_at", sa.DateTime(timezone=True), nullable=False,
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
),
|
||||||
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
sa.Column("error", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index("ix_migration_run_kind", "migration_run", ["kind"])
|
||||||
|
op.create_index("ix_migration_run_status", "migration_run", ["status"])
|
||||||
@@ -33,12 +33,6 @@ def create_app() -> Quart:
|
|||||||
|
|
||||||
app = Quart(__name__)
|
app = Quart(__name__)
|
||||||
app.secret_key = cfg.secret_key
|
app.secret_key = cfg.secret_key
|
||||||
# FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of
|
|
||||||
# thousands of image_tag_associations). Werkzeug's default form
|
|
||||||
# memory cap is 500KB; raise both ceilings so the multipart upload
|
|
||||||
# for /api/migrate/ir_ingest doesn't 413.
|
|
||||||
app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GB
|
|
||||||
app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024 # 1 GB
|
|
||||||
|
|
||||||
for bp in all_blueprints():
|
for bp in all_blueprints():
|
||||||
app.register_blueprint(bp)
|
app.register_blueprint(bp)
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .extension import extension_bp
|
from .extension import extension_bp
|
||||||
from .gallery import gallery_bp
|
from .gallery import gallery_bp
|
||||||
from .import_admin import import_admin_bp
|
from .import_admin import import_admin_bp
|
||||||
from .migrate import migrate_bp
|
|
||||||
from .ml_admin import ml_admin_bp
|
from .ml_admin import ml_admin_bp
|
||||||
from .platforms import platforms_bp
|
from .platforms import platforms_bp
|
||||||
from .posts import posts_bp
|
from .posts import posts_bp
|
||||||
@@ -54,7 +53,6 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
admin_bp,
|
admin_bp,
|
||||||
cleanup_bp,
|
cleanup_bp,
|
||||||
import_admin_bp,
|
import_admin_bp,
|
||||||
migrate_bp,
|
|
||||||
suggestions_bp,
|
suggestions_bp,
|
||||||
allowlist_bp,
|
allowlist_bp,
|
||||||
aliases_bp,
|
aliases_bp,
|
||||||
|
|||||||
@@ -1,107 +0,0 @@
|
|||||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
|
||||||
|
|
||||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
|
||||||
`export_file` field. All other kinds accept JSON. Backup + rollback
|
|
||||||
were retired in FC-3h (2026-05-24); use /api/system/backup/* instead.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from ..extensions import get_session
|
|
||||||
from ..models import MigrationRun
|
|
||||||
from ..tasks.migration import run_migration
|
|
||||||
from ._responses import error_response as _bad
|
|
||||||
|
|
||||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
|
||||||
|
|
||||||
# 'backup' + 'rollback' retired 2026-05-24 (FC-3h); see /api/system/backup/*.
|
|
||||||
_VALID_KINDS = frozenset({
|
|
||||||
"gs_ingest", "ir_ingest", "tag_apply",
|
|
||||||
"ml_queue", "verify", "cleanup",
|
|
||||||
})
|
|
||||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
|
||||||
|
|
||||||
|
|
||||||
def _run_to_dict(run: MigrationRun) -> dict:
|
|
||||||
return {
|
|
||||||
"id": run.id,
|
|
||||||
"kind": run.kind,
|
|
||||||
"status": run.status,
|
|
||||||
"dry_run": run.dry_run,
|
|
||||||
"started_at": run.started_at.isoformat(),
|
|
||||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
|
||||||
"counts": run.counts or {},
|
|
||||||
"error": run.error,
|
|
||||||
"metadata": run.metadata_ or {},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/<kind>", methods=["POST"])
|
|
||||||
async def create_run(kind: str):
|
|
||||||
if kind not in _VALID_KINDS:
|
|
||||||
return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}")
|
|
||||||
|
|
||||||
# Ingest kinds accept multipart/form-data; everything else takes JSON.
|
|
||||||
if kind in _INGEST_KINDS:
|
|
||||||
form = await request.form
|
|
||||||
files = await request.files
|
|
||||||
if "export_file" not in files:
|
|
||||||
return _bad("missing_export_file", detail="multipart export_file required")
|
|
||||||
export_file = files["export_file"]
|
|
||||||
try:
|
|
||||||
raw = export_file.read()
|
|
||||||
data = json.loads(raw.decode("utf-8"))
|
|
||||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
||||||
return _bad("invalid_export_file", detail=str(exc))
|
|
||||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
|
||||||
params: dict = {"data": data, "dry_run": dry_run}
|
|
||||||
else:
|
|
||||||
body = await request.get_json()
|
|
||||||
if body is None:
|
|
||||||
body = {}
|
|
||||||
if not isinstance(body, dict):
|
|
||||||
return _bad("invalid_body")
|
|
||||||
dry_run = bool(body.get("dry_run", False))
|
|
||||||
params = dict(body)
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
|
||||||
session.add(run)
|
|
||||||
await session.commit()
|
|
||||||
await session.refresh(run)
|
|
||||||
run_id = run.id
|
|
||||||
|
|
||||||
run_migration.delay(run_id, kind, params)
|
|
||||||
return jsonify({"run_id": run_id, "status": "pending"}), 202
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/runs/<int:run_id>", methods=["GET"])
|
|
||||||
async def get_run(run_id: int):
|
|
||||||
async with get_session() as session:
|
|
||||||
run = (await session.execute(
|
|
||||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if run is None:
|
|
||||||
return _bad("not_found", status=404)
|
|
||||||
return jsonify(_run_to_dict(run))
|
|
||||||
|
|
||||||
|
|
||||||
@migrate_bp.route("/runs", methods=["GET"])
|
|
||||||
async def list_runs():
|
|
||||||
try:
|
|
||||||
limit = int(request.args.get("limit", "10"))
|
|
||||||
except ValueError:
|
|
||||||
return _bad("invalid_limit")
|
|
||||||
if limit < 1 or limit > 100:
|
|
||||||
return _bad("invalid_limit")
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
rows = (await session.execute(
|
|
||||||
select(MigrationRun)
|
|
||||||
.order_by(MigrationRun.id.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)).scalars().all()
|
|
||||||
return jsonify([_run_to_dict(r) for r in rows])
|
|
||||||
@@ -28,7 +28,6 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.import_file",
|
"backend.app.tasks.import_file",
|
||||||
"backend.app.tasks.thumbnail",
|
"backend.app.tasks.thumbnail",
|
||||||
"backend.app.tasks.maintenance",
|
"backend.app.tasks.maintenance",
|
||||||
"backend.app.tasks.migration",
|
|
||||||
"backend.app.tasks.ml",
|
"backend.app.tasks.ml",
|
||||||
"backend.app.tasks.download",
|
"backend.app.tasks.download",
|
||||||
"backend.app.tasks.backup",
|
"backend.app.tasks.backup",
|
||||||
@@ -45,7 +44,6 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.download.*": {"queue": "download"},
|
"backend.app.tasks.download.*": {"queue": "download"},
|
||||||
"backend.app.tasks.scan.*": {"queue": "scan"},
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.migration.*": {"queue": "maintenance"},
|
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
||||||
|
|||||||
@@ -66,10 +66,7 @@ def _queue_for(task) -> str:
|
|||||||
return "download"
|
return "download"
|
||||||
if name.startswith("backend.app.tasks.scan."):
|
if name.startswith("backend.app.tasks.scan."):
|
||||||
return "scan"
|
return "scan"
|
||||||
if name.startswith((
|
if name.startswith("backend.app.tasks.maintenance."):
|
||||||
"backend.app.tasks.maintenance.",
|
|
||||||
"backend.app.tasks.migration.",
|
|
||||||
)):
|
|
||||||
return "maintenance"
|
return "maintenance"
|
||||||
return "default"
|
return "default"
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from .import_batch import ImportBatch
|
|||||||
from .import_settings import ImportSettings
|
from .import_settings import ImportSettings
|
||||||
from .import_task import ImportTask
|
from .import_task import ImportTask
|
||||||
from .library_audit_run import LibraryAuditRun
|
from .library_audit_run import LibraryAuditRun
|
||||||
from .migration_run import MigrationRun
|
|
||||||
from .ml_settings import MLSettings
|
from .ml_settings import MLSettings
|
||||||
from .post import Post
|
from .post import Post
|
||||||
from .post_attachment import PostAttachment
|
from .post_attachment import PostAttachment
|
||||||
@@ -46,7 +45,6 @@ __all__ = [
|
|||||||
"ImportSettings",
|
"ImportSettings",
|
||||||
"LibraryAuditRun",
|
"LibraryAuditRun",
|
||||||
"MLSettings",
|
"MLSettings",
|
||||||
"MigrationRun",
|
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
"TagAllowlist",
|
"TagAllowlist",
|
||||||
"TagReferenceEmbedding",
|
"TagReferenceEmbedding",
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
"""MigrationRun — tracks each FC-5 migration invocation (backup/gs/ir/etc).
|
|
||||||
|
|
||||||
kind/status are String(32) not Postgres ENUM so adding kinds later
|
|
||||||
doesn't need a schema migration. The API layer validates values.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from .base import Base
|
|
||||||
|
|
||||||
|
|
||||||
class MigrationRun(Base):
|
|
||||||
__tablename__ = "migration_run"
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
|
||||||
status: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
|
||||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
|
||||||
)
|
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=True,
|
|
||||||
)
|
|
||||||
counts: Mapped[dict] = mapped_column(
|
|
||||||
JSONB, nullable=False, default=dict, server_default=sa.text("'{}'::jsonb"),
|
|
||||||
)
|
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
metadata_: Mapped[dict] = mapped_column(
|
|
||||||
"metadata", JSONB, nullable=False, default=dict,
|
|
||||||
server_default=sa.text("'{}'::jsonb"),
|
|
||||||
)
|
|
||||||
@@ -6,9 +6,8 @@ HTTP handlers (small ops) and from Celery tasks in
|
|||||||
backend.app.tasks.admin (long ops).
|
backend.app.tasks.admin (long ops).
|
||||||
|
|
||||||
This module is the PERMANENT home of artist-cascade + image-unlink
|
This module is the PERMANENT home of artist-cascade + image-unlink
|
||||||
logic. The legacy copy at backend/app/services/migrators/cleanup.py
|
logic. (The legacy migrators/cleanup.py copy was removed with the rest of
|
||||||
stays in place until FC-3j; FC-3j will replace its body with thin
|
the one-and-done GS/IR migration tooling.)
|
||||||
re-exports from this module and then delete the wrapper.
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +0,0 @@
|
|||||||
"""FC-5 migration tooling.
|
|
||||||
|
|
||||||
One module per concern (gs/ir/overlap/ml_queue/verify/cleanup).
|
|
||||||
Each migrator returns a counts dict; the run_migration task wires
|
|
||||||
that dict into MigrationRun.counts so the UI polling shows progress.
|
|
||||||
|
|
||||||
backup + rollback were retired in FC-3h (2026-05-24); first-class
|
|
||||||
backup lives at backend/app/services/backup_service.py and exposes
|
|
||||||
its own /api/system/backup/* surface.
|
|
||||||
"""
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
"""Targeted cleanup migrator: delete every image attributed to one Artist.
|
|
||||||
|
|
||||||
Built for the IR-migration rescue case where the filesystem scan derived
|
|
||||||
a bogus 'imagerepo' artist from a mismatched bind-mount layout. Every
|
|
||||||
image attributed to that artist (40k+ rows) needs to be removed — DB
|
|
||||||
rows, original files under `/images/<bucket>/...`, and thumbnails under
|
|
||||||
`/images/thumbs/...` — before the operator remounts and re-scans.
|
|
||||||
|
|
||||||
CASCADE handles image_tag, image_provenance, series_page, and
|
|
||||||
tag_suggestion_rejection child rows; import_task.result_image_id is
|
|
||||||
SET NULL by FK. We also delete ImportTask rows whose source_path starts
|
|
||||||
with the (still-existing) IR scan prefix so the next scan isn't fooled
|
|
||||||
by them.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, ImageRecord, ImportBatch, ImportTask
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
_BATCH_SIZE = 500
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _thumb_path(images_root: Path, sha256_hex: str) -> tuple[Path, Path]:
|
|
||||||
"""Return both possible thumbnail paths (.jpg and .png). We try both
|
|
||||||
because the extension is chosen at generate-time based on the source
|
|
||||||
image's mode (alpha → .png, otherwise → .jpg)."""
|
|
||||||
bucket = sha256_hex[:3]
|
|
||||||
base = images_root / "thumbs" / bucket / sha256_hex
|
|
||||||
return base.with_suffix(".jpg"), base.with_suffix(".png")
|
|
||||||
|
|
||||||
|
|
||||||
def _delete_file(path: Path) -> bool:
|
|
||||||
"""Best-effort unlink; True if the file was actually removed."""
|
|
||||||
try:
|
|
||||||
path.unlink(missing_ok=True)
|
|
||||||
return True
|
|
||||||
except OSError as exc:
|
|
||||||
log.warning("cleanup: failed to unlink %s: %s", path, exc)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
async def cleanup_artist_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
slug: str,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
source_path_prefix: str | None = None,
|
|
||||||
) -> dict:
|
|
||||||
"""Delete every image attributed to the Artist with this slug,
|
|
||||||
along with the artist row itself and any associated import tasks.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
slug: artist.slug to target (e.g. 'imagerepo').
|
|
||||||
images_root: defaults to /images.
|
|
||||||
dry_run: skip filesystem + DB writes; still walk rows for counts.
|
|
||||||
source_path_prefix: if set, ImportTask rows whose source_path
|
|
||||||
starts with this string are deleted too (use the IR scan
|
|
||||||
mount prefix, e.g. '/import/imagerepo').
|
|
||||||
"""
|
|
||||||
root = images_root if images_root is not None else Path("/images")
|
|
||||||
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if artist is None:
|
|
||||||
raise ValueError(f"no Artist with slug={slug!r}")
|
|
||||||
|
|
||||||
artist_id = artist.id
|
|
||||||
artist_name = artist.name
|
|
||||||
|
|
||||||
total_images = (await db.execute(
|
|
||||||
select(func.count(ImageRecord.id)).where(ImageRecord.artist_id == artist_id)
|
|
||||||
)).scalar_one()
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
files_deleted = 0
|
|
||||||
thumbs_deleted = 0
|
|
||||||
images_deleted = 0
|
|
||||||
|
|
||||||
# Batched delete loop. CASCADE handles image_tag, image_provenance,
|
|
||||||
# series_page, tag_suggestion_rejection. import_task.result_image_id
|
|
||||||
# is SET NULL by FK.
|
|
||||||
while True:
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
|
||||||
.where(ImageRecord.artist_id == artist_id)
|
|
||||||
.limit(_BATCH_SIZE)
|
|
||||||
)).all()
|
|
||||||
if not rows:
|
|
||||||
break
|
|
||||||
|
|
||||||
ids = [r.id for r in rows]
|
|
||||||
counts["rows_processed"] += len(ids)
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
for r in rows:
|
|
||||||
if r.path:
|
|
||||||
if _delete_file(Path(r.path)):
|
|
||||||
files_deleted += 1
|
|
||||||
if r.sha256:
|
|
||||||
jpg, png = _thumb_path(root, r.sha256)
|
|
||||||
if _delete_file(jpg):
|
|
||||||
thumbs_deleted += 1
|
|
||||||
if _delete_file(png):
|
|
||||||
thumbs_deleted += 1
|
|
||||||
|
|
||||||
await db.execute(
|
|
||||||
delete(ImageRecord).where(ImageRecord.id.in_(ids))
|
|
||||||
)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
images_deleted += len(ids)
|
|
||||||
|
|
||||||
if dry_run:
|
|
||||||
# Nothing was actually deleted from the DB; bail after one
|
|
||||||
# pass so we don't loop forever.
|
|
||||||
break
|
|
||||||
|
|
||||||
import_tasks_deleted = 0
|
|
||||||
if source_path_prefix and not dry_run:
|
|
||||||
# Delete ImportTask rows whose source_path is under the bad mount
|
|
||||||
# prefix. These are mostly orphaned now (result_image_id was set
|
|
||||||
# NULL by CASCADE) but their presence still blocks the
|
|
||||||
# idempotency check in scan_directory if the operator remounts
|
|
||||||
# the same prefix.
|
|
||||||
like_pattern = source_path_prefix.rstrip("/") + "/%"
|
|
||||||
result = await db.execute(
|
|
||||||
delete(ImportTask).where(ImportTask.source_path.like(like_pattern))
|
|
||||||
)
|
|
||||||
import_tasks_deleted = result.rowcount or 0
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Sweep ImportBatch rows that are now empty.
|
|
||||||
empty_batches_deleted = 0
|
|
||||||
if not dry_run:
|
|
||||||
empty_batch_ids = (await db.execute(
|
|
||||||
select(ImportBatch.id).where(
|
|
||||||
~select(ImportTask.id)
|
|
||||||
.where(ImportTask.batch_id == ImportBatch.id)
|
|
||||||
.exists()
|
|
||||||
)
|
|
||||||
)).scalars().all()
|
|
||||||
if empty_batch_ids:
|
|
||||||
result = await db.execute(
|
|
||||||
delete(ImportBatch).where(ImportBatch.id.in_(empty_batch_ids))
|
|
||||||
)
|
|
||||||
empty_batches_deleted = result.rowcount or 0
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Finally, the artist row.
|
|
||||||
if not dry_run:
|
|
||||||
await db.execute(delete(Artist).where(Artist.id == artist_id))
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"counts": counts,
|
|
||||||
"artist": {"id": artist_id, "name": artist_name, "slug": slug},
|
|
||||||
"summary": {
|
|
||||||
"images_targeted": total_images,
|
|
||||||
"images_deleted": images_deleted,
|
|
||||||
"files_deleted": files_deleted,
|
|
||||||
"thumbs_deleted": thumbs_deleted,
|
|
||||||
"import_tasks_deleted": import_tasks_deleted,
|
|
||||||
"empty_batches_deleted": empty_batches_deleted,
|
|
||||||
"dry_run": dry_run,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
"""GallerySubscriber export → FabledCurator ingest.
|
|
||||||
|
|
||||||
Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection
|
|
||||||
to GS). Creates Artist (from subscriptions) + Source (nested under each
|
|
||||||
subscription) + Credential (re-encrypted with FC's key). Idempotent on
|
|
||||||
natural keys: Artist.slug, (artist_id, platform, url), Credential.platform.
|
|
||||||
|
|
||||||
Credentials arrive plaintext in the export — GS's export script
|
|
||||||
decrypts using GS's Fernet key in GS's own process. FC re-encrypts
|
|
||||||
with FC's CredentialCrypto.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, Credential, Source
|
|
||||||
from ...utils.slug import slugify
|
|
||||||
from ..credential_crypto import CredentialCrypto
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def migrate_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
data: dict,
|
|
||||||
fc_crypto: CredentialCrypto | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Ingest a parsed gallerysubscriber-export-v1.json dict."""
|
|
||||||
if data.get("source_app") != "gallerysubscriber":
|
|
||||||
raise ValueError("export source_app must be 'gallerysubscriber'")
|
|
||||||
if data.get("schema_version") != 1:
|
|
||||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
|
|
||||||
# Phase 1: subscriptions → Artist; nested sources within each.
|
|
||||||
for sub in data.get("subscriptions", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
slug = slugify(sub["name"])
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if artist is None:
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
# Continue to nested sources, but they can't link without an artist row.
|
|
||||||
continue
|
|
||||||
notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None
|
|
||||||
artist = Artist(
|
|
||||||
name=sub["name"], slug=slug,
|
|
||||||
is_subscription=True,
|
|
||||||
auto_check=bool(sub.get("enabled", True)),
|
|
||||||
notes=notes,
|
|
||||||
)
|
|
||||||
db.add(artist)
|
|
||||||
await db.flush()
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
# Nested sources under this subscription.
|
|
||||||
for src in sub.get("sources", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Source).where(
|
|
||||||
Source.artist_id == artist.id,
|
|
||||||
Source.platform == src["platform"],
|
|
||||||
Source.url == src["url"],
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Source(
|
|
||||||
artist_id=artist.id,
|
|
||||||
platform=src["platform"],
|
|
||||||
url=src["url"],
|
|
||||||
enabled=bool(src.get("enabled", True)),
|
|
||||||
check_interval_override=src.get("check_interval"),
|
|
||||||
config_overrides=src.get("metadata") or {},
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# Phase 2: credentials.
|
|
||||||
for cred in data.get("credentials", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Credential).where(Credential.platform == cred["platform"])
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
if fc_crypto is None:
|
|
||||||
# Without a crypto helper we can't encrypt — skip rather than
|
|
||||||
# store plaintext.
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
counts["conflicts"] += 1
|
|
||||||
continue
|
|
||||||
encrypted = fc_crypto.encrypt(cred["plaintext"])
|
|
||||||
db.add(Credential(
|
|
||||||
platform=cred["platform"],
|
|
||||||
credential_type=cred.get("credential_type") or "cookies",
|
|
||||||
encrypted_blob=encrypted,
|
|
||||||
expires_at=cred.get("expires_at"),
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
return counts
|
|
||||||
@@ -1,146 +0,0 @@
|
|||||||
"""ImageRepo export → FabledCurator ingest.
|
|
||||||
|
|
||||||
Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR).
|
|
||||||
Creates Tag rows (skipping artist/post kinds, resolving fandom_name to
|
|
||||||
FK). Writes the per-image-sha256 artist assignments + tag associations
|
|
||||||
+ series page assignments to /images/_migration_state/ir_tag_manifest.json
|
|
||||||
so tag_apply.py can join them to ImageRecord rows AFTER the operator
|
|
||||||
runs FC's filesystem scan.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Tag, TagKind
|
|
||||||
|
|
||||||
_SKIP_KINDS = frozenset({"artist", "post"})
|
|
||||||
_MIGRATION_STATE_DIRNAME = "_migration_state"
|
|
||||||
_IR_MANIFEST_FILENAME = "ir_tag_manifest.json"
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def manifest_path(images_root: Path | None = None) -> Path:
|
|
||||||
root = images_root if images_root is not None else Path("/images")
|
|
||||||
p = root / _MIGRATION_STATE_DIRNAME
|
|
||||||
p.mkdir(parents=True, exist_ok=True)
|
|
||||||
return p / _IR_MANIFEST_FILENAME
|
|
||||||
|
|
||||||
|
|
||||||
async def _resolve_fandom_id(
|
|
||||||
db: AsyncSession, fandom_name: str | None, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
"""Find-or-create a fandom-kind Tag by name."""
|
|
||||||
if not fandom_name:
|
|
||||||
return None
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing.id
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
t = Tag(name=fandom_name, kind=TagKind.fandom)
|
|
||||||
db.add(t)
|
|
||||||
await db.flush()
|
|
||||||
return t.id
|
|
||||||
|
|
||||||
|
|
||||||
async def migrate_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
data: dict,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Ingest a parsed imagerepo-export-v1.json dict.
|
|
||||||
|
|
||||||
Creates Tag rows + writes the IR tag manifest file. Tag-to-image
|
|
||||||
binding happens later in tag_apply.py (after FC's filesystem scan
|
|
||||||
populates image_record.sha256 → id).
|
|
||||||
"""
|
|
||||||
if data.get("source_app") != "imagerepo":
|
|
||||||
raise ValueError("export source_app must be 'imagerepo'")
|
|
||||||
if data.get("schema_version") not in (1, 2):
|
|
||||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
|
||||||
|
|
||||||
counts = _zero_counts()
|
|
||||||
|
|
||||||
# Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id).
|
|
||||||
# First pass: create all fandom-kind tags so they're available for FK resolution.
|
|
||||||
for tag in data.get("tags", []):
|
|
||||||
kind = tag.get("kind") or "general"
|
|
||||||
if kind != "fandom":
|
|
||||||
continue
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Tag(name=tag["name"], kind=TagKind.fandom))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
if not dry_run:
|
|
||||||
await db.flush()
|
|
||||||
|
|
||||||
# Second pass: every other kind.
|
|
||||||
for tag in data.get("tags", []):
|
|
||||||
kind_str = tag.get("kind") or "general"
|
|
||||||
if kind_str in _SKIP_KINDS:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if kind_str == "fandom":
|
|
||||||
continue # handled above
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
try:
|
|
||||||
kind = TagKind(kind_str)
|
|
||||||
except ValueError:
|
|
||||||
kind = TagKind.general
|
|
||||||
fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run)
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == kind)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
# Phase 2: write the per-image manifest for tag_apply.py to consume later.
|
|
||||||
# schema_version 2 (added 2026-05-24) carries `image_posts` for
|
|
||||||
# Post + Source + ImageProvenance restore; schema 1 manifests
|
|
||||||
# without it stay valid (tag_apply treats the missing field as []).
|
|
||||||
manifest = {
|
|
||||||
"schema_version": data.get("schema_version", 1),
|
|
||||||
"image_artist_assignments": data.get("image_artist_assignments", []),
|
|
||||||
"image_tag_associations": data.get("image_tag_associations", []),
|
|
||||||
"series_pages": data.get("series_pages", []),
|
|
||||||
"image_posts": data.get("image_posts", []),
|
|
||||||
}
|
|
||||||
counts["rows_processed"] += len(manifest["image_artist_assignments"])
|
|
||||||
counts["rows_processed"] += len(manifest["image_tag_associations"])
|
|
||||||
counts["rows_processed"] += len(manifest["series_pages"])
|
|
||||||
counts["rows_processed"] += len(manifest["image_posts"])
|
|
||||||
if not dry_run:
|
|
||||||
manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
|
|
||||||
|
|
||||||
return counts
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
"""Queue every migrated image_record with no embedding for ML re-processing."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import ImageRecord
|
|
||||||
|
|
||||||
|
|
||||||
async def queue_all_unprocessed_async(db: AsyncSession) -> int:
|
|
||||||
"""Find every ImageRecord with siglip_embedding IS NULL, fire
|
|
||||||
tag_and_embed.delay(id) for each. Returns count queued.
|
|
||||||
"""
|
|
||||||
from ...tasks.ml import tag_and_embed
|
|
||||||
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_(None))
|
|
||||||
)).scalars().all()
|
|
||||||
for image_id in rows:
|
|
||||||
tag_and_embed.delay(image_id)
|
|
||||||
return len(rows)
|
|
||||||
@@ -1,368 +0,0 @@
|
|||||||
"""Apply the IR tag manifest after FC's filesystem scan.
|
|
||||||
|
|
||||||
Reads /images/_migration_state/ir_tag_manifest.json and joins each entry
|
|
||||||
to an ImageRecord row by sha256 (which exists after the operator runs
|
|
||||||
FC's filesystem scan over the mounted IR images dir).
|
|
||||||
|
|
||||||
- image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug).
|
|
||||||
- image_tag_associations → image_tag insert (idempotent).
|
|
||||||
- series_pages → series_page insert (idempotent on image_id unique).
|
|
||||||
- image_posts (schema v2) → Source + Post + ImageProvenance restore.
|
|
||||||
|
|
||||||
Unmatched sha256s are logged into the result's `unmatched` list so the
|
|
||||||
Celery task can drop them into MigrationRun.metadata for the operator
|
|
||||||
to inspect.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
from datetime import datetime
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import (
|
|
||||||
Artist,
|
|
||||||
ImageProvenance,
|
|
||||||
ImageRecord,
|
|
||||||
Post,
|
|
||||||
SeriesPage,
|
|
||||||
Source,
|
|
||||||
Tag,
|
|
||||||
TagKind,
|
|
||||||
image_tag,
|
|
||||||
)
|
|
||||||
from ...utils.slug import slugify
|
|
||||||
from .ir_ingest import manifest_path
|
|
||||||
|
|
||||||
# Per-platform artist-profile URL — used as Source.url when restoring
|
|
||||||
# IR PostMetadata into FC. Must cover every platform that
|
|
||||||
# backend/app/services/extension_service.py:_PLATFORM_PATTERNS
|
|
||||||
# recognizes; an entry missing here silently drops ALL PostMetadata for
|
|
||||||
# that platform during phase 4 (operator hit this 2026-05-25:
|
|
||||||
# DeviantArt + Pixiv posts in the IR migration produced empty
|
|
||||||
# ImageProvenance because they fell through this table).
|
|
||||||
#
|
|
||||||
# Pixiv caveat: the real profile URL takes a numeric user_id
|
|
||||||
# (https://www.pixiv.net/users/12345), but IR's PostMetadata.artist
|
|
||||||
# stores the display name not the id. We use the slugified name here
|
|
||||||
# so we preserve the artist→post→image linkage; the resulting Source.url
|
|
||||||
# won't resolve in a browser and the operator may want to manually fix
|
|
||||||
# it via Settings → Subscriptions once the migration lands.
|
|
||||||
_PLATFORM_PROFILE_URL = {
|
|
||||||
"patreon": "https://www.patreon.com/{slug}",
|
|
||||||
"subscribestar": "https://www.subscribestar.com/{slug}",
|
|
||||||
"hentaifoundry": "https://www.hentai-foundry.com/user/{slug}",
|
|
||||||
"deviantart": "https://www.deviantart.com/{slug}",
|
|
||||||
"pixiv": "https://www.pixiv.net/users/{slug}",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _profile_url(platform: str, artist_slug: str) -> str | None:
|
|
||||||
fmt = _PLATFORM_PROFILE_URL.get(platform)
|
|
||||||
return fmt.format(slug=artist_slug) if fmt else None
|
|
||||||
|
|
||||||
|
|
||||||
async def _find_or_create_source(
|
|
||||||
db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Source.id).where(
|
|
||||||
Source.artist_id == artist_id,
|
|
||||||
Source.platform == platform,
|
|
||||||
Source.url == url,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False)
|
|
||||||
db.add(s)
|
|
||||||
await db.flush()
|
|
||||||
return s.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _find_or_create_post(
|
|
||||||
db: AsyncSession, *,
|
|
||||||
source_id: int, external_post_id: str,
|
|
||||||
title: str | None, description: str | None, post_url: str | None,
|
|
||||||
post_date_iso: str | None, attachment_count: int, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Post.id).where(
|
|
||||||
Post.source_id == source_id,
|
|
||||||
Post.external_post_id == external_post_id,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
post_date = None
|
|
||||||
if post_date_iso:
|
|
||||||
post_date = datetime.fromisoformat(post_date_iso)
|
|
||||||
p = Post(
|
|
||||||
source_id=source_id,
|
|
||||||
external_post_id=external_post_id,
|
|
||||||
post_title=title,
|
|
||||||
description=description,
|
|
||||||
post_url=post_url,
|
|
||||||
post_date=post_date,
|
|
||||||
attachment_count=attachment_count,
|
|
||||||
raw_metadata={"migrated_from": "imagerepo"},
|
|
||||||
)
|
|
||||||
db.add(p)
|
|
||||||
await db.flush()
|
|
||||||
return p.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _ensure_provenance(
|
|
||||||
db: AsyncSession, *,
|
|
||||||
image_id: int, post_id: int, source_id: int, dry_run: bool,
|
|
||||||
) -> bool:
|
|
||||||
"""Returns True if a new ImageProvenance row was inserted.
|
|
||||||
|
|
||||||
Also sets ImageRecord.primary_post_id to this post if the image
|
|
||||||
doesn't already have one — preserves any primary_post_id already
|
|
||||||
assigned at download time by the importer (don't clobber). This is
|
|
||||||
the linkage gallery_service.py uses to surface Post.post_date as
|
|
||||||
the image's effective date for sort/group/jump/neighbor nav.
|
|
||||||
"""
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(ImageProvenance.id).where(
|
|
||||||
ImageProvenance.image_record_id == image_id,
|
|
||||||
ImageProvenance.post_id == post_id,
|
|
||||||
ImageProvenance.source_id == source_id,
|
|
||||||
)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
|
|
||||||
# Whether-or-not the provenance row already exists, ensure the
|
|
||||||
# image's primary_post_id is set so the gallery date-coalesce works.
|
|
||||||
# Idempotent: only writes when currently NULL.
|
|
||||||
if not dry_run:
|
|
||||||
await db.execute(
|
|
||||||
ImageRecord.__table__.update()
|
|
||||||
.where(ImageRecord.id == image_id)
|
|
||||||
.where(ImageRecord.primary_post_id.is_(None))
|
|
||||||
.values(primary_post_id=post_id)
|
|
||||||
)
|
|
||||||
|
|
||||||
if existing is not None:
|
|
||||||
return False
|
|
||||||
if dry_run:
|
|
||||||
return True
|
|
||||||
db.add(ImageProvenance(
|
|
||||||
image_record_id=image_id, post_id=post_id, source_id=source_id,
|
|
||||||
))
|
|
||||||
await db.flush()
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _zero_counts() -> dict:
|
|
||||||
return {
|
|
||||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _ensure_artist_id(
|
|
||||||
db: AsyncSession, artist_name: str, dry_run: bool,
|
|
||||||
) -> int | None:
|
|
||||||
if not artist_name or not artist_name.strip():
|
|
||||||
return None
|
|
||||||
slug = slugify(artist_name)
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
return existing.id
|
|
||||||
if dry_run:
|
|
||||||
return None
|
|
||||||
a = Artist(name=artist_name, slug=slug, is_subscription=False)
|
|
||||||
db.add(a)
|
|
||||||
await db.flush()
|
|
||||||
return a.id
|
|
||||||
|
|
||||||
|
|
||||||
async def _resolve_tag_id(
|
|
||||||
db: AsyncSession, tag_name: str, tag_kind: str,
|
|
||||||
) -> int | None:
|
|
||||||
try:
|
|
||||||
kind = TagKind(tag_kind)
|
|
||||||
except ValueError:
|
|
||||||
kind = TagKind.general
|
|
||||||
row = (await db.execute(
|
|
||||||
select(Tag.id).where(Tag.name == tag_name, Tag.kind == kind)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
return row
|
|
||||||
|
|
||||||
|
|
||||||
async def _sha_to_image_id(db: AsyncSession, sha: str) -> int | None:
|
|
||||||
return (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
|
|
||||||
|
|
||||||
async def apply_async(
|
|
||||||
db: AsyncSession,
|
|
||||||
*,
|
|
||||||
images_root: Path | None = None,
|
|
||||||
dry_run: bool = False,
|
|
||||||
) -> dict:
|
|
||||||
"""Apply the manifest. Returns counts + an `unmatched` list of sha256s."""
|
|
||||||
mf_path = manifest_path(images_root)
|
|
||||||
if not mf_path.exists():
|
|
||||||
raise FileNotFoundError(f"no IR tag manifest at {mf_path}")
|
|
||||||
|
|
||||||
manifest = json.loads(mf_path.read_text())
|
|
||||||
counts = _zero_counts()
|
|
||||||
unmatched: list[dict] = []
|
|
||||||
|
|
||||||
# 1. Artist assignments.
|
|
||||||
for entry in manifest.get("image_artist_assignments", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "artist", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
aid = await _ensure_artist_id(db, entry["artist_name"], dry_run)
|
|
||||||
if aid is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
img = await db.get(ImageRecord, img_id)
|
|
||||||
if img is not None and img.artist_id != aid:
|
|
||||||
img.artist_id = aid
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
# 2. Tag associations.
|
|
||||||
for entry in manifest.get("image_tag_associations", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "tag", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
tag_id = await _resolve_tag_id(
|
|
||||||
db, entry["tag_name"], entry.get("tag_kind") or "general",
|
|
||||||
)
|
|
||||||
if tag_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
# Skip if association already exists.
|
|
||||||
already = (await db.execute(
|
|
||||||
select(image_tag.c.image_record_id).where(
|
|
||||||
image_tag.c.image_record_id == img_id,
|
|
||||||
image_tag.c.tag_id == tag_id,
|
|
||||||
)
|
|
||||||
)).first()
|
|
||||||
if already is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
await db.execute(image_tag.insert().values(
|
|
||||||
image_record_id=img_id, tag_id=tag_id, source="manual",
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# 3. Series pages.
|
|
||||||
for entry in manifest.get("series_pages", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
img_id = await _sha_to_image_id(db, entry["sha256"])
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({"kind": "series", **entry})
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
series_tag_id = await _resolve_tag_id(db, entry["series_tag_name"], "series")
|
|
||||||
if series_tag_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
existing = (await db.execute(
|
|
||||||
select(SeriesPage).where(SeriesPage.image_id == img_id)
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
if dry_run:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
continue
|
|
||||||
db.add(SeriesPage(
|
|
||||||
series_tag_id=series_tag_id,
|
|
||||||
image_id=img_id,
|
|
||||||
page_number=entry["page_number"],
|
|
||||||
))
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
|
|
||||||
# 4. Image posts (schema v2) → Source + Post + ImageProvenance.
|
|
||||||
# Restores IR PostMetadata as FC's downloader-track provenance,
|
|
||||||
# so the modal's ProvenancePanel surfaces title/description/
|
|
||||||
# source URL/publish date the same way it does for live
|
|
||||||
# gallery-dl downloads.
|
|
||||||
for entry in manifest.get("image_posts", []):
|
|
||||||
counts["rows_processed"] += 1
|
|
||||||
platform = entry.get("platform")
|
|
||||||
artist_name = entry.get("artist")
|
|
||||||
if not platform or not artist_name:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
aid = await _ensure_artist_id(db, artist_name, dry_run)
|
|
||||||
if aid is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
url = _profile_url(platform, slugify(artist_name))
|
|
||||||
if url is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
source_id = await _find_or_create_source(
|
|
||||||
db, artist_id=aid, platform=platform, url=url, dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if source_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
post_id = await _find_or_create_post(
|
|
||||||
db, source_id=source_id,
|
|
||||||
external_post_id=entry.get("post_id") or "",
|
|
||||||
title=entry.get("title"),
|
|
||||||
description=entry.get("description"),
|
|
||||||
post_url=entry.get("source_url"),
|
|
||||||
post_date_iso=entry.get("published_at"),
|
|
||||||
attachment_count=entry.get("attachment_count") or 0,
|
|
||||||
dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if post_id is None:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
for sha in entry.get("image_sha256s", []):
|
|
||||||
img_id = await _sha_to_image_id(db, sha)
|
|
||||||
if img_id is None:
|
|
||||||
unmatched.append({
|
|
||||||
"kind": "post", "sha256": sha,
|
|
||||||
"post_id": entry.get("post_id"),
|
|
||||||
})
|
|
||||||
continue
|
|
||||||
inserted = await _ensure_provenance(
|
|
||||||
db, image_id=img_id, post_id=post_id,
|
|
||||||
source_id=source_id, dry_run=dry_run,
|
|
||||||
)
|
|
||||||
if inserted:
|
|
||||||
counts["rows_inserted"] += 1
|
|
||||||
else:
|
|
||||||
counts["rows_skipped"] += 1
|
|
||||||
|
|
||||||
if not dry_run:
|
|
||||||
await db.commit()
|
|
||||||
return {"counts": counts, "unmatched": unmatched}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
"""Post-migration verification: row counts + sha256 sampling."""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ...models import Artist, Credential, ImageRecord, Source, Tag
|
|
||||||
|
|
||||||
|
|
||||||
async def verify_async(db: AsyncSession, *, expected: dict | None = None) -> dict:
|
|
||||||
"""Return per-check status dicts. `expected` is optional row-count
|
|
||||||
assertions; checks default to status='ok' when no expected provided."""
|
|
||||||
expected = expected or {}
|
|
||||||
results: dict[str, dict] = {}
|
|
||||||
|
|
||||||
checks = {
|
|
||||||
"artist_subscriptions": (
|
|
||||||
select(func.count(Artist.id)).where(Artist.is_subscription.is_(True))
|
|
||||||
),
|
|
||||||
"source_count": select(func.count(Source.id)),
|
|
||||||
"credential_count": select(func.count(Credential.id)),
|
|
||||||
"tag_count": select(func.count(Tag.id)),
|
|
||||||
"image_record_imported_or_downloaded": (
|
|
||||||
select(func.count(ImageRecord.id))
|
|
||||||
.where(ImageRecord.origin.in_(["imported_filesystem", "downloaded"]))
|
|
||||||
),
|
|
||||||
}
|
|
||||||
for name, stmt in checks.items():
|
|
||||||
actual = (await db.execute(stmt)).scalar_one()
|
|
||||||
exp = expected.get(name)
|
|
||||||
status = "ok" if exp is None or exp == actual else "mismatch"
|
|
||||||
results[name] = {"status": status, "actual": int(actual), "expected": exp}
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
async def verify_sha256_sample(
|
|
||||||
db: AsyncSession, *, sample_size: int = 20,
|
|
||||||
) -> dict:
|
|
||||||
"""Sample N image_records; verify file exists + sha256 matches."""
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(ImageRecord.id, ImageRecord.path, ImageRecord.sha256)
|
|
||||||
.order_by(func.random()).limit(sample_size)
|
|
||||||
)).all()
|
|
||||||
|
|
||||||
matched = 0
|
|
||||||
mismatched = 0
|
|
||||||
missing = 0
|
|
||||||
samples: list[dict] = []
|
|
||||||
for img_id, path, expected_sha in rows:
|
|
||||||
p = Path(path)
|
|
||||||
# Sync stdlib filesystem ops are intentional: this verify pass runs
|
|
||||||
# inside a Celery task under asyncio.run; no other awaitables compete
|
|
||||||
# for the loop. Same pattern as download_service.py.
|
|
||||||
if not p.exists(): # noqa: ASYNC240
|
|
||||||
missing += 1
|
|
||||||
samples.append({"id": img_id, "path": path, "result": "missing"})
|
|
||||||
continue
|
|
||||||
h = hashlib.sha256()
|
|
||||||
with p.open("rb") as f: # noqa: ASYNC230
|
|
||||||
for chunk in iter(lambda: f.read(65536), b""):
|
|
||||||
h.update(chunk)
|
|
||||||
if h.hexdigest() == expected_sha:
|
|
||||||
matched += 1
|
|
||||||
samples.append({"id": img_id, "result": "ok"})
|
|
||||||
else:
|
|
||||||
mismatched += 1
|
|
||||||
samples.append({
|
|
||||||
"id": img_id, "path": path, "result": "mismatch",
|
|
||||||
"expected_sha": expected_sha, "actual_sha": h.hexdigest(),
|
|
||||||
})
|
|
||||||
return {
|
|
||||||
"sample_size": len(rows),
|
|
||||||
"matched": matched,
|
|
||||||
"mismatched": mismatched,
|
|
||||||
"missing": missing,
|
|
||||||
"samples": samples,
|
|
||||||
}
|
|
||||||
@@ -1,169 +0,0 @@
|
|||||||
"""FC-5 run_migration Celery task.
|
|
||||||
|
|
||||||
Dispatches to the right migrator based on `kind`. Updates MigrationRun
|
|
||||||
row's status/counts/finished_at as it runs. Failures set status='error'
|
|
||||||
with the error message preserved.
|
|
||||||
|
|
||||||
kinds: gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup
|
|
||||||
(backup + rollback retired 2026-05-24 → see /api/system/backup/*)
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
from ..celery_app import celery
|
|
||||||
from ..models import MigrationRun
|
|
||||||
from ..services.credential_crypto import CredentialCrypto
|
|
||||||
from ..services.migrators import cleanup as cleanup_mod
|
|
||||||
from ..services.migrators import gs_ingest, ir_ingest, ml_queue, tag_apply, verify
|
|
||||||
from ._async_session import async_session_factory
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
IMAGES_ROOT = Path("/images")
|
|
||||||
_KEY_PATH = IMAGES_ROOT / "secrets" / "credential_key.b64"
|
|
||||||
|
|
||||||
|
|
||||||
async def _update_run(
|
|
||||||
db: AsyncSession, run_id: int, *,
|
|
||||||
status: str | None = None, counts: dict | None = None,
|
|
||||||
error: str | None = None, finished_at: datetime | None = None,
|
|
||||||
metadata_patch: dict | None = None,
|
|
||||||
) -> None:
|
|
||||||
run = (await db.execute(
|
|
||||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
|
||||||
)).scalar_one()
|
|
||||||
if status is not None:
|
|
||||||
run.status = status
|
|
||||||
if counts is not None:
|
|
||||||
run.counts = counts
|
|
||||||
if error is not None:
|
|
||||||
run.error = error
|
|
||||||
if finished_at is not None:
|
|
||||||
run.finished_at = finished_at
|
|
||||||
if metadata_patch:
|
|
||||||
run.metadata_ = {**(run.metadata_ or {}), **metadata_patch}
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
|
|
||||||
async def _run_async(run_id: int, kind: str, params: dict) -> dict:
|
|
||||||
factory, engine = async_session_factory()
|
|
||||||
try:
|
|
||||||
async with factory() as db:
|
|
||||||
await _update_run(db, run_id, status="running")
|
|
||||||
try:
|
|
||||||
if kind in ("backup", "rollback"):
|
|
||||||
raise ValueError(
|
|
||||||
f"kind {kind!r} retired in FC-3h; "
|
|
||||||
"use /api/system/backup/* instead"
|
|
||||||
)
|
|
||||||
|
|
||||||
elif kind == "gs_ingest":
|
|
||||||
fc_crypto = CredentialCrypto(_KEY_PATH)
|
|
||||||
counts = await gs_ingest.migrate_async(
|
|
||||||
db, data=params["data"],
|
|
||||||
fc_crypto=fc_crypto,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok", counts=counts,
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return counts
|
|
||||||
|
|
||||||
elif kind == "ir_ingest":
|
|
||||||
counts = await ir_ingest.migrate_async(
|
|
||||||
db, data=params["data"],
|
|
||||||
images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok", counts=counts,
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return counts
|
|
||||||
|
|
||||||
elif kind == "tag_apply":
|
|
||||||
result = await tag_apply.apply_async(
|
|
||||||
db, images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts=result["counts"],
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"unmatched": result["unmatched"]},
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
elif kind == "ml_queue":
|
|
||||||
count = await ml_queue.queue_all_unprocessed_async(db)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": count, "rows_inserted": 0,
|
|
||||||
"rows_skipped": 0, "files_copied": 0,
|
|
||||||
"bytes_copied": 0, "conflicts": 0},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
return {"queued": count}
|
|
||||||
|
|
||||||
elif kind == "verify":
|
|
||||||
checks = await verify.verify_async(db, expected=params.get("expected"))
|
|
||||||
sample = await verify.verify_sha256_sample(
|
|
||||||
db, sample_size=params.get("sample_size", 20),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts={"rows_processed": sample["sample_size"],
|
|
||||||
"rows_inserted": 0, "rows_skipped": 0,
|
|
||||||
"files_copied": 0, "bytes_copied": 0,
|
|
||||||
"conflicts": sample["mismatched"] + sample["missing"]},
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={"checks": checks, "sample": sample},
|
|
||||||
)
|
|
||||||
return {"checks": checks, "sample": sample}
|
|
||||||
|
|
||||||
elif kind == "cleanup":
|
|
||||||
slug = params.get("slug")
|
|
||||||
if not slug:
|
|
||||||
raise ValueError("cleanup requires params.slug")
|
|
||||||
result = await cleanup_mod.cleanup_artist_async(
|
|
||||||
db, slug=slug, images_root=IMAGES_ROOT,
|
|
||||||
dry_run=params.get("dry_run", False),
|
|
||||||
source_path_prefix=params.get("source_path_prefix"),
|
|
||||||
)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="ok",
|
|
||||||
counts=result["counts"],
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
metadata_patch={
|
|
||||||
"artist": result["artist"],
|
|
||||||
"summary": result["summary"],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError(f"unknown kind: {kind}")
|
|
||||||
|
|
||||||
except Exception as exc:
|
|
||||||
log.exception("migration kind=%s failed", kind)
|
|
||||||
await _update_run(
|
|
||||||
db, run_id, status="error", error=str(exc),
|
|
||||||
finished_at=datetime.now(UTC),
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
await engine.dispose()
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.migration.run_migration", bind=True, acks_late=True)
|
|
||||||
def run_migration(self, run_id: int, kind: str, params: dict) -> dict:
|
|
||||||
"""FC-5: dispatch a migration kind. Updates MigrationRun row as it goes."""
|
|
||||||
return asyncio.run(_run_async(run_id, kind, params))
|
|
||||||
@@ -1,245 +0,0 @@
|
|||||||
<template>
|
|
||||||
<v-card>
|
|
||||||
<v-card-title>Legacy migration</v-card-title>
|
|
||||||
<v-card-text>
|
|
||||||
<p class="text-body-2 mb-3">
|
|
||||||
Migrate existing ImageRepo and GallerySubscriber data into FabledCurator.
|
|
||||||
Each app produces a JSON export file via a small script in its own repo
|
|
||||||
(<code>scripts/export_for_fabledcurator.py</code>); upload both files
|
|
||||||
here and run the steps in order.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<ol class="text-body-2 mb-3 fc-migrate__hints">
|
|
||||||
<li>Backup FC (creates a pre-migration snapshot).</li>
|
|
||||||
<li>Upload + ingest GS export.</li>
|
|
||||||
<li>Upload + ingest IR export.</li>
|
|
||||||
<li>Switch to <strong>Import tab</strong> → trigger the existing FC filesystem scan over the bind-mounted IR images dir.</li>
|
|
||||||
<li>Return here → apply IR tag associations (joins by sha256).</li>
|
|
||||||
<li>Queue ML re-processing.</li>
|
|
||||||
<li>Verify.</li>
|
|
||||||
</ol>
|
|
||||||
|
|
||||||
<div class="fc-migrate__uploads">
|
|
||||||
<v-file-input
|
|
||||||
v-model="gsFile"
|
|
||||||
label="GallerySubscriber export (gs-export.json)"
|
|
||||||
accept="application/json,.json"
|
|
||||||
density="compact"
|
|
||||||
show-size
|
|
||||||
prepend-icon="mdi-upload"
|
|
||||||
/>
|
|
||||||
<v-file-input
|
|
||||||
v-model="irFile"
|
|
||||||
label="ImageRepo export (ir-export.json)"
|
|
||||||
accept="application/json,.json"
|
|
||||||
density="compact"
|
|
||||||
show-size
|
|
||||||
prepend-icon="mdi-upload"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="fc-migrate__steps mt-3">
|
|
||||||
<v-btn
|
|
||||||
color="accent" variant="tonal" prepend-icon="mdi-content-save"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onBackup"
|
|
||||||
>1. Backup FC</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-database-arrow-right"
|
|
||||||
:disabled="store.isRunning || !gsFile"
|
|
||||||
@click="onIngestGs"
|
|
||||||
>2. Ingest GS</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-database-arrow-right"
|
|
||||||
:disabled="store.isRunning || !irFile"
|
|
||||||
@click="onIngestIr"
|
|
||||||
>3. Ingest IR</v-btn>
|
|
||||||
<v-btn
|
|
||||||
color="primary" variant="tonal" prepend-icon="mdi-tag-multiple"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onTagApply"
|
|
||||||
>5. Apply IR tags</v-btn>
|
|
||||||
<v-btn
|
|
||||||
variant="outlined" prepend-icon="mdi-brain"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onMlQueue"
|
|
||||||
>6. Queue ML</v-btn>
|
|
||||||
<v-btn
|
|
||||||
variant="outlined" prepend-icon="mdi-check-circle"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onVerify"
|
|
||||||
>7. Verify</v-btn>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<v-btn
|
|
||||||
color="error" variant="outlined" prepend-icon="mdi-restore" class="mt-3"
|
|
||||||
:disabled="store.isRunning"
|
|
||||||
@click="onRollback"
|
|
||||||
>Rollback to pre-migration backup</v-btn>
|
|
||||||
|
|
||||||
<v-alert v-if="store.error" type="error" variant="tonal" class="mt-3" closable>
|
|
||||||
{{ String(store.error) }}
|
|
||||||
</v-alert>
|
|
||||||
|
|
||||||
<v-card v-if="store.activeRun" variant="outlined" class="mt-4">
|
|
||||||
<v-card-text>
|
|
||||||
<div class="text-subtitle-2">
|
|
||||||
#{{ store.activeRun.id }} — {{ store.activeRun.kind }}
|
|
||||||
({{ store.activeRun.status }})
|
|
||||||
</div>
|
|
||||||
<v-progress-linear
|
|
||||||
v-if="store.isRunning" indeterminate color="accent" class="my-2"
|
|
||||||
/>
|
|
||||||
<div class="text-caption">
|
|
||||||
Rows: {{ store.activeRun.counts.rows_processed || 0 }} processed,
|
|
||||||
{{ store.activeRun.counts.rows_inserted || 0 }} inserted,
|
|
||||||
{{ store.activeRun.counts.rows_skipped || 0 }} skipped.
|
|
||||||
Conflicts: {{ store.activeRun.counts.conflicts || 0 }}.
|
|
||||||
</div>
|
|
||||||
<div v-if="store.activeRun.error" class="text-error mt-1">
|
|
||||||
{{ store.activeRun.error }}
|
|
||||||
</div>
|
|
||||||
</v-card-text>
|
|
||||||
</v-card>
|
|
||||||
|
|
||||||
<div v-if="store.recentRuns.length" class="mt-4">
|
|
||||||
<div class="text-subtitle-2 mb-2">Recent runs</div>
|
|
||||||
<ul class="fc-migrate__history">
|
|
||||||
<li v-for="r in store.recentRuns" :key="r.id">
|
|
||||||
#{{ r.id }} {{ r.kind }}
|
|
||||||
<v-chip
|
|
||||||
size="x-small"
|
|
||||||
:color="r.status === 'ok' ? 'success' : (r.status === 'error' ? 'error' : undefined)"
|
|
||||||
>{{ r.status }}</v-chip>
|
|
||||||
<span class="text-caption fc-migrate__when">{{ r.started_at }}</span>
|
|
||||||
</li>
|
|
||||||
</ul>
|
|
||||||
</div>
|
|
||||||
</v-card-text>
|
|
||||||
|
|
||||||
<v-dialog v-model="confirmOpen" max-width="520">
|
|
||||||
<v-card>
|
|
||||||
<v-card-title>Confirm {{ pendingLabel }}</v-card-title>
|
|
||||||
<v-card-text>
|
|
||||||
This will modify FC's database. Make sure you've taken a backup first.
|
|
||||||
Continue?
|
|
||||||
</v-card-text>
|
|
||||||
<v-card-actions>
|
|
||||||
<v-spacer />
|
|
||||||
<v-btn variant="text" @click="confirmOpen = false">Cancel</v-btn>
|
|
||||||
<v-btn color="primary" variant="tonal" @click="onConfirm">Continue</v-btn>
|
|
||||||
</v-card-actions>
|
|
||||||
</v-card>
|
|
||||||
</v-dialog>
|
|
||||||
</v-card>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
import { onMounted, ref } from 'vue'
|
|
||||||
import { useMigrationStore } from '../../stores/migration.js'
|
|
||||||
|
|
||||||
const store = useMigrationStore()
|
|
||||||
|
|
||||||
const gsFile = ref(null)
|
|
||||||
const irFile = ref(null)
|
|
||||||
|
|
||||||
const confirmOpen = ref(false)
|
|
||||||
const pendingLabel = ref('')
|
|
||||||
const pendingAction = ref(null)
|
|
||||||
|
|
||||||
onMounted(async () => {
|
|
||||||
await store.loadRecent()
|
|
||||||
})
|
|
||||||
|
|
||||||
function _pick(model) {
|
|
||||||
if (!model) return null
|
|
||||||
// v-file-input v-model can be a single File or a [File] depending on Vuetify version.
|
|
||||||
return Array.isArray(model) ? model[0] : model
|
|
||||||
}
|
|
||||||
|
|
||||||
function requestConfirm(label, action) {
|
|
||||||
pendingLabel.value = label
|
|
||||||
pendingAction.value = action
|
|
||||||
confirmOpen.value = true
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onConfirm() {
|
|
||||||
confirmOpen.value = false
|
|
||||||
const action = pendingAction.value
|
|
||||||
pendingAction.value = null
|
|
||||||
if (action) await action()
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onBackup() {
|
|
||||||
await store.trigger('backup', { tag: 'pre_migration' })
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onIngestGs() {
|
|
||||||
const file = _pick(gsFile.value)
|
|
||||||
if (!file) return
|
|
||||||
requestConfirm('Ingest GS', async () => {
|
|
||||||
await store.trigger('gs_ingest', { dry_run: false }, file)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onIngestIr() {
|
|
||||||
const file = _pick(irFile.value)
|
|
||||||
if (!file) return
|
|
||||||
requestConfirm('Ingest IR', async () => {
|
|
||||||
await store.trigger('ir_ingest', { dry_run: false }, file)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onTagApply() {
|
|
||||||
requestConfirm('Apply IR tag associations', async () => {
|
|
||||||
await store.trigger('tag_apply', { dry_run: false })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onMlQueue() {
|
|
||||||
await store.trigger('ml_queue', {})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onVerify() {
|
|
||||||
await store.trigger('verify', {})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onRollback() {
|
|
||||||
requestConfirm('Rollback to pre-migration backup', async () => {
|
|
||||||
await store.trigger('rollback', {})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.fc-migrate__hints {
|
|
||||||
padding-left: 1.2rem;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
.fc-migrate__uploads {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: 1fr 1fr;
|
|
||||||
gap: 12px;
|
|
||||||
}
|
|
||||||
.fc-migrate__steps {
|
|
||||||
display: flex;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 8px;
|
|
||||||
}
|
|
||||||
.fc-migrate__history {
|
|
||||||
list-style: none;
|
|
||||||
padding: 0;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
.fc-migrate__history li {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 8px;
|
|
||||||
padding: 4px 0;
|
|
||||||
border-bottom: 1px solid rgb(var(--v-theme-on-surface-variant), 0.15);
|
|
||||||
}
|
|
||||||
.fc-migrate__when {
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
margin-left: auto;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -17,8 +17,8 @@
|
|||||||
<BackupCard class="mt-6" />
|
<BackupCard class="mt-6" />
|
||||||
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
<!-- TagMaintenanceCard moved to Cleanup tab (v26.05.25.7) — it
|
||||||
operates on the existing library which fits the Cleanup-tab
|
operates on the existing library which fits the Cleanup-tab
|
||||||
theme, and clusters with the other audit cards. -->
|
theme, and clusters with the other audit cards. LegacyMigrationCard
|
||||||
<LegacyMigrationCard class="mt-6" />
|
removed once the one-and-done GS/IR migration cutover completed. -->
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -32,7 +32,6 @@ import MLThresholdSliders from './MLThresholdSliders.vue'
|
|||||||
import AllowlistTable from './AllowlistTable.vue'
|
import AllowlistTable from './AllowlistTable.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import BackupCard from './BackupCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
import LegacyMigrationCard from './LegacyMigrationCard.vue'
|
|
||||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||||
|
|
||||||
// Poll queue depths so each card's QueueStatusBar shows live pending
|
// Poll queue depths so each card's QueueStatusBar shows live pending
|
||||||
|
|||||||
@@ -1,83 +0,0 @@
|
|||||||
import { defineStore } from 'pinia'
|
|
||||||
import { ref, computed } from 'vue'
|
|
||||||
import { useApi } from '../composables/useApi.js'
|
|
||||||
|
|
||||||
const POLL_MS = 2000
|
|
||||||
|
|
||||||
export const useMigrationStore = defineStore('migration', () => {
|
|
||||||
const api = useApi()
|
|
||||||
|
|
||||||
const activeRun = ref(null)
|
|
||||||
const recentRuns = ref([])
|
|
||||||
const error = ref(null)
|
|
||||||
const loading = ref(false)
|
|
||||||
|
|
||||||
let pollHandle = null
|
|
||||||
|
|
||||||
async function trigger(kind, params, file = null) {
|
|
||||||
loading.value = true
|
|
||||||
error.value = null
|
|
||||||
try {
|
|
||||||
let body
|
|
||||||
if (file) {
|
|
||||||
// Multipart upload for ingest kinds.
|
|
||||||
const form = new FormData()
|
|
||||||
form.append('export_file', file)
|
|
||||||
if (params && params.dry_run) form.append('dry_run', 'true')
|
|
||||||
if (params && params.force) form.append('force', 'true')
|
|
||||||
body = await api.post(`/api/migrate/${kind}`, { body: form })
|
|
||||||
} else {
|
|
||||||
body = await api.post(`/api/migrate/${kind}`, { body: params || {} })
|
|
||||||
}
|
|
||||||
await loadRun(body.run_id)
|
|
||||||
pollActive(body.run_id)
|
|
||||||
return body
|
|
||||||
} catch (e) {
|
|
||||||
error.value = e
|
|
||||||
throw e
|
|
||||||
} finally {
|
|
||||||
loading.value = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadRun(runId) {
|
|
||||||
activeRun.value = await api.get(`/api/migrate/runs/${runId}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadRecent() {
|
|
||||||
recentRuns.value = await api.get('/api/migrate/runs', { params: { limit: 10 } })
|
|
||||||
}
|
|
||||||
|
|
||||||
function pollActive(runId) {
|
|
||||||
stopPolling()
|
|
||||||
pollHandle = setInterval(async () => {
|
|
||||||
try {
|
|
||||||
const run = await api.get(`/api/migrate/runs/${runId}`)
|
|
||||||
activeRun.value = run
|
|
||||||
if (run.status === 'ok' || run.status === 'error') {
|
|
||||||
stopPolling()
|
|
||||||
await loadRecent()
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
error.value = e
|
|
||||||
stopPolling()
|
|
||||||
}
|
|
||||||
}, POLL_MS)
|
|
||||||
}
|
|
||||||
|
|
||||||
function stopPolling() {
|
|
||||||
if (pollHandle) {
|
|
||||||
clearInterval(pollHandle)
|
|
||||||
pollHandle = null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const isRunning = computed(
|
|
||||||
() => activeRun.value && activeRun.value.status === 'running',
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
activeRun, recentRuns, error, loading, isRunning,
|
|
||||||
trigger, loadRun, loadRecent, pollActive, stopPolling,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
@@ -1,98 +0,0 @@
|
|||||||
"""FC-5: /api/migrate API tests."""
|
|
||||||
import io
|
|
||||||
import json
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from werkzeug.datastructures import FileStorage
|
|
||||||
|
|
||||||
import backend.app.tasks.migration # noqa: F401 — register celery task
|
|
||||||
from backend.app.celery_app import celery
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
|
||||||
|
|
||||||
|
|
||||||
def _file_storage(payload: dict, filename: str = "export.json") -> FileStorage:
|
|
||||||
return FileStorage(
|
|
||||||
stream=io.BytesIO(json.dumps(payload).encode("utf-8")),
|
|
||||||
filename=filename,
|
|
||||||
content_type="application/json",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# Retired 2026-05-24 (FC-3h): `test_post_backup_returns_202_and_id`
|
|
||||||
# asserted the /api/migrate/backup endpoint, which was retired in FC-3h.
|
|
||||||
# Backup is now a first-class feature at /api/system/backup/*;
|
|
||||||
# coverage lives in tests/test_api_system_backup.py.
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_post_rejects_unknown_kind(client):
|
|
||||||
resp = await client.post("/api/migrate/destroy", json={})
|
|
||||||
assert resp.status_code == 400
|
|
||||||
body = await resp.get_json()
|
|
||||||
assert body["error"] == "unknown_kind"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_post_ingest_rejects_missing_file(client):
|
|
||||||
resp = await client.post("/api/migrate/gs_ingest", form={})
|
|
||||||
assert resp.status_code == 400
|
|
||||||
body = await resp.get_json()
|
|
||||||
assert body["error"] == "missing_export_file"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_post_ingest_accepts_multipart_file(client, monkeypatch):
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"backend.app.api.migrate.run_migration",
|
|
||||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
|
||||||
)
|
|
||||||
|
|
||||||
export = {
|
|
||||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
|
||||||
"subscriptions": [], "credentials": [],
|
|
||||||
}
|
|
||||||
resp = await client.post(
|
|
||||||
"/api/migrate/gs_ingest",
|
|
||||||
form={"dry_run": "false"},
|
|
||||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
|
||||||
)
|
|
||||||
assert resp.status_code == 202
|
|
||||||
|
|
||||||
|
|
||||||
# Retired 2026-05-24: `test_post_apply_without_backup_rejected` asserted
|
|
||||||
# the migration backup-gate rejected applies without a recent backup.
|
|
||||||
# That gate was removed in commit 5535677 (_APPLY_KINDS is now empty,
|
|
||||||
# FC-3h will own backup as a first-class feature) so the test was
|
|
||||||
# testing dead behavior. Removed rather than rewritten.
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_post_dry_run_allowed_without_backup(client, monkeypatch):
|
|
||||||
# FC-3h: the _has_recent_pre_migration_backup gate was retired; dry-run
|
|
||||||
# ingests were always allowed and now non-dry-run ingests are too.
|
|
||||||
# This test stays as regression coverage that dry_run ingest still works.
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"backend.app.api.migrate.run_migration",
|
|
||||||
type("F", (), {"delay": lambda self, *a, **k: None})(),
|
|
||||||
)
|
|
||||||
export = {
|
|
||||||
"source_app": "gallerysubscriber", "schema_version": 1,
|
|
||||||
"subscriptions": [], "credentials": [],
|
|
||||||
}
|
|
||||||
resp = await client.post(
|
|
||||||
"/api/migrate/gs_ingest",
|
|
||||||
form={"dry_run": "true"},
|
|
||||||
files={"export_file": _file_storage(export, "gs-export.json")},
|
|
||||||
)
|
|
||||||
assert resp.status_code == 202
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_get_run_404_unknown(client):
|
|
||||||
resp = await client.get("/api/migrate/runs/9999999")
|
|
||||||
assert resp.status_code == 404
|
|
||||||
|
|
||||||
|
|
||||||
def test_run_migration_task_registered():
|
|
||||||
assert "backend.app.tasks.migration.run_migration" in celery.tasks
|
|
||||||
@@ -1,122 +0,0 @@
|
|||||||
"""FC-5: gs_ingest unit tests."""
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from backend.app.models import Artist, Credential, Source
|
|
||||||
from backend.app.services.credential_crypto import CredentialCrypto
|
|
||||||
from backend.app.services.migrators import gs_ingest
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
|
||||||
|
|
||||||
|
|
||||||
def _gs_export(subscriptions=None, credentials=None) -> dict:
|
|
||||||
return {
|
|
||||||
"source_app": "gallerysubscriber",
|
|
||||||
"schema_version": 1,
|
|
||||||
"exported_at": datetime.now(UTC).isoformat(),
|
|
||||||
"subscriptions": subscriptions or [],
|
|
||||||
"credentials": credentials or [],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_subscription_creates_artist_with_slug(db):
|
|
||||||
data = _gs_export(subscriptions=[
|
|
||||||
{"name": "Alice Author", "enabled": True, "metadata": {}, "sources": []},
|
|
||||||
])
|
|
||||||
counts = await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
|
||||||
assert counts["rows_inserted"] >= 1
|
|
||||||
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.name == "Alice Author")
|
|
||||||
)).scalar_one()
|
|
||||||
assert artist.slug == "alice-author"
|
|
||||||
assert artist.is_subscription is True
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_nested_source_attached_to_artist(db):
|
|
||||||
data = _gs_export(subscriptions=[{
|
|
||||||
"name": "bob-author",
|
|
||||||
"enabled": True,
|
|
||||||
"metadata": {},
|
|
||||||
"sources": [{
|
|
||||||
"platform": "patreon",
|
|
||||||
"url": "https://patreon.com/bob",
|
|
||||||
"enabled": True,
|
|
||||||
"check_interval": 28800,
|
|
||||||
"metadata": {"k": "v"},
|
|
||||||
}],
|
|
||||||
}])
|
|
||||||
await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
|
||||||
|
|
||||||
src = (await db.execute(
|
|
||||||
select(Source).where(Source.url == "https://patreon.com/bob")
|
|
||||||
)).scalar_one()
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.id == src.artist_id)
|
|
||||||
)).scalar_one()
|
|
||||||
assert artist.name == "bob-author"
|
|
||||||
assert src.platform == "patreon"
|
|
||||||
assert src.check_interval_override == 28800
|
|
||||||
assert src.config_overrides == {"k": "v"}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_credential_re_encrypted_with_fc_key(db, tmp_path):
|
|
||||||
fc_key_path = tmp_path / "credential_key.b64"
|
|
||||||
fc_crypto = CredentialCrypto(fc_key_path)
|
|
||||||
|
|
||||||
data = _gs_export(credentials=[{
|
|
||||||
"platform": "patreon",
|
|
||||||
"credential_type": "cookies",
|
|
||||||
"plaintext": "session_cookie=abcdef",
|
|
||||||
"expires_at": None,
|
|
||||||
}])
|
|
||||||
await gs_ingest.migrate_async(db, data=data, fc_crypto=fc_crypto, dry_run=False)
|
|
||||||
|
|
||||||
cred = (await db.execute(
|
|
||||||
select(Credential).where(Credential.platform == "patreon")
|
|
||||||
)).scalar_one()
|
|
||||||
assert fc_crypto.decrypt(cred.encrypted_blob) == "session_cookie=abcdef"
|
|
||||||
assert cred.credential_type == "cookies"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_dry_run_makes_no_changes(db):
|
|
||||||
data = _gs_export(subscriptions=[
|
|
||||||
{"name": "dave", "enabled": True, "metadata": {}, "sources": []},
|
|
||||||
])
|
|
||||||
counts = await gs_ingest.migrate_async(db, data=data, dry_run=True)
|
|
||||||
assert counts["rows_processed"] >= 1
|
|
||||||
|
|
||||||
after = (await db.execute(
|
|
||||||
select(Artist).where(Artist.name == "dave")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
assert after is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_idempotent_on_rerun(db):
|
|
||||||
data = _gs_export(subscriptions=[
|
|
||||||
{"name": "eve", "enabled": True, "metadata": {}, "sources": []},
|
|
||||||
])
|
|
||||||
await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
|
||||||
counts2 = await gs_ingest.migrate_async(db, data=data, dry_run=False)
|
|
||||||
assert counts2["rows_skipped"] >= 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rejects_wrong_source_app(db):
|
|
||||||
bad = {"source_app": "wrong", "schema_version": 1, "subscriptions": [], "credentials": []}
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
await gs_ingest.migrate_async(db, data=bad, dry_run=False)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rejects_unsupported_schema_version(db):
|
|
||||||
bad = {"source_app": "gallerysubscriber", "schema_version": 99, "subscriptions": [], "credentials": []}
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
await gs_ingest.migrate_async(db, data=bad, dry_run=False)
|
|
||||||
@@ -1,129 +0,0 @@
|
|||||||
"""FC-5: ir_ingest unit tests."""
|
|
||||||
import json
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from backend.app.models import Tag
|
|
||||||
from backend.app.services.migrators import ir_ingest
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
|
||||||
|
|
||||||
|
|
||||||
def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None):
|
|
||||||
return {
|
|
||||||
"source_app": "imagerepo",
|
|
||||||
"schema_version": 1,
|
|
||||||
"exported_at": datetime.now(UTC).isoformat(),
|
|
||||||
"tags": tags or [],
|
|
||||||
"image_artist_assignments": artist_assignments or [],
|
|
||||||
"image_tag_associations": tag_associations or [],
|
|
||||||
"series_pages": series_pages or [],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_general_tag_created(db, tmp_path):
|
|
||||||
data = _ir_export(tags=[
|
|
||||||
{"name": "blue_eyes", "kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
tag = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general")
|
|
||||||
)).scalar_one()
|
|
||||||
assert tag.fandom_id is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_character_tag_resolves_fandom_name(db, tmp_path):
|
|
||||||
data = _ir_export(tags=[
|
|
||||||
{"name": "Wonderland", "kind": "fandom", "fandom_name": None},
|
|
||||||
{"name": "Alice", "kind": "character", "fandom_name": "Wonderland"},
|
|
||||||
])
|
|
||||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
fandom = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom")
|
|
||||||
)).scalar_one()
|
|
||||||
alice = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "Alice", Tag.kind == "character")
|
|
||||||
)).scalar_one()
|
|
||||||
assert alice.fandom_id == fandom.id
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_artist_and_post_kinds_skipped(db, tmp_path):
|
|
||||||
data = _ir_export(tags=[
|
|
||||||
{"name": "BobArtist", "kind": "artist", "fandom_name": None},
|
|
||||||
{"name": "PostXYZ", "kind": "post", "fandom_name": None},
|
|
||||||
{"name": "general_one", "kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
assert counts["rows_skipped"] >= 2
|
|
||||||
|
|
||||||
bob = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "BobArtist")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
post = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "PostXYZ")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
assert bob is None
|
|
||||||
assert post is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_manifest_written_to_disk(db, tmp_path):
|
|
||||||
data = _ir_export(
|
|
||||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}],
|
|
||||||
tag_associations=[
|
|
||||||
{"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
|
|
||||||
],
|
|
||||||
series_pages=[
|
|
||||||
{"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
|
||||||
assert manifest_file.exists()
|
|
||||||
manifest = json.loads(manifest_file.read_text())
|
|
||||||
assert manifest["schema_version"] == 1
|
|
||||||
assert len(manifest["image_artist_assignments"]) == 1
|
|
||||||
assert len(manifest["image_tag_associations"]) == 1
|
|
||||||
assert len(manifest["series_pages"]) == 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path):
|
|
||||||
data = _ir_export(
|
|
||||||
tags=[{"name": "dry", "kind": "general", "fandom_name": None}],
|
|
||||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}],
|
|
||||||
)
|
|
||||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True)
|
|
||||||
|
|
||||||
tag = (await db.execute(
|
|
||||||
select(Tag).where(Tag.name == "dry")
|
|
||||||
)).scalar_one_or_none()
|
|
||||||
assert tag is None
|
|
||||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
|
||||||
assert not manifest_file.exists()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_idempotent_on_rerun(db, tmp_path):
|
|
||||||
data = _ir_export(tags=[
|
|
||||||
{"name": "rerun", "kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
|
||||||
assert counts2["rows_skipped"] >= 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_rejects_wrong_source_app(db, tmp_path):
|
|
||||||
bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [],
|
|
||||||
"image_artist_assignments": [], "image_tag_associations": [], "series_pages": []}
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False)
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
"""FC-5: verify + ml_queue unit tests."""
|
|
||||||
import hashlib
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from backend.app.models import Artist, ImageRecord
|
|
||||||
from backend.app.services.migrators import ml_queue, verify
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_verify_row_counts_match(db):
|
|
||||||
db.add(Artist(name="vAlpha", slug="valpha", is_subscription=True))
|
|
||||||
db.add(Artist(name="vBeta", slug="vbeta", is_subscription=False))
|
|
||||||
await db.commit()
|
|
||||||
result = await verify.verify_async(
|
|
||||||
db, expected={"artist_subscriptions": 1},
|
|
||||||
)
|
|
||||||
assert result["artist_subscriptions"]["status"] == "ok"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_verify_sha256_sample_matches_disk(db, tmp_path):
|
|
||||||
f = tmp_path / "sample.bin"
|
|
||||||
f.write_bytes(b"hello world")
|
|
||||||
sha = hashlib.sha256(b"hello world").hexdigest()
|
|
||||||
db.add(ImageRecord(
|
|
||||||
path=str(f), sha256=sha, size_bytes=11, mime="application/octet-stream",
|
|
||||||
origin="imported_filesystem",
|
|
||||||
))
|
|
||||||
await db.commit()
|
|
||||||
result = await verify.verify_sha256_sample(db, sample_size=10)
|
|
||||||
assert result["matched"] >= 1
|
|
||||||
assert result["mismatched"] == 0
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_verify_sha256_sample_detects_missing_file(db, tmp_path):
|
|
||||||
sha = "f" * 64
|
|
||||||
db.add(ImageRecord(
|
|
||||||
path=str(tmp_path / "does-not-exist.bin"),
|
|
||||||
sha256=sha, size_bytes=1, mime="application/octet-stream",
|
|
||||||
origin="imported_filesystem",
|
|
||||||
))
|
|
||||||
await db.commit()
|
|
||||||
result = await verify.verify_sha256_sample(db, sample_size=10)
|
|
||||||
assert result["missing"] >= 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_ml_queue_returns_count_for_unprocessed(db, tmp_path, monkeypatch):
|
|
||||||
queued: list[int] = []
|
|
||||||
|
|
||||||
class _FakeTask:
|
|
||||||
def delay(self, image_id):
|
|
||||||
queued.append(image_id)
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"backend.app.tasks.ml.tag_and_embed", _FakeTask(),
|
|
||||||
)
|
|
||||||
|
|
||||||
f = tmp_path / "q.bin"
|
|
||||||
f.write_bytes(b"x")
|
|
||||||
db.add(ImageRecord(
|
|
||||||
path=str(f), sha256="q" * 64, size_bytes=1,
|
|
||||||
mime="application/octet-stream", origin="imported_filesystem",
|
|
||||||
))
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
count = await ml_queue.queue_all_unprocessed_async(db)
|
|
||||||
assert count >= 1
|
|
||||||
@@ -1,395 +0,0 @@
|
|||||||
"""FC-5: tag_apply unit tests."""
|
|
||||||
import json
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
|
|
||||||
from backend.app.models import (
|
|
||||||
Artist,
|
|
||||||
ImageProvenance,
|
|
||||||
ImageRecord,
|
|
||||||
Post,
|
|
||||||
SeriesPage,
|
|
||||||
Source,
|
|
||||||
Tag,
|
|
||||||
TagKind,
|
|
||||||
image_tag,
|
|
||||||
)
|
|
||||||
from backend.app.services.migrators import tag_apply
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
|
||||||
|
|
||||||
|
|
||||||
async def _seed_image(db, sha: str, *, suffix="x"):
|
|
||||||
img = ImageRecord(
|
|
||||||
path=f"/images/imported/aa/{suffix}.jpg",
|
|
||||||
sha256=sha, size_bytes=10, mime="image/jpeg",
|
|
||||||
width=10, height=10, origin="imported_filesystem",
|
|
||||||
)
|
|
||||||
db.add(img)
|
|
||||||
await db.flush()
|
|
||||||
return img
|
|
||||||
|
|
||||||
|
|
||||||
def _write_manifest(
|
|
||||||
tmp_path, *,
|
|
||||||
artist_assignments=None, tag_associations=None,
|
|
||||||
series_pages=None, image_posts=None, schema_version=2,
|
|
||||||
):
|
|
||||||
d = tmp_path / "_migration_state"
|
|
||||||
d.mkdir(parents=True, exist_ok=True)
|
|
||||||
(d / "ir_tag_manifest.json").write_text(json.dumps({
|
|
||||||
"schema_version": schema_version,
|
|
||||||
"image_artist_assignments": artist_assignments or [],
|
|
||||||
"image_tag_associations": tag_associations or [],
|
|
||||||
"series_pages": series_pages or [],
|
|
||||||
"image_posts": image_posts or [],
|
|
||||||
}))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_artist_assignment_sets_image_artist_id(db, tmp_path):
|
|
||||||
sha = "a" * 64
|
|
||||||
await _seed_image(db, sha)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, artist_assignments=[
|
|
||||||
{"sha256": sha, "artist_name": "BobAuthor"},
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert result["counts"]["rows_inserted"] >= 1
|
|
||||||
|
|
||||||
img = (await db.execute(
|
|
||||||
select(ImageRecord).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
artist = (await db.execute(
|
|
||||||
select(Artist).where(Artist.id == img.artist_id)
|
|
||||||
)).scalar_one()
|
|
||||||
assert artist.name == "BobAuthor"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_tag_association_creates_image_tag(db, tmp_path):
|
|
||||||
sha = "b" * 64
|
|
||||||
await _seed_image(db, sha, suffix="b")
|
|
||||||
# Seed the tag itself (ir_ingest would have created it).
|
|
||||||
tag = Tag(name="blue_eyes", kind=TagKind.general)
|
|
||||||
db.add(tag)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, tag_associations=[
|
|
||||||
{"sha256": sha, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
img_id = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
row = (await db.execute(
|
|
||||||
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
|
|
||||||
)).all()
|
|
||||||
assert len(row) == 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_unmatched_sha_appears_in_unmatched_list(db, tmp_path):
|
|
||||||
_write_manifest(tmp_path, tag_associations=[
|
|
||||||
{"sha256": "z" * 64, "tag_name": "missing_image", "tag_kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert any(e["sha256"] == "z" * 64 for e in result["unmatched"])
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_series_page_created(db, tmp_path):
|
|
||||||
sha = "c" * 64
|
|
||||||
await _seed_image(db, sha, suffix="c")
|
|
||||||
series_tag = Tag(name="MySeries", kind=TagKind.series)
|
|
||||||
db.add(series_tag)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, series_pages=[
|
|
||||||
{"sha256": sha, "series_tag_name": "MySeries", "page_number": 1},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
img_id = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
sp = (await db.execute(
|
|
||||||
select(SeriesPage).where(SeriesPage.image_id == img_id)
|
|
||||||
)).scalar_one()
|
|
||||||
assert sp.page_number == 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_idempotent_on_rerun(db, tmp_path):
|
|
||||||
sha = "d" * 64
|
|
||||||
await _seed_image(db, sha, suffix="d")
|
|
||||||
tag = Tag(name="rerun_tag", kind=TagKind.general)
|
|
||||||
db.add(tag)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, tag_associations=[
|
|
||||||
{"sha256": sha, "tag_name": "rerun_tag", "tag_kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert result2["counts"]["rows_skipped"] >= 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_missing_manifest_raises(db, tmp_path):
|
|
||||||
with pytest.raises(FileNotFoundError):
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_dry_run_makes_no_changes(db, tmp_path):
|
|
||||||
sha = "e" * 64
|
|
||||||
await _seed_image(db, sha, suffix="e")
|
|
||||||
tag = Tag(name="dry_tag", kind=TagKind.general)
|
|
||||||
db.add(tag)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, tag_associations=[
|
|
||||||
{"sha256": sha, "tag_name": "dry_tag", "tag_kind": "general", "fandom_name": None},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
|
|
||||||
|
|
||||||
img_id = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
rows = (await db.execute(
|
|
||||||
select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id)
|
|
||||||
)).all()
|
|
||||||
assert len(rows) == 0
|
|
||||||
|
|
||||||
|
|
||||||
# --- Phase 4: image_posts (schema v2) -----------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
_POST_ENTRY = {
|
|
||||||
"platform": "patreon",
|
|
||||||
"post_id": "10001",
|
|
||||||
"artist": "Maewix",
|
|
||||||
"title": "Test Post Title",
|
|
||||||
"description": "<p>HTML description</p>",
|
|
||||||
"source_url": "https://www.patreon.com/posts/test-10001",
|
|
||||||
"attachment_count": 1,
|
|
||||||
"published_at": "2026-01-15T12:00:00+00:00",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_creates_source_post_provenance(db, tmp_path):
|
|
||||||
sha = "d" * 64
|
|
||||||
await _seed_image(db, sha, suffix="d")
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert result["counts"]["rows_inserted"] >= 1
|
|
||||||
|
|
||||||
# Async Core-DML assertions — column selects, not ORM attribute access.
|
|
||||||
artist_count = (await db.execute(
|
|
||||||
select(func.count(Artist.id)).where(Artist.slug == "maewix")
|
|
||||||
)).scalar_one()
|
|
||||||
assert artist_count == 1
|
|
||||||
|
|
||||||
source_count = (await db.execute(
|
|
||||||
select(func.count(Source.id)).where(
|
|
||||||
Source.platform == "patreon",
|
|
||||||
Source.url == "https://www.patreon.com/maewix",
|
|
||||||
)
|
|
||||||
)).scalar_one()
|
|
||||||
assert source_count == 1
|
|
||||||
|
|
||||||
post_count = (await db.execute(
|
|
||||||
select(func.count(Post.id)).where(Post.external_post_id == "10001")
|
|
||||||
)).scalar_one()
|
|
||||||
assert post_count == 1
|
|
||||||
|
|
||||||
img_id = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
prov_count = (await db.execute(
|
|
||||||
select(func.count(ImageProvenance.id))
|
|
||||||
.where(ImageProvenance.image_record_id == img_id)
|
|
||||||
)).scalar_one()
|
|
||||||
assert prov_count == 1
|
|
||||||
|
|
||||||
# Phase 4 must also set ImageRecord.primary_post_id so the gallery's
|
|
||||||
# effective_date COALESCE can surface Post.post_date. Operator-flagged
|
|
||||||
# 2026-05-25: without this, IR-migrated images keep sorting by FC's
|
|
||||||
# scan date instead of the original publish date.
|
|
||||||
primary_post_id = (await db.execute(
|
|
||||||
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
|
|
||||||
)).scalar_one()
|
|
||||||
canonical_post_id = (await db.execute(
|
|
||||||
select(Post.id).where(Post.external_post_id == "10001")
|
|
||||||
)).scalar_one()
|
|
||||||
assert primary_post_id == canonical_post_id
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_primary_post_id_not_clobbered(db, tmp_path):
|
|
||||||
"""If the importer already set primary_post_id (e.g. a downloaded
|
|
||||||
image with a known provenance), phase 4 must NOT overwrite it when
|
|
||||||
re-running tag_apply against the IR migration. The existing
|
|
||||||
download-time linkage is the source of truth."""
|
|
||||||
sha = "9" * 64
|
|
||||||
await _seed_image(db, sha, suffix="9")
|
|
||||||
# Pre-set primary_post_id to a sentinel Post so we can detect a clobber.
|
|
||||||
img_id = (await db.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.sha256 == sha)
|
|
||||||
)).scalar_one()
|
|
||||||
# Build an existing Source + Post for the sentinel.
|
|
||||||
art = Artist(name="Pre-existing", slug="pre-existing")
|
|
||||||
db.add(art)
|
|
||||||
await db.flush()
|
|
||||||
src = Source(
|
|
||||||
artist_id=art.id, platform="patreon",
|
|
||||||
url="https://www.patreon.com/pre-existing",
|
|
||||||
)
|
|
||||||
db.add(src)
|
|
||||||
await db.flush()
|
|
||||||
sentinel_post = Post(
|
|
||||||
source_id=src.id, external_post_id="sentinel-99",
|
|
||||||
post_title="Pre-existing",
|
|
||||||
)
|
|
||||||
db.add(sentinel_post)
|
|
||||||
await db.flush()
|
|
||||||
await db.execute(
|
|
||||||
ImageRecord.__table__.update()
|
|
||||||
.where(ImageRecord.id == img_id)
|
|
||||||
.values(primary_post_id=sentinel_post.id)
|
|
||||||
)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
# The migration created a NEW Post (external_post_id="10001") and a
|
|
||||||
# new ImageProvenance, but primary_post_id must still point at the
|
|
||||||
# original sentinel.
|
|
||||||
primary_post_id = (await db.execute(
|
|
||||||
select(ImageRecord.primary_post_id).where(ImageRecord.id == img_id)
|
|
||||||
)).scalar_one()
|
|
||||||
assert primary_post_id == sentinel_post.id
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_idempotent_on_rerun(db, tmp_path):
|
|
||||||
sha = "e" * 64
|
|
||||||
await _seed_image(db, sha, suffix="e")
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
|
|
||||||
# Re-run: nothing should be newly inserted; all skipped.
|
|
||||||
assert result2["counts"]["rows_inserted"] == 0
|
|
||||||
assert result2["counts"]["rows_skipped"] >= 1
|
|
||||||
|
|
||||||
prov_count = (await db.execute(
|
|
||||||
select(func.count(ImageProvenance.id))
|
|
||||||
)).scalar_one()
|
|
||||||
assert prov_count == 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_unknown_sha_unmatched(db, tmp_path):
|
|
||||||
# No image seeded; sha doesn't exist.
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "image_sha256s": ["f" * 64]},
|
|
||||||
])
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
unmatched_kinds = [u["kind"] for u in result["unmatched"]]
|
|
||||||
assert "post" in unmatched_kinds
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_unknown_platform_skipped(db, tmp_path):
|
|
||||||
sha = "1" * 64
|
|
||||||
await _seed_image(db, sha, suffix="1")
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "platform": "tumblr", "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert result["counts"]["rows_skipped"] >= 1
|
|
||||||
|
|
||||||
# No Post / Source / ImageProvenance should have been created.
|
|
||||||
src_count = (await db.execute(
|
|
||||||
select(func.count(Source.id)).where(Source.platform == "tumblr")
|
|
||||||
)).scalar_one()
|
|
||||||
assert src_count == 0
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("platform,expected_url", [
|
|
||||||
("deviantart", "https://www.deviantart.com/maewix"),
|
|
||||||
("pixiv", "https://www.pixiv.net/users/maewix"),
|
|
||||||
])
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_extended_platforms_create_source(
|
|
||||||
db, tmp_path, platform, expected_url,
|
|
||||||
):
|
|
||||||
"""Regression for 2026-05-25 operator-reported bug: phase 4's
|
|
||||||
_PLATFORM_PROFILE_URL had only patreon/subscribestar/hentaifoundry,
|
|
||||||
silently dropping deviantart + pixiv PostMetadata from the IR migration."""
|
|
||||||
sha = f"{platform[0]}" * 64
|
|
||||||
await _seed_image(db, sha, suffix=f"_{platform}")
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "platform": platform, "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
|
|
||||||
result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False)
|
|
||||||
assert result["counts"]["rows_inserted"] >= 1
|
|
||||||
|
|
||||||
src_url = (await db.execute(
|
|
||||||
select(Source.url).where(Source.platform == platform)
|
|
||||||
)).scalar_one()
|
|
||||||
assert src_url == expected_url
|
|
||||||
|
|
||||||
# ImageProvenance row was created.
|
|
||||||
prov_count = (await db.execute(
|
|
||||||
select(func.count(ImageProvenance.id))
|
|
||||||
.join(Source, Source.id == ImageProvenance.source_id)
|
|
||||||
.where(Source.platform == platform)
|
|
||||||
)).scalar_one()
|
|
||||||
assert prov_count == 1
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_image_posts_dry_run_makes_no_writes(db, tmp_path):
|
|
||||||
sha = "2" * 64
|
|
||||||
await _seed_image(db, sha, suffix="2")
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
_write_manifest(tmp_path, image_posts=[
|
|
||||||
{**_POST_ENTRY, "image_sha256s": [sha]},
|
|
||||||
])
|
|
||||||
|
|
||||||
await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True)
|
|
||||||
prov_count = (await db.execute(
|
|
||||||
select(func.count(ImageProvenance.id))
|
|
||||||
)).scalar_one()
|
|
||||||
assert prov_count == 0
|
|
||||||
Reference in New Issue
Block a user