diff --git a/backend/app/services/migrators/ir_ingest.py b/backend/app/services/migrators/ir_ingest.py new file mode 100644 index 0000000..75d12cc --- /dev/null +++ b/backend/app/services/migrators/ir_ingest.py @@ -0,0 +1,141 @@ +"""ImageRepo export → FabledCurator ingest. + +Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR). +Creates Tag rows (skipping artist/post kinds, resolving fandom_name to +FK). Writes the per-image-sha256 artist assignments + tag associations ++ series page assignments to /images/_migration_state/ir_tag_manifest.json +so tag_apply.py can join them to ImageRecord rows AFTER the operator +runs FC's filesystem scan. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ...models import Tag, TagKind + +_SKIP_KINDS = frozenset({"artist", "post"}) +_MIGRATION_STATE_DIRNAME = "_migration_state" +_IR_MANIFEST_FILENAME = "ir_tag_manifest.json" + + +def _zero_counts() -> dict: + return { + "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0, + "files_copied": 0, "bytes_copied": 0, "conflicts": 0, + } + + +def manifest_path(images_root: Path | None = None) -> Path: + root = images_root if images_root is not None else Path("/images") + p = root / _MIGRATION_STATE_DIRNAME + p.mkdir(parents=True, exist_ok=True) + return p / _IR_MANIFEST_FILENAME + + +async def _resolve_fandom_id( + db: AsyncSession, fandom_name: str | None, dry_run: bool, +) -> int | None: + """Find-or-create a fandom-kind Tag by name.""" + if not fandom_name: + return None + existing = (await db.execute( + select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + return existing.id + if dry_run: + return None + t = Tag(name=fandom_name, kind=TagKind.fandom) + db.add(t) + await db.flush() + return t.id + + +async def migrate_async( + db: AsyncSession, + *, + data: dict, + images_root: Path | None = None, + dry_run: bool = False, +) -> dict: + """Ingest a parsed imagerepo-export-v1.json dict. + + Creates Tag rows + writes the IR tag manifest file. Tag-to-image + binding happens later in tag_apply.py (after FC's filesystem scan + populates image_record.sha256 → id). + """ + if data.get("source_app") != "imagerepo": + raise ValueError("export source_app must be 'imagerepo'") + if data.get("schema_version") != 1: + raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") + + counts = _zero_counts() + + # Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id). + # First pass: create all fandom-kind tags so they're available for FK resolution. + for tag in data.get("tags", []): + kind = tag.get("kind") or "general" + if kind != "fandom": + continue + counts["rows_processed"] += 1 + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom") + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=TagKind.fandom)) + counts["rows_inserted"] += 1 + if not dry_run: + await db.flush() + + # Second pass: every other kind. + for tag in data.get("tags", []): + kind_str = tag.get("kind") or "general" + if kind_str in _SKIP_KINDS: + counts["rows_skipped"] += 1 + continue + if kind_str == "fandom": + continue # handled above + counts["rows_processed"] += 1 + try: + kind = TagKind(kind_str) + except ValueError: + kind = TagKind.general + fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run) + existing = (await db.execute( + select(Tag).where(Tag.name == tag["name"], Tag.kind == kind) + )).scalar_one_or_none() + if existing is not None: + counts["rows_skipped"] += 1 + continue + if dry_run: + counts["rows_inserted"] += 1 + continue + db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id)) + counts["rows_inserted"] += 1 + + if not dry_run: + await db.commit() + + # Phase 2: write the per-image manifest for tag_apply.py to consume later. + manifest = { + "schema_version": 1, + "image_artist_assignments": data.get("image_artist_assignments", []), + "image_tag_associations": data.get("image_tag_associations", []), + "series_pages": data.get("series_pages", []), + } + counts["rows_processed"] += len(manifest["image_artist_assignments"]) + counts["rows_processed"] += len(manifest["image_tag_associations"]) + counts["rows_processed"] += len(manifest["series_pages"]) + if not dry_run: + manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) + + return counts diff --git a/tests/test_ir_ingest.py b/tests/test_ir_ingest.py new file mode 100644 index 0000000..2af8ac8 --- /dev/null +++ b/tests/test_ir_ingest.py @@ -0,0 +1,129 @@ +"""FC-5: ir_ingest unit tests.""" +import json +from datetime import UTC, datetime + +import pytest +from sqlalchemy import select + +from backend.app.models import Tag +from backend.app.services.migrators import ir_ingest + +pytestmark = pytest.mark.integration + + +def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None): + return { + "source_app": "imagerepo", + "schema_version": 1, + "exported_at": datetime.now(UTC).isoformat(), + "tags": tags or [], + "image_artist_assignments": artist_assignments or [], + "image_tag_associations": tag_associations or [], + "series_pages": series_pages or [], + } + + +@pytest.mark.asyncio +async def test_general_tag_created(db, tmp_path): + data = _ir_export(tags=[ + {"name": "blue_eyes", "kind": "general", "fandom_name": None}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + tag = (await db.execute( + select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general") + )).scalar_one() + assert tag.fandom_id is None + + +@pytest.mark.asyncio +async def test_character_tag_resolves_fandom_name(db, tmp_path): + data = _ir_export(tags=[ + {"name": "Wonderland", "kind": "fandom", "fandom_name": None}, + {"name": "Alice", "kind": "character", "fandom_name": "Wonderland"}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + fandom = (await db.execute( + select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom") + )).scalar_one() + alice = (await db.execute( + select(Tag).where(Tag.name == "Alice", Tag.kind == "character") + )).scalar_one() + assert alice.fandom_id == fandom.id + + +@pytest.mark.asyncio +async def test_artist_and_post_kinds_skipped(db, tmp_path): + data = _ir_export(tags=[ + {"name": "BobArtist", "kind": "artist", "fandom_name": None}, + {"name": "PostXYZ", "kind": "post", "fandom_name": None}, + {"name": "general_one", "kind": "general", "fandom_name": None}, + ]) + counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + assert counts["rows_skipped"] >= 2 + + bob = (await db.execute( + select(Tag).where(Tag.name == "BobArtist") + )).scalar_one_or_none() + post = (await db.execute( + select(Tag).where(Tag.name == "PostXYZ") + )).scalar_one_or_none() + assert bob is None + assert post is None + + +@pytest.mark.asyncio +async def test_manifest_written_to_disk(db, tmp_path): + data = _ir_export( + artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}], + tag_associations=[ + {"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None}, + ], + series_pages=[ + {"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1}, + ], + ) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + + manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json" + assert manifest_file.exists() + manifest = json.loads(manifest_file.read_text()) + assert manifest["schema_version"] == 1 + assert len(manifest["image_artist_assignments"]) == 1 + assert len(manifest["image_tag_associations"]) == 1 + assert len(manifest["series_pages"]) == 1 + + +@pytest.mark.asyncio +async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path): + data = _ir_export( + tags=[{"name": "dry", "kind": "general", "fandom_name": None}], + artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}], + ) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True) + + tag = (await db.execute( + select(Tag).where(Tag.name == "dry") + )).scalar_one_or_none() + assert tag is None + manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json" + assert not manifest_file.exists() + + +@pytest.mark.asyncio +async def test_idempotent_on_rerun(db, tmp_path): + data = _ir_export(tags=[ + {"name": "rerun", "kind": "general", "fandom_name": None}, + ]) + await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False) + assert counts2["rows_skipped"] >= 1 + + +@pytest.mark.asyncio +async def test_rejects_wrong_source_app(db, tmp_path): + bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [], + "image_artist_assignments": [], "image_tag_associations": [], "series_pages": []} + with pytest.raises(ValueError): + await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False)