fc5: ir_ingest — Tag rows (skip artist/post) + manifest JSON for sha256-keyed tag_apply

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/services/migrators/ir_ingest.py

@@ -0,0 +1,141 @@
+"""ImageRepo export → FabledCurator ingest.
+
+Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR).
+Creates Tag rows (skipping artist/post kinds, resolving fandom_name to
+FK). Writes the per-image-sha256 artist assignments + tag associations
++ series page assignments to /images/_migration_state/ir_tag_manifest.json
+so tag_apply.py can join them to ImageRecord rows AFTER the operator
+runs FC's filesystem scan.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ...models import Tag, TagKind
+
+_SKIP_KINDS = frozenset({"artist", "post"})
+_MIGRATION_STATE_DIRNAME = "_migration_state"
+_IR_MANIFEST_FILENAME = "ir_tag_manifest.json"
+
+
+def _zero_counts() -> dict:
+    return {
+        "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
+        "files_copied": 0, "bytes_copied": 0, "conflicts": 0,
+    }
+
+
+def manifest_path(images_root: Path | None = None) -> Path:
+    root = images_root if images_root is not None else Path("/images")
+    p = root / _MIGRATION_STATE_DIRNAME
+    p.mkdir(parents=True, exist_ok=True)
+    return p / _IR_MANIFEST_FILENAME
+
+
+async def _resolve_fandom_id(
+    db: AsyncSession, fandom_name: str | None, dry_run: bool,
+) -> int | None:
+    """Find-or-create a fandom-kind Tag by name."""
+    if not fandom_name:
+        return None
+    existing = (await db.execute(
+        select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom")
+    )).scalar_one_or_none()
+    if existing is not None:
+        return existing.id
+    if dry_run:
+        return None
+    t = Tag(name=fandom_name, kind=TagKind.fandom)
+    db.add(t)
+    await db.flush()
+    return t.id
+
+
+async def migrate_async(
+    db: AsyncSession,
+    *,
+    data: dict,
+    images_root: Path | None = None,
+    dry_run: bool = False,
+) -> dict:
+    """Ingest a parsed imagerepo-export-v1.json dict.
+
+    Creates Tag rows + writes the IR tag manifest file. Tag-to-image
+    binding happens later in tag_apply.py (after FC's filesystem scan
+    populates image_record.sha256 → id).
+    """
+    if data.get("source_app") != "imagerepo":
+        raise ValueError("export source_app must be 'imagerepo'")
+    if data.get("schema_version") != 1:
+        raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
+
+    counts = _zero_counts()
+
+    # Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id).
+    # First pass: create all fandom-kind tags so they're available for FK resolution.
+    for tag in data.get("tags", []):
+        kind = tag.get("kind") or "general"
+        if kind != "fandom":
+            continue
+        counts["rows_processed"] += 1
+        existing = (await db.execute(
+            select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom")
+        )).scalar_one_or_none()
+        if existing is not None:
+            counts["rows_skipped"] += 1
+            continue
+        if dry_run:
+            counts["rows_inserted"] += 1
+            continue
+        db.add(Tag(name=tag["name"], kind=TagKind.fandom))
+        counts["rows_inserted"] += 1
+    if not dry_run:
+        await db.flush()
+
+    # Second pass: every other kind.
+    for tag in data.get("tags", []):
+        kind_str = tag.get("kind") or "general"
+        if kind_str in _SKIP_KINDS:
+            counts["rows_skipped"] += 1
+            continue
+        if kind_str == "fandom":
+            continue  # handled above
+        counts["rows_processed"] += 1
+        try:
+            kind = TagKind(kind_str)
+        except ValueError:
+            kind = TagKind.general
+        fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run)
+        existing = (await db.execute(
+            select(Tag).where(Tag.name == tag["name"], Tag.kind == kind)
+        )).scalar_one_or_none()
+        if existing is not None:
+            counts["rows_skipped"] += 1
+            continue
+        if dry_run:
+            counts["rows_inserted"] += 1
+            continue
+        db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id))
+        counts["rows_inserted"] += 1
+
+    if not dry_run:
+        await db.commit()
+
+    # Phase 2: write the per-image manifest for tag_apply.py to consume later.
+    manifest = {
+        "schema_version": 1,
+        "image_artist_assignments": data.get("image_artist_assignments", []),
+        "image_tag_associations": data.get("image_tag_associations", []),
+        "series_pages": data.get("series_pages", []),
+    }
+    counts["rows_processed"] += len(manifest["image_artist_assignments"])
+    counts["rows_processed"] += len(manifest["image_tag_associations"])
+    counts["rows_processed"] += len(manifest["series_pages"])
+    if not dry_run:
+        manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
+
+    return counts

tests/test_ir_ingest.py

@@ -0,0 +1,129 @@
+"""FC-5: ir_ingest unit tests."""
+import json
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy import select
+
+from backend.app.models import Tag
+from backend.app.services.migrators import ir_ingest
+
+pytestmark = pytest.mark.integration
+
+
+def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None):
+    return {
+        "source_app": "imagerepo",
+        "schema_version": 1,
+        "exported_at": datetime.now(UTC).isoformat(),
+        "tags": tags or [],
+        "image_artist_assignments": artist_assignments or [],
+        "image_tag_associations": tag_associations or [],
+        "series_pages": series_pages or [],
+    }
+
+
+@pytest.mark.asyncio
+async def test_general_tag_created(db, tmp_path):
+    data = _ir_export(tags=[
+        {"name": "blue_eyes", "kind": "general", "fandom_name": None},
+    ])
+    await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+
+    tag = (await db.execute(
+        select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general")
+    )).scalar_one()
+    assert tag.fandom_id is None
+
+
+@pytest.mark.asyncio
+async def test_character_tag_resolves_fandom_name(db, tmp_path):
+    data = _ir_export(tags=[
+        {"name": "Wonderland", "kind": "fandom", "fandom_name": None},
+        {"name": "Alice", "kind": "character", "fandom_name": "Wonderland"},
+    ])
+    await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+
+    fandom = (await db.execute(
+        select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom")
+    )).scalar_one()
+    alice = (await db.execute(
+        select(Tag).where(Tag.name == "Alice", Tag.kind == "character")
+    )).scalar_one()
+    assert alice.fandom_id == fandom.id
+
+
+@pytest.mark.asyncio
+async def test_artist_and_post_kinds_skipped(db, tmp_path):
+    data = _ir_export(tags=[
+        {"name": "BobArtist", "kind": "artist", "fandom_name": None},
+        {"name": "PostXYZ", "kind": "post", "fandom_name": None},
+        {"name": "general_one", "kind": "general", "fandom_name": None},
+    ])
+    counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+    assert counts["rows_skipped"] >= 2
+
+    bob = (await db.execute(
+        select(Tag).where(Tag.name == "BobArtist")
+    )).scalar_one_or_none()
+    post = (await db.execute(
+        select(Tag).where(Tag.name == "PostXYZ")
+    )).scalar_one_or_none()
+    assert bob is None
+    assert post is None
+
+
+@pytest.mark.asyncio
+async def test_manifest_written_to_disk(db, tmp_path):
+    data = _ir_export(
+        artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}],
+        tag_associations=[
+            {"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
+        ],
+        series_pages=[
+            {"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1},
+        ],
+    )
+    await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+
+    manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
+    assert manifest_file.exists()
+    manifest = json.loads(manifest_file.read_text())
+    assert manifest["schema_version"] == 1
+    assert len(manifest["image_artist_assignments"]) == 1
+    assert len(manifest["image_tag_associations"]) == 1
+    assert len(manifest["series_pages"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path):
+    data = _ir_export(
+        tags=[{"name": "dry", "kind": "general", "fandom_name": None}],
+        artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}],
+    )
+    await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True)
+
+    tag = (await db.execute(
+        select(Tag).where(Tag.name == "dry")
+    )).scalar_one_or_none()
+    assert tag is None
+    manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
+    assert not manifest_file.exists()
+
+
+@pytest.mark.asyncio
+async def test_idempotent_on_rerun(db, tmp_path):
+    data = _ir_export(tags=[
+        {"name": "rerun", "kind": "general", "fandom_name": None},
+    ])
+    await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+    counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
+    assert counts2["rows_skipped"] >= 1
+
+
+@pytest.mark.asyncio
+async def test_rejects_wrong_source_app(db, tmp_path):
+    bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [],
+           "image_artist_assignments": [], "image_tag_associations": [], "series_pages": []}
+    with pytest.raises(ValueError):
+        await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False)