fc5: ir_ingest — Tag rows (skip artist/post) + manifest JSON for sha256-keyed tag_apply
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
"""ImageRepo export → FabledCurator ingest.
|
||||
|
||||
Reads a parsed imagerepo-export-v1.json dict (no DB connection to IR).
|
||||
Creates Tag rows (skipping artist/post kinds, resolving fandom_name to
|
||||
FK). Writes the per-image-sha256 artist assignments + tag associations
|
||||
+ series page assignments to /images/_migration_state/ir_tag_manifest.json
|
||||
so tag_apply.py can join them to ImageRecord rows AFTER the operator
|
||||
runs FC's filesystem scan.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import Tag, TagKind
|
||||
|
||||
_SKIP_KINDS = frozenset({"artist", "post"})
|
||||
_MIGRATION_STATE_DIRNAME = "_migration_state"
|
||||
_IR_MANIFEST_FILENAME = "ir_tag_manifest.json"
|
||||
|
||||
|
||||
def _zero_counts() -> dict:
|
||||
return {
|
||||
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
||||
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
||||
}
|
||||
|
||||
|
||||
def manifest_path(images_root: Path | None = None) -> Path:
|
||||
root = images_root if images_root is not None else Path("/images")
|
||||
p = root / _MIGRATION_STATE_DIRNAME
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return p / _IR_MANIFEST_FILENAME
|
||||
|
||||
|
||||
async def _resolve_fandom_id(
|
||||
db: AsyncSession, fandom_name: str | None, dry_run: bool,
|
||||
) -> int | None:
|
||||
"""Find-or-create a fandom-kind Tag by name."""
|
||||
if not fandom_name:
|
||||
return None
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == fandom_name, Tag.kind == "fandom")
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing.id
|
||||
if dry_run:
|
||||
return None
|
||||
t = Tag(name=fandom_name, kind=TagKind.fandom)
|
||||
db.add(t)
|
||||
await db.flush()
|
||||
return t.id
|
||||
|
||||
|
||||
async def migrate_async(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
data: dict,
|
||||
images_root: Path | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict:
|
||||
"""Ingest a parsed imagerepo-export-v1.json dict.
|
||||
|
||||
Creates Tag rows + writes the IR tag manifest file. Tag-to-image
|
||||
binding happens later in tag_apply.py (after FC's filesystem scan
|
||||
populates image_record.sha256 → id).
|
||||
"""
|
||||
if data.get("source_app") != "imagerepo":
|
||||
raise ValueError("export source_app must be 'imagerepo'")
|
||||
if data.get("schema_version") != 1:
|
||||
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
||||
|
||||
counts = _zero_counts()
|
||||
|
||||
# Phase 1: tags (skip artist + post kinds; resolve fandom_name → fandom_id).
|
||||
# First pass: create all fandom-kind tags so they're available for FK resolution.
|
||||
for tag in data.get("tags", []):
|
||||
kind = tag.get("kind") or "general"
|
||||
if kind != "fandom":
|
||||
continue
|
||||
counts["rows_processed"] += 1
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == "fandom")
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(Tag(name=tag["name"], kind=TagKind.fandom))
|
||||
counts["rows_inserted"] += 1
|
||||
if not dry_run:
|
||||
await db.flush()
|
||||
|
||||
# Second pass: every other kind.
|
||||
for tag in data.get("tags", []):
|
||||
kind_str = tag.get("kind") or "general"
|
||||
if kind_str in _SKIP_KINDS:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if kind_str == "fandom":
|
||||
continue # handled above
|
||||
counts["rows_processed"] += 1
|
||||
try:
|
||||
kind = TagKind(kind_str)
|
||||
except ValueError:
|
||||
kind = TagKind.general
|
||||
fandom_id = await _resolve_fandom_id(db, tag.get("fandom_name"), dry_run)
|
||||
existing = (await db.execute(
|
||||
select(Tag).where(Tag.name == tag["name"], Tag.kind == kind)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
counts["rows_skipped"] += 1
|
||||
continue
|
||||
if dry_run:
|
||||
counts["rows_inserted"] += 1
|
||||
continue
|
||||
db.add(Tag(name=tag["name"], kind=kind, fandom_id=fandom_id))
|
||||
counts["rows_inserted"] += 1
|
||||
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
|
||||
# Phase 2: write the per-image manifest for tag_apply.py to consume later.
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"image_artist_assignments": data.get("image_artist_assignments", []),
|
||||
"image_tag_associations": data.get("image_tag_associations", []),
|
||||
"series_pages": data.get("series_pages", []),
|
||||
}
|
||||
counts["rows_processed"] += len(manifest["image_artist_assignments"])
|
||||
counts["rows_processed"] += len(manifest["image_tag_associations"])
|
||||
counts["rows_processed"] += len(manifest["series_pages"])
|
||||
if not dry_run:
|
||||
manifest_path(images_root).write_text(json.dumps(manifest, indent=2))
|
||||
|
||||
return counts
|
||||
@@ -0,0 +1,129 @@
|
||||
"""FC-5: ir_ingest unit tests."""
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import Tag
|
||||
from backend.app.services.migrators import ir_ingest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _ir_export(tags=None, artist_assignments=None, tag_associations=None, series_pages=None):
|
||||
return {
|
||||
"source_app": "imagerepo",
|
||||
"schema_version": 1,
|
||||
"exported_at": datetime.now(UTC).isoformat(),
|
||||
"tags": tags or [],
|
||||
"image_artist_assignments": artist_assignments or [],
|
||||
"image_tag_associations": tag_associations or [],
|
||||
"series_pages": series_pages or [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_general_tag_created(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "blue_eyes", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
tag = (await db.execute(
|
||||
select(Tag).where(Tag.name == "blue_eyes", Tag.kind == "general")
|
||||
)).scalar_one()
|
||||
assert tag.fandom_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_character_tag_resolves_fandom_name(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "Wonderland", "kind": "fandom", "fandom_name": None},
|
||||
{"name": "Alice", "kind": "character", "fandom_name": "Wonderland"},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
fandom = (await db.execute(
|
||||
select(Tag).where(Tag.name == "Wonderland", Tag.kind == "fandom")
|
||||
)).scalar_one()
|
||||
alice = (await db.execute(
|
||||
select(Tag).where(Tag.name == "Alice", Tag.kind == "character")
|
||||
)).scalar_one()
|
||||
assert alice.fandom_id == fandom.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_artist_and_post_kinds_skipped(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "BobArtist", "kind": "artist", "fandom_name": None},
|
||||
{"name": "PostXYZ", "kind": "post", "fandom_name": None},
|
||||
{"name": "general_one", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
counts = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
assert counts["rows_skipped"] >= 2
|
||||
|
||||
bob = (await db.execute(
|
||||
select(Tag).where(Tag.name == "BobArtist")
|
||||
)).scalar_one_or_none()
|
||||
post = (await db.execute(
|
||||
select(Tag).where(Tag.name == "PostXYZ")
|
||||
)).scalar_one_or_none()
|
||||
assert bob is None
|
||||
assert post is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_written_to_disk(db, tmp_path):
|
||||
data = _ir_export(
|
||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "Alice"}],
|
||||
tag_associations=[
|
||||
{"sha256": "a" * 64, "tag_name": "blue_eyes", "tag_kind": "general", "fandom_name": None},
|
||||
],
|
||||
series_pages=[
|
||||
{"sha256": "a" * 64, "series_tag_name": "MySeries", "page_number": 1},
|
||||
],
|
||||
)
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
|
||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
||||
assert manifest_file.exists()
|
||||
manifest = json.loads(manifest_file.read_text())
|
||||
assert manifest["schema_version"] == 1
|
||||
assert len(manifest["image_artist_assignments"]) == 1
|
||||
assert len(manifest["image_tag_associations"]) == 1
|
||||
assert len(manifest["series_pages"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dry_run_writes_no_manifest_and_no_tags(db, tmp_path):
|
||||
data = _ir_export(
|
||||
tags=[{"name": "dry", "kind": "general", "fandom_name": None}],
|
||||
artist_assignments=[{"sha256": "a" * 64, "artist_name": "DryArtist"}],
|
||||
)
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=True)
|
||||
|
||||
tag = (await db.execute(
|
||||
select(Tag).where(Tag.name == "dry")
|
||||
)).scalar_one_or_none()
|
||||
assert tag is None
|
||||
manifest_file = tmp_path / "_migration_state" / "ir_tag_manifest.json"
|
||||
assert not manifest_file.exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idempotent_on_rerun(db, tmp_path):
|
||||
data = _ir_export(tags=[
|
||||
{"name": "rerun", "kind": "general", "fandom_name": None},
|
||||
])
|
||||
await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
counts2 = await ir_ingest.migrate_async(db, data=data, images_root=tmp_path, dry_run=False)
|
||||
assert counts2["rows_skipped"] >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_wrong_source_app(db, tmp_path):
|
||||
bad = {"source_app": "elsewhere", "schema_version": 1, "tags": [],
|
||||
"image_artist_assignments": [], "image_tag_associations": [], "series_pages": []}
|
||||
with pytest.raises(ValueError):
|
||||
await ir_ingest.migrate_async(db, data=bad, images_root=tmp_path, dry_run=False)
|
||||
Reference in New Issue
Block a user