diff --git a/backend/app/services/gallery_service.py b/backend/app/services/gallery_service.py index 834c6a6..8337390 100644 --- a/backend/app/services/gallery_service.py +++ b/backend/app/services/gallery_service.py @@ -237,6 +237,15 @@ class GalleryService: ) tags = (await self.session.execute(tag_stmt)).scalars().all() neighbors = await self._neighbors(record) + # Direct artist FK — used by the modal's ProvenancePanel as a + # fallback when ImageProvenance is empty (i.e., filesystem- + # imported images without a post-track provenance row). The + # source of truth for richer post-level data is still + # ImageProvenance/Post; this is just the "we at least know who + # made it" line. + artist = None + if record.artist_id is not None: + artist = await self.session.get(Artist, record.artist_id) return { "id": record.id, "path": record.path, @@ -249,6 +258,10 @@ class GalleryService: "created_at": record.created_at.isoformat(), "thumbnail_url": thumbnail_url(record.sha256, record.mime), "image_url": f"/images/{record.path.split('/images/', 1)[-1]}", + "artist": ( + {"id": artist.id, "name": artist.name, "slug": artist.slug} + if artist is not None else None + ), "tags": [ { "id": t.id, diff --git a/backend/app/services/migrators/ir_ingest.py b/backend/app/services/migrators/ir_ingest.py index 75d12cc..b501583 100644 --- a/backend/app/services/migrators/ir_ingest.py +++ b/backend/app/services/migrators/ir_ingest.py @@ -70,7 +70,7 @@ async def migrate_async( """ if data.get("source_app") != "imagerepo": raise ValueError("export source_app must be 'imagerepo'") - if data.get("schema_version") != 1: + if data.get("schema_version") not in (1, 2): raise ValueError(f"unsupported schema_version: {data.get('schema_version')}") counts = _zero_counts() @@ -126,15 +126,20 @@ async def migrate_async( await db.commit() # Phase 2: write the per-image manifest for tag_apply.py to consume later. + # schema_version 2 (added 2026-05-24) carries `image_posts` for + # Post + Source + ImageProvenance restore; schema 1 manifests + # without it stay valid (tag_apply treats the missing field as []). manifest = { - "schema_version": 1, + "schema_version": data.get("schema_version", 1), "image_artist_assignments": data.get("image_artist_assignments", []), "image_tag_associations": data.get("image_tag_associations", []), "series_pages": data.get("series_pages", []), + "image_posts": data.get("image_posts", []), } counts["rows_processed"] += len(manifest["image_artist_assignments"]) counts["rows_processed"] += len(manifest["image_tag_associations"]) counts["rows_processed"] += len(manifest["series_pages"]) + counts["rows_processed"] += len(manifest["image_posts"]) if not dry_run: manifest_path(images_root).write_text(json.dumps(manifest, indent=2)) diff --git a/backend/app/services/migrators/tag_apply.py b/backend/app/services/migrators/tag_apply.py index b02d8ab..5db92ff 100644 --- a/backend/app/services/migrators/tag_apply.py +++ b/backend/app/services/migrators/tag_apply.py @@ -7,6 +7,7 @@ FC's filesystem scan over the mounted IR images dir). - image_artist_assignments → ImageRecord.artist_id (find_or_create Artist by slug). - image_tag_associations → image_tag insert (idempotent). - series_pages → series_page insert (idempotent on image_id unique). +- image_posts (schema v2) → Source + Post + ImageProvenance restore. Unmatched sha256s are logged into the result's `unmatched` list so the Celery task can drop them into MigrationRun.metadata for the operator @@ -15,15 +16,118 @@ to inspect. from __future__ import annotations import json +from datetime import datetime from pathlib import Path from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from ...models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from ...models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + SeriesPage, + Source, + Tag, + TagKind, + image_tag, +) from ...utils.slug import slugify from .ir_ingest import manifest_path +# Per-platform artist-profile URL — used as Source.url when restoring +# IR PostMetadata into FC. Keep this table in sync with +# backend/app/services/extension_service.py:_PLATFORM_PATTERNS and +# extension/lib/platforms.js. +_PLATFORM_PROFILE_URL = { + "patreon": "https://www.patreon.com/{slug}", + "subscribestar": "https://www.subscribestar.com/{slug}", + "hentaifoundry": "https://www.hentai-foundry.com/user/{slug}", +} + + +def _profile_url(platform: str, artist_slug: str) -> str | None: + fmt = _PLATFORM_PROFILE_URL.get(platform) + return fmt.format(slug=artist_slug) if fmt else None + + +async def _find_or_create_source( + db: AsyncSession, *, artist_id: int, platform: str, url: str, dry_run: bool, +) -> int | None: + existing = (await db.execute( + select(Source.id).where( + Source.artist_id == artist_id, + Source.platform == platform, + Source.url == url, + ) + )).scalar_one_or_none() + if existing is not None: + return existing + if dry_run: + return None + s = Source(artist_id=artist_id, platform=platform, url=url, enabled=False) + db.add(s) + await db.flush() + return s.id + + +async def _find_or_create_post( + db: AsyncSession, *, + source_id: int, external_post_id: str, + title: str | None, description: str | None, post_url: str | None, + post_date_iso: str | None, attachment_count: int, dry_run: bool, +) -> int | None: + existing = (await db.execute( + select(Post.id).where( + Post.source_id == source_id, + Post.external_post_id == external_post_id, + ) + )).scalar_one_or_none() + if existing is not None: + return existing + if dry_run: + return None + post_date = None + if post_date_iso: + post_date = datetime.fromisoformat(post_date_iso) + p = Post( + source_id=source_id, + external_post_id=external_post_id, + post_title=title, + description=description, + post_url=post_url, + post_date=post_date, + attachment_count=attachment_count, + raw_metadata={"migrated_from": "imagerepo"}, + ) + db.add(p) + await db.flush() + return p.id + + +async def _ensure_provenance( + db: AsyncSession, *, + image_id: int, post_id: int, source_id: int, dry_run: bool, +) -> bool: + """Returns True if a new ImageProvenance row was inserted.""" + existing = (await db.execute( + select(ImageProvenance.id).where( + ImageProvenance.image_record_id == image_id, + ImageProvenance.post_id == post_id, + ImageProvenance.source_id == source_id, + ) + )).scalar_one_or_none() + if existing is not None: + return False + if dry_run: + return True + db.add(ImageProvenance( + image_record_id=image_id, post_id=post_id, source_id=source_id, + )) + await db.flush() + return True + def _zero_counts() -> dict: return { @@ -167,6 +271,67 @@ async def apply_async( )) counts["rows_inserted"] += 1 + # 4. Image posts (schema v2) → Source + Post + ImageProvenance. + # Restores IR PostMetadata as FC's downloader-track provenance, + # so the modal's ProvenancePanel surfaces title/description/ + # source URL/publish date the same way it does for live + # gallery-dl downloads. + for entry in manifest.get("image_posts", []): + counts["rows_processed"] += 1 + platform = entry.get("platform") + artist_name = entry.get("artist") + if not platform or not artist_name: + counts["rows_skipped"] += 1 + continue + + aid = await _ensure_artist_id(db, artist_name, dry_run) + if aid is None: + counts["rows_skipped"] += 1 + continue + + url = _profile_url(platform, slugify(artist_name)) + if url is None: + counts["rows_skipped"] += 1 + continue + + source_id = await _find_or_create_source( + db, artist_id=aid, platform=platform, url=url, dry_run=dry_run, + ) + if source_id is None: + counts["rows_skipped"] += 1 + continue + + post_id = await _find_or_create_post( + db, source_id=source_id, + external_post_id=entry.get("post_id") or "", + title=entry.get("title"), + description=entry.get("description"), + post_url=entry.get("source_url"), + post_date_iso=entry.get("published_at"), + attachment_count=entry.get("attachment_count") or 0, + dry_run=dry_run, + ) + if post_id is None: + counts["rows_skipped"] += 1 + continue + + for sha in entry.get("image_sha256s", []): + img_id = await _sha_to_image_id(db, sha) + if img_id is None: + unmatched.append({ + "kind": "post", "sha256": sha, + "post_id": entry.get("post_id"), + }) + continue + inserted = await _ensure_provenance( + db, image_id=img_id, post_id=post_id, + source_id=source_id, dry_run=dry_run, + ) + if inserted: + counts["rows_inserted"] += 1 + else: + counts["rows_skipped"] += 1 + if not dry_run: await db.commit() return {"counts": counts, "unmatched": unmatched} diff --git a/frontend/src/components/modal/ProvenancePanel.vue b/frontend/src/components/modal/ProvenancePanel.vue index 9e428c2..178c6af 100644 --- a/frontend/src/components/modal/ProvenancePanel.vue +++ b/frontend/src/components/modal/ProvenancePanel.vue @@ -43,6 +43,21 @@ class="fc-prov__desc" v-html="e.post.description_html" /> + + +
+
+ + by {{ fallbackArtist.name }} + +
+
@@ -79,13 +94,27 @@ const state = computed(() => modal.currentImageId == null ? null : prov.imageProv(modal.currentImageId) ) -// Show the panel while loading or on error, or when there is >=1 entry. -// Hide entirely when the image has zero provenance. +const fallbackArtist = computed(() => modal.current?.artist || null) + +const showArtistFallback = computed(() => { + const st = state.value + // Only show the fallback when provenance is genuinely empty (no + // entries, not loading, no error) AND the image has a direct + // artist_id we can surface. + if (!st || st.loading || st.error) return false + if (Array.isArray(st.entries) && st.entries.length > 0) return false + return fallbackArtist.value != null +}) + +// Show the panel while loading or on error, when there is >=1 entry, +// or when the artist fallback applies (image_record.artist_id set but +// no ImageProvenance row, e.g. filesystem-imported IR images). const show = computed(() => { const st = state.value if (!st) return false if (st.loading || st.error) return true - return Array.isArray(st.entries) && st.entries.length > 0 + if (Array.isArray(st.entries) && st.entries.length > 0) return true + return showArtistFallback.value }) const attachments = computed(() => state.value?.attachments || []) diff --git a/tests/test_tag_apply.py b/tests/test_tag_apply.py index 3d9ea6b..cee72c0 100644 --- a/tests/test_tag_apply.py +++ b/tests/test_tag_apply.py @@ -2,9 +2,19 @@ import json import pytest -from sqlalchemy import select +from sqlalchemy import func, select -from backend.app.models import Artist, ImageRecord, SeriesPage, Tag, TagKind, image_tag +from backend.app.models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + SeriesPage, + Source, + Tag, + TagKind, + image_tag, +) from backend.app.services.migrators import tag_apply pytestmark = pytest.mark.integration @@ -21,14 +31,19 @@ async def _seed_image(db, sha: str, *, suffix="x"): return img -def _write_manifest(tmp_path, *, artist_assignments=None, tag_associations=None, series_pages=None): +def _write_manifest( + tmp_path, *, + artist_assignments=None, tag_associations=None, + series_pages=None, image_posts=None, schema_version=2, +): d = tmp_path / "_migration_state" d.mkdir(parents=True, exist_ok=True) (d / "ir_tag_manifest.json").write_text(json.dumps({ - "schema_version": 1, + "schema_version": schema_version, "image_artist_assignments": artist_assignments or [], "image_tag_associations": tag_associations or [], "series_pages": series_pages or [], + "image_posts": image_posts or [], })) @@ -154,3 +169,130 @@ async def test_dry_run_makes_no_changes(db, tmp_path): select(image_tag.c.tag_id).where(image_tag.c.image_record_id == img_id) )).all() assert len(rows) == 0 + + +# --- Phase 4: image_posts (schema v2) ----------------------------------- + + +_POST_ENTRY = { + "platform": "patreon", + "post_id": "10001", + "artist": "Maewix", + "title": "Test Post Title", + "description": "

HTML description

", + "source_url": "https://www.patreon.com/posts/test-10001", + "attachment_count": 1, + "published_at": "2026-01-15T12:00:00+00:00", +} + + +@pytest.mark.asyncio +async def test_image_posts_creates_source_post_provenance(db, tmp_path): + sha = "d" * 64 + await _seed_image(db, sha, suffix="d") + await db.commit() + + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "image_sha256s": [sha]}, + ]) + + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result["counts"]["rows_inserted"] >= 1 + + # Async Core-DML assertions — column selects, not ORM attribute access. + artist_count = (await db.execute( + select(func.count(Artist.id)).where(Artist.slug == "maewix") + )).scalar_one() + assert artist_count == 1 + + source_count = (await db.execute( + select(func.count(Source.id)).where( + Source.platform == "patreon", + Source.url == "https://www.patreon.com/maewix", + ) + )).scalar_one() + assert source_count == 1 + + post_count = (await db.execute( + select(func.count(Post.id)).where(Post.external_post_id == "10001") + )).scalar_one() + assert post_count == 1 + + img_id = (await db.execute( + select(ImageRecord.id).where(ImageRecord.sha256 == sha) + )).scalar_one() + prov_count = (await db.execute( + select(func.count(ImageProvenance.id)) + .where(ImageProvenance.image_record_id == img_id) + )).scalar_one() + assert prov_count == 1 + + +@pytest.mark.asyncio +async def test_image_posts_idempotent_on_rerun(db, tmp_path): + sha = "e" * 64 + await _seed_image(db, sha, suffix="e") + await db.commit() + + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "image_sha256s": [sha]}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + result2 = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + + # Re-run: nothing should be newly inserted; all skipped. + assert result2["counts"]["rows_inserted"] == 0 + assert result2["counts"]["rows_skipped"] >= 1 + + prov_count = (await db.execute( + select(func.count(ImageProvenance.id)) + )).scalar_one() + assert prov_count == 1 + + +@pytest.mark.asyncio +async def test_image_posts_unknown_sha_unmatched(db, tmp_path): + # No image seeded; sha doesn't exist. + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "image_sha256s": ["f" * 64]}, + ]) + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + unmatched_kinds = [u["kind"] for u in result["unmatched"]] + assert "post" in unmatched_kinds + + +@pytest.mark.asyncio +async def test_image_posts_unknown_platform_skipped(db, tmp_path): + sha = "1" * 64 + await _seed_image(db, sha, suffix="1") + await db.commit() + + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "platform": "tumblr", "image_sha256s": [sha]}, + ]) + result = await tag_apply.apply_async(db, images_root=tmp_path, dry_run=False) + assert result["counts"]["rows_skipped"] >= 1 + + # No Post / Source / ImageProvenance should have been created. + src_count = (await db.execute( + select(func.count(Source.id)).where(Source.platform == "tumblr") + )).scalar_one() + assert src_count == 0 + + +@pytest.mark.asyncio +async def test_image_posts_dry_run_makes_no_writes(db, tmp_path): + sha = "2" * 64 + await _seed_image(db, sha, suffix="2") + await db.commit() + + _write_manifest(tmp_path, image_posts=[ + {**_POST_ENTRY, "image_sha256s": [sha]}, + ]) + + await tag_apply.apply_async(db, images_root=tmp_path, dry_run=True) + prov_count = (await db.execute( + select(func.count(ImageProvenance.id)) + )).scalar_one() + assert prov_count == 0