fc5: gs_ingest — parsed-JSON ingest, Artist/Source/Credential, re-encrypts with FC key

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-22 08:59:43 -04:00
parent e1b057f494
commit a13a54206e
2 changed files with 248 additions and 0 deletions
@@ -0,0 +1,126 @@
+"""GallerySubscriber export → FabledCurator ingest.
+
+Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection
+to GS). Creates Artist (from subscriptions) + Source (nested under each
+subscription) + Credential (re-encrypted with FC's key). Idempotent on
+natural keys: Artist.slug, (artist_id, platform, url), Credential.platform.
+
+Credentials arrive plaintext in the export — GS's export script
+decrypts using GS's Fernet key in GS's own process. FC re-encrypts
+with FC's CredentialCrypto.
+"""
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ...models import Artist, Credential, Source
+from ...utils.slug import slugify
+from ..credential_crypto import CredentialCrypto
+
+
+def _zero_counts() -> dict:
+    return {
+        "rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
+        "files_copied": 0, "bytes_copied": 0, "conflicts": 0,
+    }
+
+
+async def migrate_async(
+    db: AsyncSession,
+    *,
+    data: dict,
+    fc_crypto: CredentialCrypto | None = None,
+    dry_run: bool = False,
+) -> dict:
+    """Ingest a parsed gallerysubscriber-export-v1.json dict."""
+    if data.get("source_app") != "gallerysubscriber":
+        raise ValueError("export source_app must be 'gallerysubscriber'")
+    if data.get("schema_version") != 1:
+        raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
+
+    counts = _zero_counts()
+
+    # Phase 1: subscriptions → Artist; nested sources within each.
+    for sub in data.get("subscriptions", []):
+        counts["rows_processed"] += 1
+        slug = slugify(sub["name"])
+        artist = (await db.execute(
+            select(Artist).where(Artist.slug == slug)
+        )).scalar_one_or_none()
+        if artist is None:
+            if dry_run:
+                counts["rows_inserted"] += 1
+                # Continue to nested sources, but they can't link without an artist row.
+                continue
+            notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None
+            artist = Artist(
+                name=sub["name"], slug=slug,
+                is_subscription=True,
+                auto_check=bool(sub.get("enabled", True)),
+                notes=notes,
+            )
+            db.add(artist)
+            await db.flush()
+            counts["rows_inserted"] += 1
+        else:
+            counts["rows_skipped"] += 1
+
+        # Nested sources under this subscription.
+        for src in sub.get("sources", []):
+            counts["rows_processed"] += 1
+            existing = (await db.execute(
+                select(Source).where(
+                    Source.artist_id == artist.id,
+                    Source.platform == src["platform"],
+                    Source.url == src["url"],
+                )
+            )).scalar_one_or_none()
+            if existing is not None:
+                counts["rows_skipped"] += 1
+                continue
+            if dry_run:
+                counts["rows_inserted"] += 1
+                continue
+            db.add(Source(
+                artist_id=artist.id,
+                platform=src["platform"],
+                url=src["url"],
+                enabled=bool(src.get("enabled", True)),
+                check_interval_override=src.get("check_interval"),
+                config_overrides=src.get("metadata") or {},
+            ))
+            counts["rows_inserted"] += 1
+
+    # Phase 2: credentials.
+    for cred in data.get("credentials", []):
+        counts["rows_processed"] += 1
+        existing = (await db.execute(
+            select(Credential).where(Credential.platform == cred["platform"])
+        )).scalar_one_or_none()
+        if existing is not None:
+            counts["rows_skipped"] += 1
+            continue
+        if dry_run:
+            counts["rows_inserted"] += 1
+            continue
+        if fc_crypto is None:
+            # Without a crypto helper we can't encrypt — skip rather than
+            # store plaintext.
+            counts["rows_skipped"] += 1
+            counts["conflicts"] += 1
+            continue
+        encrypted = fc_crypto.encrypt(cred["plaintext"])
+        db.add(Credential(
+            platform=cred["platform"],
+            kind=cred.get("credential_type") or "cookies",
+            encrypted_blob=encrypted,
+            expires_at=cred.get("expires_at"),
+        ))
+        counts["rows_inserted"] += 1
+
+    if not dry_run:
+        await db.commit()
+    return counts