127 lines
4.4 KiB
Python
127 lines
4.4 KiB
Python
"""GallerySubscriber export → FabledCurator ingest.
|
|
|
|
Reads a parsed gallerysubscriber-export-v1.json dict (no DB connection
|
|
to GS). Creates Artist (from subscriptions) + Source (nested under each
|
|
subscription) + Credential (re-encrypted with FC's key). Idempotent on
|
|
natural keys: Artist.slug, (artist_id, platform, url), Credential.platform.
|
|
|
|
Credentials arrive plaintext in the export — GS's export script
|
|
decrypts using GS's Fernet key in GS's own process. FC re-encrypts
|
|
with FC's CredentialCrypto.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from ...models import Artist, Credential, Source
|
|
from ...utils.slug import slugify
|
|
from ..credential_crypto import CredentialCrypto
|
|
|
|
|
|
def _zero_counts() -> dict:
|
|
return {
|
|
"rows_processed": 0, "rows_inserted": 0, "rows_skipped": 0,
|
|
"files_copied": 0, "bytes_copied": 0, "conflicts": 0,
|
|
}
|
|
|
|
|
|
async def migrate_async(
|
|
db: AsyncSession,
|
|
*,
|
|
data: dict,
|
|
fc_crypto: CredentialCrypto | None = None,
|
|
dry_run: bool = False,
|
|
) -> dict:
|
|
"""Ingest a parsed gallerysubscriber-export-v1.json dict."""
|
|
if data.get("source_app") != "gallerysubscriber":
|
|
raise ValueError("export source_app must be 'gallerysubscriber'")
|
|
if data.get("schema_version") != 1:
|
|
raise ValueError(f"unsupported schema_version: {data.get('schema_version')}")
|
|
|
|
counts = _zero_counts()
|
|
|
|
# Phase 1: subscriptions → Artist; nested sources within each.
|
|
for sub in data.get("subscriptions", []):
|
|
counts["rows_processed"] += 1
|
|
slug = slugify(sub["name"])
|
|
artist = (await db.execute(
|
|
select(Artist).where(Artist.slug == slug)
|
|
)).scalar_one_or_none()
|
|
if artist is None:
|
|
if dry_run:
|
|
counts["rows_inserted"] += 1
|
|
# Continue to nested sources, but they can't link without an artist row.
|
|
continue
|
|
notes = json.dumps(sub.get("metadata"), indent=2) if sub.get("metadata") else None
|
|
artist = Artist(
|
|
name=sub["name"], slug=slug,
|
|
is_subscription=True,
|
|
auto_check=bool(sub.get("enabled", True)),
|
|
notes=notes,
|
|
)
|
|
db.add(artist)
|
|
await db.flush()
|
|
counts["rows_inserted"] += 1
|
|
else:
|
|
counts["rows_skipped"] += 1
|
|
|
|
# Nested sources under this subscription.
|
|
for src in sub.get("sources", []):
|
|
counts["rows_processed"] += 1
|
|
existing = (await db.execute(
|
|
select(Source).where(
|
|
Source.artist_id == artist.id,
|
|
Source.platform == src["platform"],
|
|
Source.url == src["url"],
|
|
)
|
|
)).scalar_one_or_none()
|
|
if existing is not None:
|
|
counts["rows_skipped"] += 1
|
|
continue
|
|
if dry_run:
|
|
counts["rows_inserted"] += 1
|
|
continue
|
|
db.add(Source(
|
|
artist_id=artist.id,
|
|
platform=src["platform"],
|
|
url=src["url"],
|
|
enabled=bool(src.get("enabled", True)),
|
|
check_interval_override=src.get("check_interval"),
|
|
config_overrides=src.get("metadata") or {},
|
|
))
|
|
counts["rows_inserted"] += 1
|
|
|
|
# Phase 2: credentials.
|
|
for cred in data.get("credentials", []):
|
|
counts["rows_processed"] += 1
|
|
existing = (await db.execute(
|
|
select(Credential).where(Credential.platform == cred["platform"])
|
|
)).scalar_one_or_none()
|
|
if existing is not None:
|
|
counts["rows_skipped"] += 1
|
|
continue
|
|
if dry_run:
|
|
counts["rows_inserted"] += 1
|
|
continue
|
|
if fc_crypto is None:
|
|
# Without a crypto helper we can't encrypt — skip rather than
|
|
# store plaintext.
|
|
counts["rows_skipped"] += 1
|
|
counts["conflicts"] += 1
|
|
continue
|
|
encrypted = fc_crypto.encrypt(cred["plaintext"])
|
|
db.add(Credential(
|
|
platform=cred["platform"],
|
|
credential_type=cred.get("credential_type") or "cookies",
|
|
encrypted_blob=encrypted,
|
|
expires_at=cred.get("expires_at"),
|
|
))
|
|
counts["rows_inserted"] += 1
|
|
|
|
if not dry_run:
|
|
await db.commit()
|
|
return counts
|