fc5: fix GS export script — use async engine + correct imports, designed for in-container exec
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -11,53 +11,78 @@ Credentials are decrypted in this script's process using GS's secret_key
|
|||||||
ingest with its own Fernet key. The output JSON contains plaintext
|
ingest with its own Fernet key. The output JSON contains plaintext
|
||||||
credentials — delete it after the FC ingest completes.
|
credentials — delete it after the FC ingest completes.
|
||||||
|
|
||||||
Usage:
|
Designed to run INSIDE the live GS container:
|
||||||
python scripts/export_for_fabledcurator.py --secret-key "$SECRET_KEY" > gs-export.json
|
docker cp scripts/export_for_fabledcurator.py <gs_container>:/app/scripts/
|
||||||
|
docker exec -w /app <gs_container> python scripts/export_for_fabledcurator.py \\
|
||||||
|
--secret-key "$SECRET_KEY" > ~/gs-export.json
|
||||||
|
|
||||||
|
The container has GS's Python deps (asyncpg, sqlalchemy, cryptography)
|
||||||
|
and the runtime env vars (DATABASE_URL, SECRET_KEY) already set.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
# Bootstrap GS's app context.
|
# Ensure the container's WORKDIR (/app) is on sys.path so the GS package
|
||||||
sys.path.insert(0, "backend")
|
# `app.*` imports resolve. Container layout: /app/app/ is the package.
|
||||||
from app import create_app # noqa: E402
|
sys.path.insert(0, os.getcwd())
|
||||||
from app.database import get_session # noqa: E402
|
|
||||||
|
from sqlalchemy import select # noqa: E402
|
||||||
|
from sqlalchemy.ext.asyncio import ( # noqa: E402
|
||||||
|
AsyncSession,
|
||||||
|
async_sessionmaker,
|
||||||
|
create_async_engine,
|
||||||
|
)
|
||||||
|
|
||||||
from app.models.credential import Credential # noqa: E402
|
from app.models.credential import Credential # noqa: E402
|
||||||
from app.models.source import Source # noqa: E402
|
from app.models.source import Source # noqa: E402
|
||||||
from app.models.subscription import Subscription # noqa: E402
|
from app.models.subscription import Subscription # noqa: E402
|
||||||
from app.utils.encryption import decrypt_data # noqa: E402
|
from app.utils.encryption import decrypt_data # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
def _export_subscriptions(session) -> list[dict]:
|
def _async_database_url() -> str:
|
||||||
"""Subscriptions with nested sources."""
|
"""Resolve GS's async DB URL the same way app.config.Settings does."""
|
||||||
out: list[dict] = []
|
url = os.environ.get("DATABASE_URL")
|
||||||
for sub in session.query(Subscription).all():
|
if not url:
|
||||||
sources_out: list[dict] = []
|
# GS's default fallback chain — should rarely fire in prod since
|
||||||
for src in sub.sources or []:
|
# the container's DATABASE_URL is set by compose.
|
||||||
sources_out.append({
|
url = "postgresql://gdl:password@localhost:5432/gallery_subscriber"
|
||||||
"platform": src.platform,
|
return url.replace("postgresql://", "postgresql+asyncpg://")
|
||||||
"url": src.url,
|
|
||||||
"enabled": bool(src.enabled),
|
|
||||||
"check_interval": int(src.check_interval or 28800),
|
async def _export_async(secret_key: str) -> dict:
|
||||||
"metadata": src.metadata_ or {},
|
engine = create_async_engine(_async_database_url())
|
||||||
})
|
Session = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
|
||||||
out.append({
|
try:
|
||||||
|
async with Session() as session:
|
||||||
|
subs_out: list[dict] = []
|
||||||
|
subs = (await session.execute(select(Subscription))).scalars().all()
|
||||||
|
for sub in subs:
|
||||||
|
srcs = (await session.execute(
|
||||||
|
select(Source).where(Source.subscription_id == sub.id)
|
||||||
|
)).scalars().all()
|
||||||
|
sources_out = [{
|
||||||
|
"platform": s.platform,
|
||||||
|
"url": s.url,
|
||||||
|
"enabled": bool(s.enabled),
|
||||||
|
"check_interval": int(s.check_interval or 28800),
|
||||||
|
"metadata": s.metadata_ or {},
|
||||||
|
} for s in srcs]
|
||||||
|
subs_out.append({
|
||||||
"name": sub.name,
|
"name": sub.name,
|
||||||
"enabled": bool(sub.enabled),
|
"enabled": bool(sub.enabled),
|
||||||
"metadata": sub.metadata_ or {},
|
"metadata": sub.metadata_ or {},
|
||||||
"sources": sources_out,
|
"sources": sources_out,
|
||||||
})
|
})
|
||||||
return out
|
|
||||||
|
|
||||||
|
creds_out: list[dict] = []
|
||||||
def _export_credentials(session, secret_key: str) -> list[dict]:
|
creds = (await session.execute(select(Credential))).scalars().all()
|
||||||
"""Credentials decrypted to plaintext (FC re-encrypts on ingest)."""
|
for cred in creds:
|
||||||
out: list[dict] = []
|
|
||||||
for cred in session.query(Credential).all():
|
|
||||||
try:
|
try:
|
||||||
plaintext = decrypt_data(cred.data, secret_key)
|
plaintext = decrypt_data(cred.data, secret_key)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -67,7 +92,7 @@ def _export_credentials(session, secret_key: str) -> list[dict]:
|
|||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
out.append({
|
creds_out.append({
|
||||||
"platform": cred.platform,
|
"platform": cred.platform,
|
||||||
"credential_type": cred.credential_type,
|
"credential_type": cred.credential_type,
|
||||||
"plaintext": plaintext,
|
"plaintext": plaintext,
|
||||||
@@ -75,7 +100,16 @@ def _export_credentials(session, secret_key: str) -> list[dict]:
|
|||||||
cred.expires_at.isoformat() if cred.expires_at else None
|
cred.expires_at.isoformat() if cred.expires_at else None
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
return out
|
|
||||||
|
return {
|
||||||
|
"source_app": "gallerysubscriber",
|
||||||
|
"schema_version": 1,
|
||||||
|
"exported_at": datetime.now(UTC).isoformat(),
|
||||||
|
"subscriptions": subs_out,
|
||||||
|
"credentials": creds_out,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
@@ -93,16 +127,7 @@ def main() -> None:
|
|||||||
)
|
)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
app = create_app()
|
export = asyncio.run(_export_async(args.secret_key))
|
||||||
with app.app_context():
|
|
||||||
with get_session() as session:
|
|
||||||
export = {
|
|
||||||
"source_app": "gallerysubscriber",
|
|
||||||
"schema_version": 1,
|
|
||||||
"exported_at": datetime.now(UTC).isoformat(),
|
|
||||||
"subscriptions": _export_subscriptions(session),
|
|
||||||
"credentials": _export_credentials(session, args.secret_key),
|
|
||||||
}
|
|
||||||
json.dump(export, sys.stdout, indent=2)
|
json.dump(export, sys.stdout, indent=2)
|
||||||
sys.stdout.write("\n")
|
sys.stdout.write("\n")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user