bvandeusen/GallerySubscriber
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

rapid interations of server side app and firefox extension

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-01-24 22:52:51 -05:00
commit b9b8048a2d
81 changed files with 9675 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Dockerfile.dev
+24
View File
@@ -0,0 +1,24 @@
FROM python:3.11-slim
WORKDIR /app
# Install system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
# Install Python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Copy application
COPY . .
# Create data directories
RUN mkdir -p /data/downloads /data/config /data/cookies
EXPOSE 8080
# Development mode with reload
CMD ["hypercorn", "app.main:app", "--bind", "0.0.0.0:8080", "--reload"]
backend/alembic.ini
+42
View File
@@ -0,0 +1,42 @@
[alembic]
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
backend/alembic/env.py
+75
View File
@@ -0,0 +1,75 @@
"""Alembic migration environment."""
import asyncio
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from app.config import get_settings
from app.models.base import Base
from app.models.source import Source
from app.models.download import Download
from app.models.credential import Credential
from app.models.content import ContentItem
from app.models.setting import Setting
config = context.config
settings = get_settings()
# Override sqlalchemy.url with actual database URL
config.set_main_option("sqlalchemy.url", settings.async_database_url)
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode."""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
"""Run migrations in 'online' mode with async engine."""
connectable = async_engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode."""
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
backend/alembic/script.py.mako
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
backend/alembic/versions/001_initial_schema.py
+129
View File
@@ -0,0 +1,129 @@
"""Initial schema
Revision ID: 001
Revises:
Create Date: 2025-01-24
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = '001'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Sources table
op.create_table(
'sources',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('platform', sa.String(50), nullable=False),
sa.Column('url', sa.Text(), nullable=False),
sa.Column('enabled', sa.Boolean(), default=True),
sa.Column('priority', sa.Integer(), default=0),
sa.Column('check_interval', sa.Integer(), default=3600),
sa.Column('last_check', sa.DateTime(), nullable=True),
sa.Column('last_success', sa.DateTime(), nullable=True),
sa.Column('error_count', sa.Integer(), default=0),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('platform', 'url', name='uq_source_platform_url')
)
op.create_index('ix_sources_platform', 'sources', ['platform'])
# Downloads table
op.create_table(
'downloads',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_id', sa.Integer(), nullable=True),
sa.Column('url', sa.Text(), nullable=False),
sa.Column('status', sa.String(20), default='pending'),
sa.Column('error_type', sa.String(50), nullable=True),
sa.Column('error_message', sa.Text(), nullable=True),
sa.Column('file_count', sa.Integer(), default=0),
sa.Column('total_size', sa.BigInteger(), nullable=True),
sa.Column('started_at', sa.DateTime(), nullable=True),
sa.Column('completed_at', sa.DateTime(), nullable=True),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='SET NULL')
)
op.create_index('ix_downloads_status', 'downloads', ['status'])
op.create_index('ix_downloads_source_id', 'downloads', ['source_id'])
# Content items table
op.create_table(
'content_items',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_id', sa.Integer(), nullable=False),
sa.Column('download_id', sa.Integer(), nullable=True),
sa.Column('external_id', sa.String(255), nullable=True),
sa.Column('title', sa.String(500), nullable=True),
sa.Column('content_type', sa.String(50), default='other'),
sa.Column('file_path', sa.String(500), nullable=True),
sa.Column('file_size', sa.BigInteger(), nullable=True),
sa.Column('thumbnail_path', sa.String(500), nullable=True),
sa.Column('published_at', sa.DateTime(), nullable=True),
sa.Column('discovered_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['download_id'], ['downloads.id'], ondelete='SET NULL'),
sa.UniqueConstraint('source_id', 'external_id', name='uq_content_source_external')
)
op.create_index('ix_content_items_source_id', 'content_items', ['source_id'])
op.create_index('ix_content_items_discovered_at', 'content_items', ['discovered_at'])
# Credentials table
op.create_table(
'credentials',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('platform', sa.String(50), nullable=False),
sa.Column('credential_type', sa.String(20), nullable=False),
sa.Column('data', sa.LargeBinary(), nullable=False),
sa.Column('expires_at', sa.DateTime(), nullable=True),
sa.Column('last_verified', sa.DateTime(), nullable=True),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('platform', name='uq_credential_platform')
)
op.create_index('ix_credentials_platform', 'credentials', ['platform'])
# Settings table
op.create_table(
'settings',
sa.Column('key', sa.String(100), nullable=False),
sa.Column('value', postgresql.JSONB(), nullable=False),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('key')
)
# Insert default settings
op.execute("""
INSERT INTO settings (key, value) VALUES
('download.parallel_limit', '3'),
('download.rate_limit', '3.0'),
('download.retry_count', '3'),
('download.schedule_interval', '3600'),
('notification.enabled', 'false'),
('notification.webhook_url', 'null')
""")
def downgrade() -> None:
op.drop_table('settings')
op.drop_table('credentials')
op.drop_table('content_items')
op.drop_table('downloads')
op.drop_table('sources')
backend/alembic/versions/002_add_subscriptions.py
+103
View File
@@ -0,0 +1,103 @@
"""Add subscriptions table and link sources
Revision ID: 002
Revises: 001
Create Date: 2025-01-25
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = '002'
down_revision: Union[str, None] = '001'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Create subscriptions table
op.create_table(
'subscriptions',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('enabled', sa.Boolean(), default=True),
sa.Column('priority', sa.Integer(), default=0),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id')
)
op.create_index('ix_subscriptions_name', 'subscriptions', ['name'], unique=True)
# Migrate existing sources to subscriptions
# Each unique source name becomes a subscription
conn = op.get_bind()
# Get existing sources
existing_sources = conn.execute(
sa.text("SELECT DISTINCT name FROM sources ORDER BY name")
).fetchall()
# Create subscriptions for each unique source name
for (name,) in existing_sources:
conn.execute(
sa.text("INSERT INTO subscriptions (name, enabled, priority) VALUES (:name, true, 0)"),
{"name": name}
)
# Add subscription_id column to sources
op.add_column('sources', sa.Column('subscription_id', sa.Integer(), nullable=True))
op.create_index('ix_sources_subscription_id', 'sources', ['subscription_id'])
# Update sources with their subscription_id
conn.execute(sa.text("""
UPDATE sources s
SET subscription_id = sub.id
FROM subscriptions sub
WHERE s.name = sub.name
"""))
# Make subscription_id NOT NULL after data migration
op.alter_column('sources', 'subscription_id', nullable=False)
# Add foreign key constraint
op.create_foreign_key(
'fk_sources_subscription_id',
'sources', 'subscriptions',
['subscription_id'], ['id'],
ondelete='CASCADE'
)
# Remove name and priority columns from sources (now on subscription)
op.drop_column('sources', 'name')
op.drop_column('sources', 'priority')
def downgrade() -> None:
# Add back name and priority columns to sources
op.add_column('sources', sa.Column('name', sa.String(255), nullable=True))
op.add_column('sources', sa.Column('priority', sa.Integer(), default=0))
# Populate name from subscription
conn = op.get_bind()
conn.execute(sa.text("""
UPDATE sources s
SET name = sub.name, priority = sub.priority
FROM subscriptions sub
WHERE s.subscription_id = sub.id
"""))
op.alter_column('sources', 'name', nullable=False)
# Remove foreign key and subscription_id column
op.drop_constraint('fk_sources_subscription_id', 'sources', type_='foreignkey')
op.drop_index('ix_sources_subscription_id', 'sources')
op.drop_column('sources', 'subscription_id')
# Drop subscriptions table
op.drop_index('ix_subscriptions_name', 'subscriptions')
op.drop_table('subscriptions')
backend/app/__init__.py
+1
View File
@@ -0,0 +1 @@
# Gallery Subscriber Backend
backend/app/api/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""API blueprints."""
from app.api import sources, downloads, credentials, settings, platforms, websocket
__all__ = ["sources", "downloads", "credentials", "settings", "platforms", "websocket"]
backend/app/api/credentials.py
+182
View File
@@ -0,0 +1,182 @@
"""Credential management API endpoints."""
from datetime import datetime
from quart import Blueprint, request, jsonify, current_app
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.credential import Credential, CredentialType
from app.models.setting import Setting, EXTENSION_API_KEY_SETTING, generate_api_key
from app.config import get_settings
from app.utils.encryption import encrypt_data, decrypt_data
bp = Blueprint("credentials", __name__)
VALID_PLATFORMS = ["patreon", "subscribestar", "hentaifoundry", "discord"]
@bp.route("", methods=["GET"])
async def list_credentials():
"""List all credential status (without sensitive data)."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(select(Credential))
credentials = result.scalars().all()
return jsonify({
"items": [c.to_dict(include_data=False) for c in credentials]
})
@bp.route("", methods=["POST"])
async def upload_credentials():
"""Upload credentials (cookies or token).
Can be called by:
1. Firefox extension (with X-Extension-Key header)
2. Web UI manual upload (with session auth)
"""
settings = get_settings()
# Check extension API key if present
extension_key = request.headers.get("X-Extension-Key")
if extension_key:
# Look up API key from database
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING)
)
setting = result.scalar()
stored_key = setting.value if setting else None
if not stored_key or extension_key != stored_key:
return jsonify({"error": "Invalid extension API key"}), 401
data = await request.get_json()
# Validate required fields
if not data.get("platform"):
return jsonify({"error": "Platform is required"}), 400
if not data.get("credential_type"):
return jsonify({"error": "Credential type is required"}), 400
if not data.get("data"):
return jsonify({"error": "Credential data is required"}), 400
platform = data["platform"]
cred_type = data["credential_type"]
cred_data = data["data"]
# Validate platform
if platform not in VALID_PLATFORMS:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400
# Validate credential type
if cred_type not in [CredentialType.COOKIES, CredentialType.TOKEN]:
return jsonify({"error": "Invalid credential type. Must be 'cookies' or 'token'"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Check for existing credential
result = await session.execute(
select(Credential).where(Credential.platform == platform)
)
credential = result.scalar()
# Encrypt the credential data
encrypted = encrypt_data(cred_data, settings.secret_key)
if credential:
# Update existing
credential.credential_type = cred_type
credential.data = encrypted
credential.expires_at = data.get("expires_at")
credential.last_verified = None # Reset verification
else:
# Create new
credential = Credential(
platform=platform,
credential_type=cred_type,
data=encrypted,
expires_at=data.get("expires_at"),
)
session.add(credential)
await session.commit()
await session.refresh(credential)
current_app.logger.info(f"Updated credentials for platform: {platform}")
return jsonify({
"message": "Credentials updated",
"platform": platform,
"credential_type": cred_type,
"expires_at": credential.expires_at.isoformat() if credential.expires_at else None,
})
@bp.route("/<platform>", methods=["DELETE"])
async def delete_credentials(platform: str):
"""Delete credentials for a platform."""
if platform not in VALID_PLATFORMS:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Credential).where(Credential.platform == platform)
)
credential = result.scalar()
if not credential:
return jsonify({"error": "Credentials not found for this platform"}), 404
await session.delete(credential)
await session.commit()
current_app.logger.info(f"Deleted credentials for platform: {platform}")
return "", 204
@bp.route("/verify", methods=["POST"])
async def verify_credentials():
"""Verify that credentials are working.
Attempts to make a test request to the platform.
"""
data = await request.get_json()
platform = data.get("platform")
if not platform:
return jsonify({"error": "Platform is required"}), 400
if platform not in VALID_PLATFORMS:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Credential).where(Credential.platform == platform)
)
credential = result.scalar()
if not credential:
return jsonify({
"platform": platform,
"is_valid": False,
"error": "No credentials stored for this platform",
})
# TODO: Implement actual verification by making test request
# For now, just mark as verified
credential.last_verified = datetime.utcnow()
await session.commit()
return jsonify({
"platform": platform,
"is_valid": True,
"last_verified": credential.last_verified.isoformat(),
})
backend/app/api/downloads.py
+171
View File
@@ -0,0 +1,171 @@
"""Download history API endpoints."""
from datetime import datetime
from quart import Blueprint, request, jsonify, current_app
from sqlalchemy import select, func, and_
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.models.download import Download, DownloadStatus
from app.models.source import Source
from app.tasks.downloads import process_download
bp = Blueprint("downloads", __name__)
@bp.route("", methods=["GET"])
async def list_downloads():
"""List download history with filtering and pagination."""
# Query parameters
source_id = request.args.get("source_id", type=int)
status = request.args.get("status")
from_date = request.args.get("from_date")
to_date = request.args.get("to_date")
page = int(request.args.get("page", 1))
per_page = min(int(request.args.get("per_page", 50)), 100)
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Build query
query = select(Download)
count_query = select(func.count()).select_from(Download)
filters = []
if source_id:
filters.append(Download.source_id == source_id)
if status:
filters.append(Download.status == status)
if from_date:
filters.append(Download.created_at >= datetime.fromisoformat(from_date))
if to_date:
filters.append(Download.created_at <= datetime.fromisoformat(to_date))
if filters:
query = query.where(and_(*filters))
count_query = count_query.where(and_(*filters))
# Get total count
total_result = await session.execute(count_query)
total = total_result.scalar()
# Apply pagination and ordering
query = query.order_by(Download.created_at.desc())
query = query.offset((page - 1) * per_page).limit(per_page)
result = await session.execute(query)
downloads = result.scalars().all()
return jsonify({
"items": [d.to_dict() for d in downloads],
"total": total,
"page": page,
"per_page": per_page,
"pages": (total + per_page - 1) // per_page,
})
@bp.route("/<int:download_id>", methods=["GET"])
async def get_download(download_id: int):
"""Get download details."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(select(Download).where(Download.id == download_id))
download = result.scalar()
if not download:
return jsonify({"error": "Download not found"}), 404
# Include source info if available
response = download.to_dict()
if download.source_id:
source_result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == download.source_id)
)
source = source_result.scalar()
if source:
response["source"] = {
"id": source.id,
"subscription_name": source.subscription.name if source.subscription else None,
"platform": source.platform,
}
return jsonify(response)
@bp.route("/<int:download_id>/retry", methods=["POST"])
async def retry_download(download_id: int):
"""Retry a failed download."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(select(Download).where(Download.id == download_id))
download = result.scalar()
if not download:
return jsonify({"error": "Download not found"}), 404
if download.status != DownloadStatus.FAILED:
return jsonify({"error": "Can only retry failed downloads"}), 400
# Reset status to pending
download.status = DownloadStatus.PENDING
download.error_type = None
download.error_message = None
await session.commit()
# Queue Celery task to retry the download
task = process_download.delay(download_id)
current_app.logger.info(f"Queued retry for download: {download_id}")
return jsonify({
"message": "Retry queued",
"download_id": download_id,
"task_id": task.id,
}), 202
@bp.route("/stats", methods=["GET"])
async def get_stats():
"""Get download statistics."""
period = request.args.get("period", "week") # day, week, month
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Total counts by status
status_query = select(
Download.status,
func.count(Download.id)
).group_by(Download.status)
status_result = await session.execute(status_query)
status_counts = {row[0]: row[1] for row in status_result}
# Counts by platform (through source)
platform_query = select(
Source.platform,
func.count(Download.id)
).join(Source, Download.source_id == Source.id).group_by(Source.platform)
platform_result = await session.execute(platform_query)
platform_counts = {row[0]: row[1] for row in platform_result}
# Recent downloads count
recent_query = select(func.count()).select_from(Download).where(
Download.status == DownloadStatus.COMPLETED
)
recent_result = await session.execute(recent_query)
recent_completed = recent_result.scalar()
return jsonify({
"total": sum(status_counts.values()),
"by_status": status_counts,
"by_platform": platform_counts,
"completed": status_counts.get(DownloadStatus.COMPLETED, 0),
"failed": status_counts.get(DownloadStatus.FAILED, 0),
"pending": status_counts.get(DownloadStatus.PENDING, 0),
})
backend/app/api/platforms.py
+199
View File
@@ -0,0 +1,199 @@
"""Platform information API endpoints.
Provides information about supported platforms and their configuration options.
"""
from quart import Blueprint, jsonify
from app.services.gallery_dl import GalleryDLService
bp = Blueprint("platforms", __name__)
@bp.route("", methods=["GET"])
async def list_platforms():
"""List all supported platforms with their configuration options."""
gdl_service = GalleryDLService()
platforms = {
"patreon": {
"name": "Patreon",
"description": "Download posts from Patreon creators",
"requires_auth": True,
"auth_type": "cookies",
"url_pattern": "https://www.patreon.com/{creator}",
"url_examples": [
"https://www.patreon.com/example_artist",
"https://www.patreon.com/user?u=12345678",
],
"content_types": gdl_service.get_platform_content_types("patreon"),
"content_type_descriptions": {
"images": "Standard post images",
"image_large": "High-resolution image versions",
"attachments": "File attachments (ZIP, PSD, etc.)",
"postfile": "Post-specific files",
"content": "Embedded content in post body",
},
"default_config": gdl_service.get_default_config_for_platform("patreon"),
},
"subscribestar": {
"name": "SubscribeStar",
"description": "Download posts from SubscribeStar creators",
"requires_auth": True,
"auth_type": "cookies",
"url_pattern": "https://subscribestar.{domain}/{creator}",
"url_examples": [
"https://subscribestar.adult/example_artist",
"https://www.subscribestar.com/example_artist",
],
"content_types": gdl_service.get_platform_content_types("subscribestar"),
"content_type_descriptions": {
"all": "All available content",
},
"default_config": gdl_service.get_default_config_for_platform("subscribestar"),
},
"hentaifoundry": {
"name": "Hentai Foundry",
"description": "Download artwork from Hentai Foundry artists",
"requires_auth": False,
"auth_type": "cookies", # Optional but improves access
"url_pattern": "https://www.hentai-foundry.com/user/{artist}",
"url_examples": [
"https://www.hentai-foundry.com/user/example_artist",
"https://www.hentai-foundry.com/pictures/user/example_artist",
],
"content_types": gdl_service.get_platform_content_types("hentaifoundry"),
"content_type_descriptions": {
"pictures": "Artwork/pictures",
"stories": "Written stories",
},
"default_config": gdl_service.get_default_config_for_platform("hentaifoundry"),
},
"discord": {
"name": "Discord",
"description": "Download attachments from Discord channels",
"requires_auth": True,
"auth_type": "token",
"url_pattern": "https://discord.com/channels/{server}/{channel}",
"url_examples": [
"https://discord.com/channels/123456789/987654321",
],
"content_types": gdl_service.get_platform_content_types("discord"),
"content_type_descriptions": {
"all": "All attachments and embeds",
},
"default_config": gdl_service.get_default_config_for_platform("discord"),
"notes": "Requires Discord user token (not bot token)",
},
}
return jsonify({"platforms": platforms})
@bp.route("/<platform>", methods=["GET"])
async def get_platform(platform: str):
"""Get detailed information about a specific platform."""
gdl_service = GalleryDLService()
if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]:
return jsonify({"error": f"Unknown platform: {platform}"}), 404
# Get the full platform info from list_platforms
all_platforms = (await list_platforms()).get_json()
platform_info = all_platforms["platforms"].get(platform)
if not platform_info:
return jsonify({"error": f"Platform not found: {platform}"}), 404
return jsonify(platform_info)
@bp.route("/<platform>/config-schema", methods=["GET"])
async def get_config_schema(platform: str):
"""Get the configuration schema for a platform.
This defines what options can be set per-source for this platform.
Useful for building dynamic forms in the frontend.
"""
if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]:
return jsonify({"error": f"Unknown platform: {platform}"}), 404
gdl_service = GalleryDLService()
content_types = gdl_service.get_platform_content_types(platform)
defaults = gdl_service.get_default_config_for_platform(platform)
schema = {
"platform": platform,
"fields": [
{
"name": "content_types",
"type": "multiselect",
"label": "Content Types to Download",
"description": "Select which types of content to download from this source",
"options": content_types,
"default": defaults.get("content_types", ["all"]),
},
{
"name": "sleep",
"type": "number",
"label": "Delay Between Downloads (seconds)",
"description": "Time to wait between downloading files. Higher values are safer but slower.",
"min": 0.5,
"max": 30.0,
"step": 0.5,
"default": defaults.get("sleep", 3.0),
},
{
"name": "sleep_request",
"type": "number",
"label": "Delay Between Requests (seconds)",
"description": "Time to wait between API requests. Helps avoid rate limiting.",
"min": 0.5,
"max": 15.0,
"step": 0.5,
"default": defaults.get("sleep_request", 1.5),
},
{
"name": "skip_existing",
"type": "boolean",
"label": "Skip Already Downloaded",
"description": "Skip files that have already been downloaded (uses archive database)",
"default": True,
},
{
"name": "save_metadata",
"type": "boolean",
"label": "Save Metadata",
"description": "Save JSON metadata file alongside each downloaded file",
"default": True,
},
{
"name": "timeout",
"type": "number",
"label": "Download Timeout (seconds)",
"description": "Maximum time to wait for a single download before giving up",
"min": 60,
"max": 7200,
"step": 60,
"default": 3600,
},
{
"name": "directory_pattern",
"type": "text",
"label": "Directory Pattern (Advanced)",
"description": "Custom directory structure pattern. Leave empty for platform default.",
"placeholder": defaults.get("directory_pattern", ""),
"default": None,
},
{
"name": "filename_pattern",
"type": "text",
"label": "Filename Pattern (Advanced)",
"description": "Custom filename pattern. Leave empty for platform default.",
"placeholder": defaults.get("filename_pattern", ""),
"default": None,
},
],
}
return jsonify(schema)
backend/app/api/settings.py
+165
View File
@@ -0,0 +1,165 @@
"""Settings API endpoints."""
import json
from pathlib import Path
from quart import Blueprint, request, jsonify, current_app
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.setting import Setting, DEFAULT_SETTINGS, EXTENSION_API_KEY_SETTING, generate_api_key
from app.config import get_settings
bp = Blueprint("settings", __name__)
async def get_or_create_extension_api_key(session: AsyncSession) -> str:
"""Get the extension API key, creating one if it doesn't exist."""
result = await session.execute(
select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING)
)
setting = result.scalar()
if setting:
return setting.value
# Generate new key
new_key = generate_api_key()
setting = Setting(key=EXTENSION_API_KEY_SETTING, value=new_key)
session.add(setting)
await session.commit()
return new_key
@bp.route("", methods=["GET"])
async def get_all_settings():
"""Get all application settings."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(select(Setting))
settings = result.scalars().all()
# Build settings dict with defaults
settings_dict = dict(DEFAULT_SETTINGS)
for s in settings:
settings_dict[s.key] = s.value
# Ensure extension API key exists
api_key = await get_or_create_extension_api_key(session)
settings_dict[EXTENSION_API_KEY_SETTING] = api_key
return jsonify({"settings": settings_dict})
@bp.route("", methods=["PATCH"])
async def update_settings():
"""Update application settings."""
data = await request.get_json()
if not data:
return jsonify({"error": "No settings provided"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
for key, value in data.items():
# Check if setting exists
result = await session.execute(select(Setting).where(Setting.key == key))
setting = result.scalar()
if setting:
setting.value = value
else:
setting = Setting(key=key, value=value)
session.add(setting)
await session.commit()
# Return updated settings
result = await session.execute(select(Setting))
settings = result.scalars().all()
settings_dict = dict(DEFAULT_SETTINGS)
for s in settings:
settings_dict[s.key] = s.value
current_app.logger.info(f"Updated settings: {list(data.keys())}")
return jsonify({"settings": settings_dict})
@bp.route("/api-key", methods=["GET"])
async def get_extension_api_key():
"""Get the extension API key."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
api_key = await get_or_create_extension_api_key(session)
return jsonify({"api_key": api_key})
@bp.route("/api-key/regenerate", methods=["POST"])
async def regenerate_extension_api_key():
"""Regenerate the extension API key."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Find existing key
result = await session.execute(
select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING)
)
setting = result.scalar()
# Generate new key
new_key = generate_api_key()
if setting:
setting.value = new_key
else:
setting = Setting(key=EXTENSION_API_KEY_SETTING, value=new_key)
session.add(setting)
await session.commit()
current_app.logger.info("Extension API key regenerated")
return jsonify({"api_key": new_key, "message": "API key regenerated"})
@bp.route("/gallery-dl", methods=["GET"])
async def get_gallery_dl_config():
"""Get gallery-dl configuration."""
config = get_settings()
config_path = Path(config.config_path) / "gallery-dl.conf"
if not config_path.exists():
return jsonify({"config": {}, "message": "No configuration file found"})
try:
with open(config_path) as f:
gallery_dl_config = json.load(f)
return jsonify({"config": gallery_dl_config})
except json.JSONDecodeError as e:
return jsonify({"error": f"Invalid JSON in config file: {e}"}), 500
@bp.route("/gallery-dl", methods=["PUT"])
async def update_gallery_dl_config():
"""Update gallery-dl configuration."""
data = await request.get_json()
if not data.get("config"):
return jsonify({"error": "Config object is required"}), 400
config = get_settings()
config_path = Path(config.config_path) / "gallery-dl.conf"
# Ensure config directory exists
config_path.parent.mkdir(parents=True, exist_ok=True)
try:
with open(config_path, "w") as f:
json.dump(data["config"], f, indent=4)
current_app.logger.info("Updated gallery-dl configuration")
return jsonify({"message": "Configuration updated", "config": data["config"]})
except Exception as e:
current_app.logger.error(f"Failed to write gallery-dl config: {e}")
return jsonify({"error": f"Failed to write config: {e}"}), 500
backend/app/api/sources.py
+223
View File
@@ -0,0 +1,223 @@
"""Source management API endpoints."""
from quart import Blueprint, request, jsonify, current_app
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.models.source import Source
from app.models.subscription import Subscription
from app.tasks.downloads import download_source
bp = Blueprint("sources", __name__)
@bp.route("", methods=["GET"])
async def list_sources():
"""List all sources with optional filtering and pagination."""
# Query parameters
platform = request.args.get("platform")
subscription_id = request.args.get("subscription_id")
enabled = request.args.get("enabled")
page = int(request.args.get("page", 1))
per_page = min(int(request.args.get("per_page", 50)), 100)
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Build query with subscription eager loading
query = select(Source).options(selectinload(Source.subscription))
if platform:
query = query.where(Source.platform == platform)
if subscription_id:
query = query.where(Source.subscription_id == int(subscription_id))
if enabled is not None:
query = query.where(Source.enabled == (enabled.lower() == "true"))
# Get total count
count_query = select(func.count()).select_from(Source)
if platform:
count_query = count_query.where(Source.platform == platform)
if subscription_id:
count_query = count_query.where(Source.subscription_id == int(subscription_id))
if enabled is not None:
count_query = count_query.where(Source.enabled == (enabled.lower() == "true"))
total_result = await session.execute(count_query)
total = total_result.scalar()
# Apply pagination - order by subscription name, then platform
query = query.join(Subscription).order_by(Subscription.name, Source.platform)
query = query.offset((page - 1) * per_page).limit(per_page)
result = await session.execute(query)
sources = result.scalars().all()
return jsonify({
"items": [s.to_dict() for s in sources],
"total": total,
"page": page,
"per_page": per_page,
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
})
@bp.route("", methods=["POST"])
async def create_source():
"""Create a new source for a subscription."""
data = await request.get_json()
# Validate required fields
required = ["subscription_id", "platform", "url"]
missing = [f for f in required if not data.get(f)]
if missing:
return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400
# Validate platform
valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"]
if data["platform"] not in valid_platforms:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Check subscription exists
sub_result = await session.execute(
select(Subscription).where(Subscription.id == data["subscription_id"])
)
subscription = sub_result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
# Check for duplicate platform+url
existing = await session.execute(
select(Source).where(
Source.platform == data["platform"],
Source.url == data["url"]
)
)
if existing.scalar():
return jsonify({"error": "Source with this platform and URL already exists"}), 409
source = Source(
subscription_id=data["subscription_id"],
platform=data["platform"],
url=data["url"],
enabled=data.get("enabled", True),
check_interval=data.get("check_interval", 3600),
metadata_=data.get("metadata", {}),
)
session.add(source)
await session.commit()
await session.refresh(source)
current_app.logger.info(f"Created source: {subscription.name}/{source.platform}")
return jsonify(source.to_dict()), 201
@bp.route("/<int:source_id>", methods=["GET"])
async def get_source(source_id: int):
"""Get a single source by ID."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == source_id)
)
source = result.scalar()
if not source:
return jsonify({"error": "Source not found"}), 404
return jsonify(source.to_dict())
@bp.route("/<int:source_id>", methods=["PATCH"])
async def update_source(source_id: int):
"""Update a source."""
data = await request.get_json()
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == source_id)
)
source = result.scalar()
if not source:
return jsonify({"error": "Source not found"}), 404
# Update allowed fields
allowed_fields = ["enabled", "check_interval", "metadata"]
for field in allowed_fields:
if field in data:
if field == "metadata":
source.metadata_ = data[field]
else:
setattr(source, field, data[field])
await session.commit()
await session.refresh(source)
current_app.logger.info(f"Updated source: {source.subscription.name}/{source.platform}")
return jsonify(source.to_dict())
@bp.route("/<int:source_id>", methods=["DELETE"])
async def delete_source(source_id: int):
"""Delete a source."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == source_id)
)
source = result.scalar()
if not source:
return jsonify({"error": "Source not found"}), 404
info = f"{source.subscription.name}/{source.platform}"
await session.delete(source)
await session.commit()
current_app.logger.info(f"Deleted source: {info}")
return "", 204
@bp.route("/<int:source_id>/check", methods=["POST"])
async def trigger_check(source_id: int):
"""Trigger an immediate download check for a source."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == source_id)
)
source = result.scalar()
if not source:
return jsonify({"error": "Source not found"}), 404
# Queue Celery task for immediate download
task = download_source.delay(source_id)
current_app.logger.info(f"Triggered check for source: {source.subscription.name}/{source.platform}")
return jsonify({
"message": "Check queued",
"source_id": source_id,
"subscription_name": source.subscription.name,
"platform": source.platform,
"task_id": task.id,
}), 202
backend/app/api/subscriptions.py
+307
View File
@@ -0,0 +1,307 @@
"""Subscription management API endpoints."""
from quart import Blueprint, request, jsonify, current_app
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.models.subscription import Subscription
from app.models.source import Source
from app.tasks.downloads import download_source
bp = Blueprint("subscriptions", __name__)
@bp.route("", methods=["GET"])
async def list_subscriptions():
"""List all subscriptions with optional filtering and pagination."""
# Query parameters
enabled = request.args.get("enabled")
search = request.args.get("search")
page = int(request.args.get("page", 1))
per_page = min(int(request.args.get("per_page", 50)), 100)
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Build query with eager loading of sources
query = select(Subscription).options(selectinload(Subscription.sources))
if enabled is not None:
query = query.where(Subscription.enabled == (enabled.lower() == "true"))
if search:
query = query.where(Subscription.name.ilike(f"%{search}%"))
# Get total count
count_query = select(func.count()).select_from(Subscription)
if enabled is not None:
count_query = count_query.where(Subscription.enabled == (enabled.lower() == "true"))
if search:
count_query = count_query.where(Subscription.name.ilike(f"%{search}%"))
total_result = await session.execute(count_query)
total = total_result.scalar()
# Apply pagination
query = query.order_by(Subscription.priority.desc(), Subscription.name)
query = query.offset((page - 1) * per_page).limit(per_page)
result = await session.execute(query)
subscriptions = result.scalars().unique().all()
return jsonify({
"items": [s.to_dict() for s in subscriptions],
"total": total,
"page": page,
"per_page": per_page,
"pages": (total + per_page - 1) // per_page if total > 0 else 0,
})
@bp.route("", methods=["POST"])
async def create_subscription():
"""Create a new subscription with optional sources."""
data = await request.get_json()
# Validate required fields
if not data.get("name"):
return jsonify({"error": "Name is required"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Check for duplicate name
existing = await session.execute(
select(Subscription).where(Subscription.name == data["name"])
)
if existing.scalar():
return jsonify({"error": "Subscription with this name already exists"}), 409
subscription = Subscription(
name=data["name"],
enabled=data.get("enabled", True),
priority=data.get("priority", 0),
description=data.get("description"),
metadata_=data.get("metadata", {}),
)
# Add sources if provided
sources_data = data.get("sources", [])
for source_data in sources_data:
if not source_data.get("platform") or not source_data.get("url"):
continue
source = Source(
platform=source_data["platform"],
url=source_data["url"],
enabled=source_data.get("enabled", True),
check_interval=source_data.get("check_interval", 3600),
metadata_=source_data.get("metadata", {}),
)
subscription.sources.append(source)
session.add(subscription)
await session.commit()
await session.refresh(subscription)
current_app.logger.info(f"Created subscription: {subscription.name}")
return jsonify(subscription.to_dict()), 201
@bp.route("/<int:subscription_id>", methods=["GET"])
async def get_subscription(subscription_id: int):
"""Get a single subscription by ID with its sources."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Subscription)
.options(selectinload(Subscription.sources))
.where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
return jsonify(subscription.to_dict())
@bp.route("/<int:subscription_id>", methods=["PATCH"])
async def update_subscription(subscription_id: int):
"""Update a subscription."""
data = await request.get_json()
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Subscription)
.options(selectinload(Subscription.sources))
.where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
# Check for duplicate name if changing
if "name" in data and data["name"] != subscription.name:
existing = await session.execute(
select(Subscription).where(Subscription.name == data["name"])
)
if existing.scalar():
return jsonify({"error": "Subscription with this name already exists"}), 409
# Update allowed fields
allowed_fields = ["name", "enabled", "priority", "description", "metadata"]
for field in allowed_fields:
if field in data:
if field == "metadata":
subscription.metadata_ = data[field]
else:
setattr(subscription, field, data[field])
await session.commit()
await session.refresh(subscription)
current_app.logger.info(f"Updated subscription: {subscription.name}")
return jsonify(subscription.to_dict())
@bp.route("/<int:subscription_id>", methods=["DELETE"])
async def delete_subscription(subscription_id: int):
"""Delete a subscription and all its sources."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Subscription).where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
name = subscription.name
await session.delete(subscription)
await session.commit()
current_app.logger.info(f"Deleted subscription: {name}")
return "", 204
@bp.route("/<int:subscription_id>/sources", methods=["GET"])
async def list_subscription_sources(subscription_id: int):
"""List all sources for a subscription."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Subscription)
.options(selectinload(Subscription.sources))
.where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
return jsonify({
"subscription_id": subscription_id,
"subscription_name": subscription.name,
"sources": [s.to_dict(include_subscription=False) for s in subscription.sources]
})
@bp.route("/<int:subscription_id>/sources", methods=["POST"])
async def add_source_to_subscription(subscription_id: int):
"""Add a new source to a subscription."""
data = await request.get_json()
# Validate required fields
required = ["platform", "url"]
missing = [f for f in required if not data.get(f)]
if missing:
return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400
# Validate platform
valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"]
if data["platform"] not in valid_platforms:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
# Check subscription exists
result = await session.execute(
select(Subscription).where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
# Check for duplicate platform+url
existing = await session.execute(
select(Source).where(
Source.platform == data["platform"],
Source.url == data["url"]
)
)
if existing.scalar():
return jsonify({"error": "Source with this platform and URL already exists"}), 409
# Store name before commit (attributes expire after commit)
subscription_name = subscription.name
source = Source(
subscription_id=subscription_id,
platform=data["platform"],
url=data["url"],
enabled=data.get("enabled", True),
check_interval=data.get("check_interval", 3600),
metadata_=data.get("metadata", {}),
)
session.add(source)
await session.commit()
await session.refresh(source)
current_app.logger.info(f"Added source to {subscription_name}: {source.platform}")
# Use include_subscription=False to avoid lazy loading the relationship
result = source.to_dict(include_subscription=False)
result["subscription_name"] = subscription_name
return jsonify(result), 201
@bp.route("/<int:subscription_id>/check", methods=["POST"])
async def trigger_subscription_check(subscription_id: int):
"""Trigger download check for all enabled sources in a subscription."""
async with current_app.db_engine.connect() as conn:
session = AsyncSession(bind=conn)
result = await session.execute(
select(Subscription)
.options(selectinload(Subscription.sources))
.where(Subscription.id == subscription_id)
)
subscription = result.scalar()
if not subscription:
return jsonify({"error": "Subscription not found"}), 404
enabled_sources = [s for s in subscription.sources if s.enabled]
# Queue Celery tasks for each enabled source
task_ids = []
for source in enabled_sources:
task = download_source.delay(source.id)
task_ids.append(task.id)
current_app.logger.info(f"Triggered check for subscription: {subscription.name} ({len(enabled_sources)} sources)")
return jsonify({
"message": "Checks queued",
"subscription_id": subscription_id,
"source_count": len(enabled_sources),
"task_ids": task_ids,
}), 202
backend/app/api/websocket.py
+217
View File
@@ -0,0 +1,217 @@
"""WebSocket API for real-time updates."""
import asyncio
import json
import logging
from typing import Set
from quart import Blueprint, websocket, current_app
import redis.asyncio as aioredis
from app.config import get_settings
from app.events import EVENTS_CHANNEL
logger = logging.getLogger(__name__)
bp = Blueprint("websocket", __name__)
# Connected WebSocket clients
connected_clients: Set = set()
class WebSocketManager:
"""Manages WebSocket connections and broadcasts."""
_instance = None
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance.clients = set()
return cls._instance
async def connect(self, ws):
"""Register a new WebSocket connection."""
self.clients.add(ws)
logger.info(f"WebSocket connected. Total clients: {len(self.clients)}")
async def disconnect(self, ws):
"""Remove a WebSocket connection."""
self.clients.discard(ws)
logger.info(f"WebSocket disconnected. Total clients: {len(self.clients)}")
async def broadcast(self, event_type: str, data: dict):
"""Broadcast an event to all connected clients."""
if not self.clients:
return
message = json.dumps({
"type": event_type,
"data": data,
})
disconnected = set()
for client in self.clients:
try:
await client.send(message)
except Exception as e:
logger.debug(f"Failed to send to client: {e}")
disconnected.add(client)
# Clean up disconnected clients
self.clients -= disconnected
async def send_to_client(self, ws, event_type: str, data: dict):
"""Send an event to a specific client."""
message = json.dumps({
"type": event_type,
"data": data,
})
try:
await ws.send(message)
except Exception as e:
logger.debug(f"Failed to send to client: {e}")
# Global manager instance
ws_manager = WebSocketManager()
@bp.websocket("/events")
async def events():
"""WebSocket endpoint for real-time events.
Events sent from server:
- download.started: { download_id, source_id, url }
- download.progress: { download_id, file_count }
- download.completed: { download_id, file_count, duration }
- download.failed: { download_id, error_type, error_message }
- source.updated: { source_id, name, last_check }
- credential.expiring: { platform, expires_in_hours }
"""
await ws_manager.connect(websocket._get_current_object())
try:
# Send initial connection confirmation
await websocket.send(json.dumps({
"type": "connected",
"data": {"message": "Connected to Gallery Subscriber"},
}))
# Keep connection alive and handle incoming messages
while True:
try:
# Wait for messages (ping/pong or commands)
message = await asyncio.wait_for(websocket.receive(), timeout=30)
# Handle ping
if message == "ping":
await websocket.send("pong")
else:
# Parse JSON commands if needed
try:
data = json.loads(message)
await handle_client_message(websocket._get_current_object(), data)
except json.JSONDecodeError:
pass
except asyncio.TimeoutError:
# Send keepalive ping
try:
await websocket.send(json.dumps({"type": "ping"}))
except Exception:
break
except asyncio.CancelledError:
pass
finally:
await ws_manager.disconnect(websocket._get_current_object())
async def handle_client_message(ws, data: dict):
"""Handle messages from WebSocket clients."""
msg_type = data.get("type")
if msg_type == "subscribe":
# Client wants to subscribe to specific events
# For now, all clients receive all events
await ws_manager.send_to_client(ws, "subscribed", {"status": "ok"})
elif msg_type == "ping":
await ws_manager.send_to_client(ws, "pong", {})
# Helper functions for broadcasting events from other parts of the app
async def broadcast_download_started(download_id: int, source_id: int, url: str):
"""Broadcast that a download has started."""
await ws_manager.broadcast("download.started", {
"download_id": download_id,
"source_id": source_id,
"url": url,
})
async def broadcast_download_completed(download_id: int, file_count: int, duration: float):
"""Broadcast that a download has completed."""
await ws_manager.broadcast("download.completed", {
"download_id": download_id,
"file_count": file_count,
"duration_seconds": duration,
})
async def broadcast_download_failed(download_id: int, error_type: str, error_message: str):
"""Broadcast that a download has failed."""
await ws_manager.broadcast("download.failed", {
"download_id": download_id,
"error_type": error_type,
"error_message": error_message,
})
async def broadcast_source_updated(source_id: int, name: str, last_check: str):
"""Broadcast that a source has been updated."""
await ws_manager.broadcast("source.updated", {
"source_id": source_id,
"name": name,
"last_check": last_check,
})
# Redis Pub/Sub integration for cross-process events (from Celery tasks)
async def start_redis_subscriber():
"""Start a background task that subscribes to Redis events and broadcasts them.
This should be called when the Quart app starts.
"""
settings = get_settings()
try:
redis_client = aioredis.from_url(settings.redis_url)
pubsub = redis_client.pubsub()
await pubsub.subscribe(EVENTS_CHANNEL)
logger.info(f"Started Redis subscriber on channel: {EVENTS_CHANNEL}")
async for message in pubsub.listen():
if message["type"] == "message":
try:
data = json.loads(message["data"])
event_type = data.get("type")
event_data = data.get("data", {})
# Broadcast to all WebSocket clients
await ws_manager.broadcast(event_type, event_data)
logger.debug(f"Broadcast event from Redis: {event_type}")
except json.JSONDecodeError:
logger.warning(f"Invalid JSON in Redis message: {message['data']}")
except Exception as e:
logger.error(f"Error broadcasting Redis event: {e}")
except asyncio.CancelledError:
logger.info("Redis subscriber cancelled")
await pubsub.unsubscribe(EVENTS_CHANNEL)
await redis_client.close()
except Exception as e:
logger.error(f"Redis subscriber error: {e}")
backend/app/config.py
+49
View File
@@ -0,0 +1,49 @@
"""Application configuration using pydantic-settings."""
from functools import lru_cache
from pydantic_settings import BaseSettings
class Settings(BaseSettings):
"""Application settings loaded from environment variables."""
# Database
database_url: str = "postgresql://gdl:password@localhost:5432/gallery_subscriber"
# Redis
redis_url: str = "redis://localhost:6379/0"
# Security
secret_key: str = "change-me-in-production"
# Note: extension_api_key is now stored in database and auto-generated
# Paths
download_path: str = "/data/downloads"
config_path: str = "/data/config"
cookies_path: str = "/data/cookies"
# Download settings
download_parallel_limit: int = 3
download_rate_limit: float = 3.0
default_check_interval: int = 3600 # seconds
# Server
host: str = "0.0.0.0"
port: int = 8080
debug: bool = False
log_level: str = "INFO"
@property
def async_database_url(self) -> str:
"""Convert database URL to async version for asyncpg."""
return self.database_url.replace("postgresql://", "postgresql+asyncpg://")
class Config:
env_file = ".env"
env_file_encoding = "utf-8"
@lru_cache
def get_settings() -> Settings:
"""Get cached settings instance."""
return Settings()
backend/app/events.py
+30
View File
@@ -0,0 +1,30 @@
"""Cross-process event publishing via Redis.
This module provides synchronous event publishing for use in Celery tasks.
The events are consumed by the WebSocket module which broadcasts them to clients.
"""
import json
import redis
from app.config import get_settings
# Redis pub/sub channel for cross-process events
EVENTS_CHANNEL = "gallery_subscriber:events"
def publish_event(event_type: str, data: dict):
"""Publish an event to Redis (for use in Celery tasks).
This is a sync function that can be called from Celery tasks.
The Quart app subscribes to these events and broadcasts to WebSocket clients.
Args:
event_type: Type of event (e.g., "download.started", "download.completed")
data: Event payload data
"""
settings = get_settings()
r = redis.from_url(settings.redis_url)
message = json.dumps({"type": event_type, "data": data})
r.publish(EVENTS_CHANNEL, message)
r.close()
backend/app/main.py
+116
View File
@@ -0,0 +1,116 @@
"""Main Quart application factory."""
import asyncio
import logging
from pathlib import Path
from quart import Quart, send_from_directory, send_file
from quart_cors import cors
from app.config import get_settings
from app.models import init_db
from app.api import subscriptions, sources, downloads, credentials, settings as settings_api, platforms, websocket
from app.api.websocket import start_redis_subscriber
# Path to static frontend files (built Vue app)
STATIC_DIR = Path(__file__).parent.parent / "static"
def create_app() -> Quart:
"""Create and configure the Quart application."""
app = Quart(__name__, static_folder=None) # Disable default static handling
config = get_settings()
# Configure logging
logging.basicConfig(
level=getattr(logging, config.log_level.upper()),
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
# Enable CORS for API routes only
app = cors(app, allow_origin="*")
# Register API blueprints
app.register_blueprint(subscriptions.bp, url_prefix="/api/subscriptions")
app.register_blueprint(sources.bp, url_prefix="/api/sources")
app.register_blueprint(downloads.bp, url_prefix="/api/downloads")
app.register_blueprint(credentials.bp, url_prefix="/api/credentials")
app.register_blueprint(settings_api.bp, url_prefix="/api/settings")
app.register_blueprint(platforms.bp, url_prefix="/api/platforms")
app.register_blueprint(websocket.bp, url_prefix="/ws")
@app.before_serving
async def startup():
"""Initialize database connection on startup."""
app.db_engine = await init_db(config.async_database_url)
app.logger.info("Database connection initialized")
app.logger.info(f"Static files directory: {STATIC_DIR} (exists: {STATIC_DIR.exists()})")
# Start Redis subscriber for WebSocket events from Celery tasks
app.redis_subscriber_task = asyncio.create_task(start_redis_subscriber())
app.logger.info("Redis subscriber started for WebSocket events")
@app.after_serving
async def shutdown():
"""Close database connection on shutdown."""
# Cancel Redis subscriber task
if hasattr(app, "redis_subscriber_task"):
app.redis_subscriber_task.cancel()
try:
await app.redis_subscriber_task
except asyncio.CancelledError:
pass
app.logger.info("Redis subscriber stopped")
if hasattr(app, "db_engine"):
await app.db_engine.dispose()
app.logger.info("Database connection closed")
@app.route("/health")
async def health_check():
"""Health check endpoint."""
return {"status": "healthy"}
@app.route("/api")
async def api_info():
"""API information endpoint."""
return {
"name": "Gallery Subscriber API",
"version": "1.0.0",
"endpoints": {
"subscriptions": "/api/subscriptions",
"sources": "/api/sources",
"downloads": "/api/downloads",
"credentials": "/api/credentials",
"settings": "/api/settings",
"platforms": "/api/platforms",
"websocket": "/ws/events",
},
}
# Serve static assets (JS, CSS, images, fonts)
@app.route("/assets/<path:filename>")
async def serve_assets(filename):
"""Serve static assets from the frontend build."""
return await send_from_directory(STATIC_DIR / "assets", filename)
# Serve other static files (favicon, etc.)
@app.route("/<path:filename>")
async def serve_static(filename):
"""Serve static files or fall back to index.html for Vue Router."""
file_path = STATIC_DIR / filename
if file_path.exists() and file_path.is_file():
return await send_from_directory(STATIC_DIR, filename)
# Fall back to index.html for Vue Router (SPA)
return await send_file(STATIC_DIR / "index.html")
# Serve index.html for root path
@app.route("/")
async def serve_index():
"""Serve the frontend index.html."""
return await send_file(STATIC_DIR / "index.html")
return app
# Create app instance for hypercorn
app = create_app()
backend/app/models/__init__.py
+60
View File
@@ -0,0 +1,60 @@
"""Database models and initialization."""
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import DeclarativeBase
from app.models.subscription import Subscription
from app.models.source import Source
from app.models.download import Download
from app.models.credential import Credential
from app.models.content import ContentItem
from app.models.setting import Setting
class Base(DeclarativeBase):
"""Base class for all models."""
pass
# Re-export models
__all__ = [
"Base",
"Subscription",
"Source",
"Download",
"Credential",
"ContentItem",
"Setting",
"init_db",
"get_session",
]
# Global engine and session factory
_engine = None
_session_factory = None
async def init_db(database_url: str):
"""Initialize database engine and create tables."""
global _engine, _session_factory
_engine = create_async_engine(database_url, echo=False)
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
# Import all models to register them with Base
from app.models.subscription import Subscription
from app.models.source import Source
from app.models.download import Download
from app.models.credential import Credential
from app.models.content import ContentItem
from app.models.setting import Setting
return _engine
async def get_session() -> AsyncSession:
"""Get a database session."""
if _session_factory is None:
raise RuntimeError("Database not initialized. Call init_db first.")
async with _session_factory() as session:
yield session
backend/app/models/base.py
+17
View File
@@ -0,0 +1,17 @@
"""SQLAlchemy base model."""
from datetime import datetime
from sqlalchemy import func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
"""Base class for all models."""
pass
class TimestampMixin:
"""Mixin for created_at and updated_at timestamps."""
created_at: Mapped[datetime] = mapped_column(default=func.now())
updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now())
backend/app/models/content.py
+83
View File
@@ -0,0 +1,83 @@
"""ContentItem model - discovered files/content."""
from datetime import datetime
from typing import Optional
from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base
class ContentType:
"""Content type constants."""
IMAGE = "image"
VIDEO = "video"
ATTACHMENT = "attachment"
AUDIO = "audio"
OTHER = "other"
class ContentItem(Base):
"""A content item represents a discovered/downloaded file."""
__tablename__ = "content_items"
__table_args__ = (
UniqueConstraint("source_id", "external_id", name="uq_content_source_external"),
)
id: Mapped[int] = mapped_column(primary_key=True)
source_id: Mapped[int] = mapped_column(
ForeignKey("sources.id", ondelete="CASCADE"), nullable=False, index=True
)
download_id: Mapped[Optional[int]] = mapped_column(
ForeignKey("downloads.id", ondelete="SET NULL"), nullable=True
)
# Content identification
external_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
content_type: Mapped[str] = mapped_column(String(50), default=ContentType.OTHER)
# File information
file_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
file_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
thumbnail_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
# Timestamps
published_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
discovered_at: Mapped[datetime] = mapped_column(default=datetime.utcnow)
# Flexible metadata
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
# Relationships
source: Mapped["Source"] = relationship(back_populates="content_items")
download: Mapped[Optional["Download"]] = relationship(back_populates="content_items")
def __repr__(self) -> str:
return f"<ContentItem {self.id}: {self.title or 'untitled'}>"
def to_dict(self) -> dict:
"""Convert to dictionary for API responses."""
return {
"id": self.id,
"source_id": self.source_id,
"download_id": self.download_id,
"external_id": self.external_id,
"title": self.title,
"content_type": self.content_type,
"file_path": self.file_path,
"file_size": self.file_size,
"thumbnail_path": self.thumbnail_path,
"published_at": self.published_at.isoformat() if self.published_at else None,
"discovered_at": self.discovered_at.isoformat() if self.discovered_at else None,
"metadata": self.metadata_,
}
# Import for type hints
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from app.models.source import Source
from app.models.download import Download
backend/app/models/credential.py
+53
View File
@@ -0,0 +1,53 @@
"""Credential model - encrypted cookie/token storage."""
from datetime import datetime
from typing import Optional
from sqlalchemy import String, LargeBinary, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin
class CredentialType:
"""Credential type constants."""
COOKIES = "cookies"
TOKEN = "token"
class Credential(Base, TimestampMixin):
"""Encrypted credential storage for platform authentication."""
__tablename__ = "credentials"
__table_args__ = (UniqueConstraint("platform", name="uq_credential_platform"),)
id: Mapped[int] = mapped_column(primary_key=True)
platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
credential_type: Mapped[str] = mapped_column(String(20), nullable=False)
# Encrypted credential data
data: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
# Status tracking
expires_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
last_verified: Mapped[Optional[datetime]] = mapped_column(nullable=True)
def __repr__(self) -> str:
return f"<Credential {self.platform} ({self.credential_type})>"
def to_dict(self, include_data: bool = False) -> dict:
"""Convert to dictionary for API responses.
By default, does NOT include the encrypted data for security.
"""
result = {
"id": self.id,
"platform": self.platform,
"credential_type": self.credential_type,
"expires_at": self.expires_at.isoformat() if self.expires_at else None,
"last_verified": self.last_verified.isoformat() if self.last_verified else None,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
if include_data:
result["data"] = self.data
return result
backend/app/models/download.py
+93
View File
@@ -0,0 +1,93 @@
"""Download model - individual download jobs."""
from datetime import datetime
from typing import Optional
from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin
class DownloadStatus:
"""Download status constants."""
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
SKIPPED = "skipped"
class ErrorType:
"""Error type constants matching legacy project."""
AUTH_ERROR = "auth_error"
RATE_LIMITED = "rate_limited"
NOT_FOUND = "not_found"
ACCESS_DENIED = "access_denied"
NETWORK_ERROR = "network_error"
NO_NEW_CONTENT = "no_new_content"
TIMEOUT = "timeout"
HTTP_ERROR = "http_error"
UNSUPPORTED_URL = "unsupported_url"
UNKNOWN_ERROR = "unknown_error"
class Download(Base, TimestampMixin):
"""A download represents a single download job for a source."""
__tablename__ = "downloads"
id: Mapped[int] = mapped_column(primary_key=True)
source_id: Mapped[Optional[int]] = mapped_column(
ForeignKey("sources.id", ondelete="SET NULL"), nullable=True, index=True
)
url: Mapped[str] = mapped_column(Text, nullable=False)
# Status
status: Mapped[str] = mapped_column(
String(20), default=DownloadStatus.PENDING, index=True
)
error_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
# Results
file_count: Mapped[int] = mapped_column(Integer, default=0)
total_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
# Timing
started_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
completed_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
# Flexible metadata
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
# Relationships
source: Mapped[Optional["Source"]] = relationship(back_populates="downloads")
content_items: Mapped[list["ContentItem"]] = relationship(back_populates="download")
def __repr__(self) -> str:
return f"<Download {self.id} ({self.status})>"
def to_dict(self) -> dict:
"""Convert to dictionary for API responses."""
return {
"id": self.id,
"source_id": self.source_id,
"url": self.url,
"status": self.status,
"error_type": self.error_type,
"error_message": self.error_message,
"file_count": self.file_count,
"total_size": self.total_size,
"started_at": self.started_at.isoformat() if self.started_at else None,
"completed_at": self.completed_at.isoformat() if self.completed_at else None,
"metadata": self.metadata_,
"created_at": self.created_at.isoformat() if self.created_at else None,
}
# Import for type hints
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from app.models.source import Source
from app.models.content import ContentItem
backend/app/models/setting.py
+49
View File
@@ -0,0 +1,49 @@
"""Setting model - key-value application settings."""
import secrets
from datetime import datetime
from sqlalchemy import String, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
def generate_api_key() -> str:
"""Generate a secure random API key."""
return secrets.token_urlsafe(32)
class Setting(Base):
"""Key-value setting storage."""
__tablename__ = "settings"
key: Mapped[str] = mapped_column(String(100), primary_key=True)
value: Mapped[dict] = mapped_column(JSONB, nullable=False)
updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now())
def __repr__(self) -> str:
return f"<Setting {self.key}>"
def to_dict(self) -> dict:
"""Convert to dictionary for API responses."""
return {
"key": self.key,
"value": self.value,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
# Default settings that should exist
DEFAULT_SETTINGS = {
"download.parallel_limit": 3,
"download.rate_limit": 3.0,
"download.retry_count": 3,
"download.schedule_interval": 3600,
"notification.enabled": False,
"notification.webhook_url": None,
}
# Key for the extension API key setting
EXTENSION_API_KEY_SETTING = "security.extension_api_key"
backend/app/models/source.py
+75
View File
@@ -0,0 +1,75 @@
"""Source model - platform URLs for subscriptions."""
from datetime import datetime
from typing import Optional
from sqlalchemy import String, Boolean, Integer, Text, ForeignKey, UniqueConstraint
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin
class Source(Base, TimestampMixin):
"""A source represents a platform URL for a subscription.
Sources belong to subscriptions - one subscription (artist) can have
multiple sources (Patreon, SubscribeStar, etc.).
"""
__tablename__ = "sources"
__table_args__ = (UniqueConstraint("platform", "url", name="uq_source_platform_url"),)
id: Mapped[int] = mapped_column(primary_key=True)
subscription_id: Mapped[int] = mapped_column(
ForeignKey("subscriptions.id", ondelete="CASCADE"),
nullable=False,
index=True
)
platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
url: Mapped[str] = mapped_column(Text, nullable=False)
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
check_interval: Mapped[int] = mapped_column(Integer, default=3600) # seconds
# Status tracking
last_check: Mapped[Optional[datetime]] = mapped_column(nullable=True)
last_success: Mapped[Optional[datetime]] = mapped_column(nullable=True)
error_count: Mapped[int] = mapped_column(Integer, default=0)
# Flexible metadata storage (per-source config overrides)
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
# Relationships
subscription: Mapped["Subscription"] = relationship(back_populates="sources")
downloads: Mapped[list["Download"]] = relationship(back_populates="source")
content_items: Mapped[list["ContentItem"]] = relationship(back_populates="source")
def __repr__(self) -> str:
return f"<Source {self.subscription.name if self.subscription else '?'}/{self.platform}>"
def to_dict(self, include_subscription: bool = True) -> dict:
"""Convert to dictionary for API responses."""
result = {
"id": self.id,
"subscription_id": self.subscription_id,
"platform": self.platform,
"url": self.url,
"enabled": self.enabled,
"check_interval": self.check_interval,
"last_check": self.last_check.isoformat() if self.last_check else None,
"last_success": self.last_success.isoformat() if self.last_success else None,
"error_count": self.error_count,
"metadata": self.metadata_,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
if include_subscription and self.subscription:
result["subscription_name"] = self.subscription.name
return result
# Import for type hints (avoid circular imports)
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from app.models.subscription import Subscription
from app.models.download import Download
from app.models.content import ContentItem
backend/app/models/subscription.py
+72
View File
@@ -0,0 +1,72 @@
"""Subscription model - represents an artist/creator to track."""
from datetime import datetime
from typing import Optional, TYPE_CHECKING
from sqlalchemy import String, Boolean, Integer, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin
if TYPE_CHECKING:
from app.models.source import Source
class Subscription(Base, TimestampMixin):
"""A subscription represents an artist/creator being tracked.
Each subscription can have multiple sources (platform URLs).
The subscription name is used for the directory structure:
/{download_path}/{subscription.name}/{source.platform}/{post}
"""
__tablename__ = "subscriptions"
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
priority: Mapped[int] = mapped_column(Integer, default=0)
# Optional description/notes
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
# Flexible metadata storage (tags, custom settings, etc.)
metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
# Relationships
sources: Mapped[list["Source"]] = relationship(
back_populates="subscription",
cascade="all, delete-orphan",
lazy="selectin"
)
def __repr__(self) -> str:
return f"<Subscription {self.name}>"
@property
def platform_count(self) -> int:
"""Number of platforms/sources for this subscription."""
return len(self.sources) if self.sources else 0
@property
def enabled_source_count(self) -> int:
"""Number of enabled sources."""
return sum(1 for s in self.sources if s.enabled) if self.sources else 0
def to_dict(self, include_sources: bool = True) -> dict:
"""Convert to dictionary for API responses."""
result = {
"id": self.id,
"name": self.name,
"enabled": self.enabled,
"priority": self.priority,
"description": self.description,
"metadata": self.metadata_,
"platform_count": self.platform_count,
"enabled_source_count": self.enabled_source_count,
"created_at": self.created_at.isoformat() if self.created_at else None,
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
}
if include_sources and self.sources:
result["sources"] = [s.to_dict(include_subscription=False) for s in self.sources]
return result
backend/app/services/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Service modules."""
backend/app/services/credential_manager.py
+113
View File
@@ -0,0 +1,113 @@
"""Credential management service.
Handles retrieving, decrypting, and writing cookie files for gallery-dl.
"""
import logging
from pathlib import Path
from typing import Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import get_settings
from app.models.credential import Credential
from app.utils.encryption import decrypt_data
from app.utils.cookies import parse_netscape_cookies, write_cookie_file
logger = logging.getLogger(__name__)
class CredentialManager:
"""Manages platform credentials for gallery-dl authentication."""
def __init__(self, session: AsyncSession):
self.session = session
self.settings = get_settings()
async def get_cookies_path(self, platform: str) -> Optional[str]:
"""Get or create a cookies file for a platform.
If credentials exist for the platform, decrypts them and writes
to a temporary cookies file that gallery-dl can use.
Args:
platform: Platform name (patreon, subscribestar, etc.)
Returns:
Path to cookies file, or None if no credentials stored
"""
result = await self.session.execute(
select(Credential).where(Credential.platform == platform)
)
credential = result.scalar()
if not credential:
logger.debug(f"No credentials stored for platform: {platform}")
return None
try:
# Decrypt credential data
decrypted = decrypt_data(credential.data, self.settings.secret_key)
# Write to cookies file
cookies_dir = Path(self.settings.cookies_path)
cookies_dir.mkdir(parents=True, exist_ok=True)
cookies_path = cookies_dir / f"{platform}_cookies.txt"
# If it's already in Netscape format, write directly
if decrypted.strip().startswith("#") or "\t" in decrypted:
cookies_path.write_text(decrypted)
else:
# Assume it's JSON and needs conversion
import json
try:
cookies_list = json.loads(decrypted)
if isinstance(cookies_list, list):
write_cookie_file(cookies_list, cookies_path)
else:
# Single cookie object
write_cookie_file([cookies_list], cookies_path)
except json.JSONDecodeError:
# Not JSON, write as-is
cookies_path.write_text(decrypted)
logger.debug(f"Wrote cookies file for {platform}: {cookies_path}")
return str(cookies_path)
except Exception as e:
logger.error(f"Failed to prepare cookies for {platform}: {e}")
return None
async def get_token(self, platform: str) -> Optional[str]:
"""Get a decrypted token for a platform.
Used for platforms like Discord that use token auth instead of cookies.
Args:
platform: Platform name
Returns:
Decrypted token string, or None if not stored
"""
result = await self.session.execute(
select(Credential).where(Credential.platform == platform)
)
credential = result.scalar()
if not credential or credential.credential_type != "token":
return None
try:
return decrypt_data(credential.data, self.settings.secret_key)
except Exception as e:
logger.error(f"Failed to decrypt token for {platform}: {e}")
return None
async def has_credentials(self, platform: str) -> bool:
"""Check if credentials exist for a platform."""
result = await self.session.execute(
select(Credential).where(Credential.platform == platform)
)
return result.scalar() is not None
backend/app/services/gallery_dl.py
+534
View File
@@ -0,0 +1,534 @@
"""Gallery-dl wrapper service.
This service provides a clean interface for executing gallery-dl downloads
with support for per-source configuration overrides.
"""
import json
import logging
import subprocess
import sys
import tempfile
import time
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Optional
from app.config import get_settings
logger = logging.getLogger(__name__)
class ErrorType(str, Enum):
"""Error types for categorizing download failures."""
AUTH_ERROR = "auth_error"
RATE_LIMITED = "rate_limited"
NOT_FOUND = "not_found"
ACCESS_DENIED = "access_denied"
NETWORK_ERROR = "network_error"
NO_NEW_CONTENT = "no_new_content"
TIMEOUT = "timeout"
HTTP_ERROR = "http_error"
UNSUPPORTED_URL = "unsupported_url"
UNKNOWN_ERROR = "unknown_error"
class ContentType(str, Enum):
"""Content types that can be downloaded from platforms."""
# Patreon content types
IMAGES = "images"
IMAGE_LARGE = "image_large"
ATTACHMENTS = "attachments"
POSTFILE = "postfile"
CONTENT = "content" # Embedded content in post body
# Generic types
ALL = "all"
@dataclass
class SourceConfig:
"""Per-source configuration that can override global settings.
These settings are stored in the source's metadata field and merged
with global defaults when executing downloads.
"""
# Content types to download (platform-specific)
# For Patreon: ["images", "image_large", "attachments", "postfile", "content"]
# For others: typically just ["all"] or specific types
content_types: list[str] = field(default_factory=lambda: ["all"])
# Rate limiting overrides
sleep: Optional[float] = None # Delay between downloads
sleep_request: Optional[float] = None # Delay between requests
# Directory/filename pattern overrides
directory_pattern: Optional[str] = None
filename_pattern: Optional[str] = None
# Other options
skip_existing: bool = True # Skip files already in archive
save_metadata: bool = True # Save JSON metadata alongside files
timeout: int = 3600 # Download timeout in seconds
@classmethod
def from_dict(cls, data: dict) -> "SourceConfig":
"""Create SourceConfig from a dictionary (e.g., from source.metadata)."""
return cls(
content_types=data.get("content_types", ["all"]),
sleep=data.get("sleep"),
sleep_request=data.get("sleep_request"),
directory_pattern=data.get("directory_pattern"),
filename_pattern=data.get("filename_pattern"),
skip_existing=data.get("skip_existing", True),
save_metadata=data.get("save_metadata", True),
timeout=data.get("timeout", 3600),
)
def to_dict(self) -> dict:
"""Convert to dictionary for storage."""
return {
"content_types": self.content_types,
"sleep": self.sleep,
"sleep_request": self.sleep_request,
"directory_pattern": self.directory_pattern,
"filename_pattern": self.filename_pattern,
"skip_existing": self.skip_existing,
"save_metadata": self.save_metadata,
"timeout": self.timeout,
}
@dataclass
class DownloadResult:
"""Result of a gallery-dl download execution."""
success: bool
url: str
subscription_name: str
platform: str
# Output details
files_downloaded: int = 0
stdout: str = ""
stderr: str = ""
return_code: int = 0
# Error details (if failed)
error_type: Optional[ErrorType] = None
error_message: Optional[str] = None
# Timing
duration_seconds: float = 0.0
started_at: Optional[str] = None
completed_at: Optional[str] = None
class GalleryDLService:
"""Service for executing gallery-dl downloads.
Handles:
- Building gallery-dl configuration with per-source overrides
- Executing the subprocess with proper error handling
- Parsing output and categorizing errors
- Managing temporary config files
"""
# Platform-specific default content types
# Directory patterns are relative - subscription_name/platform is prepended automatically
PLATFORM_DEFAULTS = {
"patreon": {
"content_types": ["images", "image_large", "attachments", "postfile", "content"],
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
"filename": "{num:>02}_{filename}.{extension}",
},
"subscribestar": {
"content_types": ["all"],
"directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"],
"filename": "{num:>02}_{filename}.{extension}",
},
"hentaifoundry": {
"content_types": ["pictures"],
"directory": [], # Flat structure within platform folder
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
},
"discord": {
"content_types": ["all"],
"directory": ["{channel}"],
"filename": "{date:%Y%m%d}_{message_id}_{filename}.{extension}",
},
}
def __init__(self):
self.settings = get_settings()
self._base_config = self._load_base_config()
def _load_base_config(self) -> dict:
"""Load the base gallery-dl configuration file."""
config_path = Path(self.settings.config_path) / "gallery-dl.conf"
if config_path.exists():
try:
with open(config_path) as f:
return json.load(f)
except json.JSONDecodeError as e:
logger.warning(f"Invalid gallery-dl.conf, using defaults: {e}")
# Return sensible defaults if no config exists
return self._get_default_config()
def _get_default_config(self) -> dict:
"""Get default gallery-dl configuration."""
return {
"extractor": {
"base-directory": self.settings.download_path,
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
"skip": True,
"sleep": self.settings.download_rate_limit,
"sleep-request": max(1.0, self.settings.download_rate_limit / 2),
"retries": 3,
"timeout": 30.0,
"verify": True,
"postprocessors": [
{
"name": "metadata",
"mode": "json",
"directory": ".",
"filename": "{filename}.json",
}
],
},
"downloader": {
"part": True,
"part-directory": str(Path(self.settings.config_path) / "temp"),
"retries": 3,
"timeout": 120.0,
},
"output": {
"progress": True,
},
}
def _build_config_for_source(
self,
platform: str,
source_config: SourceConfig,
destination: str,
) -> dict:
"""Build a merged configuration for a specific source.
Merges: base config → platform defaults → source overrides
"""
config = json.loads(json.dumps(self._base_config)) # Deep copy
# Get platform defaults
platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {})
# Ensure extractor section exists
if "extractor" not in config:
config["extractor"] = {}
# Apply base directory
config["extractor"]["base-directory"] = destination
# Apply rate limiting overrides
if source_config.sleep is not None:
config["extractor"]["sleep"] = source_config.sleep
if source_config.sleep_request is not None:
config["extractor"]["sleep-request"] = source_config.sleep_request
# Apply skip setting
config["extractor"]["skip"] = source_config.skip_existing
# Configure metadata postprocessor
if source_config.save_metadata:
config["extractor"]["postprocessors"] = [
{
"name": "metadata",
"mode": "json",
"directory": ".",
"filename": "{filename}.json",
}
]
else:
config["extractor"].pop("postprocessors", None)
# Build platform-specific section
platform_config = {}
# Content types (platform-specific handling)
if platform == "patreon":
# Patreon uses "files" array for content types
if "all" in source_config.content_types:
platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"]
else:
platform_config["files"] = source_config.content_types
elif platform == "hentaifoundry":
# HentaiFoundry uses "include" for content filtering
if "pictures" in source_config.content_types or "all" in source_config.content_types:
platform_config["include"] = "pictures"
# Directory pattern
if source_config.directory_pattern:
platform_config["directory"] = source_config.directory_pattern
elif "directory" in platform_defaults:
platform_config["directory"] = platform_defaults["directory"]
# Filename pattern
if source_config.filename_pattern:
platform_config["filename"] = source_config.filename_pattern
elif "filename" in platform_defaults:
platform_config["filename"] = platform_defaults["filename"]
# Always save metadata at platform level too
platform_config["metadata"] = source_config.save_metadata
# Add platform config to extractor
config["extractor"][platform] = platform_config
return config
def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]:
"""Categorize the error based on output and return code."""
combined = f"{stdout} {stderr}".lower()
if "401" in combined or "unauthorized" in combined or "login" in combined:
return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid"
if "429" in combined or "rate limit" in combined or "too many requests" in combined:
return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time"
if "404" in combined or "not found" in combined:
return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content"
if "timeout" in combined or "connection" in combined or "network" in combined:
return ErrorType.NETWORK_ERROR, "Network error - check internet connection"
if "403" in combined or "forbidden" in combined or "access denied" in combined:
return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier"
if return_code == 1 and ("no" in combined or "skip" in combined or not stdout.strip()):
return ErrorType.NO_NEW_CONTENT, "No new content to download"
if "http error" in combined:
return ErrorType.HTTP_ERROR, "HTTP error occurred"
if "no suitable" in combined or "unsupported" in combined:
return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl"
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
def _count_downloaded_files(self, stdout: str) -> int:
"""Count the number of files downloaded from stdout.
Gallery-dl outputs one line per downloaded file.
Lines starting with '#' or containing 'skip' are not downloads.
"""
if not stdout:
return 0
count = 0
for line in stdout.strip().split("\n"):
line = line.strip()
if line and not line.startswith("#") and "skip" not in line.lower():
count += 1
return count
async def download(
self,
url: str,
subscription_name: str,
platform: str,
source_config: Optional[SourceConfig] = None,
cookies_path: Optional[str] = None,
) -> DownloadResult:
"""Execute a gallery-dl download.
Args:
url: The URL to download from
subscription_name: Name of the subscription/artist (used for destination folder)
platform: Platform name (patreon, subscribestar, etc.)
source_config: Optional per-source configuration overrides
cookies_path: Optional path to cookies file
Returns:
DownloadResult with success status and details
Downloads are saved to: {download_path}/{subscription_name}/{platform}/{post_folders}
"""
import asyncio
from datetime import datetime
start_time = time.time()
started_at = datetime.utcnow().isoformat()
# Use default config if none provided
if source_config is None:
source_config = SourceConfig()
# Build destination path: subscription_name/platform
destination = str(Path(self.settings.download_path) / subscription_name / platform)
# Build merged configuration
config = self._build_config_for_source(platform, source_config, destination)
# Add cookies if provided
if cookies_path:
config["extractor"]["cookies"] = cookies_path
# Write temporary config file
with tempfile.NamedTemporaryFile(
mode="w",
suffix=".json",
delete=False,
dir=self.settings.config_path,
) as f:
json.dump(config, f, indent=2)
temp_config_path = f.name
try:
# Build command
cmd = [
sys.executable,
"-m",
"gallery_dl",
"--config",
temp_config_path,
"--verbose",
url,
]
logger.info(f"Executing gallery-dl for {subscription_name}/{platform}: {url}")
logger.debug(f"Command: {' '.join(cmd)}")
# Execute subprocess
# Run in thread pool to not block async event loop
loop = asyncio.get_event_loop()
proc = await loop.run_in_executor(
None,
lambda: subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=source_config.timeout,
),
)
duration = time.time() - start_time
completed_at = datetime.utcnow().isoformat()
# Log output
if proc.stdout:
logger.debug(f"STDOUT for {subscription_name}/{platform}:\n{proc.stdout}")
if proc.stderr:
logger.debug(f"STDERR for {subscription_name}/{platform}:\n{proc.stderr}")
# Determine success
if proc.returncode == 0:
return DownloadResult(
success=True,
url=url,
subscription_name=subscription_name,
platform=platform,
files_downloaded=self._count_downloaded_files(proc.stdout),
stdout=proc.stdout,
stderr=proc.stderr,
return_code=proc.returncode,
duration_seconds=duration,
started_at=started_at,
completed_at=completed_at,
)
# Handle errors
error_type, error_message = self._categorize_error(
proc.returncode, proc.stdout, proc.stderr
)
# NO_NEW_CONTENT is considered successful
success = error_type == ErrorType.NO_NEW_CONTENT
return DownloadResult(
success=success,
url=url,
subscription_name=subscription_name,
platform=platform,
files_downloaded=self._count_downloaded_files(proc.stdout) if success else 0,
stdout=proc.stdout,
stderr=proc.stderr,
return_code=proc.returncode,
error_type=error_type,
error_message=error_message,
duration_seconds=duration,
started_at=started_at,
completed_at=completed_at,
)
except subprocess.TimeoutExpired:
duration = time.time() - start_time
logger.error(f"Download timeout for {subscription_name}/{platform} after {duration:.1f}s")
return DownloadResult(
success=False,
url=url,
subscription_name=subscription_name,
platform=platform,
error_type=ErrorType.TIMEOUT,
error_message=f"Download timed out after {source_config.timeout} seconds",
duration_seconds=duration,
started_at=started_at,
completed_at=datetime.utcnow().isoformat(),
)
except Exception as e:
duration = time.time() - start_time
logger.exception(f"Unexpected error downloading {subscription_name}/{platform}: {e}")
return DownloadResult(
success=False,
url=url,
subscription_name=subscription_name,
platform=platform,
error_type=ErrorType.UNKNOWN_ERROR,
error_message=str(e),
duration_seconds=duration,
started_at=started_at,
completed_at=datetime.utcnow().isoformat(),
)
finally:
# Clean up temporary config file
try:
Path(temp_config_path).unlink()
except Exception:
pass
def get_platform_content_types(self, platform: str) -> list[str]:
"""Get available content types for a platform.
Useful for UI to show what options are available.
"""
if platform == "patreon":
return ["images", "image_large", "attachments", "postfile", "content"]
elif platform == "hentaifoundry":
return ["pictures", "stories"]
elif platform == "subscribestar":
return ["all"] # No granular control
elif platform == "discord":
return ["all"] # No granular control
else:
return ["all"]
def get_default_config_for_platform(self, platform: str) -> dict:
"""Get the default configuration for a platform.
Useful for showing defaults in UI.
"""
defaults = self.PLATFORM_DEFAULTS.get(platform, {})
return {
"content_types": defaults.get("content_types", ["all"]),
"directory_pattern": defaults.get("directory"),
"filename_pattern": defaults.get("filename"),
"sleep": self.settings.download_rate_limit,
"sleep_request": max(1.0, self.settings.download_rate_limit / 2),
}
backend/app/tasks/__init__.py
+5
View File
@@ -0,0 +1,5 @@
"""Celery tasks."""
from app.tasks.celery_app import celery_app
__all__ = ["celery_app"]
backend/app/tasks/celery_app.py
+40
View File
@@ -0,0 +1,40 @@
"""Celery application configuration."""
from celery import Celery
from celery.schedules import crontab
from app.config import get_settings
settings = get_settings()
celery_app = Celery(
"gallery_subscriber",
broker=settings.redis_url,
backend=settings.redis_url,
include=["app.tasks.downloads"],
)
# Celery configuration
celery_app.conf.update(
task_serializer="json",
accept_content=["json"],
result_serializer="json",
timezone="UTC",
enable_utc=True,
task_track_started=True,
task_time_limit=3600, # 1 hour max per task
worker_prefetch_multiplier=1, # Process one task at a time
worker_concurrency=settings.download_parallel_limit,
)
# Celery Beat schedule for periodic tasks
celery_app.conf.beat_schedule = {
"check-sources-hourly": {
"task": "app.tasks.downloads.scheduled_check",
"schedule": settings.default_check_interval, # Default: every hour
},
"cleanup-old-downloads-daily": {
"task": "app.tasks.downloads.cleanup_old_downloads",
"schedule": crontab(hour=3, minute=0), # Daily at 3 AM
},
}
backend/app/tasks/downloads.py
+384
View File
@@ -0,0 +1,384 @@
"""Download-related Celery tasks."""
import asyncio
import logging
from datetime import datetime, timedelta
from typing import Optional
from sqlalchemy import select, and_, or_
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import selectinload
from app.config import get_settings
from app.tasks.celery_app import celery_app
from app.models.source import Source
from app.models.subscription import Subscription
from app.models.download import Download, DownloadStatus, ErrorType as DBErrorType
from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult
from app.services.credential_manager import CredentialManager
from app.events import publish_event
logger = logging.getLogger(__name__)
# Settings for database connection
settings = get_settings()
def get_async_session():
"""Get async session factory for Celery tasks.
Creates a fresh engine and session factory each time to avoid
event loop issues when asyncio.run() creates new loops.
"""
engine = create_async_engine(settings.async_database_url, echo=False)
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
return session_factory, engine
async def _cleanup_engine(engine):
"""Properly dispose of the async engine."""
await engine.dispose()
async def _download_source_async(source_id: int) -> dict:
"""Async implementation of source download.
Args:
source_id: ID of the source to download
Returns:
Dict with download results
"""
session_factory, engine = get_async_session()
try:
async with session_factory() as session:
# Get source with subscription
result = await session.execute(
select(Source)
.options(selectinload(Source.subscription))
.where(Source.id == source_id)
)
source = result.scalar()
if not source:
logger.error(f"Source {source_id} not found")
return {"source_id": source_id, "status": "error", "error": "Source not found"}
if not source.enabled:
logger.info(f"Source {source.subscription.name}/{source.platform} is disabled, skipping")
return {"source_id": source_id, "status": "skipped", "reason": "disabled"}
subscription_name = source.subscription.name
logger.info(f"Starting download for {subscription_name}/{source.platform}")
# Create download record
download = Download(
source_id=source.id,
url=source.url,
status=DownloadStatus.RUNNING,
started_at=datetime.utcnow(),
)
session.add(download)
await session.commit()
await session.refresh(download)
# Broadcast download started event
publish_event("download.started", {
"download_id": download.id,
"source_id": source.id,
"subscription_name": subscription_name,
"platform": source.platform,
"url": source.url,
})
try:
# Get credentials
cred_manager = CredentialManager(session)
cookies_path = await cred_manager.get_cookies_path(source.platform)
if not cookies_path and source.platform in ["patreon", "subscribestar"]:
logger.warning(f"No credentials for {source.platform}, download may fail")
# Build source config from metadata
source_config = SourceConfig.from_dict(source.metadata_ or {})
# Execute download
gdl_service = GalleryDLService()
dl_result = await gdl_service.download(
url=source.url,
subscription_name=subscription_name,
platform=source.platform,
source_config=source_config,
cookies_path=cookies_path,
)
# Update download record
download.completed_at = datetime.utcnow()
download.file_count = dl_result.files_downloaded
download.metadata_ = {
"stdout": dl_result.stdout[:10000] if dl_result.stdout else None, # Truncate
"stderr": dl_result.stderr[:5000] if dl_result.stderr else None,
"duration_seconds": dl_result.duration_seconds,
}
if dl_result.success:
download.status = DownloadStatus.COMPLETED
source.last_success = datetime.utcnow()
source.error_count = 0
logger.info(f"Download completed for {subscription_name}/{source.platform}: {dl_result.files_downloaded} files")
# Broadcast download completed event
publish_event("download.completed", {
"download_id": download.id,
"source_id": source.id,
"subscription_name": subscription_name,
"platform": source.platform,
"file_count": dl_result.files_downloaded,
"duration_seconds": dl_result.duration_seconds,
})
else:
download.status = DownloadStatus.FAILED
download.error_type = dl_result.error_type.value if dl_result.error_type else None
download.error_message = dl_result.error_message
source.error_count = (source.error_count or 0) + 1
logger.warning(f"Download failed for {subscription_name}/{source.platform}: {dl_result.error_message}")
# Broadcast download failed event
publish_event("download.failed", {
"download_id": download.id,
"source_id": source.id,
"subscription_name": subscription_name,
"platform": source.platform,
"error_type": dl_result.error_type.value if dl_result.error_type else None,
"error_message": dl_result.error_message,
})
# Update source last_check
source.last_check = datetime.utcnow()
await session.commit()
return {
"source_id": source_id,
"download_id": download.id,
"status": "completed" if dl_result.success else "failed",
"files_downloaded": dl_result.files_downloaded,
"error_type": dl_result.error_type.value if dl_result.error_type else None,
"error_message": dl_result.error_message,
"duration_seconds": dl_result.duration_seconds,
}
except Exception as e:
logger.exception(f"Unexpected error downloading {subscription_name}/{source.platform}: {e}")
download.status = DownloadStatus.FAILED
download.error_type = DBErrorType.UNKNOWN_ERROR
download.error_message = str(e)
download.completed_at = datetime.utcnow()
source.error_count = (source.error_count or 0) + 1
source.last_check = datetime.utcnow()
await session.commit()
# Broadcast download failed event
publish_event("download.failed", {
"download_id": download.id,
"source_id": source.id,
"subscription_name": subscription_name,
"platform": source.platform,
"error_type": "unknown_error",
"error_message": str(e),
})
return {
"source_id": source_id,
"download_id": download.id,
"status": "error",
"error": str(e),
}
finally:
await _cleanup_engine(engine)
async def _scheduled_check_async() -> dict:
"""Async implementation of scheduled source check.
Returns:
Dict with check results
"""
session_factory, engine = get_async_session()
try:
async with session_factory() as session:
now = datetime.utcnow()
# Find sources due for checking:
# - enabled = True
# - last_check is NULL OR (now - last_check) > check_interval
# Order by subscription priority, then by when last checked
query = (
select(Source)
.options(selectinload(Source.subscription))
.join(Subscription)
.where(
and_(
Source.enabled == True,
Subscription.enabled == True, # Subscription must also be enabled
or_(
Source.last_check == None,
Source.last_check < now - timedelta(seconds=1) # Will be refined below
)
)
)
.order_by(
Subscription.priority.desc(), # Higher subscription priority first
Source.last_check.asc().nullsfirst(), # Never checked first
)
)
result = await session.execute(query)
sources = result.scalars().all()
# Filter by check_interval
due_sources = []
for source in sources:
if source.last_check is None:
due_sources.append(source)
else:
time_since_check = (now - source.last_check).total_seconds()
if time_since_check >= source.check_interval:
due_sources.append(source)
logger.info(f"Found {len(due_sources)} sources due for checking")
# Queue download tasks for each source
queued = 0
for source in due_sources:
download_source.delay(source.id)
queued += 1
logger.debug(f"Queued download for {source.subscription.name}/{source.platform}")
return {
"checked": len(sources),
"queued": queued,
"timestamp": now.isoformat(),
}
finally:
await _cleanup_engine(engine)
async def _cleanup_old_downloads_async() -> dict:
"""Async implementation of download cleanup.
Returns:
Dict with cleanup results
"""
session_factory, engine = get_async_session()
try:
async with session_factory() as session:
now = datetime.utcnow()
# Delete completed downloads older than 30 days
completed_cutoff = now - timedelta(days=30)
completed_query = select(Download).where(
and_(
Download.status == DownloadStatus.COMPLETED,
Download.created_at < completed_cutoff
)
)
completed_result = await session.execute(completed_query)
completed_old = completed_result.scalars().all()
# Delete failed downloads older than 7 days
failed_cutoff = now - timedelta(days=7)
failed_query = select(Download).where(
and_(
Download.status == DownloadStatus.FAILED,
Download.created_at < failed_cutoff
)
)
failed_result = await session.execute(failed_query)
failed_old = failed_result.scalars().all()
deleted_count = 0
for download in completed_old + failed_old:
await session.delete(download)
deleted_count += 1
await session.commit()
logger.info(f"Cleaned up {deleted_count} old download records")
return {"deleted": deleted_count}
finally:
await _cleanup_engine(engine)
# Celery tasks that wrap async functions
@celery_app.task(bind=True, max_retries=3)
def download_source(self, source_id: int):
"""Download new content from a single source.
Args:
source_id: ID of the source to download from
"""
try:
return asyncio.run(_download_source_async(source_id))
except Exception as e:
logger.exception(f"Task failed for source {source_id}: {e}")
# Retry with exponential backoff
raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries))
@celery_app.task
def scheduled_check():
"""Check all enabled sources that are due for a check.
This is called periodically by Celery Beat.
"""
return asyncio.run(_scheduled_check_async())
@celery_app.task
def cleanup_old_downloads():
"""Clean up old download records.
Keeps the database from growing indefinitely.
"""
return asyncio.run(_cleanup_old_downloads_async())
@celery_app.task(bind=True, max_retries=2)
def process_download(self, download_id: int):
"""Process a single download record (for retries).
Args:
download_id: ID of the download to process
"""
async def _process():
session_factory, engine = get_async_session()
try:
async with session_factory() as session:
result = await session.execute(
select(Download).where(Download.id == download_id)
)
download = result.scalar()
if not download:
return {"download_id": download_id, "status": "error", "error": "Download not found"}
if not download.source_id:
return {"download_id": download_id, "status": "error", "error": "No source associated"}
# Delegate to download_source
return await _download_source_async(download.source_id)
finally:
await _cleanup_engine(engine)
try:
return asyncio.run(_process())
except Exception as e:
logger.exception(f"Task failed for download {download_id}: {e}")
raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries))
backend/app/utils/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Utility modules."""
backend/app/utils/cookies.py
+102
View File
@@ -0,0 +1,102 @@
"""Cookie parsing and formatting utilities."""
from typing import Optional
from pathlib import Path
def parse_netscape_cookies(cookie_text: str) -> list[dict]:
"""Parse Netscape cookie format into a list of cookie dicts.
Netscape format: domain<TAB>flag<TAB>path<TAB>secure<TAB>expiration<TAB>name<TAB>value
Args:
cookie_text: Cookie file content in Netscape format
Returns:
List of cookie dictionaries
"""
cookies = []
for line in cookie_text.strip().split("\n"):
line = line.strip()
# Skip comments and empty lines
if not line or line.startswith("#"):
continue
parts = line.split("\t")
if len(parts) < 7:
continue
cookies.append({
"domain": parts[0],
"flag": parts[1] == "TRUE",
"path": parts[2],
"secure": parts[3] == "TRUE",
"expiration": int(parts[4]) if parts[4] != "0" else None,
"name": parts[5],
"value": parts[6],
})
return cookies
def to_netscape_format(cookies: list[dict]) -> str:
"""Convert cookie list to Netscape format.
Args:
cookies: List of cookie dictionaries
Returns:
Cookie file content in Netscape format
"""
lines = ["# Netscape HTTP Cookie File"]
for c in cookies:
line = "\t".join([
c["domain"],
"TRUE" if c.get("flag", True) else "FALSE",
c.get("path", "/"),
"TRUE" if c.get("secure", False) else "FALSE",
str(c.get("expiration", 0) or 0),
c["name"],
c["value"],
])
lines.append(line)
return "\n".join(lines)
def write_cookie_file(cookies: list[dict], path: Path) -> None:
"""Write cookies to a file in Netscape format.
Args:
cookies: List of cookie dictionaries
path: Path to write the cookie file
"""
content = to_netscape_format(cookies)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content)
def get_platform_from_domain(domain: str) -> Optional[str]:
"""Detect platform from cookie domain.
Args:
domain: Cookie domain (e.g., '.patreon.com')
Returns:
Platform name or None
"""
domain_lower = domain.lower()
if "patreon.com" in domain_lower:
return "patreon"
elif "subscribestar" in domain_lower:
return "subscribestar"
elif "hentai-foundry.com" in domain_lower:
return "hentaifoundry"
elif "discord.com" in domain_lower:
return "discord"
return None
backend/app/utils/encryption.py
+46
View File
@@ -0,0 +1,46 @@
"""Credential encryption utilities."""
import base64
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
def _get_fernet(secret_key: str) -> Fernet:
"""Derive a Fernet key from the secret key."""
kdf = PBKDF2HMAC(
algorithm=hashes.SHA256(),
length=32,
salt=b"gallery-subscriber-salt", # Static salt is OK for this use case
iterations=100000,
)
derived_key = base64.urlsafe_b64encode(kdf.derive(secret_key.encode()))
return Fernet(derived_key)
def encrypt_data(data: str, secret_key: str) -> bytes:
"""Encrypt string data using the secret key.
Args:
data: Plain text data to encrypt
secret_key: Application secret key
Returns:
Encrypted bytes
"""
fernet = _get_fernet(secret_key)
return fernet.encrypt(data.encode())
def decrypt_data(encrypted: bytes, secret_key: str) -> str:
"""Decrypt data using the secret key.
Args:
encrypted: Encrypted bytes
secret_key: Application secret key
Returns:
Decrypted string
"""
fernet = _get_fernet(secret_key)
return fernet.decrypt(encrypted).decode()
backend/requirements.txt
+28
View File
@@ -0,0 +1,28 @@
# Web framework
quart==0.19.4
quart-cors==0.7.0
hypercorn==0.16.0
flask>=3.0.0,<3.1.0
# Database
sqlalchemy[asyncio]==2.0.25
asyncpg==0.29.0
alembic==1.13.1
# Task queue
celery[redis]==5.3.6
redis==5.0.1
# Utilities
pydantic==2.5.3
pydantic-settings==2.1.0
python-dotenv==1.0.0
cryptography==41.0.7
pyyaml==6.0.1
# gallery-dl
gallery-dl>=1.31.0
# Testing
pytest==7.4.4
pytest-asyncio==0.23.3