bvandeusen/GallerySubscriber
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

rapid interations of server side app and firefox extension

Browse Source
This commit is contained in:
Bryan Van Deusen
2026-01-24 22:52:51 -05:00
commit b9b8048a2d
81 changed files with 9675 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/env.py
+75
View File
@@ -0,0 +1,75 @@
"""Alembic migration environment."""
import asyncio
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from app.config import get_settings
from app.models.base import Base
from app.models.source import Source
from app.models.download import Download
from app.models.credential import Credential
from app.models.content import ContentItem
from app.models.setting import Setting
config = context.config
settings = get_settings()
# Override sqlalchemy.url with actual database URL
config.set_main_option("sqlalchemy.url", settings.async_database_url)
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode."""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
"""Run migrations in 'online' mode with async engine."""
connectable = async_engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode."""
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
backend/alembic/script.py.mako
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
backend/alembic/versions/001_initial_schema.py
+129
View File
@@ -0,0 +1,129 @@
"""Initial schema
Revision ID: 001
Revises:
Create Date: 2025-01-24
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = '001'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Sources table
op.create_table(
'sources',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('platform', sa.String(50), nullable=False),
sa.Column('url', sa.Text(), nullable=False),
sa.Column('enabled', sa.Boolean(), default=True),
sa.Column('priority', sa.Integer(), default=0),
sa.Column('check_interval', sa.Integer(), default=3600),
sa.Column('last_check', sa.DateTime(), nullable=True),
sa.Column('last_success', sa.DateTime(), nullable=True),
sa.Column('error_count', sa.Integer(), default=0),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('platform', 'url', name='uq_source_platform_url')
)
op.create_index('ix_sources_platform', 'sources', ['platform'])
# Downloads table
op.create_table(
'downloads',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_id', sa.Integer(), nullable=True),
sa.Column('url', sa.Text(), nullable=False),
sa.Column('status', sa.String(20), default='pending'),
sa.Column('error_type', sa.String(50), nullable=True),
sa.Column('error_message', sa.Text(), nullable=True),
sa.Column('file_count', sa.Integer(), default=0),
sa.Column('total_size', sa.BigInteger(), nullable=True),
sa.Column('started_at', sa.DateTime(), nullable=True),
sa.Column('completed_at', sa.DateTime(), nullable=True),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='SET NULL')
)
op.create_index('ix_downloads_status', 'downloads', ['status'])
op.create_index('ix_downloads_source_id', 'downloads', ['source_id'])
# Content items table
op.create_table(
'content_items',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_id', sa.Integer(), nullable=False),
sa.Column('download_id', sa.Integer(), nullable=True),
sa.Column('external_id', sa.String(255), nullable=True),
sa.Column('title', sa.String(500), nullable=True),
sa.Column('content_type', sa.String(50), default='other'),
sa.Column('file_path', sa.String(500), nullable=True),
sa.Column('file_size', sa.BigInteger(), nullable=True),
sa.Column('thumbnail_path', sa.String(500), nullable=True),
sa.Column('published_at', sa.DateTime(), nullable=True),
sa.Column('discovered_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['download_id'], ['downloads.id'], ondelete='SET NULL'),
sa.UniqueConstraint('source_id', 'external_id', name='uq_content_source_external')
)
op.create_index('ix_content_items_source_id', 'content_items', ['source_id'])
op.create_index('ix_content_items_discovered_at', 'content_items', ['discovered_at'])
# Credentials table
op.create_table(
'credentials',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('platform', sa.String(50), nullable=False),
sa.Column('credential_type', sa.String(20), nullable=False),
sa.Column('data', sa.LargeBinary(), nullable=False),
sa.Column('expires_at', sa.DateTime(), nullable=True),
sa.Column('last_verified', sa.DateTime(), nullable=True),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('platform', name='uq_credential_platform')
)
op.create_index('ix_credentials_platform', 'credentials', ['platform'])
# Settings table
op.create_table(
'settings',
sa.Column('key', sa.String(100), nullable=False),
sa.Column('value', postgresql.JSONB(), nullable=False),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('key')
)
# Insert default settings
op.execute("""
INSERT INTO settings (key, value) VALUES
('download.parallel_limit', '3'),
('download.rate_limit', '3.0'),
('download.retry_count', '3'),
('download.schedule_interval', '3600'),
('notification.enabled', 'false'),
('notification.webhook_url', 'null')
""")
def downgrade() -> None:
op.drop_table('settings')
op.drop_table('credentials')
op.drop_table('content_items')
op.drop_table('downloads')
op.drop_table('sources')
backend/alembic/versions/002_add_subscriptions.py
+103
View File
@@ -0,0 +1,103 @@
"""Add subscriptions table and link sources
Revision ID: 002
Revises: 001
Create Date: 2025-01-25
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = '002'
down_revision: Union[str, None] = '001'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Create subscriptions table
op.create_table(
'subscriptions',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('enabled', sa.Boolean(), default=True),
sa.Column('priority', sa.Integer(), default=0),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('metadata', postgresql.JSONB(), default={}),
sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()),
sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()),
sa.PrimaryKeyConstraint('id')
)
op.create_index('ix_subscriptions_name', 'subscriptions', ['name'], unique=True)
# Migrate existing sources to subscriptions
# Each unique source name becomes a subscription
conn = op.get_bind()
# Get existing sources
existing_sources = conn.execute(
sa.text("SELECT DISTINCT name FROM sources ORDER BY name")
).fetchall()
# Create subscriptions for each unique source name
for (name,) in existing_sources:
conn.execute(
sa.text("INSERT INTO subscriptions (name, enabled, priority) VALUES (:name, true, 0)"),
{"name": name}
)
# Add subscription_id column to sources
op.add_column('sources', sa.Column('subscription_id', sa.Integer(), nullable=True))
op.create_index('ix_sources_subscription_id', 'sources', ['subscription_id'])
# Update sources with their subscription_id
conn.execute(sa.text("""
UPDATE sources s
SET subscription_id = sub.id
FROM subscriptions sub
WHERE s.name = sub.name
"""))
# Make subscription_id NOT NULL after data migration
op.alter_column('sources', 'subscription_id', nullable=False)
# Add foreign key constraint
op.create_foreign_key(
'fk_sources_subscription_id',
'sources', 'subscriptions',
['subscription_id'], ['id'],
ondelete='CASCADE'
)
# Remove name and priority columns from sources (now on subscription)
op.drop_column('sources', 'name')
op.drop_column('sources', 'priority')
def downgrade() -> None:
# Add back name and priority columns to sources
op.add_column('sources', sa.Column('name', sa.String(255), nullable=True))
op.add_column('sources', sa.Column('priority', sa.Integer(), default=0))
# Populate name from subscription
conn = op.get_bind()
conn.execute(sa.text("""
UPDATE sources s
SET name = sub.name, priority = sub.priority
FROM subscriptions sub
WHERE s.subscription_id = sub.id
"""))
op.alter_column('sources', 'name', nullable=False)
# Remove foreign key and subscription_id column
op.drop_constraint('fk_sources_subscription_id', 'sources', type_='foreignkey')
op.drop_index('ix_sources_subscription_id', 'sources')
op.drop_column('sources', 'subscription_id')
# Drop subscriptions table
op.drop_index('ix_subscriptions_name', 'subscriptions')
op.drop_table('subscriptions')