rapid interations of server side app and firefox extension

backend/app/models/__init__.py

@@ -0,0 +1,60 @@
+"""Database models and initialization."""
+
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
+from sqlalchemy.orm import DeclarativeBase
+
+from app.models.subscription import Subscription
+from app.models.source import Source
+from app.models.download import Download
+from app.models.credential import Credential
+from app.models.content import ContentItem
+from app.models.setting import Setting
+
+
+class Base(DeclarativeBase):
+    """Base class for all models."""
+    pass
+
+
+# Re-export models
+__all__ = [
+    "Base",
+    "Subscription",
+    "Source",
+    "Download",
+    "Credential",
+    "ContentItem",
+    "Setting",
+    "init_db",
+    "get_session",
+]
+
+# Global engine and session factory
+_engine = None
+_session_factory = None
+
+
+async def init_db(database_url: str):
+    """Initialize database engine and create tables."""
+    global _engine, _session_factory
+
+    _engine = create_async_engine(database_url, echo=False)
+    _session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
+
+    # Import all models to register them with Base
+    from app.models.subscription import Subscription
+    from app.models.source import Source
+    from app.models.download import Download
+    from app.models.credential import Credential
+    from app.models.content import ContentItem
+    from app.models.setting import Setting
+
+    return _engine
+
+
+async def get_session() -> AsyncSession:
+    """Get a database session."""
+    if _session_factory is None:
+        raise RuntimeError("Database not initialized. Call init_db first.")
+    async with _session_factory() as session:
+        yield session

backend/app/models/base.py

@@ -0,0 +1,17 @@
+"""SQLAlchemy base model."""
+
+from datetime import datetime
+from sqlalchemy import func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class Base(DeclarativeBase):
+    """Base class for all models."""
+    pass
+
+
+class TimestampMixin:
+    """Mixin for created_at and updated_at timestamps."""
+
+    created_at: Mapped[datetime] = mapped_column(default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now())

backend/app/models/content.py

@@ -0,0 +1,83 @@
+"""ContentItem model - discovered files/content."""
+
+from datetime import datetime
+from typing import Optional
+from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger, UniqueConstraint
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base
+
+
+class ContentType:
+    """Content type constants."""
+    IMAGE = "image"
+    VIDEO = "video"
+    ATTACHMENT = "attachment"
+    AUDIO = "audio"
+    OTHER = "other"
+
+
+class ContentItem(Base):
+    """A content item represents a discovered/downloaded file."""
+
+    __tablename__ = "content_items"
+    __table_args__ = (
+        UniqueConstraint("source_id", "external_id", name="uq_content_source_external"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    source_id: Mapped[int] = mapped_column(
+        ForeignKey("sources.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    download_id: Mapped[Optional[int]] = mapped_column(
+        ForeignKey("downloads.id", ondelete="SET NULL"), nullable=True
+    )
+
+    # Content identification
+    external_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
+    title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
+    content_type: Mapped[str] = mapped_column(String(50), default=ContentType.OTHER)
+
+    # File information
+    file_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
+    file_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
+    thumbnail_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
+
+    # Timestamps
+    published_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    discovered_at: Mapped[datetime] = mapped_column(default=datetime.utcnow)
+
+    # Flexible metadata
+    metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
+
+    # Relationships
+    source: Mapped["Source"] = relationship(back_populates="content_items")
+    download: Mapped[Optional["Download"]] = relationship(back_populates="content_items")
+
+    def __repr__(self) -> str:
+        return f"<ContentItem {self.id}: {self.title or 'untitled'}>"
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for API responses."""
+        return {
+            "id": self.id,
+            "source_id": self.source_id,
+            "download_id": self.download_id,
+            "external_id": self.external_id,
+            "title": self.title,
+            "content_type": self.content_type,
+            "file_path": self.file_path,
+            "file_size": self.file_size,
+            "thumbnail_path": self.thumbnail_path,
+            "published_at": self.published_at.isoformat() if self.published_at else None,
+            "discovered_at": self.discovered_at.isoformat() if self.discovered_at else None,
+            "metadata": self.metadata_,
+        }
+
+
+# Import for type hints
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from app.models.source import Source
+    from app.models.download import Download

backend/app/models/credential.py

@@ -0,0 +1,53 @@
+"""Credential model - encrypted cookie/token storage."""
+
+from datetime import datetime
+from typing import Optional
+from sqlalchemy import String, LargeBinary, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base, TimestampMixin
+
+
+class CredentialType:
+    """Credential type constants."""
+    COOKIES = "cookies"
+    TOKEN = "token"
+
+
+class Credential(Base, TimestampMixin):
+    """Encrypted credential storage for platform authentication."""
+
+    __tablename__ = "credentials"
+    __table_args__ = (UniqueConstraint("platform", name="uq_credential_platform"),)
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
+    credential_type: Mapped[str] = mapped_column(String(20), nullable=False)
+
+    # Encrypted credential data
+    data: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+
+    # Status tracking
+    expires_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    last_verified: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+
+    def __repr__(self) -> str:
+        return f"<Credential {self.platform} ({self.credential_type})>"
+
+    def to_dict(self, include_data: bool = False) -> dict:
+        """Convert to dictionary for API responses.
+
+        By default, does NOT include the encrypted data for security.
+        """
+        result = {
+            "id": self.id,
+            "platform": self.platform,
+            "credential_type": self.credential_type,
+            "expires_at": self.expires_at.isoformat() if self.expires_at else None,
+            "last_verified": self.last_verified.isoformat() if self.last_verified else None,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+        if include_data:
+            result["data"] = self.data
+        return result

backend/app/models/download.py

@@ -0,0 +1,93 @@
+"""Download model - individual download jobs."""
+
+from datetime import datetime
+from typing import Optional
+from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base, TimestampMixin
+
+
+class DownloadStatus:
+    """Download status constants."""
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+class ErrorType:
+    """Error type constants matching legacy project."""
+    AUTH_ERROR = "auth_error"
+    RATE_LIMITED = "rate_limited"
+    NOT_FOUND = "not_found"
+    ACCESS_DENIED = "access_denied"
+    NETWORK_ERROR = "network_error"
+    NO_NEW_CONTENT = "no_new_content"
+    TIMEOUT = "timeout"
+    HTTP_ERROR = "http_error"
+    UNSUPPORTED_URL = "unsupported_url"
+    UNKNOWN_ERROR = "unknown_error"
+
+
+class Download(Base, TimestampMixin):
+    """A download represents a single download job for a source."""
+
+    __tablename__ = "downloads"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    source_id: Mapped[Optional[int]] = mapped_column(
+        ForeignKey("sources.id", ondelete="SET NULL"), nullable=True, index=True
+    )
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+
+    # Status
+    status: Mapped[str] = mapped_column(
+        String(20), default=DownloadStatus.PENDING, index=True
+    )
+    error_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
+    error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    # Results
+    file_count: Mapped[int] = mapped_column(Integer, default=0)
+    total_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True)
+
+    # Timing
+    started_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    completed_at: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+
+    # Flexible metadata
+    metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
+
+    # Relationships
+    source: Mapped[Optional["Source"]] = relationship(back_populates="downloads")
+    content_items: Mapped[list["ContentItem"]] = relationship(back_populates="download")
+
+    def __repr__(self) -> str:
+        return f"<Download {self.id} ({self.status})>"
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for API responses."""
+        return {
+            "id": self.id,
+            "source_id": self.source_id,
+            "url": self.url,
+            "status": self.status,
+            "error_type": self.error_type,
+            "error_message": self.error_message,
+            "file_count": self.file_count,
+            "total_size": self.total_size,
+            "started_at": self.started_at.isoformat() if self.started_at else None,
+            "completed_at": self.completed_at.isoformat() if self.completed_at else None,
+            "metadata": self.metadata_,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+        }
+
+
+# Import for type hints
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from app.models.source import Source
+    from app.models.content import ContentItem

backend/app/models/setting.py

@@ -0,0 +1,49 @@
+"""Setting model - key-value application settings."""
+
+import secrets
+from datetime import datetime
+from sqlalchemy import String, func
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+def generate_api_key() -> str:
+    """Generate a secure random API key."""
+    return secrets.token_urlsafe(32)
+
+
+class Setting(Base):
+    """Key-value setting storage."""
+
+    __tablename__ = "settings"
+
+    key: Mapped[str] = mapped_column(String(100), primary_key=True)
+    value: Mapped[dict] = mapped_column(JSONB, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now())
+
+    def __repr__(self) -> str:
+        return f"<Setting {self.key}>"
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for API responses."""
+        return {
+            "key": self.key,
+            "value": self.value,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+
+
+# Default settings that should exist
+DEFAULT_SETTINGS = {
+    "download.parallel_limit": 3,
+    "download.rate_limit": 3.0,
+    "download.retry_count": 3,
+    "download.schedule_interval": 3600,
+    "notification.enabled": False,
+    "notification.webhook_url": None,
+}
+
+# Key for the extension API key setting
+EXTENSION_API_KEY_SETTING = "security.extension_api_key"

backend/app/models/source.py

@@ -0,0 +1,75 @@
+"""Source model - platform URLs for subscriptions."""
+
+from datetime import datetime
+from typing import Optional
+from sqlalchemy import String, Boolean, Integer, Text, ForeignKey, UniqueConstraint
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base, TimestampMixin
+
+
+class Source(Base, TimestampMixin):
+    """A source represents a platform URL for a subscription.
+
+    Sources belong to subscriptions - one subscription (artist) can have
+    multiple sources (Patreon, SubscribeStar, etc.).
+    """
+
+    __tablename__ = "sources"
+    __table_args__ = (UniqueConstraint("platform", "url", name="uq_source_platform_url"),)
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    subscription_id: Mapped[int] = mapped_column(
+        ForeignKey("subscriptions.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True
+    )
+    platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True)
+    check_interval: Mapped[int] = mapped_column(Integer, default=3600)  # seconds
+
+    # Status tracking
+    last_check: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    last_success: Mapped[Optional[datetime]] = mapped_column(nullable=True)
+    error_count: Mapped[int] = mapped_column(Integer, default=0)
+
+    # Flexible metadata storage (per-source config overrides)
+    metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
+
+    # Relationships
+    subscription: Mapped["Subscription"] = relationship(back_populates="sources")
+    downloads: Mapped[list["Download"]] = relationship(back_populates="source")
+    content_items: Mapped[list["ContentItem"]] = relationship(back_populates="source")
+
+    def __repr__(self) -> str:
+        return f"<Source {self.subscription.name if self.subscription else '?'}/{self.platform}>"
+
+    def to_dict(self, include_subscription: bool = True) -> dict:
+        """Convert to dictionary for API responses."""
+        result = {
+            "id": self.id,
+            "subscription_id": self.subscription_id,
+            "platform": self.platform,
+            "url": self.url,
+            "enabled": self.enabled,
+            "check_interval": self.check_interval,
+            "last_check": self.last_check.isoformat() if self.last_check else None,
+            "last_success": self.last_success.isoformat() if self.last_success else None,
+            "error_count": self.error_count,
+            "metadata": self.metadata_,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+        if include_subscription and self.subscription:
+            result["subscription_name"] = self.subscription.name
+        return result
+
+
+# Import for type hints (avoid circular imports)
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from app.models.subscription import Subscription
+    from app.models.download import Download
+    from app.models.content import ContentItem

backend/app/models/subscription.py

@@ -0,0 +1,72 @@
+"""Subscription model - represents an artist/creator to track."""
+
+from datetime import datetime
+from typing import Optional, TYPE_CHECKING
+from sqlalchemy import String, Boolean, Integer, Text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base, TimestampMixin
+
+if TYPE_CHECKING:
+    from app.models.source import Source
+
+
+class Subscription(Base, TimestampMixin):
+    """A subscription represents an artist/creator being tracked.
+
+    Each subscription can have multiple sources (platform URLs).
+    The subscription name is used for the directory structure:
+    /{download_path}/{subscription.name}/{source.platform}/{post}
+    """
+
+    __tablename__ = "subscriptions"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True)
+    priority: Mapped[int] = mapped_column(Integer, default=0)
+
+    # Optional description/notes
+    description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+
+    # Flexible metadata storage (tags, custom settings, etc.)
+    metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict)
+
+    # Relationships
+    sources: Mapped[list["Source"]] = relationship(
+        back_populates="subscription",
+        cascade="all, delete-orphan",
+        lazy="selectin"
+    )
+
+    def __repr__(self) -> str:
+        return f"<Subscription {self.name}>"
+
+    @property
+    def platform_count(self) -> int:
+        """Number of platforms/sources for this subscription."""
+        return len(self.sources) if self.sources else 0
+
+    @property
+    def enabled_source_count(self) -> int:
+        """Number of enabled sources."""
+        return sum(1 for s in self.sources if s.enabled) if self.sources else 0
+
+    def to_dict(self, include_sources: bool = True) -> dict:
+        """Convert to dictionary for API responses."""
+        result = {
+            "id": self.id,
+            "name": self.name,
+            "enabled": self.enabled,
+            "priority": self.priority,
+            "description": self.description,
+            "metadata": self.metadata_,
+            "platform_count": self.platform_count,
+            "enabled_source_count": self.enabled_source_count,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+        }
+        if include_sources and self.sources:
+            result["sources"] = [s.to_dict(include_subscription=False) for s in self.sources]
+        return result