commit b9b8048a2d697dcf00bdded7b76d4c7a5e01f10f Author: Bryan Van Deusen Date: Sat Jan 24 22:52:51 2026 -0500 rapid interations of server side app and firefox extension diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..d28a2d3 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,13 @@ +{ + "permissions": { + "allow": [ + "Bash(docker logs:*)", + "Bash(docker-compose up:*)", + "Bash(docker-compose exec:*)", + "Bash(docker compose logs:*)", + "Bash(docker compose exec celery sh -c \"grep -i session /data/cookies/patreon_cookies.txt || echo ''No session cookie found''\")", + "Bash(docker compose exec celery sh -c \"gallery-dl --cookies /data/cookies/patreon_cookies.txt -v ''https://www.patreon.com/knuxy'' 2>&1 | head -20\")", + "Bash(docker compose exec:*)" + ] + } +} diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..23a44b7 --- /dev/null +++ b/.env.example @@ -0,0 +1,38 @@ +# =========================================== +# Required - You MUST change these values +# =========================================== + +# Database password for PostgreSQL +DB_PASSWORD=change_me_to_secure_password + +# Application secret key (use: openssl rand -hex 32) +SECRET_KEY=change_me_to_random_string_at_least_32_characters + +# =========================================== +# Optional - These have sensible defaults +# =========================================== + +# External port to access the application +PORT=8080 + +# Download settings +DOWNLOAD_PARALLEL_LIMIT=3 +DOWNLOAD_RATE_LIMIT=3.0 +DEFAULT_CHECK_INTERVAL=3600 + +# Logging +LOG_LEVEL=INFO +DEBUG=false + +# =========================================== +# Advanced - Only change if you know what you're doing +# =========================================== + +# Paths inside container (mapped to Docker volumes) +DOWNLOAD_PATH=/data/downloads +CONFIG_PATH=/data/config +COOKIES_PATH=/data/cookies + +# These are constructed by docker-compose, only set for local development +# DATABASE_URL=postgresql://gdl:password@localhost:5432/gallery_subscriber +# REDIS_URL=redis://localhost:6379/0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8251568 --- /dev/null +++ b/.gitignore @@ -0,0 +1,85 @@ +# Environment files +.env +.env.local +.env.*.local + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +venv/ +.venv/ +ENV/ +env/ +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ + +# Node / Frontend +node_modules/ +frontend/dist/ +frontend/node_modules/ +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.npm + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.project +.classpath +.settings/ + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Docker +docker-compose.override.yml + +# Data directories (created at runtime) +data/ +*.db +*.sqlite3 + +# Logs +logs/ +*.log + +# Secrets and credentials +*.pem +*.key +cookies/ +config/ + +# Build artifacts +backend/static/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..293a65a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# Stage 1: Build frontend +FROM node:20-alpine AS frontend-build + +WORKDIR /frontend +COPY frontend/package*.json ./ +RUN npm install +COPY frontend/ . +RUN npm run build + +# Stage 2: Python backend with frontend static files +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY backend/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy backend application +COPY backend/ . + +# Copy built frontend from Stage 1 +COPY --from=frontend-build /frontend/dist /app/static + +# Create data directories +RUN mkdir -p /data/downloads /data/config /data/cookies + +# Run as non-root user +RUN useradd -m -u 1000 appuser && \ + chown -R appuser:appuser /app /data +USER appuser + +EXPOSE 8080 + +CMD ["hypercorn", "app.main:app", "--bind", "0.0.0.0:8080"] diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md new file mode 100644 index 0000000..eb14c02 --- /dev/null +++ b/PROJECT_PLAN.md @@ -0,0 +1,548 @@ +# Gallery Subscriber + +A Docker-based web application for managing gallery-dl downloads with a modern web interface. + +--- + +## Table of Contents + +1. [Project Overview](#project-overview) +2. [Architecture](#architecture) +3. [Technology Stack](#technology-stack) +4. [Project Structure](#project-structure) +5. [Database Schema](#database-schema) +6. [API Specification](#api-specification) +7. [Firefox Extension](#firefox-extension) +8. [Configuration](#configuration) +9. [Development Phases](#development-phases) +10. [Migration from Legacy Project](#migration-from-legacy-project) + +--- + +## Project Overview + +### Goals + +- **Containerized Deployment**: Run the entire download manager in Docker for portability +- **Web Interface**: Modern UI for managing subscriptions, viewing downloads, and monitoring new content +- **Flexible Authentication**: Firefox extension for cookie export OR manual cookie upload via web UI +- **Notifications**: Real-time updates when new content is downloaded +- **Reliability**: Maintain the robust scheduling and error handling from the original project + +### Non-Goals (Initial Release) + +- Multi-user support (single-user initially) +- Mobile app +- SSL termination (handled by external reverse proxy) + +--- + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ External Reverse Proxy │ +│ (Traefik/Caddy/nginx - handles SSL) │ +└──────────────────────────┬──────────────────────────────────────┘ + │ HTTP (port 8080) + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Docker Compose Stack │ +│ │ +│ ┌─────────────────────────────────────────────────────────┐ │ +│ │ Quart Backend │ │ +│ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌───────────┐ │ │ +│ │ │ REST API │ │WebSocket │ │ Static │ │ gallery-dl│ │ │ +│ │ │ /api/* │ │ /ws/* │ │ Files │ │ Wrapper │ │ │ +│ │ └──────────┘ └──────────┘ └──────────┘ └───────────┘ │ │ +│ └─────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌───────────────────────────┴───────────────────────────────┐ │ +│ │ Celery Worker │ │ +│ │ • Download tasks • Scheduled checks │ │ +│ │ • Manifest generation • Error retry logic │ │ +│ └───────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────┐ ┌─────────┴───────┐ ┌──────────────────┐ │ +│ │ PostgreSQL │ │ Redis │ │ Vue Frontend │ │ +│ │ (data) │ │ (task queue) │ │ (static build) │ │ +│ └──────────────┘ └─────────────────┘ └──────────────────┘ │ +│ │ +│ Volumes: │ +│ ├── /data/downloads (downloaded content) │ +│ ├── /data/config (gallery-dl.conf) │ +│ ├── /data/cookies (cookie files) │ +│ └── /data/db (PostgreSQL data) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Technology Stack + +### Backend +| Component | Technology | Rationale | +|-----------|------------|-----------| +| Web Framework | **Quart** | Flask-like API with native async, WebSocket support | +| Task Queue | **Celery + Redis** | Reliable background tasks, scheduling with Celery Beat | +| Database | **PostgreSQL** | JSON fields for metadata, robust concurrency | +| ORM | **SQLAlchemy 2.0** | Async support, mature ecosystem | +| Migrations | **Alembic** | Standard for SQLAlchemy | + +### Frontend +| Component | Technology | Rationale | +|-----------|------------|-----------| +| Framework | **Vue 3** | Reactive, component-based | +| UI Library | **Vuetify 3** | Material Design, comprehensive components | +| State | **Pinia** | Official Vue state management | +| HTTP Client | **Axios** | Request interceptors for auth | +| WebSocket | **Native** | Real-time updates | + +### Infrastructure +| Component | Technology | Rationale | +|-----------|------------|-----------| +| Container | **Docker Compose** | Easy deployment | +| Process Manager | **Supervisor** | Manage Quart + Celery in container | + +--- + +## Project Structure + +``` +gallery-subscriber/ +├── docker-compose.yml +├── docker-compose.dev.yml +├── .env.example +├── README.md +│ +├── backend/ +│ ├── Dockerfile +│ ├── requirements.txt +│ ├── supervisord.conf +│ │ +│ ├── app/ +│ │ ├── __init__.py +│ │ ├── main.py # Quart app factory +│ │ ├── config.py # Settings (from env) +│ │ │ +│ │ ├── api/ +│ │ │ ├── __init__.py +│ │ │ ├── sources.py # Source CRUD +│ │ │ ├── downloads.py # Download history +│ │ │ ├── credentials.py # Cookie management +│ │ │ ├── settings.py # App settings +│ │ │ └── websocket.py # Real-time events +│ │ │ +│ │ ├── models/ +│ │ │ ├── __init__.py +│ │ │ ├── source.py +│ │ │ ├── download.py +│ │ │ ├── credential.py +│ │ │ └── content.py +│ │ │ +│ │ ├── services/ +│ │ │ ├── __init__.py +│ │ │ ├── gallery_dl.py # gallery-dl subprocess wrapper +│ │ │ ├── download_manager.py +│ │ │ ├── credential_manager.py +│ │ │ └── manifest.py # Manifest generation +│ │ │ +│ │ ├── tasks/ +│ │ │ ├── __init__.py +│ │ │ ├── celery_app.py # Celery configuration +│ │ │ ├── downloads.py # Download tasks +│ │ │ └── scheduler.py # Scheduled jobs +│ │ │ +│ │ └── utils/ +│ │ ├── __init__.py +│ │ ├── cookies.py # Cookie parsing +│ │ └── encryption.py # Credential encryption +│ │ +│ ├── alembic/ +│ │ ├── env.py +│ │ └── versions/ +│ │ +│ └── tests/ +│ +├── frontend/ +│ ├── Dockerfile +│ ├── package.json +│ ├── vite.config.js +│ │ +│ └── src/ +│ ├── main.js +│ ├── App.vue +│ ├── router/ +│ ├── stores/ +│ ├── views/ +│ │ ├── Dashboard.vue +│ │ ├── Sources.vue +│ │ ├── Downloads.vue +│ │ ├── NewContent.vue +│ │ └── Settings.vue +│ ├── components/ +│ └── services/ +│ └── api.js +│ +├── extension/ +│ ├── manifest.json +│ ├── popup/ +│ │ ├── popup.html +│ │ ├── popup.css +│ │ └── popup.js +│ ├── background/ +│ │ └── background.js +│ └── icons/ +│ +└── scripts/ + ├── migrate_legacy.py # Import from old project + └── init_db.py +``` + +--- + +## Database Schema + +```sql +-- Sources (artists/creators to download from) +CREATE TABLE sources ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + platform VARCHAR(50) NOT NULL, -- patreon, subscribestar, hentaifoundry, discord + url TEXT NOT NULL, + enabled BOOLEAN DEFAULT true, + priority INTEGER DEFAULT 0, + check_interval INTEGER DEFAULT 3600, -- seconds between checks + last_check TIMESTAMP, + last_success TIMESTAMP, + error_count INTEGER DEFAULT 0, + metadata JSONB DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(platform, url) +); + +-- Downloads (individual download jobs) +CREATE TABLE downloads ( + id SERIAL PRIMARY KEY, + source_id INTEGER REFERENCES sources(id) ON DELETE SET NULL, + url TEXT NOT NULL, + status VARCHAR(20) DEFAULT 'pending', -- pending, running, completed, failed, skipped + error_type VARCHAR(50), -- auth_error, rate_limited, not_found, etc. + error_message TEXT, + file_count INTEGER DEFAULT 0, + started_at TIMESTAMP, + completed_at TIMESTAMP, + metadata JSONB DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX idx_downloads_status ON downloads(status); +CREATE INDEX idx_downloads_source ON downloads(source_id); + +-- Content Items (discovered files) +CREATE TABLE content_items ( + id SERIAL PRIMARY KEY, + source_id INTEGER REFERENCES sources(id) ON DELETE CASCADE, + download_id INTEGER REFERENCES downloads(id) ON DELETE SET NULL, + external_id VARCHAR(255), -- Platform-specific ID + title VARCHAR(500), + content_type VARCHAR(50), -- image, video, attachment + file_path VARCHAR(500), + file_size BIGINT, + thumbnail_path VARCHAR(500), + published_at TIMESTAMP, + discovered_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + metadata JSONB DEFAULT '{}', + UNIQUE(source_id, external_id) +); + +CREATE INDEX idx_content_discovered ON content_items(discovered_at DESC); + +-- Credentials (encrypted cookie/token storage) +CREATE TABLE credentials ( + id SERIAL PRIMARY KEY, + platform VARCHAR(50) NOT NULL UNIQUE, + credential_type VARCHAR(20) NOT NULL, -- cookies, token + data BYTEA NOT NULL, -- Encrypted + expires_at TIMESTAMP, + last_verified TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Settings (key-value store) +CREATE TABLE settings ( + key VARCHAR(100) PRIMARY KEY, + value JSONB NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +--- + +## API Specification + +### Sources + +``` +GET /api/sources List all sources (with pagination) +POST /api/sources Add new source +GET /api/sources/:id Get source details +PATCH /api/sources/:id Update source +DELETE /api/sources/:id Delete source +POST /api/sources/:id/check Trigger immediate download check +``` + +### Downloads + +``` +GET /api/downloads List download history +GET /api/downloads/:id Get download details with logs +POST /api/downloads/:id/retry Retry failed download +GET /api/downloads/stats Get statistics (by platform, by day) +``` + +### Content + +``` +GET /api/content List content items (filterable) +GET /api/content/new Get new content since timestamp +``` + +### Credentials + +``` +GET /api/credentials List credential status (no secrets) +POST /api/credentials Upload cookies (manual or from extension) +DELETE /api/credentials/:platform Remove credentials for platform +POST /api/credentials/verify Verify credentials are working +``` + +### Settings + +``` +GET /api/settings Get all settings +PATCH /api/settings Update settings +GET /api/settings/gallery-dl Get gallery-dl config +PUT /api/settings/gallery-dl Update gallery-dl config +``` + +### WebSocket + +``` +WS /ws/events + +Events (server -> client): + download.started { download_id, source_id, url } + download.progress { download_id, file_count } + download.completed { download_id, file_count } + download.failed { download_id, error_type, error_message } + content.new { content_id, source_name, title, thumbnail } + credential.expiring { platform, expires_in_hours } +``` + +--- + +## Firefox Extension + +### Features + +- Export cookies for supported platforms (Patreon, SubscribeStar, HentaiFoundry) +- Export Discord token +- Auto-refresh option (re-export when cookies change) +- Connection status indicator +- Manual export button per platform + +### Supported Platforms + +| Platform | Auth Type | Domains | +|----------|-----------|---------| +| Patreon | Cookies | `.patreon.com` | +| SubscribeStar | Cookies | `.subscribestar.com`, `.subscribestar.adult` | +| HentaiFoundry | Cookies | `.hentai-foundry.com` | +| Discord | Token | `.discord.com` | + +### Manual Alternative + +For users who don't want the extension: +1. Export cookies using browser dev tools or "cookies.txt" extension +2. Upload via Settings page in web UI +3. Paste cookie content or upload file + +--- + +## Configuration + +### Environment Variables (.env) + +```bash +# Database +DATABASE_URL=postgresql://gdl:password@db:5432/gallery_subscriber + +# Redis +REDIS_URL=redis://redis:6379/0 + +# Security +SECRET_KEY=your-secret-key-at-least-32-characters +EXTENSION_API_KEY=key-for-firefox-extension + +# Paths (inside container) +DOWNLOAD_PATH=/data/downloads +CONFIG_PATH=/data/config +COOKIES_PATH=/data/cookies + +# Download settings +DOWNLOAD_PARALLEL_LIMIT=3 +DOWNLOAD_RATE_LIMIT=3.0 +DEFAULT_CHECK_INTERVAL=3600 + +# Server +HOST=0.0.0.0 +PORT=8080 +``` + +### docker-compose.yml + +```yaml +services: + app: + build: ./backend + ports: + - "8080:8080" + environment: + - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=${SECRET_KEY} + - EXTENSION_API_KEY=${EXTENSION_API_KEY} + volumes: + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + depends_on: + - db + - redis + restart: unless-stopped + + celery: + build: ./backend + command: celery -A app.tasks.celery_app worker -B --loglevel=info + environment: + - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=${SECRET_KEY} + volumes: + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + depends_on: + - db + - redis + restart: unless-stopped + + db: + image: postgres:15-alpine + environment: + - POSTGRES_USER=gdl + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=gallery_subscriber + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + + redis: + image: redis:7-alpine + restart: unless-stopped + +volumes: + postgres_data: + downloads: + config: + cookies: +``` + +--- + +## Development Phases + +### Phase 1: Foundation +- [ ] Set up project structure +- [ ] Create Docker Compose development environment +- [ ] Initialize Quart app with basic routing +- [ ] Set up PostgreSQL + Alembic migrations +- [ ] Create database models +- [ ] Basic API endpoints (sources CRUD) + +### Phase 2: Download Engine +- [ ] Port gallery-dl wrapper from legacy project +- [ ] Implement Celery tasks for downloads +- [ ] Add platform-specific locking (from legacy) +- [ ] Error categorization and handling +- [ ] Download history tracking + +### Phase 3: Scheduling & State +- [ ] Celery Beat for scheduled downloads +- [ ] Round-robin scheduling logic (from legacy) +- [ ] Priority queue for failed tasks +- [ ] State persistence in PostgreSQL + +### Phase 4: Frontend +- [ ] Vue 3 + Vuetify scaffold +- [ ] Dashboard view +- [ ] Sources management UI +- [ ] Download history view +- [ ] Settings page + +### Phase 5: Authentication +- [ ] Credential storage (encrypted) +- [ ] Manual cookie upload via web UI +- [ ] Firefox extension +- [ ] Credential verification + +### Phase 6: Real-time & Content +- [ ] WebSocket event streaming +- [ ] New content feed +- [ ] Content browsing with thumbnails +- [ ] Notifications + +### Phase 7: Polish & Migration +- [ ] Migration script for legacy sources.yaml +- [ ] Import legacy gallery-dl.conf +- [ ] Testing +- [ ] Documentation + +--- + +## Migration from Legacy Project + +### What Gets Migrated + +| Legacy File | Destination | +|-------------|-------------| +| `sources.yaml` | `sources` table | +| `gallery-dl.conf` | `/data/config/gallery-dl.conf` (paths updated) | +| `state.json` | `sources` table (last_check, error_count) | +| Download history | Optional - start fresh or import logs | + +### Migration Script Usage + +```bash +python scripts/migrate_legacy.py \ + --sources /path/to/sources.yaml \ + --config /path/to/gallery-dl.conf \ + --state /path/to/state.json +``` + +### Key Patterns Preserved from Legacy + +1. **Platform locking** - One download per platform at a time +2. **Round-robin scheduling** - Fair distribution across sources +3. **Priority queue** - Failed tasks get priority on next run +4. **Error categorization** - AUTH_ERROR, RATE_LIMITED, NOT_FOUND, etc. +5. **Rate limiting** - Configurable delay between requests + +--- + +*Document Version: 2.0* +*Created: 2025-01-24* +*Project: Gallery Subscriber* diff --git a/README.md b/README.md new file mode 100644 index 0000000..f449116 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +# Gallery Subscriber + +A Docker-based web application for managing gallery-dl downloads with a modern web interface. + +## Features + +- **Web Dashboard** - Monitor downloads, view statistics, and manage sources +- **Source Management** - Add, edit, and organize download sources from multiple platforms +- **Per-Source Configuration** - Customize content types, rate limits, and patterns for each source +- **Credential Management** - Securely store cookies/tokens for authenticated platforms +- **Scheduled Downloads** - Automatic periodic checks for new content +- **Real-time Updates** - WebSocket notifications for download progress +- **Supported Platforms**: Patreon, SubscribeStar, Hentai Foundry, Discord + +## Quick Start + +1. **Copy environment file and configure:** + ```bash + cp .env.example .env + # Edit .env with your settings (especially DB_PASSWORD and SECRET_KEY) + ``` + +2. **Start the services:** + ```bash + docker-compose up -d + ``` + +3. **Run database migrations:** + ```bash + docker-compose exec app alembic upgrade head + ``` + +4. **Access the application:** + - Web UI: http://localhost:8080 + - API: http://localhost:8080/api + - Health check: http://localhost:8080/health + +## Configuration + +### Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `DB_PASSWORD` | PostgreSQL password | (required) | +| `SECRET_KEY` | App secret key (32+ chars) | (required) | +| `EXTENSION_API_KEY` | Firefox extension auth key | (required) | +| `PORT` | External port | 8080 | +| `DOWNLOAD_PARALLEL_LIMIT` | Max concurrent downloads | 3 | +| `DOWNLOAD_RATE_LIMIT` | Delay between downloads (sec) | 3.0 | +| `DEFAULT_CHECK_INTERVAL` | Check interval (sec) | 3600 | + +### Settings Page + +All configuration options are available in the web UI under Settings: +- Download settings (parallel limit, rate limit, retries) +- Notification settings (webhooks) +- Raw gallery-dl.conf editor +- Platform defaults reference + +## Screenshots + +### Dashboard +- Overview of sources and download statistics +- Recent downloads with status +- Sources needing attention + +### Sources +- List all sources with filtering +- Enable/disable sources +- Trigger manual checks +- Configure per-source options + +### Downloads +- Full download history +- Filter by status, source, date +- View detailed logs +- Retry failed downloads + +### Credentials +- Platform credential status +- Upload cookies/tokens +- Verify credentials + +### Settings +- Application configuration +- Gallery-dl config editor +- Platform documentation + +## API Endpoints + +| Endpoint | Description | +|----------|-------------| +| `GET /api/sources` | List all sources | +| `POST /api/sources` | Create a new source | +| `GET /api/sources/:id` | Get source details | +| `PATCH /api/sources/:id` | Update a source | +| `DELETE /api/sources/:id` | Delete a source | +| `POST /api/sources/:id/check` | Trigger download check | +| `GET /api/downloads` | List download history | +| `GET /api/downloads/stats` | Get download statistics | +| `POST /api/downloads/:id/retry` | Retry failed download | +| `GET /api/credentials` | List credential status | +| `POST /api/credentials` | Upload cookies/tokens | +| `DELETE /api/credentials/:platform` | Remove credentials | +| `GET /api/settings` | Get application settings | +| `PATCH /api/settings` | Update settings | +| `GET /api/platforms` | List supported platforms | +| `WS /ws/events` | Real-time event stream | + +## Project Structure + +``` +gallery-subscriber/ +├── backend/ # Quart API server +│ ├── app/ +│ │ ├── api/ # REST + WebSocket endpoints +│ │ ├── models/ # SQLAlchemy database models +│ │ ├── services/ # Business logic (gallery-dl wrapper) +│ │ ├── tasks/ # Celery background tasks +│ │ └── utils/ # Encryption, cookies, etc. +│ └── alembic/ # Database migrations +├── frontend/ # Vue 3 + Vuetify SPA +│ └── src/ +│ ├── views/ # Page components +│ ├── stores/ # Pinia state management +│ └── services/ # API client +├── extension/ # Firefox extension (planned) +├── scripts/ # Migration utilities +└── Dockerfile # Multi-stage build (frontend + backend) +``` + +## Technology Stack + +- **Backend:** Quart (async Flask) + SQLAlchemy + PostgreSQL +- **Task Queue:** Celery + Redis +- **Frontend:** Vue 3 + Vuetify 3 + Pinia (compiled to static files, served by Quart) +- **Downloader:** gallery-dl + +## Development + +For development with hot-reload: + +```bash +docker-compose -f docker-compose.yml -f docker-compose.dev.yml up +``` + +Frontend development (standalone): +```bash +cd frontend +npm install +npm run dev +``` + +## Migrating from Legacy Project + +If you have an existing Gallery-DL-scripting setup: + +```bash +# Coming soon: migration script +python scripts/migrate_legacy.py \ + --sources /path/to/sources.yaml \ + --config /path/to/gallery-dl.conf +``` + +## License + +MIT diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev new file mode 100644 index 0000000..80dc633 --- /dev/null +++ b/backend/Dockerfile.dev @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application +COPY . . + +# Create data directories +RUN mkdir -p /data/downloads /data/config /data/cookies + +EXPOSE 8080 + +# Development mode with reload +CMD ["hypercorn", "app.main:app", "--bind", "0.0.0.0:8080", "--reload"] diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..cd64a16 --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,42 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os + +sqlalchemy.url = driver://user:pass@localhost/dbname + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..1ac182e --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,75 @@ +"""Alembic migration environment.""" + +import asyncio +from logging.config import fileConfig + +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from alembic import context + +from app.config import get_settings +from app.models.base import Base +from app.models.source import Source +from app.models.download import Download +from app.models.credential import Credential +from app.models.content import ContentItem +from app.models.setting import Setting + +config = context.config +settings = get_settings() + +# Override sqlalchemy.url with actual database URL +config.set_main_option("sqlalchemy.url", settings.async_database_url) + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """Run migrations in 'online' mode with async engine.""" + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/001_initial_schema.py b/backend/alembic/versions/001_initial_schema.py new file mode 100644 index 0000000..b43fa15 --- /dev/null +++ b/backend/alembic/versions/001_initial_schema.py @@ -0,0 +1,129 @@ +"""Initial schema + +Revision ID: 001 +Revises: +Create Date: 2025-01-24 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = '001' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Sources table + op.create_table( + 'sources', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('platform', sa.String(50), nullable=False), + sa.Column('url', sa.Text(), nullable=False), + sa.Column('enabled', sa.Boolean(), default=True), + sa.Column('priority', sa.Integer(), default=0), + sa.Column('check_interval', sa.Integer(), default=3600), + sa.Column('last_check', sa.DateTime(), nullable=True), + sa.Column('last_success', sa.DateTime(), nullable=True), + sa.Column('error_count', sa.Integer(), default=0), + sa.Column('metadata', postgresql.JSONB(), default={}), + sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('platform', 'url', name='uq_source_platform_url') + ) + op.create_index('ix_sources_platform', 'sources', ['platform']) + + # Downloads table + op.create_table( + 'downloads', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('url', sa.Text(), nullable=False), + sa.Column('status', sa.String(20), default='pending'), + sa.Column('error_type', sa.String(50), nullable=True), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('file_count', sa.Integer(), default=0), + sa.Column('total_size', sa.BigInteger(), nullable=True), + sa.Column('started_at', sa.DateTime(), nullable=True), + sa.Column('completed_at', sa.DateTime(), nullable=True), + sa.Column('metadata', postgresql.JSONB(), default={}), + sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='SET NULL') + ) + op.create_index('ix_downloads_status', 'downloads', ['status']) + op.create_index('ix_downloads_source_id', 'downloads', ['source_id']) + + # Content items table + op.create_table( + 'content_items', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('download_id', sa.Integer(), nullable=True), + sa.Column('external_id', sa.String(255), nullable=True), + sa.Column('title', sa.String(500), nullable=True), + sa.Column('content_type', sa.String(50), default='other'), + sa.Column('file_path', sa.String(500), nullable=True), + sa.Column('file_size', sa.BigInteger(), nullable=True), + sa.Column('thumbnail_path', sa.String(500), nullable=True), + sa.Column('published_at', sa.DateTime(), nullable=True), + sa.Column('discovered_at', sa.DateTime(), server_default=sa.func.now()), + sa.Column('metadata', postgresql.JSONB(), default={}), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['download_id'], ['downloads.id'], ondelete='SET NULL'), + sa.UniqueConstraint('source_id', 'external_id', name='uq_content_source_external') + ) + op.create_index('ix_content_items_source_id', 'content_items', ['source_id']) + op.create_index('ix_content_items_discovered_at', 'content_items', ['discovered_at']) + + # Credentials table + op.create_table( + 'credentials', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('platform', sa.String(50), nullable=False), + sa.Column('credential_type', sa.String(20), nullable=False), + sa.Column('data', sa.LargeBinary(), nullable=False), + sa.Column('expires_at', sa.DateTime(), nullable=True), + sa.Column('last_verified', sa.DateTime(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('platform', name='uq_credential_platform') + ) + op.create_index('ix_credentials_platform', 'credentials', ['platform']) + + # Settings table + op.create_table( + 'settings', + sa.Column('key', sa.String(100), nullable=False), + sa.Column('value', postgresql.JSONB(), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()), + sa.PrimaryKeyConstraint('key') + ) + + # Insert default settings + op.execute(""" + INSERT INTO settings (key, value) VALUES + ('download.parallel_limit', '3'), + ('download.rate_limit', '3.0'), + ('download.retry_count', '3'), + ('download.schedule_interval', '3600'), + ('notification.enabled', 'false'), + ('notification.webhook_url', 'null') + """) + + +def downgrade() -> None: + op.drop_table('settings') + op.drop_table('credentials') + op.drop_table('content_items') + op.drop_table('downloads') + op.drop_table('sources') diff --git a/backend/alembic/versions/002_add_subscriptions.py b/backend/alembic/versions/002_add_subscriptions.py new file mode 100644 index 0000000..aaacb22 --- /dev/null +++ b/backend/alembic/versions/002_add_subscriptions.py @@ -0,0 +1,103 @@ +"""Add subscriptions table and link sources + +Revision ID: 002 +Revises: 001 +Create Date: 2025-01-25 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = '002' +down_revision: Union[str, None] = '001' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Create subscriptions table + op.create_table( + 'subscriptions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('enabled', sa.Boolean(), default=True), + sa.Column('priority', sa.Integer(), default=0), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('metadata', postgresql.JSONB(), default={}), + sa.Column('created_at', sa.DateTime(), server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime(), server_default=sa.func.now(), onupdate=sa.func.now()), + sa.PrimaryKeyConstraint('id') + ) + op.create_index('ix_subscriptions_name', 'subscriptions', ['name'], unique=True) + + # Migrate existing sources to subscriptions + # Each unique source name becomes a subscription + conn = op.get_bind() + + # Get existing sources + existing_sources = conn.execute( + sa.text("SELECT DISTINCT name FROM sources ORDER BY name") + ).fetchall() + + # Create subscriptions for each unique source name + for (name,) in existing_sources: + conn.execute( + sa.text("INSERT INTO subscriptions (name, enabled, priority) VALUES (:name, true, 0)"), + {"name": name} + ) + + # Add subscription_id column to sources + op.add_column('sources', sa.Column('subscription_id', sa.Integer(), nullable=True)) + op.create_index('ix_sources_subscription_id', 'sources', ['subscription_id']) + + # Update sources with their subscription_id + conn.execute(sa.text(""" + UPDATE sources s + SET subscription_id = sub.id + FROM subscriptions sub + WHERE s.name = sub.name + """)) + + # Make subscription_id NOT NULL after data migration + op.alter_column('sources', 'subscription_id', nullable=False) + + # Add foreign key constraint + op.create_foreign_key( + 'fk_sources_subscription_id', + 'sources', 'subscriptions', + ['subscription_id'], ['id'], + ondelete='CASCADE' + ) + + # Remove name and priority columns from sources (now on subscription) + op.drop_column('sources', 'name') + op.drop_column('sources', 'priority') + + +def downgrade() -> None: + # Add back name and priority columns to sources + op.add_column('sources', sa.Column('name', sa.String(255), nullable=True)) + op.add_column('sources', sa.Column('priority', sa.Integer(), default=0)) + + # Populate name from subscription + conn = op.get_bind() + conn.execute(sa.text(""" + UPDATE sources s + SET name = sub.name, priority = sub.priority + FROM subscriptions sub + WHERE s.subscription_id = sub.id + """)) + + op.alter_column('sources', 'name', nullable=False) + + # Remove foreign key and subscription_id column + op.drop_constraint('fk_sources_subscription_id', 'sources', type_='foreignkey') + op.drop_index('ix_sources_subscription_id', 'sources') + op.drop_column('sources', 'subscription_id') + + # Drop subscriptions table + op.drop_index('ix_subscriptions_name', 'subscriptions') + op.drop_table('subscriptions') diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..c91c3c7 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1 @@ +# Gallery Subscriber Backend diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..624da7f --- /dev/null +++ b/backend/app/api/__init__.py @@ -0,0 +1,5 @@ +"""API blueprints.""" + +from app.api import sources, downloads, credentials, settings, platforms, websocket + +__all__ = ["sources", "downloads", "credentials", "settings", "platforms", "websocket"] diff --git a/backend/app/api/credentials.py b/backend/app/api/credentials.py new file mode 100644 index 0000000..c96aa0a --- /dev/null +++ b/backend/app/api/credentials.py @@ -0,0 +1,182 @@ +"""Credential management API endpoints.""" + +from datetime import datetime +from quart import Blueprint, request, jsonify, current_app +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.credential import Credential, CredentialType +from app.models.setting import Setting, EXTENSION_API_KEY_SETTING, generate_api_key +from app.config import get_settings +from app.utils.encryption import encrypt_data, decrypt_data + +bp = Blueprint("credentials", __name__) + +VALID_PLATFORMS = ["patreon", "subscribestar", "hentaifoundry", "discord"] + + +@bp.route("", methods=["GET"]) +async def list_credentials(): + """List all credential status (without sensitive data).""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute(select(Credential)) + credentials = result.scalars().all() + + return jsonify({ + "items": [c.to_dict(include_data=False) for c in credentials] + }) + + +@bp.route("", methods=["POST"]) +async def upload_credentials(): + """Upload credentials (cookies or token). + + Can be called by: + 1. Firefox extension (with X-Extension-Key header) + 2. Web UI manual upload (with session auth) + """ + settings = get_settings() + + # Check extension API key if present + extension_key = request.headers.get("X-Extension-Key") + if extension_key: + # Look up API key from database + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + result = await session.execute( + select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING) + ) + setting = result.scalar() + stored_key = setting.value if setting else None + + if not stored_key or extension_key != stored_key: + return jsonify({"error": "Invalid extension API key"}), 401 + + data = await request.get_json() + + # Validate required fields + if not data.get("platform"): + return jsonify({"error": "Platform is required"}), 400 + if not data.get("credential_type"): + return jsonify({"error": "Credential type is required"}), 400 + if not data.get("data"): + return jsonify({"error": "Credential data is required"}), 400 + + platform = data["platform"] + cred_type = data["credential_type"] + cred_data = data["data"] + + # Validate platform + if platform not in VALID_PLATFORMS: + return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400 + + # Validate credential type + if cred_type not in [CredentialType.COOKIES, CredentialType.TOKEN]: + return jsonify({"error": "Invalid credential type. Must be 'cookies' or 'token'"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Check for existing credential + result = await session.execute( + select(Credential).where(Credential.platform == platform) + ) + credential = result.scalar() + + # Encrypt the credential data + encrypted = encrypt_data(cred_data, settings.secret_key) + + if credential: + # Update existing + credential.credential_type = cred_type + credential.data = encrypted + credential.expires_at = data.get("expires_at") + credential.last_verified = None # Reset verification + else: + # Create new + credential = Credential( + platform=platform, + credential_type=cred_type, + data=encrypted, + expires_at=data.get("expires_at"), + ) + session.add(credential) + + await session.commit() + await session.refresh(credential) + + current_app.logger.info(f"Updated credentials for platform: {platform}") + return jsonify({ + "message": "Credentials updated", + "platform": platform, + "credential_type": cred_type, + "expires_at": credential.expires_at.isoformat() if credential.expires_at else None, + }) + + +@bp.route("/", methods=["DELETE"]) +async def delete_credentials(platform: str): + """Delete credentials for a platform.""" + if platform not in VALID_PLATFORMS: + return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Credential).where(Credential.platform == platform) + ) + credential = result.scalar() + + if not credential: + return jsonify({"error": "Credentials not found for this platform"}), 404 + + await session.delete(credential) + await session.commit() + + current_app.logger.info(f"Deleted credentials for platform: {platform}") + return "", 204 + + +@bp.route("/verify", methods=["POST"]) +async def verify_credentials(): + """Verify that credentials are working. + + Attempts to make a test request to the platform. + """ + data = await request.get_json() + platform = data.get("platform") + + if not platform: + return jsonify({"error": "Platform is required"}), 400 + + if platform not in VALID_PLATFORMS: + return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(VALID_PLATFORMS)}"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Credential).where(Credential.platform == platform) + ) + credential = result.scalar() + + if not credential: + return jsonify({ + "platform": platform, + "is_valid": False, + "error": "No credentials stored for this platform", + }) + + # TODO: Implement actual verification by making test request + # For now, just mark as verified + credential.last_verified = datetime.utcnow() + await session.commit() + + return jsonify({ + "platform": platform, + "is_valid": True, + "last_verified": credential.last_verified.isoformat(), + }) diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py new file mode 100644 index 0000000..07d30c9 --- /dev/null +++ b/backend/app/api/downloads.py @@ -0,0 +1,171 @@ +"""Download history API endpoints.""" + +from datetime import datetime +from quart import Blueprint, request, jsonify, current_app +from sqlalchemy import select, func, and_ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.download import Download, DownloadStatus +from app.models.source import Source +from app.tasks.downloads import process_download + +bp = Blueprint("downloads", __name__) + + +@bp.route("", methods=["GET"]) +async def list_downloads(): + """List download history with filtering and pagination.""" + # Query parameters + source_id = request.args.get("source_id", type=int) + status = request.args.get("status") + from_date = request.args.get("from_date") + to_date = request.args.get("to_date") + page = int(request.args.get("page", 1)) + per_page = min(int(request.args.get("per_page", 50)), 100) + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Build query + query = select(Download) + count_query = select(func.count()).select_from(Download) + + filters = [] + if source_id: + filters.append(Download.source_id == source_id) + if status: + filters.append(Download.status == status) + if from_date: + filters.append(Download.created_at >= datetime.fromisoformat(from_date)) + if to_date: + filters.append(Download.created_at <= datetime.fromisoformat(to_date)) + + if filters: + query = query.where(and_(*filters)) + count_query = count_query.where(and_(*filters)) + + # Get total count + total_result = await session.execute(count_query) + total = total_result.scalar() + + # Apply pagination and ordering + query = query.order_by(Download.created_at.desc()) + query = query.offset((page - 1) * per_page).limit(per_page) + + result = await session.execute(query) + downloads = result.scalars().all() + + return jsonify({ + "items": [d.to_dict() for d in downloads], + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page, + }) + + +@bp.route("/", methods=["GET"]) +async def get_download(download_id: int): + """Get download details.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute(select(Download).where(Download.id == download_id)) + download = result.scalar() + + if not download: + return jsonify({"error": "Download not found"}), 404 + + # Include source info if available + response = download.to_dict() + if download.source_id: + source_result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == download.source_id) + ) + source = source_result.scalar() + if source: + response["source"] = { + "id": source.id, + "subscription_name": source.subscription.name if source.subscription else None, + "platform": source.platform, + } + + return jsonify(response) + + +@bp.route("//retry", methods=["POST"]) +async def retry_download(download_id: int): + """Retry a failed download.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute(select(Download).where(Download.id == download_id)) + download = result.scalar() + + if not download: + return jsonify({"error": "Download not found"}), 404 + + if download.status != DownloadStatus.FAILED: + return jsonify({"error": "Can only retry failed downloads"}), 400 + + # Reset status to pending + download.status = DownloadStatus.PENDING + download.error_type = None + download.error_message = None + await session.commit() + + # Queue Celery task to retry the download + task = process_download.delay(download_id) + + current_app.logger.info(f"Queued retry for download: {download_id}") + return jsonify({ + "message": "Retry queued", + "download_id": download_id, + "task_id": task.id, + }), 202 + + +@bp.route("/stats", methods=["GET"]) +async def get_stats(): + """Get download statistics.""" + period = request.args.get("period", "week") # day, week, month + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Total counts by status + status_query = select( + Download.status, + func.count(Download.id) + ).group_by(Download.status) + + status_result = await session.execute(status_query) + status_counts = {row[0]: row[1] for row in status_result} + + # Counts by platform (through source) + platform_query = select( + Source.platform, + func.count(Download.id) + ).join(Source, Download.source_id == Source.id).group_by(Source.platform) + + platform_result = await session.execute(platform_query) + platform_counts = {row[0]: row[1] for row in platform_result} + + # Recent downloads count + recent_query = select(func.count()).select_from(Download).where( + Download.status == DownloadStatus.COMPLETED + ) + recent_result = await session.execute(recent_query) + recent_completed = recent_result.scalar() + + return jsonify({ + "total": sum(status_counts.values()), + "by_status": status_counts, + "by_platform": platform_counts, + "completed": status_counts.get(DownloadStatus.COMPLETED, 0), + "failed": status_counts.get(DownloadStatus.FAILED, 0), + "pending": status_counts.get(DownloadStatus.PENDING, 0), + }) diff --git a/backend/app/api/platforms.py b/backend/app/api/platforms.py new file mode 100644 index 0000000..54de77b --- /dev/null +++ b/backend/app/api/platforms.py @@ -0,0 +1,199 @@ +"""Platform information API endpoints. + +Provides information about supported platforms and their configuration options. +""" + +from quart import Blueprint, jsonify + +from app.services.gallery_dl import GalleryDLService + +bp = Blueprint("platforms", __name__) + + +@bp.route("", methods=["GET"]) +async def list_platforms(): + """List all supported platforms with their configuration options.""" + gdl_service = GalleryDLService() + + platforms = { + "patreon": { + "name": "Patreon", + "description": "Download posts from Patreon creators", + "requires_auth": True, + "auth_type": "cookies", + "url_pattern": "https://www.patreon.com/{creator}", + "url_examples": [ + "https://www.patreon.com/example_artist", + "https://www.patreon.com/user?u=12345678", + ], + "content_types": gdl_service.get_platform_content_types("patreon"), + "content_type_descriptions": { + "images": "Standard post images", + "image_large": "High-resolution image versions", + "attachments": "File attachments (ZIP, PSD, etc.)", + "postfile": "Post-specific files", + "content": "Embedded content in post body", + }, + "default_config": gdl_service.get_default_config_for_platform("patreon"), + }, + "subscribestar": { + "name": "SubscribeStar", + "description": "Download posts from SubscribeStar creators", + "requires_auth": True, + "auth_type": "cookies", + "url_pattern": "https://subscribestar.{domain}/{creator}", + "url_examples": [ + "https://subscribestar.adult/example_artist", + "https://www.subscribestar.com/example_artist", + ], + "content_types": gdl_service.get_platform_content_types("subscribestar"), + "content_type_descriptions": { + "all": "All available content", + }, + "default_config": gdl_service.get_default_config_for_platform("subscribestar"), + }, + "hentaifoundry": { + "name": "Hentai Foundry", + "description": "Download artwork from Hentai Foundry artists", + "requires_auth": False, + "auth_type": "cookies", # Optional but improves access + "url_pattern": "https://www.hentai-foundry.com/user/{artist}", + "url_examples": [ + "https://www.hentai-foundry.com/user/example_artist", + "https://www.hentai-foundry.com/pictures/user/example_artist", + ], + "content_types": gdl_service.get_platform_content_types("hentaifoundry"), + "content_type_descriptions": { + "pictures": "Artwork/pictures", + "stories": "Written stories", + }, + "default_config": gdl_service.get_default_config_for_platform("hentaifoundry"), + }, + "discord": { + "name": "Discord", + "description": "Download attachments from Discord channels", + "requires_auth": True, + "auth_type": "token", + "url_pattern": "https://discord.com/channels/{server}/{channel}", + "url_examples": [ + "https://discord.com/channels/123456789/987654321", + ], + "content_types": gdl_service.get_platform_content_types("discord"), + "content_type_descriptions": { + "all": "All attachments and embeds", + }, + "default_config": gdl_service.get_default_config_for_platform("discord"), + "notes": "Requires Discord user token (not bot token)", + }, + } + + return jsonify({"platforms": platforms}) + + +@bp.route("/", methods=["GET"]) +async def get_platform(platform: str): + """Get detailed information about a specific platform.""" + gdl_service = GalleryDLService() + + if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]: + return jsonify({"error": f"Unknown platform: {platform}"}), 404 + + # Get the full platform info from list_platforms + all_platforms = (await list_platforms()).get_json() + platform_info = all_platforms["platforms"].get(platform) + + if not platform_info: + return jsonify({"error": f"Platform not found: {platform}"}), 404 + + return jsonify(platform_info) + + +@bp.route("//config-schema", methods=["GET"]) +async def get_config_schema(platform: str): + """Get the configuration schema for a platform. + + This defines what options can be set per-source for this platform. + Useful for building dynamic forms in the frontend. + """ + if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]: + return jsonify({"error": f"Unknown platform: {platform}"}), 404 + + gdl_service = GalleryDLService() + content_types = gdl_service.get_platform_content_types(platform) + defaults = gdl_service.get_default_config_for_platform(platform) + + schema = { + "platform": platform, + "fields": [ + { + "name": "content_types", + "type": "multiselect", + "label": "Content Types to Download", + "description": "Select which types of content to download from this source", + "options": content_types, + "default": defaults.get("content_types", ["all"]), + }, + { + "name": "sleep", + "type": "number", + "label": "Delay Between Downloads (seconds)", + "description": "Time to wait between downloading files. Higher values are safer but slower.", + "min": 0.5, + "max": 30.0, + "step": 0.5, + "default": defaults.get("sleep", 3.0), + }, + { + "name": "sleep_request", + "type": "number", + "label": "Delay Between Requests (seconds)", + "description": "Time to wait between API requests. Helps avoid rate limiting.", + "min": 0.5, + "max": 15.0, + "step": 0.5, + "default": defaults.get("sleep_request", 1.5), + }, + { + "name": "skip_existing", + "type": "boolean", + "label": "Skip Already Downloaded", + "description": "Skip files that have already been downloaded (uses archive database)", + "default": True, + }, + { + "name": "save_metadata", + "type": "boolean", + "label": "Save Metadata", + "description": "Save JSON metadata file alongside each downloaded file", + "default": True, + }, + { + "name": "timeout", + "type": "number", + "label": "Download Timeout (seconds)", + "description": "Maximum time to wait for a single download before giving up", + "min": 60, + "max": 7200, + "step": 60, + "default": 3600, + }, + { + "name": "directory_pattern", + "type": "text", + "label": "Directory Pattern (Advanced)", + "description": "Custom directory structure pattern. Leave empty for platform default.", + "placeholder": defaults.get("directory_pattern", ""), + "default": None, + }, + { + "name": "filename_pattern", + "type": "text", + "label": "Filename Pattern (Advanced)", + "description": "Custom filename pattern. Leave empty for platform default.", + "placeholder": defaults.get("filename_pattern", ""), + "default": None, + }, + ], + } + + return jsonify(schema) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py new file mode 100644 index 0000000..64f185b --- /dev/null +++ b/backend/app/api/settings.py @@ -0,0 +1,165 @@ +"""Settings API endpoints.""" + +import json +from pathlib import Path +from quart import Blueprint, request, jsonify, current_app +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.setting import Setting, DEFAULT_SETTINGS, EXTENSION_API_KEY_SETTING, generate_api_key +from app.config import get_settings + +bp = Blueprint("settings", __name__) + + +async def get_or_create_extension_api_key(session: AsyncSession) -> str: + """Get the extension API key, creating one if it doesn't exist.""" + result = await session.execute( + select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING) + ) + setting = result.scalar() + + if setting: + return setting.value + + # Generate new key + new_key = generate_api_key() + setting = Setting(key=EXTENSION_API_KEY_SETTING, value=new_key) + session.add(setting) + await session.commit() + + return new_key + + +@bp.route("", methods=["GET"]) +async def get_all_settings(): + """Get all application settings.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute(select(Setting)) + settings = result.scalars().all() + + # Build settings dict with defaults + settings_dict = dict(DEFAULT_SETTINGS) + for s in settings: + settings_dict[s.key] = s.value + + # Ensure extension API key exists + api_key = await get_or_create_extension_api_key(session) + settings_dict[EXTENSION_API_KEY_SETTING] = api_key + + return jsonify({"settings": settings_dict}) + + +@bp.route("", methods=["PATCH"]) +async def update_settings(): + """Update application settings.""" + data = await request.get_json() + + if not data: + return jsonify({"error": "No settings provided"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + for key, value in data.items(): + # Check if setting exists + result = await session.execute(select(Setting).where(Setting.key == key)) + setting = result.scalar() + + if setting: + setting.value = value + else: + setting = Setting(key=key, value=value) + session.add(setting) + + await session.commit() + + # Return updated settings + result = await session.execute(select(Setting)) + settings = result.scalars().all() + settings_dict = dict(DEFAULT_SETTINGS) + for s in settings: + settings_dict[s.key] = s.value + + current_app.logger.info(f"Updated settings: {list(data.keys())}") + return jsonify({"settings": settings_dict}) + + +@bp.route("/api-key", methods=["GET"]) +async def get_extension_api_key(): + """Get the extension API key.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + api_key = await get_or_create_extension_api_key(session) + return jsonify({"api_key": api_key}) + + +@bp.route("/api-key/regenerate", methods=["POST"]) +async def regenerate_extension_api_key(): + """Regenerate the extension API key.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Find existing key + result = await session.execute( + select(Setting).where(Setting.key == EXTENSION_API_KEY_SETTING) + ) + setting = result.scalar() + + # Generate new key + new_key = generate_api_key() + + if setting: + setting.value = new_key + else: + setting = Setting(key=EXTENSION_API_KEY_SETTING, value=new_key) + session.add(setting) + + await session.commit() + + current_app.logger.info("Extension API key regenerated") + return jsonify({"api_key": new_key, "message": "API key regenerated"}) + + +@bp.route("/gallery-dl", methods=["GET"]) +async def get_gallery_dl_config(): + """Get gallery-dl configuration.""" + config = get_settings() + config_path = Path(config.config_path) / "gallery-dl.conf" + + if not config_path.exists(): + return jsonify({"config": {}, "message": "No configuration file found"}) + + try: + with open(config_path) as f: + gallery_dl_config = json.load(f) + return jsonify({"config": gallery_dl_config}) + except json.JSONDecodeError as e: + return jsonify({"error": f"Invalid JSON in config file: {e}"}), 500 + + +@bp.route("/gallery-dl", methods=["PUT"]) +async def update_gallery_dl_config(): + """Update gallery-dl configuration.""" + data = await request.get_json() + + if not data.get("config"): + return jsonify({"error": "Config object is required"}), 400 + + config = get_settings() + config_path = Path(config.config_path) / "gallery-dl.conf" + + # Ensure config directory exists + config_path.parent.mkdir(parents=True, exist_ok=True) + + try: + with open(config_path, "w") as f: + json.dump(data["config"], f, indent=4) + + current_app.logger.info("Updated gallery-dl configuration") + return jsonify({"message": "Configuration updated", "config": data["config"]}) + except Exception as e: + current_app.logger.error(f"Failed to write gallery-dl config: {e}") + return jsonify({"error": f"Failed to write config: {e}"}), 500 diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py new file mode 100644 index 0000000..f01f8b4 --- /dev/null +++ b/backend/app/api/sources.py @@ -0,0 +1,223 @@ +"""Source management API endpoints.""" + +from quart import Blueprint, request, jsonify, current_app +from sqlalchemy import select, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.source import Source +from app.models.subscription import Subscription +from app.tasks.downloads import download_source + +bp = Blueprint("sources", __name__) + + +@bp.route("", methods=["GET"]) +async def list_sources(): + """List all sources with optional filtering and pagination.""" + # Query parameters + platform = request.args.get("platform") + subscription_id = request.args.get("subscription_id") + enabled = request.args.get("enabled") + page = int(request.args.get("page", 1)) + per_page = min(int(request.args.get("per_page", 50)), 100) + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Build query with subscription eager loading + query = select(Source).options(selectinload(Source.subscription)) + + if platform: + query = query.where(Source.platform == platform) + if subscription_id: + query = query.where(Source.subscription_id == int(subscription_id)) + if enabled is not None: + query = query.where(Source.enabled == (enabled.lower() == "true")) + + # Get total count + count_query = select(func.count()).select_from(Source) + if platform: + count_query = count_query.where(Source.platform == platform) + if subscription_id: + count_query = count_query.where(Source.subscription_id == int(subscription_id)) + if enabled is not None: + count_query = count_query.where(Source.enabled == (enabled.lower() == "true")) + + total_result = await session.execute(count_query) + total = total_result.scalar() + + # Apply pagination - order by subscription name, then platform + query = query.join(Subscription).order_by(Subscription.name, Source.platform) + query = query.offset((page - 1) * per_page).limit(per_page) + + result = await session.execute(query) + sources = result.scalars().all() + + return jsonify({ + "items": [s.to_dict() for s in sources], + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if total > 0 else 0, + }) + + +@bp.route("", methods=["POST"]) +async def create_source(): + """Create a new source for a subscription.""" + data = await request.get_json() + + # Validate required fields + required = ["subscription_id", "platform", "url"] + missing = [f for f in required if not data.get(f)] + if missing: + return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400 + + # Validate platform + valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"] + if data["platform"] not in valid_platforms: + return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Check subscription exists + sub_result = await session.execute( + select(Subscription).where(Subscription.id == data["subscription_id"]) + ) + subscription = sub_result.scalar() + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + # Check for duplicate platform+url + existing = await session.execute( + select(Source).where( + Source.platform == data["platform"], + Source.url == data["url"] + ) + ) + if existing.scalar(): + return jsonify({"error": "Source with this platform and URL already exists"}), 409 + + source = Source( + subscription_id=data["subscription_id"], + platform=data["platform"], + url=data["url"], + enabled=data.get("enabled", True), + check_interval=data.get("check_interval", 3600), + metadata_=data.get("metadata", {}), + ) + + session.add(source) + await session.commit() + await session.refresh(source) + + current_app.logger.info(f"Created source: {subscription.name}/{source.platform}") + return jsonify(source.to_dict()), 201 + + +@bp.route("/", methods=["GET"]) +async def get_source(source_id: int): + """Get a single source by ID.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == source_id) + ) + source = result.scalar() + + if not source: + return jsonify({"error": "Source not found"}), 404 + + return jsonify(source.to_dict()) + + +@bp.route("/", methods=["PATCH"]) +async def update_source(source_id: int): + """Update a source.""" + data = await request.get_json() + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == source_id) + ) + source = result.scalar() + + if not source: + return jsonify({"error": "Source not found"}), 404 + + # Update allowed fields + allowed_fields = ["enabled", "check_interval", "metadata"] + for field in allowed_fields: + if field in data: + if field == "metadata": + source.metadata_ = data[field] + else: + setattr(source, field, data[field]) + + await session.commit() + await session.refresh(source) + + current_app.logger.info(f"Updated source: {source.subscription.name}/{source.platform}") + return jsonify(source.to_dict()) + + +@bp.route("/", methods=["DELETE"]) +async def delete_source(source_id: int): + """Delete a source.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == source_id) + ) + source = result.scalar() + + if not source: + return jsonify({"error": "Source not found"}), 404 + + info = f"{source.subscription.name}/{source.platform}" + await session.delete(source) + await session.commit() + + current_app.logger.info(f"Deleted source: {info}") + return "", 204 + + +@bp.route("//check", methods=["POST"]) +async def trigger_check(source_id: int): + """Trigger an immediate download check for a source.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == source_id) + ) + source = result.scalar() + + if not source: + return jsonify({"error": "Source not found"}), 404 + + # Queue Celery task for immediate download + task = download_source.delay(source_id) + + current_app.logger.info(f"Triggered check for source: {source.subscription.name}/{source.platform}") + return jsonify({ + "message": "Check queued", + "source_id": source_id, + "subscription_name": source.subscription.name, + "platform": source.platform, + "task_id": task.id, + }), 202 diff --git a/backend/app/api/subscriptions.py b/backend/app/api/subscriptions.py new file mode 100644 index 0000000..8bacb07 --- /dev/null +++ b/backend/app/api/subscriptions.py @@ -0,0 +1,307 @@ +"""Subscription management API endpoints.""" + +from quart import Blueprint, request, jsonify, current_app +from sqlalchemy import select, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.models.subscription import Subscription +from app.models.source import Source +from app.tasks.downloads import download_source + +bp = Blueprint("subscriptions", __name__) + + +@bp.route("", methods=["GET"]) +async def list_subscriptions(): + """List all subscriptions with optional filtering and pagination.""" + # Query parameters + enabled = request.args.get("enabled") + search = request.args.get("search") + page = int(request.args.get("page", 1)) + per_page = min(int(request.args.get("per_page", 50)), 100) + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Build query with eager loading of sources + query = select(Subscription).options(selectinload(Subscription.sources)) + + if enabled is not None: + query = query.where(Subscription.enabled == (enabled.lower() == "true")) + if search: + query = query.where(Subscription.name.ilike(f"%{search}%")) + + # Get total count + count_query = select(func.count()).select_from(Subscription) + if enabled is not None: + count_query = count_query.where(Subscription.enabled == (enabled.lower() == "true")) + if search: + count_query = count_query.where(Subscription.name.ilike(f"%{search}%")) + + total_result = await session.execute(count_query) + total = total_result.scalar() + + # Apply pagination + query = query.order_by(Subscription.priority.desc(), Subscription.name) + query = query.offset((page - 1) * per_page).limit(per_page) + + result = await session.execute(query) + subscriptions = result.scalars().unique().all() + + return jsonify({ + "items": [s.to_dict() for s in subscriptions], + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if total > 0 else 0, + }) + + +@bp.route("", methods=["POST"]) +async def create_subscription(): + """Create a new subscription with optional sources.""" + data = await request.get_json() + + # Validate required fields + if not data.get("name"): + return jsonify({"error": "Name is required"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Check for duplicate name + existing = await session.execute( + select(Subscription).where(Subscription.name == data["name"]) + ) + if existing.scalar(): + return jsonify({"error": "Subscription with this name already exists"}), 409 + + subscription = Subscription( + name=data["name"], + enabled=data.get("enabled", True), + priority=data.get("priority", 0), + description=data.get("description"), + metadata_=data.get("metadata", {}), + ) + + # Add sources if provided + sources_data = data.get("sources", []) + for source_data in sources_data: + if not source_data.get("platform") or not source_data.get("url"): + continue + source = Source( + platform=source_data["platform"], + url=source_data["url"], + enabled=source_data.get("enabled", True), + check_interval=source_data.get("check_interval", 3600), + metadata_=source_data.get("metadata", {}), + ) + subscription.sources.append(source) + + session.add(subscription) + await session.commit() + await session.refresh(subscription) + + current_app.logger.info(f"Created subscription: {subscription.name}") + return jsonify(subscription.to_dict()), 201 + + +@bp.route("/", methods=["GET"]) +async def get_subscription(subscription_id: int): + """Get a single subscription by ID with its sources.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Subscription) + .options(selectinload(Subscription.sources)) + .where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + return jsonify(subscription.to_dict()) + + +@bp.route("/", methods=["PATCH"]) +async def update_subscription(subscription_id: int): + """Update a subscription.""" + data = await request.get_json() + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Subscription) + .options(selectinload(Subscription.sources)) + .where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + # Check for duplicate name if changing + if "name" in data and data["name"] != subscription.name: + existing = await session.execute( + select(Subscription).where(Subscription.name == data["name"]) + ) + if existing.scalar(): + return jsonify({"error": "Subscription with this name already exists"}), 409 + + # Update allowed fields + allowed_fields = ["name", "enabled", "priority", "description", "metadata"] + for field in allowed_fields: + if field in data: + if field == "metadata": + subscription.metadata_ = data[field] + else: + setattr(subscription, field, data[field]) + + await session.commit() + await session.refresh(subscription) + + current_app.logger.info(f"Updated subscription: {subscription.name}") + return jsonify(subscription.to_dict()) + + +@bp.route("/", methods=["DELETE"]) +async def delete_subscription(subscription_id: int): + """Delete a subscription and all its sources.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Subscription).where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + name = subscription.name + await session.delete(subscription) + await session.commit() + + current_app.logger.info(f"Deleted subscription: {name}") + return "", 204 + + +@bp.route("//sources", methods=["GET"]) +async def list_subscription_sources(subscription_id: int): + """List all sources for a subscription.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Subscription) + .options(selectinload(Subscription.sources)) + .where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + return jsonify({ + "subscription_id": subscription_id, + "subscription_name": subscription.name, + "sources": [s.to_dict(include_subscription=False) for s in subscription.sources] + }) + + +@bp.route("//sources", methods=["POST"]) +async def add_source_to_subscription(subscription_id: int): + """Add a new source to a subscription.""" + data = await request.get_json() + + # Validate required fields + required = ["platform", "url"] + missing = [f for f in required if not data.get(f)] + if missing: + return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400 + + # Validate platform + valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"] + if data["platform"] not in valid_platforms: + return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400 + + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + # Check subscription exists + result = await session.execute( + select(Subscription).where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + # Check for duplicate platform+url + existing = await session.execute( + select(Source).where( + Source.platform == data["platform"], + Source.url == data["url"] + ) + ) + if existing.scalar(): + return jsonify({"error": "Source with this platform and URL already exists"}), 409 + + # Store name before commit (attributes expire after commit) + subscription_name = subscription.name + + source = Source( + subscription_id=subscription_id, + platform=data["platform"], + url=data["url"], + enabled=data.get("enabled", True), + check_interval=data.get("check_interval", 3600), + metadata_=data.get("metadata", {}), + ) + + session.add(source) + await session.commit() + await session.refresh(source) + + current_app.logger.info(f"Added source to {subscription_name}: {source.platform}") + # Use include_subscription=False to avoid lazy loading the relationship + result = source.to_dict(include_subscription=False) + result["subscription_name"] = subscription_name + return jsonify(result), 201 + + +@bp.route("//check", methods=["POST"]) +async def trigger_subscription_check(subscription_id: int): + """Trigger download check for all enabled sources in a subscription.""" + async with current_app.db_engine.connect() as conn: + session = AsyncSession(bind=conn) + + result = await session.execute( + select(Subscription) + .options(selectinload(Subscription.sources)) + .where(Subscription.id == subscription_id) + ) + subscription = result.scalar() + + if not subscription: + return jsonify({"error": "Subscription not found"}), 404 + + enabled_sources = [s for s in subscription.sources if s.enabled] + + # Queue Celery tasks for each enabled source + task_ids = [] + for source in enabled_sources: + task = download_source.delay(source.id) + task_ids.append(task.id) + + current_app.logger.info(f"Triggered check for subscription: {subscription.name} ({len(enabled_sources)} sources)") + return jsonify({ + "message": "Checks queued", + "subscription_id": subscription_id, + "source_count": len(enabled_sources), + "task_ids": task_ids, + }), 202 diff --git a/backend/app/api/websocket.py b/backend/app/api/websocket.py new file mode 100644 index 0000000..7d26edc --- /dev/null +++ b/backend/app/api/websocket.py @@ -0,0 +1,217 @@ +"""WebSocket API for real-time updates.""" + +import asyncio +import json +import logging +from typing import Set +from quart import Blueprint, websocket, current_app +import redis.asyncio as aioredis + +from app.config import get_settings +from app.events import EVENTS_CHANNEL + +logger = logging.getLogger(__name__) + +bp = Blueprint("websocket", __name__) + +# Connected WebSocket clients +connected_clients: Set = set() + + +class WebSocketManager: + """Manages WebSocket connections and broadcasts.""" + + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance.clients = set() + return cls._instance + + async def connect(self, ws): + """Register a new WebSocket connection.""" + self.clients.add(ws) + logger.info(f"WebSocket connected. Total clients: {len(self.clients)}") + + async def disconnect(self, ws): + """Remove a WebSocket connection.""" + self.clients.discard(ws) + logger.info(f"WebSocket disconnected. Total clients: {len(self.clients)}") + + async def broadcast(self, event_type: str, data: dict): + """Broadcast an event to all connected clients.""" + if not self.clients: + return + + message = json.dumps({ + "type": event_type, + "data": data, + }) + + disconnected = set() + for client in self.clients: + try: + await client.send(message) + except Exception as e: + logger.debug(f"Failed to send to client: {e}") + disconnected.add(client) + + # Clean up disconnected clients + self.clients -= disconnected + + async def send_to_client(self, ws, event_type: str, data: dict): + """Send an event to a specific client.""" + message = json.dumps({ + "type": event_type, + "data": data, + }) + try: + await ws.send(message) + except Exception as e: + logger.debug(f"Failed to send to client: {e}") + + +# Global manager instance +ws_manager = WebSocketManager() + + +@bp.websocket("/events") +async def events(): + """WebSocket endpoint for real-time events. + + Events sent from server: + - download.started: { download_id, source_id, url } + - download.progress: { download_id, file_count } + - download.completed: { download_id, file_count, duration } + - download.failed: { download_id, error_type, error_message } + - source.updated: { source_id, name, last_check } + - credential.expiring: { platform, expires_in_hours } + """ + await ws_manager.connect(websocket._get_current_object()) + + try: + # Send initial connection confirmation + await websocket.send(json.dumps({ + "type": "connected", + "data": {"message": "Connected to Gallery Subscriber"}, + })) + + # Keep connection alive and handle incoming messages + while True: + try: + # Wait for messages (ping/pong or commands) + message = await asyncio.wait_for(websocket.receive(), timeout=30) + + # Handle ping + if message == "ping": + await websocket.send("pong") + else: + # Parse JSON commands if needed + try: + data = json.loads(message) + await handle_client_message(websocket._get_current_object(), data) + except json.JSONDecodeError: + pass + + except asyncio.TimeoutError: + # Send keepalive ping + try: + await websocket.send(json.dumps({"type": "ping"})) + except Exception: + break + + except asyncio.CancelledError: + pass + finally: + await ws_manager.disconnect(websocket._get_current_object()) + + +async def handle_client_message(ws, data: dict): + """Handle messages from WebSocket clients.""" + msg_type = data.get("type") + + if msg_type == "subscribe": + # Client wants to subscribe to specific events + # For now, all clients receive all events + await ws_manager.send_to_client(ws, "subscribed", {"status": "ok"}) + + elif msg_type == "ping": + await ws_manager.send_to_client(ws, "pong", {}) + + +# Helper functions for broadcasting events from other parts of the app + +async def broadcast_download_started(download_id: int, source_id: int, url: str): + """Broadcast that a download has started.""" + await ws_manager.broadcast("download.started", { + "download_id": download_id, + "source_id": source_id, + "url": url, + }) + + +async def broadcast_download_completed(download_id: int, file_count: int, duration: float): + """Broadcast that a download has completed.""" + await ws_manager.broadcast("download.completed", { + "download_id": download_id, + "file_count": file_count, + "duration_seconds": duration, + }) + + +async def broadcast_download_failed(download_id: int, error_type: str, error_message: str): + """Broadcast that a download has failed.""" + await ws_manager.broadcast("download.failed", { + "download_id": download_id, + "error_type": error_type, + "error_message": error_message, + }) + + +async def broadcast_source_updated(source_id: int, name: str, last_check: str): + """Broadcast that a source has been updated.""" + await ws_manager.broadcast("source.updated", { + "source_id": source_id, + "name": name, + "last_check": last_check, + }) + + +# Redis Pub/Sub integration for cross-process events (from Celery tasks) + +async def start_redis_subscriber(): + """Start a background task that subscribes to Redis events and broadcasts them. + + This should be called when the Quart app starts. + """ + settings = get_settings() + + try: + redis_client = aioredis.from_url(settings.redis_url) + pubsub = redis_client.pubsub() + await pubsub.subscribe(EVENTS_CHANNEL) + + logger.info(f"Started Redis subscriber on channel: {EVENTS_CHANNEL}") + + async for message in pubsub.listen(): + if message["type"] == "message": + try: + data = json.loads(message["data"]) + event_type = data.get("type") + event_data = data.get("data", {}) + + # Broadcast to all WebSocket clients + await ws_manager.broadcast(event_type, event_data) + logger.debug(f"Broadcast event from Redis: {event_type}") + except json.JSONDecodeError: + logger.warning(f"Invalid JSON in Redis message: {message['data']}") + except Exception as e: + logger.error(f"Error broadcasting Redis event: {e}") + + except asyncio.CancelledError: + logger.info("Redis subscriber cancelled") + await pubsub.unsubscribe(EVENTS_CHANNEL) + await redis_client.close() + except Exception as e: + logger.error(f"Redis subscriber error: {e}") diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..a52235b --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,49 @@ +"""Application configuration using pydantic-settings.""" + +from functools import lru_cache +from pydantic_settings import BaseSettings + + +class Settings(BaseSettings): + """Application settings loaded from environment variables.""" + + # Database + database_url: str = "postgresql://gdl:password@localhost:5432/gallery_subscriber" + + # Redis + redis_url: str = "redis://localhost:6379/0" + + # Security + secret_key: str = "change-me-in-production" + # Note: extension_api_key is now stored in database and auto-generated + + # Paths + download_path: str = "/data/downloads" + config_path: str = "/data/config" + cookies_path: str = "/data/cookies" + + # Download settings + download_parallel_limit: int = 3 + download_rate_limit: float = 3.0 + default_check_interval: int = 3600 # seconds + + # Server + host: str = "0.0.0.0" + port: int = 8080 + debug: bool = False + log_level: str = "INFO" + + @property + def async_database_url(self) -> str: + """Convert database URL to async version for asyncpg.""" + return self.database_url.replace("postgresql://", "postgresql+asyncpg://") + + class Config: + env_file = ".env" + env_file_encoding = "utf-8" + + +@lru_cache +def get_settings() -> Settings: + """Get cached settings instance.""" + return Settings() diff --git a/backend/app/events.py b/backend/app/events.py new file mode 100644 index 0000000..fd4e0ba --- /dev/null +++ b/backend/app/events.py @@ -0,0 +1,30 @@ +"""Cross-process event publishing via Redis. + +This module provides synchronous event publishing for use in Celery tasks. +The events are consumed by the WebSocket module which broadcasts them to clients. +""" + +import json +import redis + +from app.config import get_settings + +# Redis pub/sub channel for cross-process events +EVENTS_CHANNEL = "gallery_subscriber:events" + + +def publish_event(event_type: str, data: dict): + """Publish an event to Redis (for use in Celery tasks). + + This is a sync function that can be called from Celery tasks. + The Quart app subscribes to these events and broadcasts to WebSocket clients. + + Args: + event_type: Type of event (e.g., "download.started", "download.completed") + data: Event payload data + """ + settings = get_settings() + r = redis.from_url(settings.redis_url) + message = json.dumps({"type": event_type, "data": data}) + r.publish(EVENTS_CHANNEL, message) + r.close() diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..e1b68c8 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,116 @@ +"""Main Quart application factory.""" + +import asyncio +import logging +from pathlib import Path +from quart import Quart, send_from_directory, send_file +from quart_cors import cors + +from app.config import get_settings +from app.models import init_db +from app.api import subscriptions, sources, downloads, credentials, settings as settings_api, platforms, websocket +from app.api.websocket import start_redis_subscriber + +# Path to static frontend files (built Vue app) +STATIC_DIR = Path(__file__).parent.parent / "static" + + +def create_app() -> Quart: + """Create and configure the Quart application.""" + app = Quart(__name__, static_folder=None) # Disable default static handling + config = get_settings() + + # Configure logging + logging.basicConfig( + level=getattr(logging, config.log_level.upper()), + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + + # Enable CORS for API routes only + app = cors(app, allow_origin="*") + + # Register API blueprints + app.register_blueprint(subscriptions.bp, url_prefix="/api/subscriptions") + app.register_blueprint(sources.bp, url_prefix="/api/sources") + app.register_blueprint(downloads.bp, url_prefix="/api/downloads") + app.register_blueprint(credentials.bp, url_prefix="/api/credentials") + app.register_blueprint(settings_api.bp, url_prefix="/api/settings") + app.register_blueprint(platforms.bp, url_prefix="/api/platforms") + app.register_blueprint(websocket.bp, url_prefix="/ws") + + @app.before_serving + async def startup(): + """Initialize database connection on startup.""" + app.db_engine = await init_db(config.async_database_url) + app.logger.info("Database connection initialized") + app.logger.info(f"Static files directory: {STATIC_DIR} (exists: {STATIC_DIR.exists()})") + + # Start Redis subscriber for WebSocket events from Celery tasks + app.redis_subscriber_task = asyncio.create_task(start_redis_subscriber()) + app.logger.info("Redis subscriber started for WebSocket events") + + @app.after_serving + async def shutdown(): + """Close database connection on shutdown.""" + # Cancel Redis subscriber task + if hasattr(app, "redis_subscriber_task"): + app.redis_subscriber_task.cancel() + try: + await app.redis_subscriber_task + except asyncio.CancelledError: + pass + app.logger.info("Redis subscriber stopped") + + if hasattr(app, "db_engine"): + await app.db_engine.dispose() + app.logger.info("Database connection closed") + + @app.route("/health") + async def health_check(): + """Health check endpoint.""" + return {"status": "healthy"} + + @app.route("/api") + async def api_info(): + """API information endpoint.""" + return { + "name": "Gallery Subscriber API", + "version": "1.0.0", + "endpoints": { + "subscriptions": "/api/subscriptions", + "sources": "/api/sources", + "downloads": "/api/downloads", + "credentials": "/api/credentials", + "settings": "/api/settings", + "platforms": "/api/platforms", + "websocket": "/ws/events", + }, + } + + # Serve static assets (JS, CSS, images, fonts) + @app.route("/assets/") + async def serve_assets(filename): + """Serve static assets from the frontend build.""" + return await send_from_directory(STATIC_DIR / "assets", filename) + + # Serve other static files (favicon, etc.) + @app.route("/") + async def serve_static(filename): + """Serve static files or fall back to index.html for Vue Router.""" + file_path = STATIC_DIR / filename + if file_path.exists() and file_path.is_file(): + return await send_from_directory(STATIC_DIR, filename) + # Fall back to index.html for Vue Router (SPA) + return await send_file(STATIC_DIR / "index.html") + + # Serve index.html for root path + @app.route("/") + async def serve_index(): + """Serve the frontend index.html.""" + return await send_file(STATIC_DIR / "index.html") + + return app + + +# Create app instance for hypercorn +app = create_app() diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..dc9e8f6 --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,60 @@ +"""Database models and initialization.""" + +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker +from sqlalchemy.orm import DeclarativeBase + +from app.models.subscription import Subscription +from app.models.source import Source +from app.models.download import Download +from app.models.credential import Credential +from app.models.content import ContentItem +from app.models.setting import Setting + + +class Base(DeclarativeBase): + """Base class for all models.""" + pass + + +# Re-export models +__all__ = [ + "Base", + "Subscription", + "Source", + "Download", + "Credential", + "ContentItem", + "Setting", + "init_db", + "get_session", +] + +# Global engine and session factory +_engine = None +_session_factory = None + + +async def init_db(database_url: str): + """Initialize database engine and create tables.""" + global _engine, _session_factory + + _engine = create_async_engine(database_url, echo=False) + _session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False) + + # Import all models to register them with Base + from app.models.subscription import Subscription + from app.models.source import Source + from app.models.download import Download + from app.models.credential import Credential + from app.models.content import ContentItem + from app.models.setting import Setting + + return _engine + + +async def get_session() -> AsyncSession: + """Get a database session.""" + if _session_factory is None: + raise RuntimeError("Database not initialized. Call init_db first.") + async with _session_factory() as session: + yield session diff --git a/backend/app/models/base.py b/backend/app/models/base.py new file mode 100644 index 0000000..19b0a95 --- /dev/null +++ b/backend/app/models/base.py @@ -0,0 +1,17 @@ +"""SQLAlchemy base model.""" + +from datetime import datetime +from sqlalchemy import func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + """Base class for all models.""" + pass + + +class TimestampMixin: + """Mixin for created_at and updated_at timestamps.""" + + created_at: Mapped[datetime] = mapped_column(default=func.now()) + updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now()) diff --git a/backend/app/models/content.py b/backend/app/models/content.py new file mode 100644 index 0000000..5b7fa12 --- /dev/null +++ b/backend/app/models/content.py @@ -0,0 +1,83 @@ +"""ContentItem model - discovered files/content.""" + +from datetime import datetime +from typing import Optional +from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base + + +class ContentType: + """Content type constants.""" + IMAGE = "image" + VIDEO = "video" + ATTACHMENT = "attachment" + AUDIO = "audio" + OTHER = "other" + + +class ContentItem(Base): + """A content item represents a discovered/downloaded file.""" + + __tablename__ = "content_items" + __table_args__ = ( + UniqueConstraint("source_id", "external_id", name="uq_content_source_external"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + source_id: Mapped[int] = mapped_column( + ForeignKey("sources.id", ondelete="CASCADE"), nullable=False, index=True + ) + download_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("downloads.id", ondelete="SET NULL"), nullable=True + ) + + # Content identification + external_id: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + title: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + content_type: Mapped[str] = mapped_column(String(50), default=ContentType.OTHER) + + # File information + file_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + file_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True) + thumbnail_path: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + + # Timestamps + published_at: Mapped[Optional[datetime]] = mapped_column(nullable=True) + discovered_at: Mapped[datetime] = mapped_column(default=datetime.utcnow) + + # Flexible metadata + metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + + # Relationships + source: Mapped["Source"] = relationship(back_populates="content_items") + download: Mapped[Optional["Download"]] = relationship(back_populates="content_items") + + def __repr__(self) -> str: + return f"" + + def to_dict(self) -> dict: + """Convert to dictionary for API responses.""" + return { + "id": self.id, + "source_id": self.source_id, + "download_id": self.download_id, + "external_id": self.external_id, + "title": self.title, + "content_type": self.content_type, + "file_path": self.file_path, + "file_size": self.file_size, + "thumbnail_path": self.thumbnail_path, + "published_at": self.published_at.isoformat() if self.published_at else None, + "discovered_at": self.discovered_at.isoformat() if self.discovered_at else None, + "metadata": self.metadata_, + } + + +# Import for type hints +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from app.models.source import Source + from app.models.download import Download diff --git a/backend/app/models/credential.py b/backend/app/models/credential.py new file mode 100644 index 0000000..c3bf1a1 --- /dev/null +++ b/backend/app/models/credential.py @@ -0,0 +1,53 @@ +"""Credential model - encrypted cookie/token storage.""" + +from datetime import datetime +from typing import Optional +from sqlalchemy import String, LargeBinary, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base, TimestampMixin + + +class CredentialType: + """Credential type constants.""" + COOKIES = "cookies" + TOKEN = "token" + + +class Credential(Base, TimestampMixin): + """Encrypted credential storage for platform authentication.""" + + __tablename__ = "credentials" + __table_args__ = (UniqueConstraint("platform", name="uq_credential_platform"),) + + id: Mapped[int] = mapped_column(primary_key=True) + platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + credential_type: Mapped[str] = mapped_column(String(20), nullable=False) + + # Encrypted credential data + data: Mapped[bytes] = mapped_column(LargeBinary, nullable=False) + + # Status tracking + expires_at: Mapped[Optional[datetime]] = mapped_column(nullable=True) + last_verified: Mapped[Optional[datetime]] = mapped_column(nullable=True) + + def __repr__(self) -> str: + return f"" + + def to_dict(self, include_data: bool = False) -> dict: + """Convert to dictionary for API responses. + + By default, does NOT include the encrypted data for security. + """ + result = { + "id": self.id, + "platform": self.platform, + "credential_type": self.credential_type, + "expires_at": self.expires_at.isoformat() if self.expires_at else None, + "last_verified": self.last_verified.isoformat() if self.last_verified else None, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + if include_data: + result["data"] = self.data + return result diff --git a/backend/app/models/download.py b/backend/app/models/download.py new file mode 100644 index 0000000..2069590 --- /dev/null +++ b/backend/app/models/download.py @@ -0,0 +1,93 @@ +"""Download model - individual download jobs.""" + +from datetime import datetime +from typing import Optional +from sqlalchemy import String, Integer, Text, ForeignKey, BigInteger +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin + + +class DownloadStatus: + """Download status constants.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + SKIPPED = "skipped" + + +class ErrorType: + """Error type constants matching legacy project.""" + AUTH_ERROR = "auth_error" + RATE_LIMITED = "rate_limited" + NOT_FOUND = "not_found" + ACCESS_DENIED = "access_denied" + NETWORK_ERROR = "network_error" + NO_NEW_CONTENT = "no_new_content" + TIMEOUT = "timeout" + HTTP_ERROR = "http_error" + UNSUPPORTED_URL = "unsupported_url" + UNKNOWN_ERROR = "unknown_error" + + +class Download(Base, TimestampMixin): + """A download represents a single download job for a source.""" + + __tablename__ = "downloads" + + id: Mapped[int] = mapped_column(primary_key=True) + source_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("sources.id", ondelete="SET NULL"), nullable=True, index=True + ) + url: Mapped[str] = mapped_column(Text, nullable=False) + + # Status + status: Mapped[str] = mapped_column( + String(20), default=DownloadStatus.PENDING, index=True + ) + error_type: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) + error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # Results + file_count: Mapped[int] = mapped_column(Integer, default=0) + total_size: Mapped[Optional[int]] = mapped_column(BigInteger, nullable=True) + + # Timing + started_at: Mapped[Optional[datetime]] = mapped_column(nullable=True) + completed_at: Mapped[Optional[datetime]] = mapped_column(nullable=True) + + # Flexible metadata + metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + + # Relationships + source: Mapped[Optional["Source"]] = relationship(back_populates="downloads") + content_items: Mapped[list["ContentItem"]] = relationship(back_populates="download") + + def __repr__(self) -> str: + return f"" + + def to_dict(self) -> dict: + """Convert to dictionary for API responses.""" + return { + "id": self.id, + "source_id": self.source_id, + "url": self.url, + "status": self.status, + "error_type": self.error_type, + "error_message": self.error_message, + "file_count": self.file_count, + "total_size": self.total_size, + "started_at": self.started_at.isoformat() if self.started_at else None, + "completed_at": self.completed_at.isoformat() if self.completed_at else None, + "metadata": self.metadata_, + "created_at": self.created_at.isoformat() if self.created_at else None, + } + + +# Import for type hints +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from app.models.source import Source + from app.models.content import ContentItem diff --git a/backend/app/models/setting.py b/backend/app/models/setting.py new file mode 100644 index 0000000..5430094 --- /dev/null +++ b/backend/app/models/setting.py @@ -0,0 +1,49 @@ +"""Setting model - key-value application settings.""" + +import secrets +from datetime import datetime +from sqlalchemy import String, func +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +def generate_api_key() -> str: + """Generate a secure random API key.""" + return secrets.token_urlsafe(32) + + +class Setting(Base): + """Key-value setting storage.""" + + __tablename__ = "settings" + + key: Mapped[str] = mapped_column(String(100), primary_key=True) + value: Mapped[dict] = mapped_column(JSONB, nullable=False) + updated_at: Mapped[datetime] = mapped_column(default=func.now(), onupdate=func.now()) + + def __repr__(self) -> str: + return f"" + + def to_dict(self) -> dict: + """Convert to dictionary for API responses.""" + return { + "key": self.key, + "value": self.value, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + + +# Default settings that should exist +DEFAULT_SETTINGS = { + "download.parallel_limit": 3, + "download.rate_limit": 3.0, + "download.retry_count": 3, + "download.schedule_interval": 3600, + "notification.enabled": False, + "notification.webhook_url": None, +} + +# Key for the extension API key setting +EXTENSION_API_KEY_SETTING = "security.extension_api_key" diff --git a/backend/app/models/source.py b/backend/app/models/source.py new file mode 100644 index 0000000..d520daa --- /dev/null +++ b/backend/app/models/source.py @@ -0,0 +1,75 @@ +"""Source model - platform URLs for subscriptions.""" + +from datetime import datetime +from typing import Optional +from sqlalchemy import String, Boolean, Integer, Text, ForeignKey, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin + + +class Source(Base, TimestampMixin): + """A source represents a platform URL for a subscription. + + Sources belong to subscriptions - one subscription (artist) can have + multiple sources (Patreon, SubscribeStar, etc.). + """ + + __tablename__ = "sources" + __table_args__ = (UniqueConstraint("platform", "url", name="uq_source_platform_url"),) + + id: Mapped[int] = mapped_column(primary_key=True) + subscription_id: Mapped[int] = mapped_column( + ForeignKey("subscriptions.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + platform: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + url: Mapped[str] = mapped_column(Text, nullable=False) + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + check_interval: Mapped[int] = mapped_column(Integer, default=3600) # seconds + + # Status tracking + last_check: Mapped[Optional[datetime]] = mapped_column(nullable=True) + last_success: Mapped[Optional[datetime]] = mapped_column(nullable=True) + error_count: Mapped[int] = mapped_column(Integer, default=0) + + # Flexible metadata storage (per-source config overrides) + metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + + # Relationships + subscription: Mapped["Subscription"] = relationship(back_populates="sources") + downloads: Mapped[list["Download"]] = relationship(back_populates="source") + content_items: Mapped[list["ContentItem"]] = relationship(back_populates="source") + + def __repr__(self) -> str: + return f"" + + def to_dict(self, include_subscription: bool = True) -> dict: + """Convert to dictionary for API responses.""" + result = { + "id": self.id, + "subscription_id": self.subscription_id, + "platform": self.platform, + "url": self.url, + "enabled": self.enabled, + "check_interval": self.check_interval, + "last_check": self.last_check.isoformat() if self.last_check else None, + "last_success": self.last_success.isoformat() if self.last_success else None, + "error_count": self.error_count, + "metadata": self.metadata_, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + if include_subscription and self.subscription: + result["subscription_name"] = self.subscription.name + return result + + +# Import for type hints (avoid circular imports) +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from app.models.subscription import Subscription + from app.models.download import Download + from app.models.content import ContentItem diff --git a/backend/app/models/subscription.py b/backend/app/models/subscription.py new file mode 100644 index 0000000..7a16b58 --- /dev/null +++ b/backend/app/models/subscription.py @@ -0,0 +1,72 @@ +"""Subscription model - represents an artist/creator to track.""" + +from datetime import datetime +from typing import Optional, TYPE_CHECKING +from sqlalchemy import String, Boolean, Integer, Text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.base import Base, TimestampMixin + +if TYPE_CHECKING: + from app.models.source import Source + + +class Subscription(Base, TimestampMixin): + """A subscription represents an artist/creator being tracked. + + Each subscription can have multiple sources (platform URLs). + The subscription name is used for the directory structure: + /{download_path}/{subscription.name}/{source.platform}/{post} + """ + + __tablename__ = "subscriptions" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) + enabled: Mapped[bool] = mapped_column(Boolean, default=True) + priority: Mapped[int] = mapped_column(Integer, default=0) + + # Optional description/notes + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # Flexible metadata storage (tags, custom settings, etc.) + metadata_: Mapped[dict] = mapped_column("metadata", JSONB, default=dict) + + # Relationships + sources: Mapped[list["Source"]] = relationship( + back_populates="subscription", + cascade="all, delete-orphan", + lazy="selectin" + ) + + def __repr__(self) -> str: + return f"" + + @property + def platform_count(self) -> int: + """Number of platforms/sources for this subscription.""" + return len(self.sources) if self.sources else 0 + + @property + def enabled_source_count(self) -> int: + """Number of enabled sources.""" + return sum(1 for s in self.sources if s.enabled) if self.sources else 0 + + def to_dict(self, include_sources: bool = True) -> dict: + """Convert to dictionary for API responses.""" + result = { + "id": self.id, + "name": self.name, + "enabled": self.enabled, + "priority": self.priority, + "description": self.description, + "metadata": self.metadata_, + "platform_count": self.platform_count, + "enabled_source_count": self.enabled_source_count, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + if include_sources and self.sources: + result["sources"] = [s.to_dict(include_subscription=False) for s in self.sources] + return result diff --git a/backend/app/services/__init__.py b/backend/app/services/__init__.py new file mode 100644 index 0000000..436b05b --- /dev/null +++ b/backend/app/services/__init__.py @@ -0,0 +1 @@ +"""Service modules.""" diff --git a/backend/app/services/credential_manager.py b/backend/app/services/credential_manager.py new file mode 100644 index 0000000..1042fe3 --- /dev/null +++ b/backend/app/services/credential_manager.py @@ -0,0 +1,113 @@ +"""Credential management service. + +Handles retrieving, decrypting, and writing cookie files for gallery-dl. +""" + +import logging +from pathlib import Path +from typing import Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import get_settings +from app.models.credential import Credential +from app.utils.encryption import decrypt_data +from app.utils.cookies import parse_netscape_cookies, write_cookie_file + +logger = logging.getLogger(__name__) + + +class CredentialManager: + """Manages platform credentials for gallery-dl authentication.""" + + def __init__(self, session: AsyncSession): + self.session = session + self.settings = get_settings() + + async def get_cookies_path(self, platform: str) -> Optional[str]: + """Get or create a cookies file for a platform. + + If credentials exist for the platform, decrypts them and writes + to a temporary cookies file that gallery-dl can use. + + Args: + platform: Platform name (patreon, subscribestar, etc.) + + Returns: + Path to cookies file, or None if no credentials stored + """ + result = await self.session.execute( + select(Credential).where(Credential.platform == platform) + ) + credential = result.scalar() + + if not credential: + logger.debug(f"No credentials stored for platform: {platform}") + return None + + try: + # Decrypt credential data + decrypted = decrypt_data(credential.data, self.settings.secret_key) + + # Write to cookies file + cookies_dir = Path(self.settings.cookies_path) + cookies_dir.mkdir(parents=True, exist_ok=True) + + cookies_path = cookies_dir / f"{platform}_cookies.txt" + + # If it's already in Netscape format, write directly + if decrypted.strip().startswith("#") or "\t" in decrypted: + cookies_path.write_text(decrypted) + else: + # Assume it's JSON and needs conversion + import json + try: + cookies_list = json.loads(decrypted) + if isinstance(cookies_list, list): + write_cookie_file(cookies_list, cookies_path) + else: + # Single cookie object + write_cookie_file([cookies_list], cookies_path) + except json.JSONDecodeError: + # Not JSON, write as-is + cookies_path.write_text(decrypted) + + logger.debug(f"Wrote cookies file for {platform}: {cookies_path}") + return str(cookies_path) + + except Exception as e: + logger.error(f"Failed to prepare cookies for {platform}: {e}") + return None + + async def get_token(self, platform: str) -> Optional[str]: + """Get a decrypted token for a platform. + + Used for platforms like Discord that use token auth instead of cookies. + + Args: + platform: Platform name + + Returns: + Decrypted token string, or None if not stored + """ + result = await self.session.execute( + select(Credential).where(Credential.platform == platform) + ) + credential = result.scalar() + + if not credential or credential.credential_type != "token": + return None + + try: + return decrypt_data(credential.data, self.settings.secret_key) + except Exception as e: + logger.error(f"Failed to decrypt token for {platform}: {e}") + return None + + async def has_credentials(self, platform: str) -> bool: + """Check if credentials exist for a platform.""" + result = await self.session.execute( + select(Credential).where(Credential.platform == platform) + ) + return result.scalar() is not None diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py new file mode 100644 index 0000000..1dd40ae --- /dev/null +++ b/backend/app/services/gallery_dl.py @@ -0,0 +1,534 @@ +"""Gallery-dl wrapper service. + +This service provides a clean interface for executing gallery-dl downloads +with support for per-source configuration overrides. +""" + +import json +import logging +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Optional + +from app.config import get_settings + +logger = logging.getLogger(__name__) + + +class ErrorType(str, Enum): + """Error types for categorizing download failures.""" + AUTH_ERROR = "auth_error" + RATE_LIMITED = "rate_limited" + NOT_FOUND = "not_found" + ACCESS_DENIED = "access_denied" + NETWORK_ERROR = "network_error" + NO_NEW_CONTENT = "no_new_content" + TIMEOUT = "timeout" + HTTP_ERROR = "http_error" + UNSUPPORTED_URL = "unsupported_url" + UNKNOWN_ERROR = "unknown_error" + + +class ContentType(str, Enum): + """Content types that can be downloaded from platforms.""" + # Patreon content types + IMAGES = "images" + IMAGE_LARGE = "image_large" + ATTACHMENTS = "attachments" + POSTFILE = "postfile" + CONTENT = "content" # Embedded content in post body + + # Generic types + ALL = "all" + + +@dataclass +class SourceConfig: + """Per-source configuration that can override global settings. + + These settings are stored in the source's metadata field and merged + with global defaults when executing downloads. + """ + # Content types to download (platform-specific) + # For Patreon: ["images", "image_large", "attachments", "postfile", "content"] + # For others: typically just ["all"] or specific types + content_types: list[str] = field(default_factory=lambda: ["all"]) + + # Rate limiting overrides + sleep: Optional[float] = None # Delay between downloads + sleep_request: Optional[float] = None # Delay between requests + + # Directory/filename pattern overrides + directory_pattern: Optional[str] = None + filename_pattern: Optional[str] = None + + # Other options + skip_existing: bool = True # Skip files already in archive + save_metadata: bool = True # Save JSON metadata alongside files + timeout: int = 3600 # Download timeout in seconds + + @classmethod + def from_dict(cls, data: dict) -> "SourceConfig": + """Create SourceConfig from a dictionary (e.g., from source.metadata).""" + return cls( + content_types=data.get("content_types", ["all"]), + sleep=data.get("sleep"), + sleep_request=data.get("sleep_request"), + directory_pattern=data.get("directory_pattern"), + filename_pattern=data.get("filename_pattern"), + skip_existing=data.get("skip_existing", True), + save_metadata=data.get("save_metadata", True), + timeout=data.get("timeout", 3600), + ) + + def to_dict(self) -> dict: + """Convert to dictionary for storage.""" + return { + "content_types": self.content_types, + "sleep": self.sleep, + "sleep_request": self.sleep_request, + "directory_pattern": self.directory_pattern, + "filename_pattern": self.filename_pattern, + "skip_existing": self.skip_existing, + "save_metadata": self.save_metadata, + "timeout": self.timeout, + } + + +@dataclass +class DownloadResult: + """Result of a gallery-dl download execution.""" + success: bool + url: str + subscription_name: str + platform: str + + # Output details + files_downloaded: int = 0 + stdout: str = "" + stderr: str = "" + return_code: int = 0 + + # Error details (if failed) + error_type: Optional[ErrorType] = None + error_message: Optional[str] = None + + # Timing + duration_seconds: float = 0.0 + started_at: Optional[str] = None + completed_at: Optional[str] = None + + +class GalleryDLService: + """Service for executing gallery-dl downloads. + + Handles: + - Building gallery-dl configuration with per-source overrides + - Executing the subprocess with proper error handling + - Parsing output and categorizing errors + - Managing temporary config files + """ + + # Platform-specific default content types + # Directory patterns are relative - subscription_name/platform is prepended automatically + PLATFORM_DEFAULTS = { + "patreon": { + "content_types": ["images", "image_large", "attachments", "postfile", "content"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + }, + "subscribestar": { + "content_types": ["all"], + "directory": ["{date:%Y-%m-%d}_{id}_{title[:40]}"], + "filename": "{num:>02}_{filename}.{extension}", + }, + "hentaifoundry": { + "content_types": ["pictures"], + "directory": [], # Flat structure within platform folder + "filename": "{category}_{index:>03}_{title[:50]}.{extension}", + }, + "discord": { + "content_types": ["all"], + "directory": ["{channel}"], + "filename": "{date:%Y%m%d}_{message_id}_{filename}.{extension}", + }, + } + + def __init__(self): + self.settings = get_settings() + self._base_config = self._load_base_config() + + def _load_base_config(self) -> dict: + """Load the base gallery-dl configuration file.""" + config_path = Path(self.settings.config_path) / "gallery-dl.conf" + + if config_path.exists(): + try: + with open(config_path) as f: + return json.load(f) + except json.JSONDecodeError as e: + logger.warning(f"Invalid gallery-dl.conf, using defaults: {e}") + + # Return sensible defaults if no config exists + return self._get_default_config() + + def _get_default_config(self) -> dict: + """Get default gallery-dl configuration.""" + return { + "extractor": { + "base-directory": self.settings.download_path, + "archive": str(Path(self.settings.config_path) / "archive.sqlite3"), + "skip": True, + "sleep": self.settings.download_rate_limit, + "sleep-request": max(1.0, self.settings.download_rate_limit / 2), + "retries": 3, + "timeout": 30.0, + "verify": True, + "postprocessors": [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ], + }, + "downloader": { + "part": True, + "part-directory": str(Path(self.settings.config_path) / "temp"), + "retries": 3, + "timeout": 120.0, + }, + "output": { + "progress": True, + }, + } + + def _build_config_for_source( + self, + platform: str, + source_config: SourceConfig, + destination: str, + ) -> dict: + """Build a merged configuration for a specific source. + + Merges: base config → platform defaults → source overrides + """ + config = json.loads(json.dumps(self._base_config)) # Deep copy + + # Get platform defaults + platform_defaults = self.PLATFORM_DEFAULTS.get(platform, {}) + + # Ensure extractor section exists + if "extractor" not in config: + config["extractor"] = {} + + # Apply base directory + config["extractor"]["base-directory"] = destination + + # Apply rate limiting overrides + if source_config.sleep is not None: + config["extractor"]["sleep"] = source_config.sleep + if source_config.sleep_request is not None: + config["extractor"]["sleep-request"] = source_config.sleep_request + + # Apply skip setting + config["extractor"]["skip"] = source_config.skip_existing + + # Configure metadata postprocessor + if source_config.save_metadata: + config["extractor"]["postprocessors"] = [ + { + "name": "metadata", + "mode": "json", + "directory": ".", + "filename": "{filename}.json", + } + ] + else: + config["extractor"].pop("postprocessors", None) + + # Build platform-specific section + platform_config = {} + + # Content types (platform-specific handling) + if platform == "patreon": + # Patreon uses "files" array for content types + if "all" in source_config.content_types: + platform_config["files"] = ["images", "image_large", "attachments", "postfile", "content"] + else: + platform_config["files"] = source_config.content_types + elif platform == "hentaifoundry": + # HentaiFoundry uses "include" for content filtering + if "pictures" in source_config.content_types or "all" in source_config.content_types: + platform_config["include"] = "pictures" + + # Directory pattern + if source_config.directory_pattern: + platform_config["directory"] = source_config.directory_pattern + elif "directory" in platform_defaults: + platform_config["directory"] = platform_defaults["directory"] + + # Filename pattern + if source_config.filename_pattern: + platform_config["filename"] = source_config.filename_pattern + elif "filename" in platform_defaults: + platform_config["filename"] = platform_defaults["filename"] + + # Always save metadata at platform level too + platform_config["metadata"] = source_config.save_metadata + + # Add platform config to extractor + config["extractor"][platform] = platform_config + + return config + + def _categorize_error(self, return_code: int, stdout: str, stderr: str) -> tuple[ErrorType, str]: + """Categorize the error based on output and return code.""" + combined = f"{stdout} {stderr}".lower() + + if "401" in combined or "unauthorized" in combined or "login" in combined: + return ErrorType.AUTH_ERROR, "Authentication failed - cookies may be expired or invalid" + + if "429" in combined or "rate limit" in combined or "too many requests" in combined: + return ErrorType.RATE_LIMITED, "Rate limited by server - try increasing sleep time" + + if "404" in combined or "not found" in combined: + return ErrorType.NOT_FOUND, "URL not found - artist may have changed username or deleted content" + + if "timeout" in combined or "connection" in combined or "network" in combined: + return ErrorType.NETWORK_ERROR, "Network error - check internet connection" + + if "403" in combined or "forbidden" in combined or "access denied" in combined: + return ErrorType.ACCESS_DENIED, "Access denied - may need higher subscription tier" + + if return_code == 1 and ("no" in combined or "skip" in combined or not stdout.strip()): + return ErrorType.NO_NEW_CONTENT, "No new content to download" + + if "http error" in combined: + return ErrorType.HTTP_ERROR, "HTTP error occurred" + + if "no suitable" in combined or "unsupported" in combined: + return ErrorType.UNSUPPORTED_URL, "URL format not supported by gallery-dl" + + return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})" + + def _count_downloaded_files(self, stdout: str) -> int: + """Count the number of files downloaded from stdout. + + Gallery-dl outputs one line per downloaded file. + Lines starting with '#' or containing 'skip' are not downloads. + """ + if not stdout: + return 0 + + count = 0 + for line in stdout.strip().split("\n"): + line = line.strip() + if line and not line.startswith("#") and "skip" not in line.lower(): + count += 1 + + return count + + async def download( + self, + url: str, + subscription_name: str, + platform: str, + source_config: Optional[SourceConfig] = None, + cookies_path: Optional[str] = None, + ) -> DownloadResult: + """Execute a gallery-dl download. + + Args: + url: The URL to download from + subscription_name: Name of the subscription/artist (used for destination folder) + platform: Platform name (patreon, subscribestar, etc.) + source_config: Optional per-source configuration overrides + cookies_path: Optional path to cookies file + + Returns: + DownloadResult with success status and details + + Downloads are saved to: {download_path}/{subscription_name}/{platform}/{post_folders} + """ + import asyncio + from datetime import datetime + + start_time = time.time() + started_at = datetime.utcnow().isoformat() + + # Use default config if none provided + if source_config is None: + source_config = SourceConfig() + + # Build destination path: subscription_name/platform + destination = str(Path(self.settings.download_path) / subscription_name / platform) + + # Build merged configuration + config = self._build_config_for_source(platform, source_config, destination) + + # Add cookies if provided + if cookies_path: + config["extractor"]["cookies"] = cookies_path + + # Write temporary config file + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".json", + delete=False, + dir=self.settings.config_path, + ) as f: + json.dump(config, f, indent=2) + temp_config_path = f.name + + try: + # Build command + cmd = [ + sys.executable, + "-m", + "gallery_dl", + "--config", + temp_config_path, + "--verbose", + url, + ] + + logger.info(f"Executing gallery-dl for {subscription_name}/{platform}: {url}") + logger.debug(f"Command: {' '.join(cmd)}") + + # Execute subprocess + # Run in thread pool to not block async event loop + loop = asyncio.get_event_loop() + proc = await loop.run_in_executor( + None, + lambda: subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=source_config.timeout, + ), + ) + + duration = time.time() - start_time + completed_at = datetime.utcnow().isoformat() + + # Log output + if proc.stdout: + logger.debug(f"STDOUT for {subscription_name}/{platform}:\n{proc.stdout}") + if proc.stderr: + logger.debug(f"STDERR for {subscription_name}/{platform}:\n{proc.stderr}") + + # Determine success + if proc.returncode == 0: + return DownloadResult( + success=True, + url=url, + subscription_name=subscription_name, + platform=platform, + files_downloaded=self._count_downloaded_files(proc.stdout), + stdout=proc.stdout, + stderr=proc.stderr, + return_code=proc.returncode, + duration_seconds=duration, + started_at=started_at, + completed_at=completed_at, + ) + + # Handle errors + error_type, error_message = self._categorize_error( + proc.returncode, proc.stdout, proc.stderr + ) + + # NO_NEW_CONTENT is considered successful + success = error_type == ErrorType.NO_NEW_CONTENT + + return DownloadResult( + success=success, + url=url, + subscription_name=subscription_name, + platform=platform, + files_downloaded=self._count_downloaded_files(proc.stdout) if success else 0, + stdout=proc.stdout, + stderr=proc.stderr, + return_code=proc.returncode, + error_type=error_type, + error_message=error_message, + duration_seconds=duration, + started_at=started_at, + completed_at=completed_at, + ) + + except subprocess.TimeoutExpired: + duration = time.time() - start_time + logger.error(f"Download timeout for {subscription_name}/{platform} after {duration:.1f}s") + + return DownloadResult( + success=False, + url=url, + subscription_name=subscription_name, + platform=platform, + error_type=ErrorType.TIMEOUT, + error_message=f"Download timed out after {source_config.timeout} seconds", + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.utcnow().isoformat(), + ) + + except Exception as e: + duration = time.time() - start_time + logger.exception(f"Unexpected error downloading {subscription_name}/{platform}: {e}") + + return DownloadResult( + success=False, + url=url, + subscription_name=subscription_name, + platform=platform, + error_type=ErrorType.UNKNOWN_ERROR, + error_message=str(e), + duration_seconds=duration, + started_at=started_at, + completed_at=datetime.utcnow().isoformat(), + ) + + finally: + # Clean up temporary config file + try: + Path(temp_config_path).unlink() + except Exception: + pass + + def get_platform_content_types(self, platform: str) -> list[str]: + """Get available content types for a platform. + + Useful for UI to show what options are available. + """ + if platform == "patreon": + return ["images", "image_large", "attachments", "postfile", "content"] + elif platform == "hentaifoundry": + return ["pictures", "stories"] + elif platform == "subscribestar": + return ["all"] # No granular control + elif platform == "discord": + return ["all"] # No granular control + else: + return ["all"] + + def get_default_config_for_platform(self, platform: str) -> dict: + """Get the default configuration for a platform. + + Useful for showing defaults in UI. + """ + defaults = self.PLATFORM_DEFAULTS.get(platform, {}) + return { + "content_types": defaults.get("content_types", ["all"]), + "directory_pattern": defaults.get("directory"), + "filename_pattern": defaults.get("filename"), + "sleep": self.settings.download_rate_limit, + "sleep_request": max(1.0, self.settings.download_rate_limit / 2), + } diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py new file mode 100644 index 0000000..e1c4fbf --- /dev/null +++ b/backend/app/tasks/__init__.py @@ -0,0 +1,5 @@ +"""Celery tasks.""" + +from app.tasks.celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/backend/app/tasks/celery_app.py b/backend/app/tasks/celery_app.py new file mode 100644 index 0000000..2eab98e --- /dev/null +++ b/backend/app/tasks/celery_app.py @@ -0,0 +1,40 @@ +"""Celery application configuration.""" + +from celery import Celery +from celery.schedules import crontab + +from app.config import get_settings + +settings = get_settings() + +celery_app = Celery( + "gallery_subscriber", + broker=settings.redis_url, + backend=settings.redis_url, + include=["app.tasks.downloads"], +) + +# Celery configuration +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_track_started=True, + task_time_limit=3600, # 1 hour max per task + worker_prefetch_multiplier=1, # Process one task at a time + worker_concurrency=settings.download_parallel_limit, +) + +# Celery Beat schedule for periodic tasks +celery_app.conf.beat_schedule = { + "check-sources-hourly": { + "task": "app.tasks.downloads.scheduled_check", + "schedule": settings.default_check_interval, # Default: every hour + }, + "cleanup-old-downloads-daily": { + "task": "app.tasks.downloads.cleanup_old_downloads", + "schedule": crontab(hour=3, minute=0), # Daily at 3 AM + }, +} diff --git a/backend/app/tasks/downloads.py b/backend/app/tasks/downloads.py new file mode 100644 index 0000000..042289d --- /dev/null +++ b/backend/app/tasks/downloads.py @@ -0,0 +1,384 @@ +"""Download-related Celery tasks.""" + +import asyncio +import logging +from datetime import datetime, timedelta +from typing import Optional + +from sqlalchemy import select, and_, or_ +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker +from sqlalchemy.orm import selectinload + +from app.config import get_settings +from app.tasks.celery_app import celery_app +from app.models.source import Source +from app.models.subscription import Subscription +from app.models.download import Download, DownloadStatus, ErrorType as DBErrorType +from app.services.gallery_dl import GalleryDLService, SourceConfig, DownloadResult +from app.services.credential_manager import CredentialManager +from app.events import publish_event + +logger = logging.getLogger(__name__) + +# Settings for database connection +settings = get_settings() + + +def get_async_session(): + """Get async session factory for Celery tasks. + + Creates a fresh engine and session factory each time to avoid + event loop issues when asyncio.run() creates new loops. + """ + engine = create_async_engine(settings.async_database_url, echo=False) + session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + return session_factory, engine + + +async def _cleanup_engine(engine): + """Properly dispose of the async engine.""" + await engine.dispose() + + +async def _download_source_async(source_id: int) -> dict: + """Async implementation of source download. + + Args: + source_id: ID of the source to download + + Returns: + Dict with download results + """ + session_factory, engine = get_async_session() + + try: + async with session_factory() as session: + # Get source with subscription + result = await session.execute( + select(Source) + .options(selectinload(Source.subscription)) + .where(Source.id == source_id) + ) + source = result.scalar() + + if not source: + logger.error(f"Source {source_id} not found") + return {"source_id": source_id, "status": "error", "error": "Source not found"} + + if not source.enabled: + logger.info(f"Source {source.subscription.name}/{source.platform} is disabled, skipping") + return {"source_id": source_id, "status": "skipped", "reason": "disabled"} + + subscription_name = source.subscription.name + logger.info(f"Starting download for {subscription_name}/{source.platform}") + + # Create download record + download = Download( + source_id=source.id, + url=source.url, + status=DownloadStatus.RUNNING, + started_at=datetime.utcnow(), + ) + session.add(download) + await session.commit() + await session.refresh(download) + + # Broadcast download started event + publish_event("download.started", { + "download_id": download.id, + "source_id": source.id, + "subscription_name": subscription_name, + "platform": source.platform, + "url": source.url, + }) + + try: + # Get credentials + cred_manager = CredentialManager(session) + cookies_path = await cred_manager.get_cookies_path(source.platform) + + if not cookies_path and source.platform in ["patreon", "subscribestar"]: + logger.warning(f"No credentials for {source.platform}, download may fail") + + # Build source config from metadata + source_config = SourceConfig.from_dict(source.metadata_ or {}) + + # Execute download + gdl_service = GalleryDLService() + dl_result = await gdl_service.download( + url=source.url, + subscription_name=subscription_name, + platform=source.platform, + source_config=source_config, + cookies_path=cookies_path, + ) + + # Update download record + download.completed_at = datetime.utcnow() + download.file_count = dl_result.files_downloaded + download.metadata_ = { + "stdout": dl_result.stdout[:10000] if dl_result.stdout else None, # Truncate + "stderr": dl_result.stderr[:5000] if dl_result.stderr else None, + "duration_seconds": dl_result.duration_seconds, + } + + if dl_result.success: + download.status = DownloadStatus.COMPLETED + source.last_success = datetime.utcnow() + source.error_count = 0 + logger.info(f"Download completed for {subscription_name}/{source.platform}: {dl_result.files_downloaded} files") + + # Broadcast download completed event + publish_event("download.completed", { + "download_id": download.id, + "source_id": source.id, + "subscription_name": subscription_name, + "platform": source.platform, + "file_count": dl_result.files_downloaded, + "duration_seconds": dl_result.duration_seconds, + }) + else: + download.status = DownloadStatus.FAILED + download.error_type = dl_result.error_type.value if dl_result.error_type else None + download.error_message = dl_result.error_message + source.error_count = (source.error_count or 0) + 1 + logger.warning(f"Download failed for {subscription_name}/{source.platform}: {dl_result.error_message}") + + # Broadcast download failed event + publish_event("download.failed", { + "download_id": download.id, + "source_id": source.id, + "subscription_name": subscription_name, + "platform": source.platform, + "error_type": dl_result.error_type.value if dl_result.error_type else None, + "error_message": dl_result.error_message, + }) + + # Update source last_check + source.last_check = datetime.utcnow() + await session.commit() + + return { + "source_id": source_id, + "download_id": download.id, + "status": "completed" if dl_result.success else "failed", + "files_downloaded": dl_result.files_downloaded, + "error_type": dl_result.error_type.value if dl_result.error_type else None, + "error_message": dl_result.error_message, + "duration_seconds": dl_result.duration_seconds, + } + + except Exception as e: + logger.exception(f"Unexpected error downloading {subscription_name}/{source.platform}: {e}") + + download.status = DownloadStatus.FAILED + download.error_type = DBErrorType.UNKNOWN_ERROR + download.error_message = str(e) + download.completed_at = datetime.utcnow() + source.error_count = (source.error_count or 0) + 1 + source.last_check = datetime.utcnow() + + await session.commit() + + # Broadcast download failed event + publish_event("download.failed", { + "download_id": download.id, + "source_id": source.id, + "subscription_name": subscription_name, + "platform": source.platform, + "error_type": "unknown_error", + "error_message": str(e), + }) + + return { + "source_id": source_id, + "download_id": download.id, + "status": "error", + "error": str(e), + } + finally: + await _cleanup_engine(engine) + + +async def _scheduled_check_async() -> dict: + """Async implementation of scheduled source check. + + Returns: + Dict with check results + """ + session_factory, engine = get_async_session() + + try: + async with session_factory() as session: + now = datetime.utcnow() + + # Find sources due for checking: + # - enabled = True + # - last_check is NULL OR (now - last_check) > check_interval + # Order by subscription priority, then by when last checked + query = ( + select(Source) + .options(selectinload(Source.subscription)) + .join(Subscription) + .where( + and_( + Source.enabled == True, + Subscription.enabled == True, # Subscription must also be enabled + or_( + Source.last_check == None, + Source.last_check < now - timedelta(seconds=1) # Will be refined below + ) + ) + ) + .order_by( + Subscription.priority.desc(), # Higher subscription priority first + Source.last_check.asc().nullsfirst(), # Never checked first + ) + ) + + result = await session.execute(query) + sources = result.scalars().all() + + # Filter by check_interval + due_sources = [] + for source in sources: + if source.last_check is None: + due_sources.append(source) + else: + time_since_check = (now - source.last_check).total_seconds() + if time_since_check >= source.check_interval: + due_sources.append(source) + + logger.info(f"Found {len(due_sources)} sources due for checking") + + # Queue download tasks for each source + queued = 0 + for source in due_sources: + download_source.delay(source.id) + queued += 1 + logger.debug(f"Queued download for {source.subscription.name}/{source.platform}") + + return { + "checked": len(sources), + "queued": queued, + "timestamp": now.isoformat(), + } + finally: + await _cleanup_engine(engine) + + +async def _cleanup_old_downloads_async() -> dict: + """Async implementation of download cleanup. + + Returns: + Dict with cleanup results + """ + session_factory, engine = get_async_session() + + try: + async with session_factory() as session: + now = datetime.utcnow() + + # Delete completed downloads older than 30 days + completed_cutoff = now - timedelta(days=30) + completed_query = select(Download).where( + and_( + Download.status == DownloadStatus.COMPLETED, + Download.created_at < completed_cutoff + ) + ) + completed_result = await session.execute(completed_query) + completed_old = completed_result.scalars().all() + + # Delete failed downloads older than 7 days + failed_cutoff = now - timedelta(days=7) + failed_query = select(Download).where( + and_( + Download.status == DownloadStatus.FAILED, + Download.created_at < failed_cutoff + ) + ) + failed_result = await session.execute(failed_query) + failed_old = failed_result.scalars().all() + + deleted_count = 0 + for download in completed_old + failed_old: + await session.delete(download) + deleted_count += 1 + + await session.commit() + + logger.info(f"Cleaned up {deleted_count} old download records") + return {"deleted": deleted_count} + finally: + await _cleanup_engine(engine) + + +# Celery tasks that wrap async functions + +@celery_app.task(bind=True, max_retries=3) +def download_source(self, source_id: int): + """Download new content from a single source. + + Args: + source_id: ID of the source to download from + """ + try: + return asyncio.run(_download_source_async(source_id)) + except Exception as e: + logger.exception(f"Task failed for source {source_id}: {e}") + # Retry with exponential backoff + raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries)) + + +@celery_app.task +def scheduled_check(): + """Check all enabled sources that are due for a check. + + This is called periodically by Celery Beat. + """ + return asyncio.run(_scheduled_check_async()) + + +@celery_app.task +def cleanup_old_downloads(): + """Clean up old download records. + + Keeps the database from growing indefinitely. + """ + return asyncio.run(_cleanup_old_downloads_async()) + + +@celery_app.task(bind=True, max_retries=2) +def process_download(self, download_id: int): + """Process a single download record (for retries). + + Args: + download_id: ID of the download to process + """ + async def _process(): + session_factory, engine = get_async_session() + + try: + async with session_factory() as session: + result = await session.execute( + select(Download).where(Download.id == download_id) + ) + download = result.scalar() + + if not download: + return {"download_id": download_id, "status": "error", "error": "Download not found"} + + if not download.source_id: + return {"download_id": download_id, "status": "error", "error": "No source associated"} + + # Delegate to download_source + return await _download_source_async(download.source_id) + finally: + await _cleanup_engine(engine) + + try: + return asyncio.run(_process()) + except Exception as e: + logger.exception(f"Task failed for download {download_id}: {e}") + raise self.retry(exc=e, countdown=60 * (2 ** self.request.retries)) diff --git a/backend/app/utils/__init__.py b/backend/app/utils/__init__.py new file mode 100644 index 0000000..183c974 --- /dev/null +++ b/backend/app/utils/__init__.py @@ -0,0 +1 @@ +"""Utility modules.""" diff --git a/backend/app/utils/cookies.py b/backend/app/utils/cookies.py new file mode 100644 index 0000000..bde0186 --- /dev/null +++ b/backend/app/utils/cookies.py @@ -0,0 +1,102 @@ +"""Cookie parsing and formatting utilities.""" + +from typing import Optional +from pathlib import Path + + +def parse_netscape_cookies(cookie_text: str) -> list[dict]: + """Parse Netscape cookie format into a list of cookie dicts. + + Netscape format: domainflagpathsecureexpirationnamevalue + + Args: + cookie_text: Cookie file content in Netscape format + + Returns: + List of cookie dictionaries + """ + cookies = [] + + for line in cookie_text.strip().split("\n"): + line = line.strip() + + # Skip comments and empty lines + if not line or line.startswith("#"): + continue + + parts = line.split("\t") + if len(parts) < 7: + continue + + cookies.append({ + "domain": parts[0], + "flag": parts[1] == "TRUE", + "path": parts[2], + "secure": parts[3] == "TRUE", + "expiration": int(parts[4]) if parts[4] != "0" else None, + "name": parts[5], + "value": parts[6], + }) + + return cookies + + +def to_netscape_format(cookies: list[dict]) -> str: + """Convert cookie list to Netscape format. + + Args: + cookies: List of cookie dictionaries + + Returns: + Cookie file content in Netscape format + """ + lines = ["# Netscape HTTP Cookie File"] + + for c in cookies: + line = "\t".join([ + c["domain"], + "TRUE" if c.get("flag", True) else "FALSE", + c.get("path", "/"), + "TRUE" if c.get("secure", False) else "FALSE", + str(c.get("expiration", 0) or 0), + c["name"], + c["value"], + ]) + lines.append(line) + + return "\n".join(lines) + + +def write_cookie_file(cookies: list[dict], path: Path) -> None: + """Write cookies to a file in Netscape format. + + Args: + cookies: List of cookie dictionaries + path: Path to write the cookie file + """ + content = to_netscape_format(cookies) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + +def get_platform_from_domain(domain: str) -> Optional[str]: + """Detect platform from cookie domain. + + Args: + domain: Cookie domain (e.g., '.patreon.com') + + Returns: + Platform name or None + """ + domain_lower = domain.lower() + + if "patreon.com" in domain_lower: + return "patreon" + elif "subscribestar" in domain_lower: + return "subscribestar" + elif "hentai-foundry.com" in domain_lower: + return "hentaifoundry" + elif "discord.com" in domain_lower: + return "discord" + + return None diff --git a/backend/app/utils/encryption.py b/backend/app/utils/encryption.py new file mode 100644 index 0000000..56b806b --- /dev/null +++ b/backend/app/utils/encryption.py @@ -0,0 +1,46 @@ +"""Credential encryption utilities.""" + +import base64 +from cryptography.fernet import Fernet +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + + +def _get_fernet(secret_key: str) -> Fernet: + """Derive a Fernet key from the secret key.""" + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=b"gallery-subscriber-salt", # Static salt is OK for this use case + iterations=100000, + ) + derived_key = base64.urlsafe_b64encode(kdf.derive(secret_key.encode())) + return Fernet(derived_key) + + +def encrypt_data(data: str, secret_key: str) -> bytes: + """Encrypt string data using the secret key. + + Args: + data: Plain text data to encrypt + secret_key: Application secret key + + Returns: + Encrypted bytes + """ + fernet = _get_fernet(secret_key) + return fernet.encrypt(data.encode()) + + +def decrypt_data(encrypted: bytes, secret_key: str) -> str: + """Decrypt data using the secret key. + + Args: + encrypted: Encrypted bytes + secret_key: Application secret key + + Returns: + Decrypted string + """ + fernet = _get_fernet(secret_key) + return fernet.decrypt(encrypted).decode() diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..60c1906 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,28 @@ +# Web framework +quart==0.19.4 +quart-cors==0.7.0 +hypercorn==0.16.0 +flask>=3.0.0,<3.1.0 + +# Database +sqlalchemy[asyncio]==2.0.25 +asyncpg==0.29.0 +alembic==1.13.1 + +# Task queue +celery[redis]==5.3.6 +redis==5.0.1 + +# Utilities +pydantic==2.5.3 +pydantic-settings==2.1.0 +python-dotenv==1.0.0 +cryptography==41.0.7 +pyyaml==6.0.1 + +# gallery-dl +gallery-dl>=1.31.0 + +# Testing +pytest==7.4.4 +pytest-asyncio==0.23.3 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..cc543b8 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,35 @@ +# Development overrides - use with: docker-compose -f docker-compose.yml -f docker-compose.dev.yml up +services: + app: + build: + context: ./backend + dockerfile: Dockerfile.dev + volumes: + - ./backend/app:/app/app:ro + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + environment: + - DEBUG=true + - LOG_LEVEL=DEBUG + + celery: + build: + context: ./backend + dockerfile: Dockerfile.dev + volumes: + - ./backend/app:/app/app:ro + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + environment: + - DEBUG=true + - LOG_LEVEL=DEBUG + + db: + ports: + - "5432:5432" + + redis: + ports: + - "6379:6379" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b17c220 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,81 @@ +services: + # Combined backend API + frontend static files + app: + build: . + ports: + - "${PORT:-8080}:8080" + environment: + - TZ=${TZ:-America/New_York} + - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=${SECRET_KEY} + - DOWNLOAD_PATH=/data/downloads + - CONFIG_PATH=/data/config + - COOKIES_PATH=/data/cookies + - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} + - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} + - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + volumes: + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + restart: unless-stopped + + # Celery worker for background tasks + celery: + build: . + command: celery -A app.tasks.celery_app worker -B --loglevel=info + environment: + - TZ=${TZ:-America/New_York} + - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber + - REDIS_URL=redis://redis:6379/0 + - SECRET_KEY=${SECRET_KEY} + - DOWNLOAD_PATH=/data/downloads + - CONFIG_PATH=/data/config + - COOKIES_PATH=/data/cookies + - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} + - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} + - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + volumes: + - downloads:/data/downloads + - config:/data/config + - cookies:/data/cookies + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + restart: unless-stopped + + # PostgreSQL database + db: + image: postgres:15-alpine + environment: + - TZ=${TZ:-America/New_York} + - POSTGRES_USER=gdl + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=gallery_subscriber + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U gdl -d gallery_subscriber"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # Redis for Celery task queue + redis: + image: redis:7-alpine + restart: unless-stopped + +volumes: + postgres_data: + downloads: + config: + cookies: diff --git a/extension/background/background.js b/extension/background/background.js new file mode 100644 index 0000000..a8d7405 --- /dev/null +++ b/extension/background/background.js @@ -0,0 +1,215 @@ +/** + * Background script - handles cookie operations and API calls + */ + +// Initialize API on startup +browser.runtime.onInstalled.addListener(async () => { + console.log('GallerySubscriber extension installed'); + await api.init(); +}); + +browser.runtime.onStartup.addListener(async () => { + await api.init(); +}); + +// Handle messages from popup and options pages +browser.runtime.onMessage.addListener((message, sender, sendResponse) => { + handleMessage(message) + .then(sendResponse) + .catch(error => { + console.error('Message handler error:', error); + sendResponse({ error: error.message }); + }); + return true; // Keep channel open for async response +}); + +/** + * Route messages to appropriate handlers + * @param {object} message - Message object with type property + * @returns {Promise} - Response data + */ +async function handleMessage(message) { + switch (message.type) { + case 'EXPORT_COOKIES': + return await exportCookies(message.platform); + + case 'EXPORT_ALL_COOKIES': + return await exportAllCookies(); + + case 'GET_PLATFORM_STATUS': + return await getPlatformStatus(); + + case 'TEST_CONNECTION': + return await testConnection(); + + case 'GET_CONFIG': + return await getConfig(); + + case 'SAVE_CONFIG': + return await saveConfig(message.config); + + default: + throw new Error(`Unknown message type: ${message.type}`); + } +} + +/** + * Export cookies for a single platform to the backend + * @param {string} platformKey - Platform identifier + * @returns {Promise} - Result object + */ +async function exportCookies(platformKey) { + await api.init(); + + const platform = PLATFORMS[platformKey]; + if (!platform) { + throw new Error(`Unknown platform: ${platformKey}`); + } + + let credentialData; + let credentialType; + + if (platform.authType === 'token') { + // Discord and other token-based platforms require manual entry + throw new Error(`${platform.name} requires manual token entry in the web UI. Token extraction is not supported for security reasons.`); + } + + // Cookie-based auth + const cookies = await extractCookiesForPlatform(platformKey); + + if (cookies.length === 0) { + throw new Error(`No cookies found for ${platform.name}. Please ensure you are logged in.`); + } + + credentialData = toNetscapeFormat(cookies); + credentialType = 'cookies'; + + // Send to backend + const result = await api.uploadCredentials(platformKey, credentialType, credentialData); + + return { + success: true, + platform: platformKey, + cookieCount: cookies.length, + message: result.message || 'Credentials exported successfully' + }; +} + +/** + * Export cookies for all cookie-based platforms + * @returns {Promise} - Results for each platform + */ +async function exportAllCookies() { + await api.init(); + + const results = {}; + const platforms = Object.keys(PLATFORMS); + + for (const platformKey of platforms) { + const platform = PLATFORMS[platformKey]; + + // Skip token-based platforms + if (platform.authType === 'token') { + results[platformKey] = { + success: false, + platform: platformKey, + skipped: true, + message: 'Token-based auth requires manual entry' + }; + continue; + } + + try { + results[platformKey] = await exportCookies(platformKey); + } catch (error) { + results[platformKey] = { + success: false, + platform: platformKey, + error: error.message + }; + } + } + + return results; +} + +/** + * Get status information for all platforms + * @returns {Promise} - Status for each platform + */ +async function getPlatformStatus() { + const status = {}; + + for (const [key, platform] of Object.entries(PLATFORMS)) { + try { + if (platform.authType === 'token') { + // Token-based platforms + status[key] = { + name: platform.name, + authType: 'token', + hasCookies: false, + cookieCount: 0, + note: platform.note || 'Requires manual token entry' + }; + } else { + // Cookie-based platforms + const cookies = await extractCookiesForPlatform(key); + status[key] = { + name: platform.name, + authType: 'cookies', + hasCookies: cookies.length > 0, + cookieCount: cookies.length + }; + } + } catch (error) { + status[key] = { + name: platform.name, + authType: platform.authType, + hasCookies: false, + cookieCount: 0, + error: error.message + }; + } + } + + return status; +} + +/** + * Test connection to the backend + * @returns {Promise} - Connection status + */ +async function testConnection() { + await api.init(); + + if (!api.isConfigured()) { + return { connected: false, error: 'Not configured' }; + } + + try { + await api.testConnection(); + return { connected: true }; + } catch (error) { + return { connected: false, error: error.message }; + } +} + +/** + * Get current configuration + * @returns {Promise} - Config object + */ +async function getConfig() { + const config = await browser.storage.local.get(['apiUrl', 'apiKey']); + return config; +} + +/** + * Save configuration + * @param {object} config - Config object with apiUrl and apiKey + * @returns {Promise} - Success status + */ +async function saveConfig(config) { + await browser.storage.local.set(config); + await api.init(); + return { success: true }; +} diff --git a/extension/create-icons.py b/extension/create-icons.py new file mode 100644 index 0000000..2ecd5c9 --- /dev/null +++ b/extension/create-icons.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +Create minimal placeholder PNG icons using only standard library. +These are simple solid-color squares that Firefox will accept. +""" + +import struct +import zlib +from pathlib import Path + +def create_png(width, height, rgb_color): + """Create a minimal PNG file with a solid color.""" + r, g, b = rgb_color + + def png_chunk(chunk_type, data): + """Create a PNG chunk with CRC.""" + chunk = chunk_type + data + crc = zlib.crc32(chunk) & 0xffffffff + return struct.pack('>I', len(data)) + chunk + struct.pack('>I', crc) + + # PNG signature + signature = b'\x89PNG\r\n\x1a\n' + + # IHDR chunk (image header) + ihdr_data = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + ihdr = png_chunk(b'IHDR', ihdr_data) + + # IDAT chunk (image data) + # Create raw pixel data: each row starts with filter byte 0, then RGB pixels + raw_data = b'' + for y in range(height): + raw_data += b'\x00' # Filter byte (none) + for x in range(width): + raw_data += bytes([r, g, b]) + + compressed = zlib.compress(raw_data, 9) + idat = png_chunk(b'IDAT', compressed) + + # IEND chunk + iend = png_chunk(b'IEND', b'') + + return signature + ihdr + idat + iend + +def main(): + icons_dir = Path(__file__).parent / 'icons' + icons_dir.mkdir(exist_ok=True) + + # Primary blue color (#1976d2) + color = (25, 118, 210) + + sizes = [16, 32, 48, 128] + + for size in sizes: + png_data = create_png(size, size, color) + output_path = icons_dir / f'icon-{size}.png' + with open(output_path, 'wb') as f: + f.write(png_data) + print(f'Created {output_path} ({len(png_data)} bytes)') + + print('\nPlaceholder icons created!') + print('Replace with custom icons for a better look.') + +if __name__ == '__main__': + main() diff --git a/extension/generate-icons.py b/extension/generate-icons.py new file mode 100644 index 0000000..f8631c6 --- /dev/null +++ b/extension/generate-icons.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Generate placeholder icons for the GallerySubscriber extension. +Run this script from the extension directory. + +Requirements: Pillow (pip install Pillow) +""" + +import os +from pathlib import Path + +try: + from PIL import Image, ImageDraw +except ImportError: + print("Pillow not installed. Install with: pip install Pillow") + print("Or manually create icon-16.png, icon-32.png, icon-48.png, icon-128.png") + exit(1) + +# Icon configuration +BACKGROUND_COLOR = '#1976d2' # Primary blue +SIZES = [16, 32, 48, 128] + +def create_icon(size): + """Create a simple colored square icon with rounded corners effect.""" + img = Image.new('RGBA', (size, size), (0, 0, 0, 0)) + draw = ImageDraw.Draw(img) + + # Draw filled rounded rectangle (approximated with circle corners for small sizes) + margin = max(1, size // 16) + radius = max(2, size // 8) + + # For simplicity, just draw a solid square with the primary color + draw.rectangle([margin, margin, size - margin - 1, size - margin - 1], + fill=BACKGROUND_COLOR) + + # Draw a simple "G" or download arrow for larger sizes + if size >= 32: + # Draw a simple download arrow + center_x = size // 2 + center_y = size // 2 + arrow_size = size // 3 + + # Arrow body (vertical line) + line_width = max(2, size // 12) + draw.rectangle([ + center_x - line_width // 2, + center_y - arrow_size // 2, + center_x + line_width // 2, + center_y + arrow_size // 4 + ], fill='white') + + # Arrow head (triangle pointing down) + head_size = arrow_size // 2 + draw.polygon([ + (center_x - head_size, center_y), + (center_x + head_size, center_y), + (center_x, center_y + head_size) + ], fill='white') + + # Base line + draw.rectangle([ + center_x - arrow_size // 2, + center_y + arrow_size // 2, + center_x + arrow_size // 2, + center_y + arrow_size // 2 + line_width + ], fill='white') + + return img + +def main(): + icons_dir = Path(__file__).parent / 'icons' + icons_dir.mkdir(exist_ok=True) + + for size in SIZES: + icon = create_icon(size) + output_path = icons_dir / f'icon-{size}.png' + icon.save(output_path, 'PNG') + print(f'Created {output_path}') + + print('\nDone! Icons created successfully.') + print('You can replace these with custom icons later.') + +if __name__ == '__main__': + main() diff --git a/extension/icons/README.md b/extension/icons/README.md new file mode 100644 index 0000000..8888f1b --- /dev/null +++ b/extension/icons/README.md @@ -0,0 +1,40 @@ +# Extension Icons + +This directory should contain PNG icons for the extension in the following sizes: +- icon-16.png (16x16 pixels) +- icon-32.png (32x32 pixels) +- icon-48.png (48x48 pixels) +- icon-128.png (128x128 pixels) + +## Quick Setup + +You can generate placeholder icons using any image editor or online tool. +The icons should ideally be a simple "GS" logo or download icon. + +## Recommended Design + +- Background: #1976d2 (primary blue) +- Foreground: White +- Style: Simple, recognizable at small sizes +- Consider a download arrow or gallery grid icon + +## Generate with ImageMagick + +If you have ImageMagick installed: + +```bash +# Create a simple colored square as placeholder +for size in 16 32 48 128; do + convert -size ${size}x${size} xc:#1976d2 icon-${size}.png +done +``` + +## Generate with Python/Pillow + +```python +from PIL import Image, ImageDraw + +for size in [16, 32, 48, 128]: + img = Image.new('RGB', (size, size), '#1976d2') + img.save(f'icon-{size}.png') +``` diff --git a/extension/icons/icon-128.png b/extension/icons/icon-128.png new file mode 100644 index 0000000..993c9c5 Binary files /dev/null and b/extension/icons/icon-128.png differ diff --git a/extension/icons/icon-16.png b/extension/icons/icon-16.png new file mode 100644 index 0000000..52cf670 Binary files /dev/null and b/extension/icons/icon-16.png differ diff --git a/extension/icons/icon-32.png b/extension/icons/icon-32.png new file mode 100644 index 0000000..1eeb512 Binary files /dev/null and b/extension/icons/icon-32.png differ diff --git a/extension/icons/icon-48.png b/extension/icons/icon-48.png new file mode 100644 index 0000000..a135808 Binary files /dev/null and b/extension/icons/icon-48.png differ diff --git a/extension/manifest.json b/extension/manifest.json new file mode 100644 index 0000000..405f44c --- /dev/null +++ b/extension/manifest.json @@ -0,0 +1,51 @@ +{ + "manifest_version": 2, + "name": "GallerySubscriber", + "version": "1.0.0", + "description": "Export cookies from supported platforms to GallerySubscriber for automated downloads", + + "permissions": [ + "cookies", + "storage", + "tabs", + "activeTab", + "*://*.patreon.com/*", + "*://*.subscribestar.com/*", + "*://*.subscribestar.adult/*", + "*://*.hentai-foundry.com/*", + "*://*.discord.com/*" + ], + + "browser_action": { + "default_popup": "popup/popup.html", + "default_icon": { + "16": "icons/icon-16.png", + "32": "icons/icon-32.png", + "48": "icons/icon-48.png", + "128": "icons/icon-128.png" + }, + "default_title": "GallerySubscriber" + }, + + "background": { + "scripts": [ + "lib/platforms.js", + "lib/cookies.js", + "lib/api.js", + "background/background.js" + ], + "persistent": false + }, + + "options_ui": { + "page": "options/options.html", + "browser_style": true + }, + + "icons": { + "16": "icons/icon-16.png", + "32": "icons/icon-32.png", + "48": "icons/icon-48.png", + "128": "icons/icon-128.png" + } +} diff --git a/extension/options/options.html b/extension/options/options.html new file mode 100644 index 0000000..1c5c84f --- /dev/null +++ b/extension/options/options.html @@ -0,0 +1,287 @@ + + + + + GallerySubscriber Settings + + + +
+

GallerySubscriber Extension Settings

+ +
+

Setup Instructions

+
    +
  1. Open your GallerySubscriber web interface
    2. +
  2. Go to Settings page
    3. +
  3. Copy the Extension API Key
    4. +
  4. Paste the API URL and key below
    5. +
+
+ +
+

Backend Connection

+ +
+ + +

+ The URL of your GallerySubscriber backend. Include /api at the end. +

+
+ +
+ +
+ + +
+

+ Find this in GallerySubscriber: Settings > Extension API Key +

+
+ +
+ + + +
+
+ +
+ + +
+
+ + + + diff --git a/extension/options/options.js b/extension/options/options.js new file mode 100644 index 0000000..9c6d924 --- /dev/null +++ b/extension/options/options.js @@ -0,0 +1,183 @@ +/** + * Options page script + */ + +document.addEventListener('DOMContentLoaded', async () => { + await loadSettings(); + setupEventListeners(); +}); + +/** + * Load saved settings into form + */ +async function loadSettings() { + const config = await browser.storage.local.get(['apiUrl', 'apiKey']); + + document.getElementById('api-url').value = config.apiUrl || ''; + document.getElementById('api-key').value = config.apiKey || ''; +} + +/** + * Setup event listeners + */ +function setupEventListeners() { + // Toggle API key visibility + const toggleBtn = document.getElementById('toggle-key-btn'); + const apiKeyInput = document.getElementById('api-key'); + + toggleBtn.addEventListener('click', () => { + if (apiKeyInput.type === 'password') { + apiKeyInput.type = 'text'; + toggleBtn.textContent = 'Hide'; + } else { + apiKeyInput.type = 'password'; + toggleBtn.textContent = 'Show'; + } + }); + + // Test connection + document.getElementById('test-btn').addEventListener('click', testConnection); + + // Save settings + document.getElementById('save-btn').addEventListener('click', saveSettings); + + // Auto-save on enter key + document.getElementById('api-url').addEventListener('keypress', (e) => { + if (e.key === 'Enter') saveSettings(); + }); + document.getElementById('api-key').addEventListener('keypress', (e) => { + if (e.key === 'Enter') saveSettings(); + }); +} + +/** + * Test connection to backend + */ +async function testConnection() { + const testBtn = document.getElementById('test-btn'); + const statusDot = document.getElementById('test-status'); + const testMessage = document.getElementById('test-message'); + + testBtn.disabled = true; + testBtn.textContent = 'Testing...'; + statusDot.className = 'status-dot'; + testMessage.textContent = ''; + + // Get current values + const apiUrl = normalizeApiUrl(document.getElementById('api-url').value.trim()); + const apiKey = document.getElementById('api-key').value.trim(); + + if (!apiUrl || !apiKey) { + statusDot.className = 'status-dot error'; + testMessage.textContent = 'Please fill in both fields'; + testBtn.disabled = false; + testBtn.textContent = 'Test Connection'; + return; + } + + // Temporarily save for testing + await browser.storage.local.set({ apiUrl, apiKey }); + + try { + const result = await browser.runtime.sendMessage({ type: 'TEST_CONNECTION' }); + + if (result.connected) { + statusDot.className = 'status-dot connected'; + testMessage.textContent = 'Connected successfully!'; + } else { + statusDot.className = 'status-dot error'; + testMessage.textContent = result.error || 'Connection failed'; + } + } catch (error) { + statusDot.className = 'status-dot error'; + testMessage.textContent = error.message; + } finally { + testBtn.disabled = false; + testBtn.textContent = 'Test Connection'; + } +} + +/** + * Save settings + */ +async function saveSettings() { + const saveBtn = document.getElementById('save-btn'); + const saveResult = document.getElementById('save-result'); + + saveBtn.disabled = true; + saveResult.textContent = ''; + saveResult.className = 'result'; + + let apiUrl = document.getElementById('api-url').value.trim(); + const apiKey = document.getElementById('api-key').value.trim(); + + // Validate + if (!apiUrl) { + showSaveResult('API URL is required', 'error'); + saveBtn.disabled = false; + return; + } + + if (!apiKey) { + showSaveResult('API Key is required', 'error'); + saveBtn.disabled = false; + return; + } + + // Normalize URL + apiUrl = normalizeApiUrl(apiUrl); + document.getElementById('api-url').value = apiUrl; + + // Save + try { + await browser.storage.local.set({ apiUrl, apiKey }); + await browser.runtime.sendMessage({ + type: 'SAVE_CONFIG', + config: { apiUrl, apiKey } + }); + + showSaveResult('Settings saved!', 'success'); + } catch (error) { + showSaveResult(`Failed to save: ${error.message}`, 'error'); + } finally { + saveBtn.disabled = false; + } +} + +/** + * Normalize API URL to ensure it ends with /api + * @param {string} url - Input URL + * @returns {string} - Normalized URL + */ +function normalizeApiUrl(url) { + if (!url) return ''; + + // Remove trailing slash + url = url.replace(/\/+$/, ''); + + // Ensure it ends with /api + if (!url.endsWith('/api')) { + url = url + '/api'; + } + + return url; +} + +/** + * Show save result message + * @param {string} message - Message to display + * @param {string} type - 'success' or 'error' + */ +function showSaveResult(message, type) { + const saveResult = document.getElementById('save-result'); + saveResult.textContent = message; + saveResult.className = `result ${type}`; + + // Auto-hide success messages + if (type === 'success') { + setTimeout(() => { + saveResult.textContent = ''; + saveResult.className = 'result'; + }, 3000); + } +} diff --git a/extension/popup/popup.css b/extension/popup/popup.css new file mode 100644 index 0000000..a301720 --- /dev/null +++ b/extension/popup/popup.css @@ -0,0 +1,282 @@ +:root { + --primary-color: #1976d2; + --success-color: #4caf50; + --warning-color: #ff9800; + --error-color: #f44336; + --bg-color: #ffffff; + --text-color: #212121; + --text-secondary: #757575; + --border-color: #e0e0e0; +} + +* { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + font-size: 14px; + color: var(--text-color); + background: var(--bg-color); + min-width: 320px; + max-width: 400px; +} + +.popup-container { + padding: 12px; +} + +.header { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 1px solid var(--border-color); +} + +.header h1 { + font-size: 16px; + font-weight: 600; + flex: 1; +} + +.logo { + width: 24px; + height: 24px; +} + +.status-indicator { + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--error-color); + cursor: help; +} + +.status-indicator.connected { + background: var(--success-color); +} + +.section { + margin-bottom: 16px; +} + +.section h2 { + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + color: var(--text-secondary); + margin-bottom: 8px; + letter-spacing: 0.5px; +} + +.platforms-grid { + display: grid; + gap: 8px; +} + +.platform-card { + display: flex; + align-items: center; + padding: 10px 12px; + border: 1px solid var(--border-color); + border-radius: 6px; + cursor: pointer; + transition: all 0.15s ease; + background: var(--bg-color); +} + +.platform-card:hover:not(.disabled) { + border-color: var(--primary-color); + background: #f5f9ff; +} + +.platform-card.disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.platform-card .platform-icon { + width: 32px; + height: 32px; + border-radius: 6px; + display: flex; + align-items: center; + justify-content: center; + margin-right: 10px; + color: white; + font-weight: bold; + font-size: 14px; +} + +.platform-card .info { + flex: 1; + min-width: 0; +} + +.platform-card .name { + font-weight: 500; + font-size: 14px; +} + +.platform-card .status { + font-size: 12px; + color: var(--text-secondary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.platform-card .status.ready { + color: var(--success-color); +} + +.platform-card .status.no-cookies { + color: var(--warning-color); +} + +.platform-card .status.error { + color: var(--error-color); +} + +.platform-card .action-icon { + width: 20px; + height: 20px; + opacity: 0.5; + flex-shrink: 0; +} + +.platform-card.loading .action-icon { + animation: spin 1s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.btn { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 10px 16px; + border: none; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + transition: background 0.15s ease; +} + +.btn-primary { + background: var(--primary-color); + color: white; +} + +.btn-primary:hover { + background: #1565c0; +} + +.btn-secondary { + background: #f5f5f5; + color: var(--text-color); + border: 1px solid var(--border-color); +} + +.btn-secondary:hover { + background: #eeeeee; +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-icon { + background: none; + border: none; + padding: 6px; + cursor: pointer; + border-radius: 4px; + color: var(--text-secondary); + transition: background 0.15s ease; +} + +.btn-icon:hover { + background: #f5f5f5; + color: var(--text-color); +} + +.full-width { + width: 100%; +} + +.mt-2 { + margin-top: 8px; +} + +.alert { + padding: 12px; + border-radius: 6px; + margin-bottom: 12px; +} + +.alert strong { + display: block; + margin-bottom: 4px; +} + +.alert p { + font-size: 13px; + margin: 0; +} + +.alert-warning { + background: #fff8e1; + border: 1px solid #ffcc02; + color: #795548; +} + +.status-message { + padding: 10px 12px; + border-radius: 6px; + margin-bottom: 12px; + font-size: 13px; +} + +.status-message.success { + background: #e8f5e9; + color: #2e7d32; + border: 1px solid #a5d6a7; +} + +.status-message.error { + background: #ffebee; + color: #c62828; + border: 1px solid #ef9a9a; +} + +.status-message.warning { + background: #fff8e1; + color: #795548; + border: 1px solid #ffcc02; +} + +.footer { + display: flex; + align-items: center; + justify-content: space-between; + padding-top: 12px; + border-top: 1px solid var(--border-color); +} + +.version { + font-size: 11px; + color: #9e9e9e; +} + +.hidden { + display: none !important; +} diff --git a/extension/popup/popup.html b/extension/popup/popup.html new file mode 100644 index 0000000..81f9847 --- /dev/null +++ b/extension/popup/popup.html @@ -0,0 +1,56 @@ + + + + + + + +
+
+ +

GallerySubscriber

+ +
+ + + + + + + +
+ + v1.0.0 +
+
+ + + + + diff --git a/extension/popup/popup.js b/extension/popup/popup.js new file mode 100644 index 0000000..626596b --- /dev/null +++ b/extension/popup/popup.js @@ -0,0 +1,272 @@ +/** + * Popup script - handles UI interactions + */ + +document.addEventListener('DOMContentLoaded', async () => { + await init(); +}); + +/** + * Initialize the popup + */ +async function init() { + // Check configuration + const config = await browser.runtime.sendMessage({ type: 'GET_CONFIG' }); + + if (!config.apiUrl || !config.apiKey) { + showSetupRequired(); + return; + } + + // Test connection + const connectionStatus = await browser.runtime.sendMessage({ type: 'TEST_CONNECTION' }); + updateConnectionIndicator(connectionStatus.connected); + + // Show main content + document.getElementById('setup-required').classList.add('hidden'); + document.getElementById('main-content').classList.remove('hidden'); + + if (!connectionStatus.connected) { + showError(`Cannot connect to backend: ${connectionStatus.error}`); + } + + // Load platform status + await loadPlatformStatus(); + + // Setup event listeners + setupEventListeners(); +} + +/** + * Show the setup required message + */ +function showSetupRequired() { + document.getElementById('setup-required').classList.remove('hidden'); + document.getElementById('main-content').classList.add('hidden'); + + document.getElementById('open-settings-btn').addEventListener('click', () => { + browser.runtime.openOptionsPage(); + }); +} + +/** + * Update the connection status indicator + * @param {boolean} connected - Whether connected to backend + */ +function updateConnectionIndicator(connected) { + const indicator = document.getElementById('connection-status'); + indicator.classList.toggle('connected', connected); + indicator.title = connected ? 'Connected to GallerySubscriber' : 'Disconnected'; +} + +/** + * Load and display platform status + */ +async function loadPlatformStatus() { + const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' }); + const container = document.getElementById('platforms-list'); + container.innerHTML = ''; + + for (const [key, platform] of Object.entries(PLATFORMS)) { + const platformStatus = status[key] || {}; + const card = createPlatformCard(key, platform, platformStatus); + container.appendChild(card); + } +} + +/** + * Create a platform card element + * @param {string} key - Platform key + * @param {object} platform - Platform definition + * @param {object} status - Platform status + * @returns {HTMLElement} + */ +function createPlatformCard(key, platform, status) { + const card = document.createElement('div'); + card.className = 'platform-card'; + card.dataset.platform = key; + + const isDisabled = platform.authType === 'token'; + if (isDisabled) { + card.classList.add('disabled'); + } + + const statusText = getStatusText(status, platform); + const statusClass = getStatusClass(status, platform); + + card.innerHTML = ` +
+ ${platform.name.charAt(0)} +
+
+
${platform.name}
+
${statusText}
+
+ + + + + + `; + + if (!isDisabled) { + card.addEventListener('click', () => exportPlatformCookies(key, card)); + } + + return card; +} + +/** + * Get status text for display + * @param {object} status - Platform status + * @param {object} platform - Platform definition + * @returns {string} + */ +function getStatusText(status, platform) { + if (platform.authType === 'token') { + return 'Manual token entry required'; + } + if (status.error) { + return 'Error checking cookies'; + } + if (!status.hasCookies || status.cookieCount === 0) { + return 'No cookies found - log in first'; + } + return `${status.cookieCount} cookies ready`; +} + +/** + * Get status CSS class + * @param {object} status - Platform status + * @param {object} platform - Platform definition + * @returns {string} + */ +function getStatusClass(status, platform) { + if (platform.authType === 'token') { + return 'no-cookies'; + } + if (status.error) { + return 'error'; + } + if (!status.hasCookies || status.cookieCount === 0) { + return 'no-cookies'; + } + return 'ready'; +} + +/** + * Export cookies for a single platform + * @param {string} platformKey - Platform key + * @param {HTMLElement} cardElement - The card element + */ +async function exportPlatformCookies(platformKey, cardElement) { + cardElement.classList.add('loading'); + hideStatusMessage(); + + try { + const result = await browser.runtime.sendMessage({ + type: 'EXPORT_COOKIES', + platform: platformKey + }); + + if (result.error) { + showError(result.error); + } else { + showSuccess(`${PLATFORMS[platformKey].name}: ${result.cookieCount} cookies exported!`); + // Refresh status + await loadPlatformStatus(); + } + } catch (error) { + showError(error.message); + } finally { + cardElement.classList.remove('loading'); + } +} + +/** + * Export cookies for all platforms + */ +async function exportAllCookies() { + const btn = document.getElementById('export-all-btn'); + btn.disabled = true; + btn.textContent = 'Exporting...'; + hideStatusMessage(); + + try { + const results = await browser.runtime.sendMessage({ type: 'EXPORT_ALL_COOKIES' }); + + const successes = Object.values(results).filter(r => r.success).length; + const skipped = Object.values(results).filter(r => r.skipped).length; + const failures = Object.values(results).filter(r => !r.success && !r.skipped).length; + + if (failures === 0 && successes > 0) { + showSuccess(`Exported ${successes} platform(s) successfully!`); + } else if (successes > 0) { + showWarning(`${successes} succeeded, ${failures} failed`); + } else if (failures > 0) { + showError('All exports failed. Are you logged in?'); + } else { + showWarning('No platforms to export'); + } + + await loadPlatformStatus(); + } catch (error) { + showError(error.message); + } finally { + btn.disabled = false; + btn.textContent = 'Export All Platforms'; + } +} + +/** + * Setup event listeners + */ +function setupEventListeners() { + document.getElementById('export-all-btn').addEventListener('click', exportAllCookies); + document.getElementById('settings-btn').addEventListener('click', () => { + browser.runtime.openOptionsPage(); + }); +} + +/** + * Show success message + * @param {string} message - Message to display + */ +function showSuccess(message) { + showStatusMessage(message, 'success'); +} + +/** + * Show error message + * @param {string} message - Message to display + */ +function showError(message) { + showStatusMessage(message, 'error'); +} + +/** + * Show warning message + * @param {string} message - Message to display + */ +function showWarning(message) { + showStatusMessage(message, 'warning'); +} + +/** + * Show a status message + * @param {string} message - Message text + * @param {string} type - Message type (success, error, warning) + */ +function showStatusMessage(message, type) { + const el = document.getElementById('status-message'); + el.textContent = message; + el.className = `status-message ${type}`; + el.classList.remove('hidden'); +} + +/** + * Hide the status message + */ +function hideStatusMessage() { + document.getElementById('status-message').classList.add('hidden'); +} diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..f16f588 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + Gallery Subscriber + + +
+ + + diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..a232996 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,24 @@ +{ + "name": "gallery-subscriber-frontend", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "vue": "^3.4.15", + "vue-router": "^4.2.5", + "vuetify": "^3.5.1", + "pinia": "^2.1.7", + "axios": "^1.6.5", + "@mdi/font": "^7.4.47" + }, + "devDependencies": { + "@vitejs/plugin-vue": "^5.0.3", + "vite": "^5.0.11", + "sass": "^1.70.0" + } +} diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg new file mode 100644 index 0000000..1fcf038 --- /dev/null +++ b/frontend/public/favicon.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..9dd9735 --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,88 @@ + + + diff --git a/frontend/src/composables/useWebSocket.js b/frontend/src/composables/useWebSocket.js new file mode 100644 index 0000000..11bc234 --- /dev/null +++ b/frontend/src/composables/useWebSocket.js @@ -0,0 +1,136 @@ +import { ref, onMounted, onUnmounted } from 'vue' + +const WS_URL = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws/events` + +// Singleton WebSocket connection +let ws = null +let reconnectTimeout = null +const listeners = new Map() +const isConnected = ref(false) +const lastEvent = ref(null) + +function connect() { + if (ws && ws.readyState === WebSocket.OPEN) { + return + } + + try { + ws = new WebSocket(WS_URL) + + ws.onopen = () => { + console.log('WebSocket connected') + isConnected.value = true + } + + ws.onclose = () => { + console.log('WebSocket disconnected') + isConnected.value = false + ws = null + + // Reconnect after 5 seconds + if (!reconnectTimeout) { + reconnectTimeout = setTimeout(() => { + reconnectTimeout = null + connect() + }, 5000) + } + } + + ws.onerror = (error) => { + console.error('WebSocket error:', error) + } + + ws.onmessage = (event) => { + try { + const data = JSON.parse(event.data) + lastEvent.value = data + + // Handle ping + if (data.type === 'ping') { + ws.send(JSON.stringify({ type: 'pong' })) + return + } + + // Notify all listeners for this event type + const typeListeners = listeners.get(data.type) || [] + typeListeners.forEach(callback => { + try { + callback(data.data) + } catch (e) { + console.error('Error in WebSocket listener:', e) + } + }) + + // Also notify wildcard listeners + const wildcardListeners = listeners.get('*') || [] + wildcardListeners.forEach(callback => { + try { + callback(data.type, data.data) + } catch (e) { + console.error('Error in WebSocket listener:', e) + } + }) + } catch (e) { + console.error('Failed to parse WebSocket message:', e) + } + } + } catch (e) { + console.error('Failed to create WebSocket:', e) + } +} + +function disconnect() { + if (reconnectTimeout) { + clearTimeout(reconnectTimeout) + reconnectTimeout = null + } + if (ws) { + ws.close() + ws = null + } + isConnected.value = false +} + +function subscribe(eventType, callback) { + if (!listeners.has(eventType)) { + listeners.set(eventType, []) + } + listeners.get(eventType).push(callback) + + // Return unsubscribe function + return () => { + const typeListeners = listeners.get(eventType) + if (typeListeners) { + const index = typeListeners.indexOf(callback) + if (index > -1) { + typeListeners.splice(index, 1) + } + } + } +} + +function send(type, data = {}) { + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type, data })) + } +} + +export function useWebSocket() { + onMounted(() => { + connect() + }) + + return { + isConnected, + lastEvent, + subscribe, + send, + connect, + disconnect, + } +} + +// Auto-connect on import +if (typeof window !== 'undefined') { + connect() +} diff --git a/frontend/src/main.js b/frontend/src/main.js new file mode 100644 index 0000000..5921819 --- /dev/null +++ b/frontend/src/main.js @@ -0,0 +1,13 @@ +import { createApp } from 'vue' +import { createPinia } from 'pinia' +import App from './App.vue' +import router from './router' +import vuetify from './plugins/vuetify' + +const app = createApp(App) + +app.use(createPinia()) +app.use(router) +app.use(vuetify) + +app.mount('#app') diff --git a/frontend/src/plugins/vuetify.js b/frontend/src/plugins/vuetify.js new file mode 100644 index 0000000..e1805fa --- /dev/null +++ b/frontend/src/plugins/vuetify.js @@ -0,0 +1,45 @@ +import 'vuetify/styles' +import '@mdi/font/css/materialdesignicons.css' +import { createVuetify } from 'vuetify' +import * as components from 'vuetify/components' +import * as directives from 'vuetify/directives' + +export default createVuetify({ + components, + directives, + theme: { + defaultTheme: 'dark', + themes: { + dark: { + colors: { + primary: '#1976D2', + secondary: '#424242', + accent: '#82B1FF', + error: '#FF5252', + info: '#2196F3', + success: '#4CAF50', + warning: '#FFC107', + }, + }, + light: { + colors: { + primary: '#1976D2', + secondary: '#424242', + accent: '#82B1FF', + error: '#FF5252', + info: '#2196F3', + success: '#4CAF50', + warning: '#FFC107', + }, + }, + }, + }, + defaults: { + VCard: { + elevation: 2, + }, + VBtn: { + variant: 'flat', + }, + }, +}) diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js new file mode 100644 index 0000000..e5e4ecd --- /dev/null +++ b/frontend/src/router/index.js @@ -0,0 +1,41 @@ +import { createRouter, createWebHistory } from 'vue-router' + +const routes = [ + { + path: '/', + name: 'Dashboard', + component: () => import('../views/Dashboard.vue'), + }, + { + path: '/subscriptions', + name: 'Subscriptions', + component: () => import('../views/Subscriptions.vue'), + }, + // Legacy route redirects + { + path: '/sources', + redirect: '/subscriptions', + }, + { + path: '/downloads', + name: 'Downloads', + component: () => import('../views/Downloads.vue'), + }, + { + path: '/credentials', + name: 'Credentials', + component: () => import('../views/Credentials.vue'), + }, + { + path: '/settings', + name: 'Settings', + component: () => import('../views/Settings.vue'), + }, +] + +const router = createRouter({ + history: createWebHistory(), + routes, +}) + +export default router diff --git a/frontend/src/services/api.js b/frontend/src/services/api.js new file mode 100644 index 0000000..3ef0a8d --- /dev/null +++ b/frontend/src/services/api.js @@ -0,0 +1,66 @@ +import axios from 'axios' + +const api = axios.create({ + baseURL: '/api', + timeout: 30000, + headers: { + 'Content-Type': 'application/json', + }, +}) + +// Subscriptions API +export const subscriptionsApi = { + list: (params = {}) => api.get('/subscriptions', { params }), + get: (id) => api.get(`/subscriptions/${id}`), + create: (data) => api.post('/subscriptions', data), + update: (id, data) => api.patch(`/subscriptions/${id}`, data), + delete: (id) => api.delete(`/subscriptions/${id}`), + triggerCheck: (id) => api.post(`/subscriptions/${id}/check`), + listSources: (id) => api.get(`/subscriptions/${id}/sources`), + addSource: (id, data) => api.post(`/subscriptions/${id}/sources`, data), +} + +// Sources API +export const sourcesApi = { + list: (params = {}) => api.get('/sources', { params }), + get: (id) => api.get(`/sources/${id}`), + create: (data) => api.post('/sources', data), + update: (id, data) => api.patch(`/sources/${id}`, data), + delete: (id) => api.delete(`/sources/${id}`), + triggerCheck: (id) => api.post(`/sources/${id}/check`), +} + +// Downloads API +export const downloadsApi = { + list: (params = {}) => api.get('/downloads', { params }), + get: (id) => api.get(`/downloads/${id}`), + retry: (id) => api.post(`/downloads/${id}/retry`), + stats: (params = {}) => api.get('/downloads/stats', { params }), +} + +// Credentials API +export const credentialsApi = { + list: () => api.get('/credentials'), + upload: (data) => api.post('/credentials', data), + delete: (platform) => api.delete(`/credentials/${platform}`), + verify: (platform) => api.post('/credentials/verify', { platform }), +} + +// Settings API +export const settingsApi = { + get: () => api.get('/settings'), + update: (data) => api.patch('/settings', data), + getGalleryDL: () => api.get('/settings/gallery-dl'), + updateGalleryDL: (config) => api.put('/settings/gallery-dl', { config }), + getApiKey: () => api.get('/settings/api-key'), + regenerateApiKey: () => api.post('/settings/api-key/regenerate'), +} + +// Platforms API +export const platformsApi = { + list: () => api.get('/platforms'), + get: (platform) => api.get(`/platforms/${platform}`), + getConfigSchema: (platform) => api.get(`/platforms/${platform}/config-schema`), +} + +export default api diff --git a/frontend/src/stores/downloads.js b/frontend/src/stores/downloads.js new file mode 100644 index 0000000..d97e282 --- /dev/null +++ b/frontend/src/stores/downloads.js @@ -0,0 +1,50 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { downloadsApi } from '../services/api' + +export const useDownloadsStore = defineStore('downloads', () => { + const downloads = ref([]) + const stats = ref(null) + const loading = ref(false) + const total = ref(0) + const currentPage = ref(1) + const perPage = ref(20) + + async function fetchDownloads(params = {}) { + loading.value = true + try { + const response = await downloadsApi.list({ + page: currentPage.value, + per_page: perPage.value, + ...params, + }) + downloads.value = response.data.items + total.value = response.data.total + return response.data + } finally { + loading.value = false + } + } + + async function fetchStats(params = {}) { + const response = await downloadsApi.stats(params) + stats.value = response.data + return response.data + } + + async function retryDownload(id) { + return await downloadsApi.retry(id) + } + + return { + downloads, + stats, + loading, + total, + currentPage, + perPage, + fetchDownloads, + fetchStats, + retryDownload, + } +}) diff --git a/frontend/src/stores/notifications.js b/frontend/src/stores/notifications.js new file mode 100644 index 0000000..619e610 --- /dev/null +++ b/frontend/src/stores/notifications.js @@ -0,0 +1,41 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' + +export const useNotificationStore = defineStore('notifications', () => { + const show = ref(false) + const message = ref('') + const color = ref('info') + + function notify(msg, type = 'info') { + message.value = msg + color.value = type + show.value = true + } + + function success(msg) { + notify(msg, 'success') + } + + function error(msg) { + notify(msg, 'error') + } + + function warning(msg) { + notify(msg, 'warning') + } + + function info(msg) { + notify(msg, 'info') + } + + return { + show, + message, + color, + notify, + success, + error, + warning, + info, + } +}) diff --git a/frontend/src/stores/settings.js b/frontend/src/stores/settings.js new file mode 100644 index 0000000..3078f17 --- /dev/null +++ b/frontend/src/stores/settings.js @@ -0,0 +1,48 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { settingsApi } from '../services/api' + +export const useSettingsStore = defineStore('settings', () => { + const settings = ref({}) + const galleryDLConfig = ref({}) + const loading = ref(false) + + async function fetchSettings() { + loading.value = true + try { + const response = await settingsApi.get() + settings.value = response.data.settings + return response.data.settings + } finally { + loading.value = false + } + } + + async function updateSettings(data) { + const response = await settingsApi.update(data) + settings.value = response.data.settings + return response.data.settings + } + + async function fetchGalleryDLConfig() { + const response = await settingsApi.getGalleryDL() + galleryDLConfig.value = response.data.config + return response.data.config + } + + async function updateGalleryDLConfig(config) { + const response = await settingsApi.updateGalleryDL(config) + galleryDLConfig.value = response.data.config + return response.data.config + } + + return { + settings, + galleryDLConfig, + loading, + fetchSettings, + updateSettings, + fetchGalleryDLConfig, + updateGalleryDLConfig, + } +}) diff --git a/frontend/src/stores/sources.js b/frontend/src/stores/sources.js new file mode 100644 index 0000000..88d977e --- /dev/null +++ b/frontend/src/stores/sources.js @@ -0,0 +1,69 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { sourcesApi } from '../services/api' + +export const useSourcesStore = defineStore('sources', () => { + const sources = ref([]) + const loading = ref(false) + const total = ref(0) + const currentPage = ref(1) + const perPage = ref(20) + + const enabledSources = computed(() => sources.value.filter(s => s.enabled)) + const disabledSources = computed(() => sources.value.filter(s => !s.enabled)) + + async function fetchSources(params = {}) { + loading.value = true + try { + const response = await sourcesApi.list({ + page: currentPage.value, + per_page: perPage.value, + ...params, + }) + sources.value = response.data.items + total.value = response.data.total + return response.data + } finally { + loading.value = false + } + } + + async function createSource(data) { + const response = await sourcesApi.create(data) + sources.value.push(response.data) + return response.data + } + + async function updateSource(id, data) { + const response = await sourcesApi.update(id, data) + const index = sources.value.findIndex(s => s.id === id) + if (index !== -1) { + sources.value[index] = response.data + } + return response.data + } + + async function deleteSource(id) { + await sourcesApi.delete(id) + sources.value = sources.value.filter(s => s.id !== id) + } + + async function triggerCheck(id) { + return await sourcesApi.triggerCheck(id) + } + + return { + sources, + loading, + total, + currentPage, + perPage, + enabledSources, + disabledSources, + fetchSources, + createSource, + updateSource, + deleteSource, + triggerCheck, + } +}) diff --git a/frontend/src/stores/subscriptions.js b/frontend/src/stores/subscriptions.js new file mode 100644 index 0000000..f68c537 --- /dev/null +++ b/frontend/src/stores/subscriptions.js @@ -0,0 +1,112 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import { subscriptionsApi, sourcesApi } from '../services/api' + +export const useSubscriptionsStore = defineStore('subscriptions', () => { + const subscriptions = ref([]) + const loading = ref(false) + const total = ref(0) + const currentPage = ref(1) + const perPage = ref(20) + + const enabledSubscriptions = computed(() => subscriptions.value.filter(s => s.enabled)) + const disabledSubscriptions = computed(() => subscriptions.value.filter(s => !s.enabled)) + + async function fetchSubscriptions(params = {}) { + loading.value = true + try { + const response = await subscriptionsApi.list({ + page: currentPage.value, + per_page: perPage.value, + ...params, + }) + subscriptions.value = response.data.items + total.value = response.data.total + return response.data + } finally { + loading.value = false + } + } + + async function getSubscription(id) { + const response = await subscriptionsApi.get(id) + return response.data + } + + async function createSubscription(data) { + const response = await subscriptionsApi.create(data) + subscriptions.value.push(response.data) + return response.data + } + + async function updateSubscription(id, data) { + const response = await subscriptionsApi.update(id, data) + const index = subscriptions.value.findIndex(s => s.id === id) + if (index !== -1) { + subscriptions.value[index] = response.data + } + return response.data + } + + async function deleteSubscription(id) { + await subscriptionsApi.delete(id) + subscriptions.value = subscriptions.value.filter(s => s.id !== id) + } + + async function triggerCheck(id) { + return await subscriptionsApi.triggerCheck(id) + } + + async function addSource(subscriptionId, sourceData) { + const response = await subscriptionsApi.addSource(subscriptionId, sourceData) + // Update the subscription in the store + const index = subscriptions.value.findIndex(s => s.id === subscriptionId) + if (index !== -1 && subscriptions.value[index].sources) { + subscriptions.value[index].sources.push(response.data) + } + return response.data + } + + async function deleteSource(sourceId) { + await sourcesApi.delete(sourceId) + // Update subscriptions that contain this source + for (const sub of subscriptions.value) { + if (sub.sources) { + sub.sources = sub.sources.filter(s => s.id !== sourceId) + } + } + } + + async function updateSource(sourceId, data) { + const response = await sourcesApi.update(sourceId, data) + // Update in subscriptions + for (const sub of subscriptions.value) { + if (sub.sources) { + const idx = sub.sources.findIndex(s => s.id === sourceId) + if (idx !== -1) { + sub.sources[idx] = response.data + } + } + } + return response.data + } + + return { + subscriptions, + loading, + total, + currentPage, + perPage, + enabledSubscriptions, + disabledSubscriptions, + fetchSubscriptions, + getSubscription, + createSubscription, + updateSubscription, + deleteSubscription, + triggerCheck, + addSource, + deleteSource, + updateSource, + } +}) diff --git a/frontend/src/stores/websocket.js b/frontend/src/stores/websocket.js new file mode 100644 index 0000000..812c674 --- /dev/null +++ b/frontend/src/stores/websocket.js @@ -0,0 +1,52 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' +import { useWebSocket } from '../composables/useWebSocket' +import { useNotificationStore } from './notifications' +import { useDownloadsStore } from './downloads' +import { useSourcesStore } from './sources' + +export const useWebSocketStore = defineStore('websocket', () => { + const { isConnected, subscribe } = useWebSocket() + const initialized = ref(false) + + function init() { + if (initialized.value) return + initialized.value = true + + const notifications = useNotificationStore() + const downloadsStore = useDownloadsStore() + const sourcesStore = useSourcesStore() + + // Subscribe to download events + subscribe('download.started', (data) => { + notifications.info(`Download started for source ${data.source_id}`) + }) + + subscribe('download.completed', (data) => { + notifications.success(`Download completed: ${data.file_count} files`) + // Refresh downloads list + downloadsStore.fetchDownloads() + downloadsStore.fetchStats() + }) + + subscribe('download.failed', (data) => { + notifications.error(`Download failed: ${data.error_message}`) + downloadsStore.fetchDownloads() + }) + + // Subscribe to source events + subscribe('source.updated', (data) => { + sourcesStore.fetchSources() + }) + + // Subscribe to credential events + subscribe('credential.expiring', (data) => { + notifications.warning(`${data.platform} credentials expiring in ${data.expires_in_hours} hours`) + }) + } + + return { + isConnected, + init, + } +}) diff --git a/frontend/src/views/Credentials.vue b/frontend/src/views/Credentials.vue new file mode 100644 index 0000000..c967231 --- /dev/null +++ b/frontend/src/views/Credentials.vue @@ -0,0 +1,339 @@ + + + diff --git a/frontend/src/views/Dashboard.vue b/frontend/src/views/Dashboard.vue new file mode 100644 index 0000000..7d72e80 --- /dev/null +++ b/frontend/src/views/Dashboard.vue @@ -0,0 +1,283 @@ + + + diff --git a/frontend/src/views/Downloads.vue b/frontend/src/views/Downloads.vue new file mode 100644 index 0000000..51e9b60 --- /dev/null +++ b/frontend/src/views/Downloads.vue @@ -0,0 +1,329 @@ + + + diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue new file mode 100644 index 0000000..571850d --- /dev/null +++ b/frontend/src/views/Settings.vue @@ -0,0 +1,474 @@ + + + + + diff --git a/frontend/src/views/SourceForm.vue b/frontend/src/views/SourceForm.vue new file mode 100644 index 0000000..0d29e95 --- /dev/null +++ b/frontend/src/views/SourceForm.vue @@ -0,0 +1,316 @@ + + + diff --git a/frontend/src/views/Sources.vue b/frontend/src/views/Sources.vue new file mode 100644 index 0000000..54b8769 --- /dev/null +++ b/frontend/src/views/Sources.vue @@ -0,0 +1,246 @@ + + + diff --git a/frontend/src/views/Subscriptions.vue b/frontend/src/views/Subscriptions.vue new file mode 100644 index 0000000..352d096 --- /dev/null +++ b/frontend/src/views/Subscriptions.vue @@ -0,0 +1,502 @@ + + + + + diff --git a/frontend/vite.config.js b/frontend/vite.config.js new file mode 100644 index 0000000..3e3013f --- /dev/null +++ b/frontend/vite.config.js @@ -0,0 +1,23 @@ +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' + +export default defineConfig({ + plugins: [vue()], + server: { + port: 3000, + proxy: { + '/api': { + target: 'http://localhost:8080', + changeOrigin: true, + }, + '/ws': { + target: 'ws://localhost:8080', + ws: true, + }, + }, + }, + build: { + outDir: 'dist', + assetsDir: 'assets', + }, +})