diff --git a/.env.example b/.env.example index 23a44b7..6da7f80 100644 --- a/.env.example +++ b/.env.example @@ -1,38 +1,87 @@ +# =========================================== +# GallerySubscriber Environment Configuration +# =========================================== +# Copy this file to .env and configure the required values. + # =========================================== # Required - You MUST change these values # =========================================== # Database password for PostgreSQL +# Use a strong, unique password DB_PASSWORD=change_me_to_secure_password -# Application secret key (use: openssl rand -hex 32) +# Application secret key for encryption +# Generate with: openssl rand -hex 32 SECRET_KEY=change_me_to_random_string_at_least_32_characters # =========================================== # Optional - These have sensible defaults # =========================================== -# External port to access the application +# Timezone (used by app, celery, and database) +# See: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +TZ=America/New_York + +# External port to access the web UI PORT=8080 # Download settings +# DOWNLOAD_PARALLEL_LIMIT: Max concurrent downloads +# DOWNLOAD_RATE_LIMIT: Seconds between requests (to avoid rate limiting) +# DEFAULT_CHECK_INTERVAL: Seconds between automatic checks for new content DOWNLOAD_PARALLEL_LIMIT=3 DOWNLOAD_RATE_LIMIT=3.0 DEFAULT_CHECK_INTERVAL=3600 -# Logging +# Logging level (DEBUG, INFO, WARNING, ERROR) LOG_LEVEL=INFO + +# Debug mode (enables verbose output, not for production) DEBUG=false +# =========================================== +# Database Configuration +# =========================================== +# These compose the DATABASE_URL automatically. +# Defaults work with the included PostgreSQL container. +# Change these if using an external database. + +DB_USER=gdl +DB_HOST=db +DB_PORT=5432 +DB_NAME=gallery_subscriber + +# =========================================== +# Redis Configuration +# =========================================== +# Defaults work with the included Redis container. +# Change these if using an external Redis instance. + +REDIS_HOST=redis +REDIS_PORT=6379 + +# =========================================== +# Data Storage Paths +# =========================================== +# Host paths for persistent data. By default, Docker named volumes are used. +# Set these to use bind mounts instead (e.g., for NAS storage). +# +# Examples: +# DATA_DOWNLOADS=/mnt/nas/gallery-subscriber/downloads +# DATA_CONFIG=./data/config +# +# DATA_DOWNLOADS: Where downloaded files are stored +# DATA_CONFIG: Gallery-dl config and download archive (for deduplication) + +# DATA_DOWNLOADS=downloads +# DATA_CONFIG=config + # =========================================== # Advanced - Only change if you know what you're doing # =========================================== -# Paths inside container (mapped to Docker volumes) -DOWNLOAD_PATH=/data/downloads -CONFIG_PATH=/data/config -COOKIES_PATH=/data/cookies - -# These are constructed by docker-compose, only set for local development +# Full connection URLs (constructed automatically from components above) +# Only set these directly for special configurations # DATABASE_URL=postgresql://gdl:password@localhost:5432/gallery_subscriber # REDIS_URL=redis://localhost:6379/0 diff --git a/README.md b/README.md index f449116..96661f8 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,84 @@ -# Gallery Subscriber +# GallerySubscriber -A Docker-based web application for managing gallery-dl downloads with a modern web interface. +A Docker-based web application for managing gallery-dl downloads with a modern web interface and browser extension. + +## How It Works + +GallerySubscriber automates downloading content from subscription platforms like Patreon, SubscribeStar, Discord, and more. Here's the typical workflow: + +1. **Install the Firefox extension** to export your login cookies/tokens from supported platforms +2. **Add subscriptions** in the web UI (e.g., "ArtistName") and attach sources (platform URLs) +3. **Export credentials** from the extension to authenticate downloads +4. **Trigger a check** or wait for automatic scheduled downloads +5. **Content is downloaded** to organized folders: `{subscription}/{platform}/{post}/` + +The app handles rate limiting, retries, and tracks what's already been downloaded so you never get duplicates. ## Features -- **Web Dashboard** - Monitor downloads, view statistics, and manage sources -- **Source Management** - Add, edit, and organize download sources from multiple platforms -- **Per-Source Configuration** - Customize content types, rate limits, and patterns for each source -- **Credential Management** - Securely store cookies/tokens for authenticated platforms +- **Web Dashboard** - Monitor downloads, view statistics, and manage subscriptions +- **Subscription Management** - Organize artists/creators with multiple platform sources +- **Firefox Extension** - One-click cookie/token export from Patreon, SubscribeStar, Discord +- **Per-Source Configuration** - Customize content types, rate limits, and patterns +- **Credential Management** - Securely encrypted storage for cookies/tokens - **Scheduled Downloads** - Automatic periodic checks for new content - **Real-time Updates** - WebSocket notifications for download progress - **Supported Platforms**: Patreon, SubscribeStar, Hentai Foundry, Discord ## Quick Start -1. **Copy environment file and configure:** - ```bash - cp .env.example .env - # Edit .env with your settings (especially DB_PASSWORD and SECRET_KEY) - ``` +### 1. Configure Environment -2. **Start the services:** - ```bash - docker-compose up -d - ``` +```bash +# Download the docker-compose and example env +curl -O https://git.fabledsword.com/bvandeusen/gallerysubscriber/raw/branch/main/docker-compose.yml +curl -O https://git.fabledsword.com/bvandeusen/gallerysubscriber/raw/branch/main/.env.example -3. **Run database migrations:** - ```bash - docker-compose exec app alembic upgrade head - ``` +# Create your .env file +cp .env.example .env -4. **Access the application:** - - Web UI: http://localhost:8080 - - API: http://localhost:8080/api - - Health check: http://localhost:8080/health +# Edit .env - you MUST change DB_PASSWORD and SECRET_KEY +# Generate a secret key with: openssl rand -hex 32 +``` + +### 2. Start the Services + +```bash +docker compose up -d +``` + +### 3. Run Database Migrations + +```bash +docker compose exec app alembic upgrade head +``` + +### 4. Access the Application + +- **Web UI**: http://localhost:8080 +- **API**: http://localhost:8080/api +- **Health Check**: http://localhost:8080/health + +### 5. Install the Firefox Extension + +1. Download the extension from the `extension/` folder +2. Go to `about:debugging#/runtime/this-firefox` in Firefox +3. Click "Load Temporary Add-on" and select `extension/manifest.json` +4. Click the extension icon and configure the API URL and key (found in Settings) + +### 6. Export Your Credentials + +1. Log into your subscription platforms (Patreon, etc.) in Firefox +2. Click the GallerySubscriber extension icon +3. Click "Export" for each platform to send cookies to the backend +4. For Discord: just browse Discord normally - the token is captured automatically + +### 7. Add Subscriptions + +1. Go to **Subscriptions** in the web UI +2. Click **Add Subscription** and enter the creator's name +3. Add sources (platform URLs) like `https://www.patreon.com/artistname` +4. Click the refresh icon to trigger a download check ## Configuration @@ -42,75 +87,119 @@ A Docker-based web application for managing gallery-dl downloads with a modern w | Variable | Description | Default | |----------|-------------|---------| | `DB_PASSWORD` | PostgreSQL password | (required) | -| `SECRET_KEY` | App secret key (32+ chars) | (required) | -| `EXTENSION_API_KEY` | Firefox extension auth key | (required) | -| `PORT` | External port | 8080 | -| `DOWNLOAD_PARALLEL_LIMIT` | Max concurrent downloads | 3 | -| `DOWNLOAD_RATE_LIMIT` | Delay between downloads (sec) | 3.0 | -| `DEFAULT_CHECK_INTERVAL` | Check interval (sec) | 3600 | +| `SECRET_KEY` | App secret key for encryption (32+ chars) | (required) | +| `TZ` | Timezone | `America/New_York` | +| `PORT` | External port | `8080` | +| `DOWNLOAD_PARALLEL_LIMIT` | Max concurrent downloads | `3` | +| `DOWNLOAD_RATE_LIMIT` | Delay between requests (seconds) | `3.0` | +| `DEFAULT_CHECK_INTERVAL` | Auto-check interval (seconds) | `3600` | +| `LOG_LEVEL` | Logging verbosity | `INFO` | + +See `.env.example` for the complete list with descriptions. ### Settings Page -All configuration options are available in the web UI under Settings: +All configuration options are available in the web UI under **Settings**: - Download settings (parallel limit, rate limit, retries) -- Notification settings (webhooks) +- Extension API key (for connecting the Firefox extension) - Raw gallery-dl.conf editor - Platform defaults reference -## Screenshots +## Firefox Extension -### Dashboard -- Overview of sources and download statistics -- Recent downloads with status -- Sources needing attention +The extension exports authentication cookies/tokens from supported platforms to the backend. -### Sources -- List all sources with filtering -- Enable/disable sources -- Trigger manual checks -- Configure per-source options +### Supported Platforms -### Downloads -- Full download history -- Filter by status, source, date -- View detailed logs -- Retry failed downloads +| Platform | Auth Type | How It Works | +|----------|-----------|--------------| +| Patreon | Cookies | Exports session cookies | +| SubscribeStar | Cookies | Exports session cookies | +| Hentai Foundry | Cookies | Exports session cookies | +| Discord | Token | Automatically captures Authorization header | -### Credentials -- Platform credential status -- Upload cookies/tokens -- Verify credentials +### Extension Setup -### Settings -- Application configuration -- Gallery-dl config editor -- Platform documentation +1. Load the extension in Firefox (`about:debugging` → Load Temporary Add-on) +2. Click the extension icon → Settings (gear icon) +3. Enter your backend API URL (e.g., `http://localhost:8080/api`) +4. Enter the Extension API Key from Settings page in the web UI +5. Click "Test Connection" to verify + +### Exporting Credentials + +- **Cookie-based platforms**: Log in, then click "Export" in the extension popup +- **Discord**: Just open Discord in Firefox - the token is captured automatically from API requests + +## Download Organization + +Files are downloaded to: `/data/downloads/{subscription}/{platform}/{post}/` + +Example structure: +``` +downloads/ +├── ArtistName/ +│ ├── patreon/ +│ │ ├── 2024-01-15_12345_Post Title/ +│ │ │ ├── 01_image.png +│ │ │ ├── 02_image.jpg +│ │ │ └── 01_image.json (metadata) +│ │ └── 2024-01-10_12340_Another Post/ +│ └── discord/ +│ └── art-channel/ +│ └── 20240115_987654321_photo.png +``` ## API Endpoints | Endpoint | Description | |----------|-------------| +| `GET /api/subscriptions` | List all subscriptions | +| `POST /api/subscriptions` | Create a subscription | +| `POST /api/subscriptions/:id/sources` | Add source to subscription | +| `POST /api/subscriptions/:id/check` | Trigger download check | | `GET /api/sources` | List all sources | -| `POST /api/sources` | Create a new source | -| `GET /api/sources/:id` | Get source details | | `PATCH /api/sources/:id` | Update a source | -| `DELETE /api/sources/:id` | Delete a source | -| `POST /api/sources/:id/check` | Trigger download check | | `GET /api/downloads` | List download history | | `GET /api/downloads/stats` | Get download statistics | | `POST /api/downloads/:id/retry` | Retry failed download | | `GET /api/credentials` | List credential status | | `POST /api/credentials` | Upload cookies/tokens | -| `DELETE /api/credentials/:platform` | Remove credentials | | `GET /api/settings` | Get application settings | | `PATCH /api/settings` | Update settings | -| `GET /api/platforms` | List supported platforms | | `WS /ws/events` | Real-time event stream | +## Development + +For development with hot-reload and debug logging: + +```bash +# Clone the repository +git clone https://git.fabledsword.com/bvandeusen/gallerysubscriber.git +cd gallerysubscriber + +# Start development environment +docker compose -f docker-compose.dev.yml up --build +``` + +This provides: +- Live code reloading (backend changes apply automatically) +- Debug logging enabled +- PostgreSQL exposed on port 5432 +- Redis exposed on port 6379 + +### Frontend Development (standalone) + +```bash +cd frontend +npm install +npm run dev +``` + ## Project Structure ``` -gallery-subscriber/ +gallerysubscriber/ ├── backend/ # Quart API server │ ├── app/ │ │ ├── api/ # REST + WebSocket endpoints @@ -124,43 +213,37 @@ gallery-subscriber/ │ ├── views/ # Page components │ ├── stores/ # Pinia state management │ └── services/ # API client -├── extension/ # Firefox extension (planned) -├── scripts/ # Migration utilities -└── Dockerfile # Multi-stage build (frontend + backend) +├── extension/ # Firefox extension +│ ├── manifest.json # Extension manifest (v2) +│ ├── popup/ # Extension popup UI +│ ├── options/ # Extension settings page +│ ├── background/ # Background script +│ └── lib/ # Shared utilities +└── Dockerfile # Multi-stage build ``` ## Technology Stack -- **Backend:** Quart (async Flask) + SQLAlchemy + PostgreSQL -- **Task Queue:** Celery + Redis -- **Frontend:** Vue 3 + Vuetify 3 + Pinia (compiled to static files, served by Quart) -- **Downloader:** gallery-dl +- **Backend**: Quart (async Flask) + SQLAlchemy + PostgreSQL +- **Task Queue**: Celery + Redis +- **Frontend**: Vue 3 + Vuetify 3 + Pinia +- **Extension**: Firefox WebExtension (Manifest V2) +- **Downloader**: gallery-dl -## Development +## Troubleshooting -For development with hot-reload: +### Downloads failing with "Authentication failed" +- Re-export cookies from the Firefox extension +- Check that you're still logged into the platform +- For Discord: browse Discord to refresh the captured token -```bash -docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -``` +### "No cookies found" in extension +- Make sure you're logged into the platform in Firefox +- Check that cookies aren't being blocked by privacy extensions -Frontend development (standalone): -```bash -cd frontend -npm install -npm run dev -``` - -## Migrating from Legacy Project - -If you have an existing Gallery-DL-scripting setup: - -```bash -# Coming soon: migration script -python scripts/migrate_legacy.py \ - --sources /path/to/sources.yaml \ - --config /path/to/gallery-dl.conf -``` +### Downloads not starting +- Check the Celery worker logs: `docker compose logs celery` +- Verify credentials are uploaded: check the Credentials page ## License diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cc543b8..14b0e4d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,35 +1,93 @@ -# Development overrides - use with: docker-compose -f docker-compose.yml -f docker-compose.dev.yml up +# GallerySubscriber - Development Docker Compose +# +# This builds from source with hot-reloading and debug mode. +# Usage: docker compose -f docker-compose.dev.yml up --build +# +# Features: +# - Builds from local Dockerfile +# - Volume mounts for live code reloading +# - Debug logging enabled +# - Database and Redis ports exposed for local tools + services: + # Combined backend API + frontend static files app: - build: - context: ./backend - dockerfile: Dockerfile.dev + build: . + ports: + - "${PORT:-8080}:8080" + environment: + - TZ=${TZ:-America/New_York} + - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 + - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production} + - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} + - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} + - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + - LOG_LEVEL=DEBUG + - DEBUG=true volumes: - ./backend/app:/app/app:ro - - downloads:/data/downloads - - config:/data/config - - cookies:/data/cookies - environment: - - DEBUG=true - - LOG_LEVEL=DEBUG + - ${DATA_DOWNLOADS:-downloads}:/data/downloads + - ${DATA_CONFIG:-config}:/data/config + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + restart: unless-stopped + # Celery worker for background tasks celery: - build: - context: ./backend - dockerfile: Dockerfile.dev + build: . + command: celery -A app.tasks.celery_app worker -B --loglevel=debug + environment: + - TZ=${TZ:-America/New_York} + - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 + - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production} + - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} + - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} + - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + - LOG_LEVEL=DEBUG + - DEBUG=true volumes: - ./backend/app:/app/app:ro - - downloads:/data/downloads - - config:/data/config - - cookies:/data/cookies - environment: - - DEBUG=true - - LOG_LEVEL=DEBUG + - ${DATA_DOWNLOADS:-downloads}:/data/downloads + - ${DATA_CONFIG:-config}:/data/config + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + restart: unless-stopped + # PostgreSQL database db: + image: postgres:15-alpine ports: - - "5432:5432" + - "${DB_PORT:-5432}:5432" + environment: + - TZ=${TZ:-America/New_York} + - POSTGRES_USER=${DB_USER:-gdl} + - POSTGRES_PASSWORD=${DB_PASSWORD:-devpassword} + - POSTGRES_DB=${DB_NAME:-gallery_subscriber} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-gdl} -d ${DB_NAME:-gallery_subscriber}"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + # Redis for Celery task queue redis: + image: redis:7-alpine ports: - - "6379:6379" + - "${REDIS_PORT:-6379}:6379" + restart: unless-stopped + +volumes: + postgres_data: + downloads: + config: diff --git a/docker-compose.yml b/docker-compose.yml index b17c220..00979de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,30 @@ +# GallerySubscriber - Production Docker Compose +# +# Quick Start: +# 1. Copy .env.example to .env and configure required values +# 2. Run: docker compose up -d +# 3. Access the web UI at http://localhost:8080 +# +# For development, see docker-compose.dev.yml + services: # Combined backend API + frontend static files app: - build: . + image: git.fabledsword.com/bvandeusen/gallerysubscriber:latest ports: - "${PORT:-8080}:8080" environment: - TZ=${TZ:-America/New_York} - - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber - - REDIS_URL=redis://redis:6379/0 + - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 - SECRET_KEY=${SECRET_KEY} - - DOWNLOAD_PATH=/data/downloads - - CONFIG_PATH=/data/config - - COOKIES_PATH=/data/cookies - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + - LOG_LEVEL=${LOG_LEVEL:-INFO} volumes: - - downloads:/data/downloads - - config:/data/config - - cookies:/data/cookies + - ${DATA_DOWNLOADS:-downloads}:/data/downloads + - ${DATA_CONFIG:-config}:/data/config depends_on: db: condition: service_healthy @@ -28,23 +34,20 @@ services: # Celery worker for background tasks celery: - build: . + image: git.fabledsword.com/bvandeusen/gallerysubscriber:latest command: celery -A app.tasks.celery_app worker -B --loglevel=info environment: - TZ=${TZ:-America/New_York} - - DATABASE_URL=postgresql://gdl:${DB_PASSWORD}@db:5432/gallery_subscriber - - REDIS_URL=redis://redis:6379/0 + - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber} + - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0 - SECRET_KEY=${SECRET_KEY} - - DOWNLOAD_PATH=/data/downloads - - CONFIG_PATH=/data/config - - COOKIES_PATH=/data/cookies - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3} - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0} - DEFAULT_CHECK_INTERVAL=${DEFAULT_CHECK_INTERVAL:-3600} + - LOG_LEVEL=${LOG_LEVEL:-INFO} volumes: - - downloads:/data/downloads - - config:/data/config - - cookies:/data/cookies + - ${DATA_DOWNLOADS:-downloads}:/data/downloads + - ${DATA_CONFIG:-config}:/data/config depends_on: db: condition: service_healthy @@ -57,13 +60,13 @@ services: image: postgres:15-alpine environment: - TZ=${TZ:-America/New_York} - - POSTGRES_USER=gdl + - POSTGRES_USER=${DB_USER:-gdl} - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_DB=gallery_subscriber + - POSTGRES_DB=${DB_NAME:-gallery_subscriber} volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U gdl -d gallery_subscriber"] + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-gdl} -d ${DB_NAME:-gallery_subscriber}"] interval: 5s timeout: 5s retries: 5 @@ -78,4 +81,3 @@ volumes: postgres_data: downloads: config: - cookies: