added functionality for pixiv and deviantart

This commit is contained in:
Bryan Van Deusen
2026-02-02 20:17:13 -05:00
parent 3ee7de7ecd
commit cc19bbd372
16 changed files with 158 additions and 27 deletions
+1 -1
View File
@@ -12,7 +12,7 @@ from app.utils.encryption import encrypt_data, decrypt_data
bp = Blueprint("credentials", __name__) bp = Blueprint("credentials", __name__)
VALID_PLATFORMS = ["patreon", "subscribestar", "hentaifoundry", "discord"] VALID_PLATFORMS = ["patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"]
@bp.route("", methods=["GET"]) @bp.route("", methods=["GET"])
+40 -2
View File
@@ -85,6 +85,44 @@ async def list_platforms():
"default_config": gdl_service.get_default_config_for_platform("discord"), "default_config": gdl_service.get_default_config_for_platform("discord"),
"notes": "Requires Discord user token (not bot token)", "notes": "Requires Discord user token (not bot token)",
}, },
"pixiv": {
"name": "Pixiv",
"description": "Download artwork from Pixiv artists",
"requires_auth": True,
"auth_type": "cookies",
"url_pattern": "https://www.pixiv.net/users/{user_id}",
"url_examples": [
"https://www.pixiv.net/users/12345678",
"https://www.pixiv.net/en/users/12345678",
],
"content_types": gdl_service.get_platform_content_types("pixiv"),
"content_type_descriptions": {
"all": "All artwork (illustrations and manga)",
"illusts": "Illustrations only",
"manga": "Manga only",
"bookmarks": "Bookmarked works",
},
"default_config": gdl_service.get_default_config_for_platform("pixiv"),
},
"deviantart": {
"name": "DeviantArt",
"description": "Download artwork from DeviantArt artists",
"requires_auth": False,
"auth_type": "cookies",
"url_pattern": "https://www.deviantart.com/{username}",
"url_examples": [
"https://www.deviantart.com/example-artist",
"https://www.deviantart.com/example-artist/gallery",
],
"content_types": gdl_service.get_platform_content_types("deviantart"),
"content_type_descriptions": {
"all": "All deviations",
"gallery": "Gallery artwork",
"scraps": "Scraps folder",
"favorites": "Favorited works",
},
"default_config": gdl_service.get_default_config_for_platform("deviantart"),
},
} }
return jsonify({"platforms": platforms}) return jsonify({"platforms": platforms})
@@ -95,7 +133,7 @@ async def get_platform(platform: str):
"""Get detailed information about a specific platform.""" """Get detailed information about a specific platform."""
gdl_service = GalleryDLService() gdl_service = GalleryDLService()
if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]: if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"]:
return jsonify({"error": f"Unknown platform: {platform}"}), 404 return jsonify({"error": f"Unknown platform: {platform}"}), 404
# Get the full platform info from list_platforms # Get the full platform info from list_platforms
@@ -115,7 +153,7 @@ async def get_config_schema(platform: str):
This defines what options can be set per-source for this platform. This defines what options can be set per-source for this platform.
Useful for building dynamic forms in the frontend. Useful for building dynamic forms in the frontend.
""" """
if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord"]: if platform not in ["patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"]:
return jsonify({"error": f"Unknown platform: {platform}"}), 404 return jsonify({"error": f"Unknown platform: {platform}"}), 404
gdl_service = GalleryDLService() gdl_service = GalleryDLService()
+1 -1
View File
@@ -78,7 +78,7 @@ async def create_source():
return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400 return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400
# Validate platform # Validate platform
valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"] valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"]
if data["platform"] not in valid_platforms: if data["platform"] not in valid_platforms:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400 return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400
+1 -1
View File
@@ -232,7 +232,7 @@ async def add_source_to_subscription(subscription_id: int):
return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400 return jsonify({"error": f"Missing required fields: {', '.join(missing)}"}), 400
# Validate platform # Validate platform
valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord"] valid_platforms = ["patreon", "subscribestar", "hentaifoundry", "discord", "pixiv", "deviantart"]
if data["platform"] not in valid_platforms: if data["platform"] not in valid_platforms:
return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400 return jsonify({"error": f"Invalid platform. Must be one of: {', '.join(valid_platforms)}"}), 400
+21
View File
@@ -171,6 +171,23 @@ class GalleryDLService:
# Deep crawling options # Deep crawling options
"threads": True, # Include thread content for complete history "threads": True, # Include thread content for complete history
}, },
"pixiv": {
"content_types": ["all"],
"directory": ["{category}"],
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
# Content options
"ugoira": True, # Download ugoira (animated) as WebM
},
"deviantart": {
"content_types": ["all"],
"directory": [], # Flat structure within platform folder
"filename": "{index:>03}_{title[:50]}.{extension}",
# Content options
"flat": True, # Flatten folder structure
"original": True, # Download original quality
"mature": True, # Include mature content
"metadata": True, # Include metadata
},
} }
def __init__(self, rate_limit: Optional[float] = None): def __init__(self, rate_limit: Optional[float] = None):
@@ -682,6 +699,10 @@ class GalleryDLService:
return ["all"] # No granular control return ["all"] # No granular control
elif platform == "discord": elif platform == "discord":
return ["all"] # No granular control return ["all"] # No granular control
elif platform == "pixiv":
return ["all", "illusts", "manga", "bookmarks"]
elif platform == "deviantart":
return ["all", "gallery", "scraps", "favorites"]
else: else:
return ["all"] return ["all"]
+8 -3
View File
@@ -1,12 +1,15 @@
{ {
"manifest_version": 2, "manifest_version": 2,
"name": "GallerySubscriber", "name": "GallerySubscriber",
"version": "1.0.0", "version": "1.1.0",
"description": "Export cookies from supported platforms to GallerySubscriber for automated downloads", "description": "Export cookies from supported platforms to GallerySubscriber for automated downloads",
"browser_specific_settings": { "browser_specific_settings": {
"gecko": { "gecko": {
"id": "gallerysubscriber@fabledsword.com" "id": "gallerysubscriber@fabledsword.com",
"data_collection_permissions": {
"required": ["none"]
}
} }
}, },
@@ -20,7 +23,9 @@
"*://*.subscribestar.com/*", "*://*.subscribestar.com/*",
"*://*.subscribestar.adult/*", "*://*.subscribestar.adult/*",
"*://*.hentai-foundry.com/*", "*://*.hentai-foundry.com/*",
"*://*.discord.com/*" "*://*.discord.com/*",
"*://*.pixiv.net/*",
"*://*.deviantart.com/*"
], ],
"browser_action": { "browser_action": {
+48 -17
View File
@@ -11,7 +11,13 @@ document.addEventListener('DOMContentLoaded', async () => {
document.getElementById('setup-required').classList.remove('hidden'); document.getElementById('setup-required').classList.remove('hidden');
const alertDiv = document.querySelector('#setup-required .alert'); const alertDiv = document.querySelector('#setup-required .alert');
if (alertDiv) { if (alertDiv) {
alertDiv.innerHTML = `<strong>Error</strong><p>${error.message}</p>`; alertDiv.textContent = '';
const strong = document.createElement('strong');
strong.textContent = 'Error';
const p = document.createElement('p');
p.textContent = error.message;
alertDiv.appendChild(strong);
alertDiv.appendChild(p);
alertDiv.className = 'alert alert-error'; alertDiv.className = 'alert alert-error';
} }
} }
@@ -51,7 +57,11 @@ async function init() {
// Show loading state for platforms // Show loading state for platforms
const platformsContainer = document.getElementById('platforms-list'); const platformsContainer = document.getElementById('platforms-list');
platformsContainer.innerHTML = '<div style="text-align: center; padding: 20px; color: var(--text-secondary);">Loading platforms...</div>'; platformsContainer.textContent = '';
const loadingDiv = document.createElement('div');
loadingDiv.style.cssText = 'text-align: center; padding: 20px; color: var(--text-secondary);';
loadingDiv.textContent = 'Loading platforms...';
platformsContainer.appendChild(loadingDiv);
// Now do background tasks in parallel (non-blocking) // Now do background tasks in parallel (non-blocking)
// Check if we should test connection (rate limited) // Check if we should test connection (rate limited)
@@ -134,7 +144,7 @@ function updateConnectionIndicator(connected) {
async function loadPlatformStatus() { async function loadPlatformStatus() {
const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' }); const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' });
const container = document.getElementById('platforms-list'); const container = document.getElementById('platforms-list');
container.innerHTML = ''; container.textContent = '';
for (const [key, platform] of Object.entries(PLATFORMS)) { for (const [key, platform] of Object.entries(PLATFORMS)) {
const platformStatus = status[key] || {}; const platformStatus = status[key] || {};
@@ -167,20 +177,41 @@ function createPlatformCard(key, platform, status) {
const statusText = getStatusText(status, platform, key); const statusText = getStatusText(status, platform, key);
const statusClass = getStatusClass(status, platform, key); const statusClass = getStatusClass(status, platform, key);
card.innerHTML = ` // Build card content using DOM methods (avoids innerHTML security warnings)
<div class="platform-icon" style="background-color: ${platform.color}"> const iconDiv = document.createElement('div');
${platform.name.charAt(0)} iconDiv.className = 'platform-icon';
</div> iconDiv.style.backgroundColor = platform.color;
<div class="info"> iconDiv.textContent = platform.name.charAt(0);
<div class="name">${platform.name}</div>
<div class="status ${statusClass}">${statusText}</div> const infoDiv = document.createElement('div');
</div> infoDiv.className = 'info';
<span class="action-icon">
<svg width="20" height="20" viewBox="0 0 24 24"> const nameDiv = document.createElement('div');
<path fill="currentColor" d="M5,20H19V18H5M19,9H15V3H9V9H5L12,16L19,9Z"/> nameDiv.className = 'name';
</svg> nameDiv.textContent = platform.name;
</span>
`; const statusDiv = document.createElement('div');
statusDiv.className = `status ${statusClass}`;
statusDiv.textContent = statusText;
infoDiv.appendChild(nameDiv);
infoDiv.appendChild(statusDiv);
const actionSpan = document.createElement('span');
actionSpan.className = 'action-icon';
const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
svg.setAttribute('width', '20');
svg.setAttribute('height', '20');
svg.setAttribute('viewBox', '0 0 24 24');
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
path.setAttribute('fill', 'currentColor');
path.setAttribute('d', 'M5,20H19V18H5M19,9H15V3H9V9H5L12,16L19,9Z');
svg.appendChild(path);
actionSpan.appendChild(svg);
card.appendChild(iconDiv);
card.appendChild(infoDiv);
card.appendChild(actionSpan);
if (!isDisabled && !isDiscordWithoutToken) { if (!isDisabled && !isDiscordWithoutToken) {
card.addEventListener('click', () => exportPlatformCookies(key, card)); card.addEventListener('click', () => exportPlatformCookies(key, card));
+10
View File
@@ -0,0 +1,10 @@
module.exports = {
ignoreFiles: [
"create-icons.py",
"generate-icons.py",
"web-ext-artifacts",
"web-ext-config.js",
".amo-upload-uuid",
"*.md"
]
};
+4
View File
@@ -235,6 +235,8 @@ function getPlatformIcon(platform) {
subscribestar: 'mdi-star', subscribestar: 'mdi-star',
hentaifoundry: 'mdi-palette', hentaifoundry: 'mdi-palette',
discord: 'mdi-discord', discord: 'mdi-discord',
pixiv: 'mdi-alpha-p-box',
deviantart: 'mdi-deviantart',
} }
return icons[platform] || 'mdi-web' return icons[platform] || 'mdi-web'
} }
@@ -245,6 +247,8 @@ function getPlatformColor(platform) {
subscribestar: '#FFD700', subscribestar: '#FFD700',
hentaifoundry: '#9C27B0', hentaifoundry: '#9C27B0',
discord: '#5865F2', discord: '#5865F2',
pixiv: '#0096FA',
deviantart: '#05CC47',
} }
return colors[platform] || 'grey' return colors[platform] || 'grey'
} }
+2
View File
@@ -520,6 +520,8 @@ function getPlatformIcon(platform) {
subscribestar: 'mdi-star', subscribestar: 'mdi-star',
hentaifoundry: 'mdi-palette', hentaifoundry: 'mdi-palette',
discord: 'mdi-discord', discord: 'mdi-discord',
pixiv: 'mdi-alpha-p-box',
deviantart: 'mdi-deviantart',
} }
return icons[platform] || 'mdi-web' return icons[platform] || 'mdi-web'
} }
+4
View File
@@ -539,6 +539,8 @@ function getPlatformIcon(platform) {
subscribestar: 'mdi-star', subscribestar: 'mdi-star',
hentaifoundry: 'mdi-palette', hentaifoundry: 'mdi-palette',
discord: 'mdi-discord', discord: 'mdi-discord',
pixiv: 'mdi-alpha-p-box',
deviantart: 'mdi-deviantart',
} }
return icons[platform] || 'mdi-web' return icons[platform] || 'mdi-web'
} }
@@ -549,6 +551,8 @@ function getPlatformColor(platform) {
subscribestar: '#FFD700', subscribestar: '#FFD700',
hentaifoundry: '#9C27B0', hentaifoundry: '#9C27B0',
discord: '#5865F2', discord: '#5865F2',
pixiv: '#0096FA',
deviantart: '#05CC47',
} }
return colors[platform] || 'grey' return colors[platform] || 'grey'
} }
+2
View File
@@ -199,6 +199,8 @@ const platformOptions = [
{ title: 'SubscribeStar', value: 'subscribestar' }, { title: 'SubscribeStar', value: 'subscribestar' },
{ title: 'Hentai Foundry', value: 'hentaifoundry' }, { title: 'Hentai Foundry', value: 'hentaifoundry' },
{ title: 'Discord', value: 'discord' }, { title: 'Discord', value: 'discord' },
{ title: 'Pixiv', value: 'pixiv' },
{ title: 'DeviantArt', value: 'deviantart' },
] ]
const urlHint = computed(() => { const urlHint = computed(() => {
+6
View File
@@ -150,6 +150,8 @@ const platformOptions = [
{ title: 'SubscribeStar', value: 'subscribestar' }, { title: 'SubscribeStar', value: 'subscribestar' },
{ title: 'Hentai Foundry', value: 'hentaifoundry' }, { title: 'Hentai Foundry', value: 'hentaifoundry' },
{ title: 'Discord', value: 'discord' }, { title: 'Discord', value: 'discord' },
{ title: 'Pixiv', value: 'pixiv' },
{ title: 'DeviantArt', value: 'deviantart' },
] ]
const enabledOptions = [ const enabledOptions = [
@@ -189,6 +191,8 @@ function getPlatformIcon(platform) {
subscribestar: 'mdi-star', subscribestar: 'mdi-star',
hentaifoundry: 'mdi-palette', hentaifoundry: 'mdi-palette',
discord: 'mdi-discord', discord: 'mdi-discord',
pixiv: 'mdi-alpha-p-box',
deviantart: 'mdi-deviantart',
} }
return icons[platform] || 'mdi-web' return icons[platform] || 'mdi-web'
} }
@@ -199,6 +203,8 @@ function getPlatformColor(platform) {
subscribestar: 'amber', subscribestar: 'amber',
hentaifoundry: 'purple', hentaifoundry: 'purple',
discord: 'indigo', discord: 'indigo',
pixiv: 'blue',
deviantart: 'green',
} }
return colors[platform] || 'grey' return colors[platform] || 'grey'
} }
+6
View File
@@ -381,6 +381,8 @@ const platformOptions = [
{ title: 'SubscribeStar', value: 'subscribestar' }, { title: 'SubscribeStar', value: 'subscribestar' },
{ title: 'Hentai Foundry', value: 'hentaifoundry' }, { title: 'Hentai Foundry', value: 'hentaifoundry' },
{ title: 'Discord', value: 'discord' }, { title: 'Discord', value: 'discord' },
{ title: 'Pixiv', value: 'pixiv' },
{ title: 'DeviantArt', value: 'deviantart' },
] ]
const headers = [ const headers = [
@@ -423,6 +425,8 @@ function getPlatformIcon(platform) {
subscribestar: 'mdi-star', subscribestar: 'mdi-star',
hentaifoundry: 'mdi-palette', hentaifoundry: 'mdi-palette',
discord: 'mdi-discord', discord: 'mdi-discord',
pixiv: 'mdi-alpha-p-box',
deviantart: 'mdi-deviantart',
} }
return icons[platform] || 'mdi-web' return icons[platform] || 'mdi-web'
} }
@@ -433,6 +437,8 @@ function getPlatformColor(platform) {
subscribestar: 'amber', subscribestar: 'amber',
hentaifoundry: 'purple', hentaifoundry: 'purple',
discord: 'indigo', discord: 'indigo',
pixiv: 'blue',
deviantart: 'green',
} }
return colors[platform] || 'grey' return colors[platform] || 'grey'
} }
+4 -2
View File
@@ -1,10 +1,10 @@
# GallerySubscriber - Project Summary # GallerySubscriber - Project Summary
**Updated:** 2026-01-30 (Error recognition improvements, analysis tooling) **Updated:** 2026-02-02 (Added Pixiv and DeviantArt platform support)
## Overview ## Overview
GallerySubscriber is a Docker-based web application that automates downloading content from subscription platforms (Patreon, SubscribeStar, Discord, Hentai Foundry). It combines a Vue 3 web dashboard, Firefox extension for credential export, and a Python backend with Celery for background task scheduling. GallerySubscriber is a Docker-based web application that automates downloading content from subscription platforms (Patreon, SubscribeStar, Discord, Hentai Foundry, Pixiv, DeviantArt). It combines a Vue 3 web dashboard, Firefox extension for credential export, and a Python backend with Celery for background task scheduling.
**Core Purpose:** Automatically download and organize content from multiple subscription platforms using gallery-dl as the underlying download engine. **Core Purpose:** Automatically download and organize content from multiple subscription platforms using gallery-dl as the underlying download engine.
@@ -159,6 +159,8 @@ Download (1) ──→ (many) ContentItem
| SubscribeStar Adult | Cookies | images, attachments | Infinite scroll (complete) | | SubscribeStar Adult | Cookies | images, attachments | Infinite scroll (complete) |
| Hentai Foundry | Cookies | pictures, scraps, stories | All pages (complete) | | Hentai Foundry | Cookies | pictures, scraps, stories | All pages (complete) |
| Discord | Token | messages, attachments, embeds | All threads included | | Discord | Token | messages, attachments, embeds | All threads included |
| Pixiv | Cookies | illusts, manga, bookmarks | All works (complete) |
| DeviantArt | Cookies (optional) | gallery, scraps, favorites | All deviations (complete) |
**Note:** All platforms fetch complete history by default - no depth limits. Gallery-dl paginates through ALL available content. **Note:** All platforms fetch complete history by default - no depth limits. Gallery-dl paginates through ALL available content.