more UI polish and schedule safe guards and optimization
This commit is contained in:
Bryan Van Deusen
@@ -13,7 +13,8 @@
|
||||
"Bash(where:*)",
|
||||
"Bash(where npm:*)",
|
||||
"Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")",
|
||||
"WebFetch(domain:github.com)"
|
||||
"WebFetch(domain:github.com)",
|
||||
"WebFetch(domain:raw.githubusercontent.com)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,10 +154,10 @@ class GalleryDLService:
|
||||
# Automatic infinite scroll pagination is built-in
|
||||
},
|
||||
"hentaifoundry": {
|
||||
"content_types": ["pictures"],
|
||||
"content_types": ["all"], # Download all content types
|
||||
"directory": [], # Flat structure within platform folder
|
||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||
"include": "all", # Include all content types (pictures, scraps, stories)
|
||||
"include": "all", # Include pictures, scraps, stories - complete content
|
||||
# Automatic pagination through all pages (25 items/page)
|
||||
},
|
||||
"discord": {
|
||||
@@ -206,7 +206,8 @@ class GalleryDLService:
|
||||
"archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
|
||||
"skip": True,
|
||||
"sleep": self._rate_limit,
|
||||
"sleep-request": max(1.0, self._rate_limit / 2),
|
||||
# Faster request rate for pagination/checking (1/4 of rate_limit, min 0.5s)
|
||||
"sleep-request": max(0.5, self._rate_limit / 4),
|
||||
"retries": 3,
|
||||
"timeout": 30.0,
|
||||
"verify": True,
|
||||
@@ -624,7 +625,7 @@ class GalleryDLService:
|
||||
if platform == "patreon":
|
||||
return ["images", "image_large", "attachments", "postfile", "content"]
|
||||
elif platform == "hentaifoundry":
|
||||
return ["pictures", "stories"]
|
||||
return ["all", "pictures", "scraps", "stories"]
|
||||
elif platform == "subscribestar":
|
||||
return ["all"] # No granular control
|
||||
elif platform == "discord":
|
||||
@@ -643,5 +644,5 @@ class GalleryDLService:
|
||||
"directory_pattern": defaults.get("directory"),
|
||||
"filename_pattern": defaults.get("filename"),
|
||||
"sleep": self._rate_limit,
|
||||
"sleep_request": max(1.0, self._rate_limit / 2),
|
||||
"sleep_request": max(0.5, self._rate_limit / 4),
|
||||
}
|
||||
|
||||
@@ -287,9 +287,10 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
|
||||
# Update download record with results
|
||||
download.completed_at = utcnow()
|
||||
download.file_count = dl_result.files_downloaded
|
||||
# Store full logs - cleanup task will age out old records
|
||||
download.metadata_ = {
|
||||
"stdout": dl_result.stdout[:10000] if dl_result.stdout else None,
|
||||
"stderr": dl_result.stderr[:5000] if dl_result.stderr else None,
|
||||
"stdout": dl_result.stdout if dl_result.stdout else None,
|
||||
"stderr": dl_result.stderr if dl_result.stderr else None,
|
||||
"duration_seconds": dl_result.duration_seconds,
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,23 @@
|
||||
Reset Stuck ({{ stuckRunningCount }})
|
||||
</v-btn>
|
||||
<v-spacer />
|
||||
<!-- Credential Status -->
|
||||
<div class="d-flex align-center ga-2">
|
||||
<v-icon size="small" class="mr-1">mdi-key</v-icon>
|
||||
<span class="text-body-2 text-grey mr-2">Auth:</span>
|
||||
<v-chip
|
||||
v-for="platform in supportedPlatforms"
|
||||
:key="platform"
|
||||
:color="credentialsStore.hasPlatformCredentials(platform) ? 'success' : 'grey'"
|
||||
:variant="credentialsStore.hasPlatformCredentials(platform) ? 'tonal' : 'outlined'"
|
||||
size="x-small"
|
||||
:to="credentialsStore.hasPlatformCredentials(platform) ? undefined : '/credentials'"
|
||||
>
|
||||
<v-icon start size="x-small">{{ getPlatformIcon(platform) }}</v-icon>
|
||||
{{ platform }}
|
||||
</v-chip>
|
||||
</div>
|
||||
<v-divider vertical class="mx-2" />
|
||||
<div class="text-body-2 text-grey d-flex align-center">
|
||||
<v-icon size="small" class="mr-1">mdi-harddisk</v-icon>
|
||||
Storage: {{ storageStats?.total_size_formatted || 'Calculating...' }}
|
||||
@@ -267,9 +284,10 @@
|
||||
<v-btn
|
||||
size="small"
|
||||
variant="text"
|
||||
:to="`/sources/${source.id}/edit`"
|
||||
to="/subscriptions"
|
||||
>
|
||||
Edit
|
||||
View
|
||||
<v-tooltip activator="parent">View in Subscriptions</v-tooltip>
|
||||
</v-btn>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -290,13 +308,18 @@
|
||||
import { ref, computed, onMounted, onUnmounted } from 'vue'
|
||||
import { useSourcesStore } from '../stores/sources'
|
||||
import { useDownloadsStore } from '../stores/downloads'
|
||||
import { useCredentialsStore } from '../stores/credentials'
|
||||
import { useNotificationStore } from '../stores/notifications'
|
||||
import { settingsApi, downloadsApi } from '../services/api'
|
||||
|
||||
const sourcesStore = useSourcesStore()
|
||||
const downloadsStore = useDownloadsStore()
|
||||
const credentialsStore = useCredentialsStore()
|
||||
const notifications = useNotificationStore()
|
||||
|
||||
// Supported platforms for credential status
|
||||
const supportedPlatforms = ['patreon', 'subscribestar', 'hentaifoundry', 'discord']
|
||||
|
||||
// State
|
||||
const checkingAll = ref(false)
|
||||
const retryingAll = ref(false)
|
||||
@@ -337,6 +360,7 @@ function loadDashboardData() {
|
||||
// Fire off all requests in parallel, don't block UI
|
||||
// Each section handles its own loading state
|
||||
sourcesStore.fetchSources().catch(e => console.error('Failed to fetch sources:', e))
|
||||
credentialsStore.fetchCredentials().catch(e => console.error('Failed to fetch credentials:', e))
|
||||
|
||||
downloadsStore.fetchStats()
|
||||
.catch(e => console.error('Failed to fetch stats:', e))
|
||||
@@ -490,6 +514,16 @@ function getStatusIcon(status) {
|
||||
return icons[status] || 'mdi-help-circle'
|
||||
}
|
||||
|
||||
function getPlatformIcon(platform) {
|
||||
const icons = {
|
||||
patreon: 'mdi-patreon',
|
||||
subscribestar: 'mdi-star',
|
||||
hentaifoundry: 'mdi-palette',
|
||||
discord: 'mdi-discord',
|
||||
}
|
||||
return icons[platform] || 'mdi-web'
|
||||
}
|
||||
|
||||
function getStatusColor(status) {
|
||||
const colors = {
|
||||
completed: 'success',
|
||||
|
||||
@@ -197,7 +197,7 @@
|
||||
</td>
|
||||
<td>
|
||||
<div>{{ formatDate(source.last_check) }}</div>
|
||||
<div v-if="source.enabled && source.last_check" class="text-caption text-grey">
|
||||
<div v-if="source.enabled" class="text-caption text-grey">
|
||||
Next: {{ formatNextCheck(source) }}
|
||||
</div>
|
||||
</td>
|
||||
@@ -323,15 +323,22 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch, onMounted } from 'vue'
|
||||
import { ref, watch, onMounted, computed } from 'vue'
|
||||
import { useSubscriptionsStore } from '../stores/subscriptions'
|
||||
import { useCredentialsStore } from '../stores/credentials'
|
||||
import { useSettingsStore } from '../stores/settings'
|
||||
import { useNotificationStore } from '../stores/notifications'
|
||||
|
||||
const subscriptionsStore = useSubscriptionsStore()
|
||||
const credentialsStore = useCredentialsStore()
|
||||
const settingsStore = useSettingsStore()
|
||||
const notifications = useNotificationStore()
|
||||
|
||||
// Get global schedule interval from settings (default 8 hours)
|
||||
const scheduleInterval = computed(() =>
|
||||
settingsStore.settings['download.schedule_interval'] || 28800
|
||||
)
|
||||
|
||||
const searchQuery = ref('')
|
||||
const filterEnabled = ref(null)
|
||||
const expandedIds = ref([])
|
||||
@@ -387,6 +394,7 @@ const headers = [
|
||||
onMounted(() => {
|
||||
loadSubscriptions()
|
||||
credentialsStore.fetchCredentials()
|
||||
settingsStore.fetchSettings() // For schedule_interval
|
||||
})
|
||||
|
||||
watch([searchQuery, filterEnabled], () => {
|
||||
@@ -435,9 +443,10 @@ function formatDate(dateStr) {
|
||||
}
|
||||
|
||||
function formatNextCheck(source) {
|
||||
if (!source.last_check || !source.check_interval) return 'Unknown'
|
||||
if (!source.last_check) return 'Due now'
|
||||
const lastCheck = new Date(source.last_check)
|
||||
const nextCheck = new Date(lastCheck.getTime() + source.check_interval * 1000)
|
||||
// Use global schedule_interval from settings
|
||||
const nextCheck = new Date(lastCheck.getTime() + scheduleInterval.value * 1000)
|
||||
const now = new Date()
|
||||
|
||||
if (nextCheck <= now) return 'Due now'
|
||||
|
||||
+73
-11
@@ -1,6 +1,6 @@
|
||||
# GallerySubscriber - Project Summary
|
||||
|
||||
**Updated:** 2026-01-29 (task routing, scheduler/worker separation, 8-hour default interval)
|
||||
**Updated:** 2026-01-29 (Redis DB config, mismatch detection, stale job recovery, Downloads UI improvements)
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -152,24 +152,29 @@ Download (1) ──→ (many) ContentItem
|
||||
|
||||
## Supported Platforms
|
||||
|
||||
| Platform | Auth Type | Content Types |
|
||||
|----------|-----------|---------------|
|
||||
| Patreon | Cookies | images, attachments, posts |
|
||||
| SubscribeStar | Cookies | images, attachments |
|
||||
| SubscribeStar Adult | Cookies | images, attachments |
|
||||
| Hentai Foundry | Cookies | images |
|
||||
| Discord | Token | messages, attachments |
|
||||
| Platform | Auth Type | Content Types | Pagination |
|
||||
|----------|-----------|---------------|------------|
|
||||
| Patreon | Cookies | images, attachments, posts, content | Cursor-based (complete history) |
|
||||
| SubscribeStar | Cookies | images, attachments | Infinite scroll (complete) |
|
||||
| SubscribeStar Adult | Cookies | images, attachments | Infinite scroll (complete) |
|
||||
| Hentai Foundry | Cookies | pictures, scraps, stories | All pages (complete) |
|
||||
| Discord | Token | messages, attachments, embeds | All threads included |
|
||||
|
||||
**Note:** All platforms fetch complete history by default - no depth limits. Gallery-dl paginates through ALL available content.
|
||||
|
||||
## Configuration
|
||||
|
||||
**Environment Variables (.env):**
|
||||
- `DB_PASSWORD` (required) - PostgreSQL password
|
||||
- `SECRET_KEY` (required) - Encryption key (32+ chars)
|
||||
- `REDIS_DB` (default: 0) - Redis database number (must be same for all services)
|
||||
- `DOWNLOAD_PARALLEL_LIMIT` (default: 3) - Concurrent downloads
|
||||
- `DOWNLOAD_RATE_LIMIT` (default: 3.0) - Seconds between requests
|
||||
- `LOG_LEVEL` (default: INFO)
|
||||
- `TZ` (default: America/New_York)
|
||||
|
||||
**IMPORTANT:** All services (app, scheduler, worker) must use the same Redis database. A mismatch causes Celery tasks to be lost (jobs stuck in "queued" status). The system includes automatic detection and recovery for this scenario.
|
||||
|
||||
## Key Patterns
|
||||
|
||||
1. **Async/Await** - Quart + asyncpg for non-blocking I/O
|
||||
@@ -180,6 +185,9 @@ Download (1) ──→ (many) ContentItem
|
||||
6. **Task Routing** - Separate queues for scheduling vs download tasks
|
||||
7. **Database-Driven Settings** - Worker reads rate_limit, retry_count, parallel_limit, schedule_interval from DB
|
||||
8. **Orphan Job Cleanup** - Scheduler startup + periodic task resets stuck "running" jobs
|
||||
9. **Redis DB Mismatch Detection** - Workers log markers to detect configuration inconsistencies
|
||||
10. **Stale Job Recovery** - Automatic re-queuing of lost Celery tasks with duplicate prevention
|
||||
11. **Race Condition Guards** - Downloads check status before claiming to prevent duplicate processing
|
||||
|
||||
## Docker Services
|
||||
|
||||
@@ -226,7 +234,7 @@ Tasks are routed to separate queues for clean separation of concerns:
|
||||
```
|
||||
|
||||
**Task Routing Config** (in `celery_app.py`):
|
||||
- `scheduled_check`, `cleanup_old_downloads`, `update_storage_stats`, `reset_orphaned_jobs` → "scheduling" queue
|
||||
- `scheduled_check`, `cleanup_old_downloads`, `update_storage_stats`, `reset_orphaned_jobs`, `requeue_stale_jobs` → "scheduling" queue
|
||||
- `download_source`, `process_download` → "downloads" queue
|
||||
|
||||
## Common Operations
|
||||
@@ -302,7 +310,61 @@ Download workers do NOT perform these startup tasks to avoid duplicates.
|
||||
| Key | Default | Purpose |
|
||||
|-----|---------|---------|
|
||||
| `download.schedule_interval` | 28800 | Seconds between source checks (8 hours, applies to all sources) |
|
||||
| `download.rate_limit` | 3.0 | Seconds between gallery-dl requests |
|
||||
| `download.rate_limit` | 3.0 | Seconds between file downloads |
|
||||
| `download.parallel_limit` | 3 | Concurrent Celery workers (requires restart) |
|
||||
| `download.retry_count` | 3 | Max retries for failed downloads |
|
||||
| `download.crawl_depth` | 0 | gallery-dl post crawl depth |
|
||||
|
||||
**Note:** `sleep-request` (delay between HTTP requests during pagination) is automatically set to 1/4 of `rate_limit` (minimum 0.5s) to speed up archive checks while maintaining safe download pacing.
|
||||
|
||||
## Downloads Page Features
|
||||
|
||||
The Downloads view (`/downloads`) provides comprehensive download monitoring:
|
||||
|
||||
**Header Section:**
|
||||
- Live stats chips showing counts for Queued, Running, Completed, Failed
|
||||
- Refresh button to manually update data
|
||||
- Maintenance dropdown with recovery actions
|
||||
|
||||
**Filters:**
|
||||
- Status filter (queued, running, completed, failed, skipped)
|
||||
- Source filter showing "Subscription:Platform" format for easy identification
|
||||
- Date range filters (from/to)
|
||||
|
||||
**Table Columns:**
|
||||
- Status with color-coded chips and animated spinner for running jobs
|
||||
- Source showing "Subscription:Platform" label
|
||||
- URL (truncated with full URL on hover)
|
||||
- Error type (formatted for readability)
|
||||
- File count
|
||||
- Date created
|
||||
- Duration (calculated from started_at to completed_at)
|
||||
- Actions (retry for failed, details for all)
|
||||
|
||||
**Details Dialog:**
|
||||
- Full download metadata
|
||||
- Output log (stdout from gallery-dl)
|
||||
- Error log (stderr)
|
||||
- Retry button for failed downloads
|
||||
|
||||
**Maintenance Actions:**
|
||||
- **Reset Orphaned Running Jobs**: Resets jobs stuck in "running" status back to "queued"
|
||||
- **Re-queue Stale Queued Jobs**: Re-sends Celery tasks for jobs stuck in "queued" status
|
||||
|
||||
**Auto-Refresh:** Automatically refreshes every 30 seconds when there are running or queued jobs.
|
||||
|
||||
## Duplicate Download Prevention
|
||||
|
||||
Multiple layers prevent duplicate downloads for the same source:
|
||||
|
||||
1. **Re-queue Functions** (`maintenance.py`):
|
||||
- Track `queued_source_ids` set during re-queuing
|
||||
- Skip sources that already have a task re-queued
|
||||
- Mark duplicate jobs as SKIPPED with descriptive error message
|
||||
|
||||
2. **Download Task** (`downloads.py`):
|
||||
- Check download status is still QUEUED before claiming
|
||||
- Check if source already has a RUNNING download
|
||||
- Mark duplicates as SKIPPED to prevent reprocessing
|
||||
|
||||
3. **Scheduler Logic** (`downloads.py`):
|
||||
- `scheduled_check` skips sources with existing QUEUED or RUNNING downloads
|
||||
|
||||
Reference in New Issue
Block a user