From 097ab16c50a20caed941a8b8871264d71b6be978 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bvandeusen@cognitheurge.com>
Date: Fri, 30 Jan 2026 00:01:45 -0500
Subject: [PATCH] more UI polish and schedule safe guards and optimization

---
 .claude/settings.local.json          |  3 +-
 backend/app/services/gallery_dl.py   | 11 ++--
 backend/app/tasks/downloads.py       |  5 +-
 frontend/src/views/Dashboard.vue     | 38 ++++++++++++-
 frontend/src/views/Subscriptions.vue | 17 ++++--
 summary.md                           | 84 ++++++++++++++++++++++++----
 6 files changed, 133 insertions(+), 25 deletions(-)

diff --git a/.claude/settings.local.json b/.claude/settings.local.json
index edca24f..8278838 100644
--- a/.claude/settings.local.json
+++ b/.claude/settings.local.json
@@ -13,7 +13,8 @@
       "Bash(where:*)",
       "Bash(where npm:*)",
       "Bash(powershell -Command \"Copy-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\extension\\\\web-ext-artifacts\\\\be3706e9e89640a5b70b-1.0.0.xpi'' ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' -Force; Get-Item ''c:\\\\Users\\\\bvandeusen\\\\Nextcloud\\\\Projects\\\\GallerySubscriber\\\\gallerysubscriber-signed.xpi'' | Select-Object Name, Length\")",
-      "WebFetch(domain:github.com)"
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:raw.githubusercontent.com)"
     ]
   }
 }
diff --git a/backend/app/services/gallery_dl.py b/backend/app/services/gallery_dl.py
index 0171a71..06a2619 100644
--- a/backend/app/services/gallery_dl.py
+++ b/backend/app/services/gallery_dl.py
@@ -154,10 +154,10 @@ class GalleryDLService:
             # Automatic infinite scroll pagination is built-in
         },
         "hentaifoundry": {
-            "content_types": ["pictures"],
+            "content_types": ["all"],  # Download all content types
             "directory": [],  # Flat structure within platform folder
             "filename": "{category}_{index:>03}_{title[:50]}.{extension}",
-            "include": "all",  # Include all content types (pictures, scraps, stories)
+            "include": "all",  # Include pictures, scraps, stories - complete content
             # Automatic pagination through all pages (25 items/page)
         },
         "discord": {
@@ -206,7 +206,8 @@ class GalleryDLService:
                 "archive": str(Path(self.settings.config_path) / "archive.sqlite3"),
                 "skip": True,
                 "sleep": self._rate_limit,
-                "sleep-request": max(1.0, self._rate_limit / 2),
+                # Faster request rate for pagination/checking (1/4 of rate_limit, min 0.5s)
+                "sleep-request": max(0.5, self._rate_limit / 4),
                 "retries": 3,
                 "timeout": 30.0,
                 "verify": True,
@@ -624,7 +625,7 @@ class GalleryDLService:
         if platform == "patreon":
             return ["images", "image_large", "attachments", "postfile", "content"]
         elif platform == "hentaifoundry":
-            return ["pictures", "stories"]
+            return ["all", "pictures", "scraps", "stories"]
         elif platform == "subscribestar":
             return ["all"]  # No granular control
         elif platform == "discord":
@@ -643,5 +644,5 @@ class GalleryDLService:
             "directory_pattern": defaults.get("directory"),
             "filename_pattern": defaults.get("filename"),
             "sleep": self._rate_limit,
-            "sleep_request": max(1.0, self._rate_limit / 2),
+            "sleep_request": max(0.5, self._rate_limit / 4),
         }
diff --git a/backend/app/tasks/downloads.py b/backend/app/tasks/downloads.py
index 280ed9e..43a121d 100644
--- a/backend/app/tasks/downloads.py
+++ b/backend/app/tasks/downloads.py
@@ -287,9 +287,10 @@ async def _download_source_async(source_id: int, download_id: int = None) -> dic
             # Update download record with results
             download.completed_at = utcnow()
             download.file_count = dl_result.files_downloaded
+            # Store full logs - cleanup task will age out old records
             download.metadata_ = {
-                "stdout": dl_result.stdout[:10000] if dl_result.stdout else None,
-                "stderr": dl_result.stderr[:5000] if dl_result.stderr else None,
+                "stdout": dl_result.stdout if dl_result.stdout else None,
+                "stderr": dl_result.stderr if dl_result.stderr else None,
                 "duration_seconds": dl_result.duration_seconds,
             }
 
diff --git a/frontend/src/views/Dashboard.vue b/frontend/src/views/Dashboard.vue
index 1bea648..536b3cf 100644
--- a/frontend/src/views/Dashboard.vue
+++ b/frontend/src/views/Dashboard.vue
@@ -95,6 +95,23 @@
               Reset Stuck ({{ stuckRunningCount }})
             </v-btn>
             <v-spacer />
+            <!-- Credential Status -->
+            <div class="d-flex align-center ga-2">
+              <v-icon size="small" class="mr-1">mdi-key</v-icon>
+              <span class="text-body-2 text-grey mr-2">Auth:</span>
+              <v-chip
+                v-for="platform in supportedPlatforms"
+                :key="platform"
+                :color="credentialsStore.hasPlatformCredentials(platform) ? 'success' : 'grey'"
+                :variant="credentialsStore.hasPlatformCredentials(platform) ? 'tonal' : 'outlined'"
+                size="x-small"
+                :to="credentialsStore.hasPlatformCredentials(platform) ? undefined : '/credentials'"
+              >
+                <v-icon start size="x-small">{{ getPlatformIcon(platform) }}</v-icon>
+                {{ platform }}
+              </v-chip>
+            </div>
+            <v-divider vertical class="mx-2" />
             <div class="text-body-2 text-grey d-flex align-center">
               <v-icon size="small" class="mr-1">mdi-harddisk</v-icon>
               Storage: {{ storageStats?.total_size_formatted || 'Calculating...' }}
@@ -267,9 +284,10 @@
                     <v-btn
                       size="small"
                       variant="text"
-                      :to="`/sources/${source.id}/edit`"
+                      to="/subscriptions"
                     >
-                      Edit
+                      View
+                      <v-tooltip activator="parent">View in Subscriptions</v-tooltip>
                     </v-btn>
                   </td>
                 </tr>
@@ -290,13 +308,18 @@
 import { ref, computed, onMounted, onUnmounted } from 'vue'
 import { useSourcesStore } from '../stores/sources'
 import { useDownloadsStore } from '../stores/downloads'
+import { useCredentialsStore } from '../stores/credentials'
 import { useNotificationStore } from '../stores/notifications'
 import { settingsApi, downloadsApi } from '../services/api'
 
 const sourcesStore = useSourcesStore()
 const downloadsStore = useDownloadsStore()
+const credentialsStore = useCredentialsStore()
 const notifications = useNotificationStore()
 
+// Supported platforms for credential status
+const supportedPlatforms = ['patreon', 'subscribestar', 'hentaifoundry', 'discord']
+
 // State
 const checkingAll = ref(false)
 const retryingAll = ref(false)
@@ -337,6 +360,7 @@ function loadDashboardData() {
   // Fire off all requests in parallel, don't block UI
   // Each section handles its own loading state
   sourcesStore.fetchSources().catch(e => console.error('Failed to fetch sources:', e))
+  credentialsStore.fetchCredentials().catch(e => console.error('Failed to fetch credentials:', e))
 
   downloadsStore.fetchStats()
     .catch(e => console.error('Failed to fetch stats:', e))
@@ -490,6 +514,16 @@ function getStatusIcon(status) {
   return icons[status] || 'mdi-help-circle'
 }
 
+function getPlatformIcon(platform) {
+  const icons = {
+    patreon: 'mdi-patreon',
+    subscribestar: 'mdi-star',
+    hentaifoundry: 'mdi-palette',
+    discord: 'mdi-discord',
+  }
+  return icons[platform] || 'mdi-web'
+}
+
 function getStatusColor(status) {
   const colors = {
     completed: 'success',
diff --git a/frontend/src/views/Subscriptions.vue b/frontend/src/views/Subscriptions.vue
index 9bc64b8..bdf456b 100644
--- a/frontend/src/views/Subscriptions.vue
+++ b/frontend/src/views/Subscriptions.vue
@@ -197,7 +197,7 @@
                     </td>
                     <td>
                       <div>{{ formatDate(source.last_check) }}</div>
-                      <div v-if="source.enabled && source.last_check" class="text-caption text-grey">
+                      <div v-if="source.enabled" class="text-caption text-grey">
                         Next: {{ formatNextCheck(source) }}
                       </div>
                     </td>
@@ -323,15 +323,22 @@
 </template>
 
 <script setup>
-import { ref, watch, onMounted } from 'vue'
+import { ref, watch, onMounted, computed } from 'vue'
 import { useSubscriptionsStore } from '../stores/subscriptions'
 import { useCredentialsStore } from '../stores/credentials'
+import { useSettingsStore } from '../stores/settings'
 import { useNotificationStore } from '../stores/notifications'
 
 const subscriptionsStore = useSubscriptionsStore()
 const credentialsStore = useCredentialsStore()
+const settingsStore = useSettingsStore()
 const notifications = useNotificationStore()
 
+// Get global schedule interval from settings (default 8 hours)
+const scheduleInterval = computed(() =>
+  settingsStore.settings['download.schedule_interval'] || 28800
+)
+
 const searchQuery = ref('')
 const filterEnabled = ref(null)
 const expandedIds = ref([])
@@ -387,6 +394,7 @@ const headers = [
 onMounted(() => {
   loadSubscriptions()
   credentialsStore.fetchCredentials()
+  settingsStore.fetchSettings()  // For schedule_interval
 })
 
 watch([searchQuery, filterEnabled], () => {
@@ -435,9 +443,10 @@ function formatDate(dateStr) {
 }
 
 function formatNextCheck(source) {
-  if (!source.last_check || !source.check_interval) return 'Unknown'
+  if (!source.last_check) return 'Due now'
   const lastCheck = new Date(source.last_check)
-  const nextCheck = new Date(lastCheck.getTime() + source.check_interval * 1000)
+  // Use global schedule_interval from settings
+  const nextCheck = new Date(lastCheck.getTime() + scheduleInterval.value * 1000)
   const now = new Date()
 
   if (nextCheck <= now) return 'Due now'
diff --git a/summary.md b/summary.md
index 4e4a958..21d411f 100644
--- a/summary.md
+++ b/summary.md
@@ -1,6 +1,6 @@
 # GallerySubscriber - Project Summary
 
-**Updated:** 2026-01-29 (task routing, scheduler/worker separation, 8-hour default interval)
+**Updated:** 2026-01-29 (Redis DB config, mismatch detection, stale job recovery, Downloads UI improvements)
 
 ## Overview
 
@@ -152,24 +152,29 @@ Download (1) ──→ (many) ContentItem
 
 ## Supported Platforms
 
-| Platform | Auth Type | Content Types |
-|----------|-----------|---------------|
-| Patreon | Cookies | images, attachments, posts |
-| SubscribeStar | Cookies | images, attachments |
-| SubscribeStar Adult | Cookies | images, attachments |
-| Hentai Foundry | Cookies | images |
-| Discord | Token | messages, attachments |
+| Platform | Auth Type | Content Types | Pagination |
+|----------|-----------|---------------|------------|
+| Patreon | Cookies | images, attachments, posts, content | Cursor-based (complete history) |
+| SubscribeStar | Cookies | images, attachments | Infinite scroll (complete) |
+| SubscribeStar Adult | Cookies | images, attachments | Infinite scroll (complete) |
+| Hentai Foundry | Cookies | pictures, scraps, stories | All pages (complete) |
+| Discord | Token | messages, attachments, embeds | All threads included |
+
+**Note:** All platforms fetch complete history by default - no depth limits. Gallery-dl paginates through ALL available content.
 
 ## Configuration
 
 **Environment Variables (.env):**
 - `DB_PASSWORD` (required) - PostgreSQL password
 - `SECRET_KEY` (required) - Encryption key (32+ chars)
+- `REDIS_DB` (default: 0) - Redis database number (must be same for all services)
 - `DOWNLOAD_PARALLEL_LIMIT` (default: 3) - Concurrent downloads
 - `DOWNLOAD_RATE_LIMIT` (default: 3.0) - Seconds between requests
 - `LOG_LEVEL` (default: INFO)
 - `TZ` (default: America/New_York)
 
+**IMPORTANT:** All services (app, scheduler, worker) must use the same Redis database. A mismatch causes Celery tasks to be lost (jobs stuck in "queued" status). The system includes automatic detection and recovery for this scenario.
+
 ## Key Patterns
 
 1. **Async/Await** - Quart + asyncpg for non-blocking I/O
@@ -180,6 +185,9 @@ Download (1) ──→ (many) ContentItem
 6. **Task Routing** - Separate queues for scheduling vs download tasks
 7. **Database-Driven Settings** - Worker reads rate_limit, retry_count, parallel_limit, schedule_interval from DB
 8. **Orphan Job Cleanup** - Scheduler startup + periodic task resets stuck "running" jobs
+9. **Redis DB Mismatch Detection** - Workers log markers to detect configuration inconsistencies
+10. **Stale Job Recovery** - Automatic re-queuing of lost Celery tasks with duplicate prevention
+11. **Race Condition Guards** - Downloads check status before claiming to prevent duplicate processing
 
 ## Docker Services
 
@@ -226,7 +234,7 @@ Tasks are routed to separate queues for clean separation of concerns:
 ```
 
 **Task Routing Config** (in `celery_app.py`):
-- `scheduled_check`, `cleanup_old_downloads`, `update_storage_stats`, `reset_orphaned_jobs` → "scheduling" queue
+- `scheduled_check`, `cleanup_old_downloads`, `update_storage_stats`, `reset_orphaned_jobs`, `requeue_stale_jobs` → "scheduling" queue
 - `download_source`, `process_download` → "downloads" queue
 
 ## Common Operations
@@ -302,7 +310,61 @@ Download workers do NOT perform these startup tasks to avoid duplicates.
 | Key | Default | Purpose |
 |-----|---------|---------|
 | `download.schedule_interval` | 28800 | Seconds between source checks (8 hours, applies to all sources) |
-| `download.rate_limit` | 3.0 | Seconds between gallery-dl requests |
+| `download.rate_limit` | 3.0 | Seconds between file downloads |
 | `download.parallel_limit` | 3 | Concurrent Celery workers (requires restart) |
 | `download.retry_count` | 3 | Max retries for failed downloads |
-| `download.crawl_depth` | 0 | gallery-dl post crawl depth |
+
+**Note:** `sleep-request` (delay between HTTP requests during pagination) is automatically set to 1/4 of `rate_limit` (minimum 0.5s) to speed up archive checks while maintaining safe download pacing.
+
+## Downloads Page Features
+
+The Downloads view (`/downloads`) provides comprehensive download monitoring:
+
+**Header Section:**
+- Live stats chips showing counts for Queued, Running, Completed, Failed
+- Refresh button to manually update data
+- Maintenance dropdown with recovery actions
+
+**Filters:**
+- Status filter (queued, running, completed, failed, skipped)
+- Source filter showing "Subscription:Platform" format for easy identification
+- Date range filters (from/to)
+
+**Table Columns:**
+- Status with color-coded chips and animated spinner for running jobs
+- Source showing "Subscription:Platform" label
+- URL (truncated with full URL on hover)
+- Error type (formatted for readability)
+- File count
+- Date created
+- Duration (calculated from started_at to completed_at)
+- Actions (retry for failed, details for all)
+
+**Details Dialog:**
+- Full download metadata
+- Output log (stdout from gallery-dl)
+- Error log (stderr)
+- Retry button for failed downloads
+
+**Maintenance Actions:**
+- **Reset Orphaned Running Jobs**: Resets jobs stuck in "running" status back to "queued"
+- **Re-queue Stale Queued Jobs**: Re-sends Celery tasks for jobs stuck in "queued" status
+
+**Auto-Refresh:** Automatically refreshes every 30 seconds when there are running or queued jobs.
+
+## Duplicate Download Prevention
+
+Multiple layers prevent duplicate downloads for the same source:
+
+1. **Re-queue Functions** (`maintenance.py`):
+   - Track `queued_source_ids` set during re-queuing
+   - Skip sources that already have a task re-queued
+   - Mark duplicate jobs as SKIPPED with descriptive error message
+
+2. **Download Task** (`downloads.py`):
+   - Check download status is still QUEUED before claiming
+   - Check if source already has a RUNNING download
+   - Mark duplicates as SKIPPED to prevent reprocessing
+
+3. **Scheduler Logic** (`downloads.py`):
+   - `scheduled_check` skips sources with existing QUEUED or RUNNING downloads