polish downloads view

docker-compose.dev.yml

@@ -8,6 +8,7 @@
 #   - Volume mounts for live code reloading
 #   - Debug logging enabled
 #   - Database and Redis ports exposed for local tools
+#   - Separate scheduler and worker services (matches production)
 
 services:
   # Combined backend API + frontend static files
@@ -18,7 +19,7 @@ services:
     environment:
       - TZ=${TZ:-America/New_York}
       - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
-      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
       - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
       - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3}
       - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0}
@@ -35,14 +36,14 @@ services:
         condition: service_started
     restart: unless-stopped
 
-  # Celery worker for background tasks
+  # Celery download workers - only process download tasks (heavy work)
-  celery:
+  worker:
     build: .
-    command: celery -A app.tasks.celery_app worker -B --loglevel=debug
+    command: celery -A app.tasks.celery_app worker --queues=downloads --loglevel=debug
     environment:
       - TZ=${TZ:-America/New_York}
       - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
-      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
       - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
       - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3}
       - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0}
@@ -59,6 +60,26 @@ services:
         condition: service_started
     restart: unless-stopped
 
+  # Celery scheduler - runs beat + handles lightweight scheduling tasks
+  scheduler:
+    build: .
+    command: celery -A app.tasks.celery_app worker --queues=scheduling --beat --loglevel=debug --concurrency=1
+    environment:
+      - TZ=${TZ:-America/New_York}
+      - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
+      - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
+      - LOG_LEVEL=DEBUG
+      - DEBUG=true
+    volumes:
+      - ./backend/app:/app/app:ro
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    restart: unless-stopped
+
   # PostgreSQL database
   db:
     image: postgres:15-alpine

frontend/src/services/api.js

@@ -38,6 +38,7 @@ export const downloadsApi = {
   stats: (params = {}) => api.get('/downloads/stats', { params }),
   recentActivity: (params = {}) => api.get('/downloads/recent-activity', { params }),
   resetOrphaned: (thresholdMinutes = 0) => api.post('/downloads/reset-orphaned', { threshold_minutes: thresholdMinutes }),
+  requeueStale: (thresholdMinutes = 0) => api.post('/downloads/requeue-stale', { threshold_minutes: thresholdMinutes }),
 }
 
 // Credentials API

frontend/src/stores/downloads.js

@@ -43,6 +43,14 @@ export const useDownloadsStore = defineStore('downloads', () => {
     return await downloadsApi.retry(id)
   }
 
+  async function resetOrphaned(thresholdMinutes = 0) {
+    return await downloadsApi.resetOrphaned(thresholdMinutes)
+  }
+
+  async function requeueStale(thresholdMinutes = 0) {
+    return await downloadsApi.requeueStale(thresholdMinutes)
+  }
+
   return {
     downloads,
     recentActivity,
@@ -55,5 +63,7 @@ export const useDownloadsStore = defineStore('downloads', () => {
     fetchRecentActivity,
     fetchStats,
     retryDownload,
+    resetOrphaned,
+    requeueStale,
   }
 })

frontend/src/views/Downloads.vue

@@ -1,5 +1,65 @@
 <template>
   <div>
+    <!-- Header with Stats and Actions -->
+    <v-row class="mb-4" align="center">
+      <v-col cols="12" md="6">
+        <h1 class="text-h5 mb-2">Downloads</h1>
+        <div v-if="downloadsStore.stats" class="d-flex flex-wrap ga-2">
+          <v-chip color="warning" variant="tonal" size="small">
+            <v-icon start size="small">mdi-clock-outline</v-icon>
+            {{ downloadsStore.stats.queued || 0 }} Queued
+          </v-chip>
+          <v-chip color="info" variant="tonal" size="small">
+            <v-icon start size="small" class="mdi-spin">mdi-loading</v-icon>
+            {{ downloadsStore.stats.running || 0 }} Running
+          </v-chip>
+          <v-chip color="success" variant="tonal" size="small">
+            <v-icon start size="small">mdi-check-circle</v-icon>
+            {{ downloadsStore.stats.completed || 0 }} Completed
+          </v-chip>
+          <v-chip color="error" variant="tonal" size="small">
+            <v-icon start size="small">mdi-alert-circle</v-icon>
+            {{ downloadsStore.stats.failed || 0 }} Failed
+          </v-chip>
+        </div>
+      </v-col>
+      <v-col cols="12" md="6" class="d-flex justify-end ga-2 flex-wrap">
+        <v-btn
+          variant="outlined"
+          size="small"
+          @click="refresh"
+          :loading="downloadsStore.loading"
+        >
+          <v-icon start>mdi-refresh</v-icon>
+          Refresh
+        </v-btn>
+        <v-menu>
+          <template v-slot:activator="{ props }">
+            <v-btn
+              variant="outlined"
+              size="small"
+              color="warning"
+              v-bind="props"
+            >
+              <v-icon start>mdi-wrench</v-icon>
+              Maintenance
+              <v-icon end>mdi-chevron-down</v-icon>
+            </v-btn>
+          </template>
+          <v-list density="compact">
+            <v-list-item @click="resetOrphanedJobs">
+              <v-list-item-title>Reset Orphaned Running Jobs</v-list-item-title>
+              <v-list-item-subtitle>Reset jobs stuck in "running" status</v-list-item-subtitle>
+            </v-list-item>
+            <v-list-item @click="requeueStaleJobs">
+              <v-list-item-title>Re-queue Stale Queued Jobs</v-list-item-title>
+              <v-list-item-subtitle>Re-send lost Celery tasks for queued jobs</v-list-item-subtitle>
+            </v-list-item>
+          </v-list>
+        </v-menu>
+      </v-col>
+    </v-row>
+
     <!-- Filters -->
     <v-row class="mb-4">
       <v-col cols="12" md="3">
@@ -59,13 +119,19 @@
             size="small"
             variant="tonal"
           >
-            <v-icon start size="small">{{ getStatusIcon(item.status) }}</v-icon>
+            <v-icon start size="small" :class="{ 'mdi-spin': item.status === 'running' }">
+              {{ getStatusIcon(item.status) }}
+            </v-icon>
             {{ item.status }}
           </v-chip>
         </template>
 
+        <template v-slot:item.source="{ item }">
+          <span class="text-caption">{{ getSourceLabel(item.source_id) }}</span>
+        </template>
+
         <template v-slot:item.url="{ item }">
-          <span :title="item.url">{{ truncateUrl(item.url) }}</span>
+          <span :title="item.url" class="text-caption">{{ truncateUrl(item.url) }}</span>
         </template>
 
         <template v-slot:item.error_type="{ item }">
@@ -118,6 +184,13 @@
             <v-tooltip activator="parent">Details</v-tooltip>
           </v-btn>
         </template>
+
+        <template v-slot:no-data>
+          <div class="text-center pa-4">
+            <v-icon size="48" color="grey-lighten-1">mdi-download-off</v-icon>
+            <p class="text-grey mt-2">No downloads found</p>
+          </div>
+        </template>
       </v-data-table>
     </v-card>
 
@@ -139,17 +212,21 @@
           <v-table density="compact">
             <tbody>
               <tr>
-                <td class="font-weight-bold" width="150">URL</td>
+                <td class="font-weight-bold" width="150">Source</td>
-                <td style="word-break: break-all">{{ selectedDownload.url }}</td>
+                <td>{{ getSourceLabel(selectedDownload.source_id) }}</td>
               </tr>
               <tr>
-                <td class="font-weight-bold">Source</td>
+                <td class="font-weight-bold">URL</td>
-                <td>{{ getSourceLabel(selectedDownload.source_id) }}</td>
+                <td style="word-break: break-all">{{ selectedDownload.url }}</td>
               </tr>
               <tr>
                 <td class="font-weight-bold">Files Downloaded</td>
                 <td>{{ selectedDownload.file_count }}</td>
               </tr>
+              <tr>
+                <td class="font-weight-bold">Created</td>
+                <td>{{ formatDate(selectedDownload.created_at) }}</td>
+              </tr>
               <tr>
                 <td class="font-weight-bold">Started</td>
                 <td>{{ formatDate(selectedDownload.started_at) }}</td>
@@ -202,7 +279,7 @@
 </template>
 
 <script setup>
-import { ref, computed, onMounted, watch } from 'vue'
+import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
 import { useDownloadsStore } from '../stores/downloads'
 import { useSourcesStore } from '../stores/sources'
 import { useNotificationStore } from '../stores/notifications'
@@ -218,9 +295,10 @@ const filterToDate = ref(null)
 const detailsDialog = ref(false)
 const selectedDownload = ref(null)
 const retryingIds = ref([])
+let refreshInterval = null
 
 const statusOptions = [
-  { title: 'Pending', value: 'pending' },
+  { title: 'Queued', value: 'queued' },
   { title: 'Running', value: 'running' },
   { title: 'Completed', value: 'completed' },
   { title: 'Failed', value: 'failed' },
@@ -236,24 +314,45 @@ const sourceOptions = computed(() =>
 
 const headers = [
   { title: 'Status', key: 'status', width: 120 },
+  { title: 'Source', key: 'source', width: 180 },
   { title: 'URL', key: 'url' },
-  { title: 'Error', key: 'error_type', width: 150 },
+  { title: 'Error', key: 'error_type', width: 130 },
-  { title: 'Files', key: 'file_count', width: 100 },
+  { title: 'Files', key: 'file_count', width: 80 },
-  { title: 'Date', key: 'created_at', width: 180 },
+  { title: 'Date', key: 'created_at', width: 160 },
-  { title: 'Duration', key: 'duration', width: 100 },
+  { title: 'Duration', key: 'duration', width: 90 },
-  { title: 'Actions', key: 'actions', width: 100, sortable: false },
+  { title: 'Actions', key: 'actions', width: 90, sortable: false },
 ]
 
 onMounted(() => {
-  // Load data in background - don't block UI rendering
+  refresh()
-  loadDownloads()
   sourcesStore.fetchSources()
+
+  // Auto-refresh every 30 seconds if there are running/queued jobs
+  refreshInterval = setInterval(() => {
+    if (downloadsStore.stats?.running > 0 || downloadsStore.stats?.queued > 0) {
+      loadDownloads()
+      downloadsStore.fetchStats()
+    }
+  }, 30000)
+})
+
+onUnmounted(() => {
+  if (refreshInterval) {
+    clearInterval(refreshInterval)
+  }
 })
 
 watch([filterStatus, filterSourceId, filterFromDate, filterToDate], () => {
   loadDownloads()
 })
 
+async function refresh() {
+  await Promise.all([
+    loadDownloads(),
+    downloadsStore.fetchStats()
+  ])
+}
+
 async function loadDownloads() {
   const params = {}
   if (filterStatus.value) params.status = filterStatus.value
@@ -263,12 +362,32 @@ async function loadDownloads() {
   await downloadsStore.fetchDownloads(params)
 }
 
+async function resetOrphanedJobs() {
+  try {
+    const result = await downloadsStore.resetOrphaned()
+    notifications.success(`Reset ${result.data.reset_count} orphaned jobs`)
+    await refresh()
+  } catch (error) {
+    notifications.error(`Failed to reset orphaned jobs: ${error.message}`)
+  }
+}
+
+async function requeueStaleJobs() {
+  try {
+    const result = await downloadsStore.requeueStale()
+    notifications.success(`Re-queued ${result.data.requeued_count} stale jobs`)
+    await refresh()
+  } catch (error) {
+    notifications.error(`Failed to re-queue stale jobs: ${error.message}`)
+  }
+}
+
 function getStatusIcon(status) {
   const icons = {
     completed: 'mdi-check-circle',
     failed: 'mdi-alert-circle',
     running: 'mdi-loading',
-    pending: 'mdi-clock-outline',
+    queued: 'mdi-clock-outline',
     skipped: 'mdi-skip-next',
   }
   return icons[status] || 'mdi-help-circle'
@@ -279,7 +398,7 @@ function getStatusColor(status) {
     completed: 'success',
     failed: 'error',
     running: 'info',
-    pending: 'warning',
+    queued: 'warning',
     skipped: 'grey',
   }
   return colors[status] || 'grey'
@@ -307,8 +426,8 @@ function formatErrorType(errorType) {
 
 function truncateUrl(url) {
   if (!url) return ''
-  if (url.length <= 50) return url
+  if (url.length <= 40) return url
-  return url.substring(0, 47) + '...'
+  return url.substring(0, 37) + '...'
 }
 
 function getSourceLabel(sourceId) {
@@ -330,7 +449,7 @@ async function retryDownload(download) {
   try {
     await downloadsStore.retryDownload(download.id)
     notifications.success('Download queued for retry')
-    await loadDownloads()
+    await refresh()
   } catch (error) {
     notifications.error(`Failed to retry: ${error.message}`)
   } finally {
@@ -338,3 +457,14 @@ async function retryDownload(download) {
   }
 }
 </script>
+
+<style scoped>
+.mdi-spin {
+  animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+  from { transform: rotate(0deg); }
+  to { transform: rotate(360deg); }
+}
+</style>