polish downloads view

2026-01-29 19:42:14 -05:00
parent fcccfb24ee
commit c1fe8148b8
4 changed files with 187 additions and 25 deletions
@@ -8,6 +8,7 @@
 #   - Volume mounts for live code reloading
 #   - Debug logging enabled
 #   - Database and Redis ports exposed for local tools
+#   - Separate scheduler and worker services (matches production)

 services:
  # Combined backend API + frontend static files
@@ -18,7 +19,7 @@ services:
    environment:
      - TZ=${TZ:-America/New_York}
      - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
-      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
      - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
      - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3}
      - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0}
@@ -35,14 +36,14 @@ services:
        condition: service_started
    restart: unless-stopped

-  # Celery worker for background tasks
-  celery:
+  # Celery download workers - only process download tasks (heavy work)
+  worker:
    build: .
-    command: celery -A app.tasks.celery_app worker -B --loglevel=debug
+    command: celery -A app.tasks.celery_app worker --queues=downloads --loglevel=debug
    environment:
      - TZ=${TZ:-America/New_York}
      - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
-      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/0
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
      - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
      - DOWNLOAD_PARALLEL_LIMIT=${DOWNLOAD_PARALLEL_LIMIT:-3}
      - DOWNLOAD_RATE_LIMIT=${DOWNLOAD_RATE_LIMIT:-3.0}
@@ -59,6 +60,26 @@ services:
        condition: service_started
    restart: unless-stopped

+  # Celery scheduler - runs beat + handles lightweight scheduling tasks
+  scheduler:
+    build: .
+    command: celery -A app.tasks.celery_app worker --queues=scheduling --beat --loglevel=debug --concurrency=1
+    environment:
+      - TZ=${TZ:-America/New_York}
+      - DATABASE_URL=postgresql://${DB_USER:-gdl}:${DB_PASSWORD:-devpassword}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-gallery_subscriber}
+      - REDIS_URL=redis://${REDIS_HOST:-redis}:${REDIS_PORT:-6379}/${REDIS_DB:-0}
+      - SECRET_KEY=${SECRET_KEY:-dev-secret-key-not-for-production}
+      - LOG_LEVEL=DEBUG
+      - DEBUG=true
+    volumes:
+      - ./backend/app:/app/app:ro
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    restart: unless-stopped
+
  # PostgreSQL database
  db:
    image: postgres:15-alpine
@@ -38,6 +38,7 @@ export const downloadsApi = {
  stats: (params = {}) => api.get('/downloads/stats', { params }),
  recentActivity: (params = {}) => api.get('/downloads/recent-activity', { params }),
  resetOrphaned: (thresholdMinutes = 0) => api.post('/downloads/reset-orphaned', { threshold_minutes: thresholdMinutes }),
+  requeueStale: (thresholdMinutes = 0) => api.post('/downloads/requeue-stale', { threshold_minutes: thresholdMinutes }),
 }

 // Credentials API
@@ -43,6 +43,14 @@ export const useDownloadsStore = defineStore('downloads', () => {
    return await downloadsApi.retry(id)
  }

+  async function resetOrphaned(thresholdMinutes = 0) {
+    return await downloadsApi.resetOrphaned(thresholdMinutes)
+  }
+
+  async function requeueStale(thresholdMinutes = 0) {
+    return await downloadsApi.requeueStale(thresholdMinutes)
+  }
+
  return {
    downloads,
    recentActivity,
@@ -55,5 +63,7 @@ export const useDownloadsStore = defineStore('downloads', () => {
    fetchRecentActivity,
    fetchStats,
    retryDownload,
+    resetOrphaned,
+    requeueStale,
  }
 })
@@ -1,5 +1,65 @@
 <template>
  <div>
+    <!-- Header with Stats and Actions -->
+    <v-row class="mb-4" align="center">
+      <v-col cols="12" md="6">
+        <h1 class="text-h5 mb-2">Downloads</h1>
+        <div v-if="downloadsStore.stats" class="d-flex flex-wrap ga-2">
+          <v-chip color="warning" variant="tonal" size="small">
+            <v-icon start size="small">mdi-clock-outline</v-icon>
+            {{ downloadsStore.stats.queued || 0 }} Queued
+          </v-chip>
+          <v-chip color="info" variant="tonal" size="small">
+            <v-icon start size="small" class="mdi-spin">mdi-loading</v-icon>
+            {{ downloadsStore.stats.running || 0 }} Running
+          </v-chip>
+          <v-chip color="success" variant="tonal" size="small">
+            <v-icon start size="small">mdi-check-circle</v-icon>
+            {{ downloadsStore.stats.completed || 0 }} Completed
+          </v-chip>
+          <v-chip color="error" variant="tonal" size="small">
+            <v-icon start size="small">mdi-alert-circle</v-icon>
+            {{ downloadsStore.stats.failed || 0 }} Failed
+          </v-chip>
+        </div>
+      </v-col>
+      <v-col cols="12" md="6" class="d-flex justify-end ga-2 flex-wrap">
+        <v-btn
+          variant="outlined"
+          size="small"
+          @click="refresh"
+          :loading="downloadsStore.loading"
+        >
+          <v-icon start>mdi-refresh</v-icon>
+          Refresh
+        </v-btn>
+        <v-menu>
+          <template v-slot:activator="{ props }">
+            <v-btn
+              variant="outlined"
+              size="small"
+              color="warning"
+              v-bind="props"
+            >
+              <v-icon start>mdi-wrench</v-icon>
+              Maintenance
+              <v-icon end>mdi-chevron-down</v-icon>
+            </v-btn>
+          </template>
+          <v-list density="compact">
+            <v-list-item @click="resetOrphanedJobs">
+              <v-list-item-title>Reset Orphaned Running Jobs</v-list-item-title>
+              <v-list-item-subtitle>Reset jobs stuck in "running" status</v-list-item-subtitle>
+            </v-list-item>
+            <v-list-item @click="requeueStaleJobs">
+              <v-list-item-title>Re-queue Stale Queued Jobs</v-list-item-title>
+              <v-list-item-subtitle>Re-send lost Celery tasks for queued jobs</v-list-item-subtitle>
+            </v-list-item>
+          </v-list>
+        </v-menu>
+      </v-col>
+    </v-row>
+
    <!-- Filters -->
    <v-row class="mb-4">
      <v-col cols="12" md="3">
@@ -59,13 +119,19 @@
            size="small"
            variant="tonal"
          >
-            <v-icon start size="small">{{ getStatusIcon(item.status) }}</v-icon>
+            <v-icon start size="small" :class="{ 'mdi-spin': item.status === 'running' }">
+              {{ getStatusIcon(item.status) }}
+            </v-icon>
            {{ item.status }}
          </v-chip>
        </template>

+        <template v-slot:item.source="{ item }">
+          <span class="text-caption">{{ getSourceLabel(item.source_id) }}</span>
+        </template>
+
        <template v-slot:item.url="{ item }">
-          <span :title="item.url">{{ truncateUrl(item.url) }}</span>
+          <span :title="item.url" class="text-caption">{{ truncateUrl(item.url) }}</span>
        </template>

        <template v-slot:item.error_type="{ item }">
@@ -118,6 +184,13 @@
            <v-tooltip activator="parent">Details</v-tooltip>
          </v-btn>
        </template>
+
+        <template v-slot:no-data>
+          <div class="text-center pa-4">
+            <v-icon size="48" color="grey-lighten-1">mdi-download-off</v-icon>
+            <p class="text-grey mt-2">No downloads found</p>
+          </div>
+        </template>
      </v-data-table>
    </v-card>

@@ -139,17 +212,21 @@
          <v-table density="compact">
            <tbody>
              <tr>
-                <td class="font-weight-bold" width="150">URL</td>
-                <td style="word-break: break-all">{{ selectedDownload.url }}</td>
+                <td class="font-weight-bold" width="150">Source</td>
+                <td>{{ getSourceLabel(selectedDownload.source_id) }}</td>
              </tr>
              <tr>
-                <td class="font-weight-bold">Source</td>
-                <td>{{ getSourceLabel(selectedDownload.source_id) }}</td>
+                <td class="font-weight-bold">URL</td>
+                <td style="word-break: break-all">{{ selectedDownload.url }}</td>
              </tr>
              <tr>
                <td class="font-weight-bold">Files Downloaded</td>
                <td>{{ selectedDownload.file_count }}</td>
              </tr>
+              <tr>
+                <td class="font-weight-bold">Created</td>
+                <td>{{ formatDate(selectedDownload.created_at) }}</td>
+              </tr>
              <tr>
                <td class="font-weight-bold">Started</td>
                <td>{{ formatDate(selectedDownload.started_at) }}</td>
@@ -202,7 +279,7 @@
 </template>

 <script setup>
-import { ref, computed, onMounted, watch } from 'vue'
+import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
 import { useDownloadsStore } from '../stores/downloads'
 import { useSourcesStore } from '../stores/sources'
 import { useNotificationStore } from '../stores/notifications'
@@ -218,9 +295,10 @@ const filterToDate = ref(null)
 const detailsDialog = ref(false)
 const selectedDownload = ref(null)
 const retryingIds = ref([])
+let refreshInterval = null

 const statusOptions = [
-  { title: 'Pending', value: 'pending' },
+  { title: 'Queued', value: 'queued' },
  { title: 'Running', value: 'running' },
  { title: 'Completed', value: 'completed' },
  { title: 'Failed', value: 'failed' },
@@ -236,24 +314,45 @@ const sourceOptions = computed(() =>

 const headers = [
  { title: 'Status', key: 'status', width: 120 },
+  { title: 'Source', key: 'source', width: 180 },
  { title: 'URL', key: 'url' },
-  { title: 'Error', key: 'error_type', width: 150 },
-  { title: 'Files', key: 'file_count', width: 100 },
-  { title: 'Date', key: 'created_at', width: 180 },
-  { title: 'Duration', key: 'duration', width: 100 },
-  { title: 'Actions', key: 'actions', width: 100, sortable: false },
+  { title: 'Error', key: 'error_type', width: 130 },
+  { title: 'Files', key: 'file_count', width: 80 },
+  { title: 'Date', key: 'created_at', width: 160 },
+  { title: 'Duration', key: 'duration', width: 90 },
+  { title: 'Actions', key: 'actions', width: 90, sortable: false },
 ]

 onMounted(() => {
-  // Load data in background - don't block UI rendering
-  loadDownloads()
+  refresh()
  sourcesStore.fetchSources()
+
+  // Auto-refresh every 30 seconds if there are running/queued jobs
+  refreshInterval = setInterval(() => {
+    if (downloadsStore.stats?.running > 0 || downloadsStore.stats?.queued > 0) {
+      loadDownloads()
+      downloadsStore.fetchStats()
+    }
+  }, 30000)
+})
+
+onUnmounted(() => {
+  if (refreshInterval) {
+    clearInterval(refreshInterval)
+  }
 })

 watch([filterStatus, filterSourceId, filterFromDate, filterToDate], () => {
  loadDownloads()
 })

+async function refresh() {
+  await Promise.all([
+    loadDownloads(),
+    downloadsStore.fetchStats()
+  ])
+}
+
 async function loadDownloads() {
  const params = {}
  if (filterStatus.value) params.status = filterStatus.value
@@ -263,12 +362,32 @@ async function loadDownloads() {
  await downloadsStore.fetchDownloads(params)
 }

+async function resetOrphanedJobs() {
+  try {
+    const result = await downloadsStore.resetOrphaned()
+    notifications.success(`Reset ${result.data.reset_count} orphaned jobs`)
+    await refresh()
+  } catch (error) {
+    notifications.error(`Failed to reset orphaned jobs: ${error.message}`)
+  }
+}
+
+async function requeueStaleJobs() {
+  try {
+    const result = await downloadsStore.requeueStale()
+    notifications.success(`Re-queued ${result.data.requeued_count} stale jobs`)
+    await refresh()
+  } catch (error) {
+    notifications.error(`Failed to re-queue stale jobs: ${error.message}`)
+  }
+}
+
 function getStatusIcon(status) {
  const icons = {
    completed: 'mdi-check-circle',
    failed: 'mdi-alert-circle',
    running: 'mdi-loading',
-    pending: 'mdi-clock-outline',
+    queued: 'mdi-clock-outline',
    skipped: 'mdi-skip-next',
  }
  return icons[status] || 'mdi-help-circle'
@@ -279,7 +398,7 @@ function getStatusColor(status) {
    completed: 'success',
    failed: 'error',
    running: 'info',
-    pending: 'warning',
+    queued: 'warning',
    skipped: 'grey',
  }
  return colors[status] || 'grey'
@@ -307,8 +426,8 @@ function formatErrorType(errorType) {

 function truncateUrl(url) {
  if (!url) return ''
-  if (url.length <= 50) return url
-  return url.substring(0, 47) + '...'
+  if (url.length <= 40) return url
+  return url.substring(0, 37) + '...'
 }

 function getSourceLabel(sourceId) {
@@ -330,7 +449,7 @@ async function retryDownload(download) {
  try {
    await downloadsStore.retryDownload(download.id)
    notifications.success('Download queued for retry')
-    await loadDownloads()
+    await refresh()
  } catch (error) {
    notifications.error(`Failed to retry: ${error.message}`)
  } finally {
@@ -338,3 +457,14 @@ async function retryDownload(download) {
  }
 }
 </script>
+
+<style scoped>
+.mdi-spin {
+  animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+  from { transform: rotate(0deg); }
+  to { transform: rotate(360deg); }
+}
+</style>