Merge pull request 'Showcase cadence tuning + cooldown-aware bulk retry' (#38) from dev into main

backend/app/api/sources.py

@@ -5,7 +5,7 @@ from sqlalchemy import select
 
 from ..extensions import get_session
 from ..models import DownloadEvent, Source
-from ..services.scheduler_service import scheduler_status
+from ..services.scheduler_service import active_platform_cooldowns, scheduler_status
 from ..services.source_service import (
     KNOWN_PLATFORMS,
     ArtistNotFoundError,
@@ -125,7 +125,16 @@ async def check_source(source_id: int):
     """FC-3c: enqueue a download for this source.
 
     Returns 202 with the new DownloadEvent id. If a pending/running
-    event already exists for this source, returns 409 with that id."""
+    event already exists for this source, returns 409 with that id. If
+    the source's platform is currently in a rate-limit cooldown, returns
+    **202 with `{status: "deferred", cooldown_until, platform}`** and
+    does NOT create an event or dispatch — the bulk retry path uses this
+    to avoid bowling N sources right back into the rate limit the
+    cooldown is preventing. Single-click "retry this one source" passes
+    `?force=true` to override the cooldown (operator-explicit, useful
+    for rapid auth-fix testing). The in-flight guard always applies.
+    """
+    force = (request.args.get("force") or "").lower() in ("1", "true", "yes")
     async with get_session() as session:
         source = (await session.execute(
             select(Source).where(Source.id == source_id)
@@ -135,6 +144,19 @@ async def check_source(source_id: int):
         if not source.enabled:
             return _bad("source_disabled", detail="enable the source first")
 
+        # Cooldown gate (unless explicitly overridden). Checked before the
+        # in-flight guard because a deferred retry doesn't need to create
+        # or check for an event at all.
+        if not force:
+            cooldowns = await active_platform_cooldowns(session)
+            expires_at = cooldowns.get(source.platform)
+            if expires_at is not None:
+                return jsonify({
+                    "status": "deferred",
+                    "platform": source.platform,
+                    "cooldown_until": expires_at.isoformat(),
+                }), 202
+
         in_flight = (await session.execute(
             select(DownloadEvent.id).where(
                 DownloadEvent.source_id == source_id,

backend/app/services/scheduler_service.py

@@ -87,10 +87,15 @@ async def set_platform_cooldown(
     await session.execute(stmt)
 
 
-async def _platforms_in_cooldown(session: AsyncSession) -> dict[str, datetime]:
+async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime]:
     """Return {platform: expires_at} for platforms whose cooldown is still
     in the future. Expired rows are ignored (a future maintenance sweep can
-    delete them; they don't affect routing decisions on their own)."""
+    delete them; they don't affect routing decisions on their own).
+
+    Exposed beyond scheduler_service so the manual check endpoint
+    (`/api/sources/<id>/check`) can defer bulk retries that would bowl
+    into the same rate limit the cooldown is preventing.
+    """
     rows = (await session.execute(
         select(AppSetting.key, AppSetting.value)
         .where(AppSetting.key.startswith(PLATFORM_COOLDOWN_KEY_PREFIX))
@@ -133,7 +138,7 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
         .order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
     )).scalars().all()
 
-    cooldowns = await _platforms_in_cooldown(session)
+    cooldowns = await active_platform_cooldowns(session)
     settings = await ImportSettings.load(session)
 
     now = datetime.now(UTC)
@@ -212,7 +217,7 @@ async def scheduler_status(session: AsyncSession) -> dict:
         elif next_due_at is None or nca < next_due_at:
             next_due_at = nca
 
-    cooldowns = await _platforms_in_cooldown(session)
+    cooldowns = await active_platform_cooldowns(session)
 
     return {
         "last_tick_at": last_tick_at,

frontend/src/components/discovery/MasonryGrid.vue

@@ -79,9 +79,18 @@ function aspectStyle(item) {
   return { aspectRatio: `${w} / ${h}` }
 }
 
+// Larger rootMargin than the composable default (600px) because the
+// sentinel sits at the BOTTOM of the masonry container, whose height is
+// the MAX of the column heights. A single tall image (long manga page,
+// panorama) in one column pushes the sentinel way past the visible
+// bottom of the SHORTER columns — the user reads the short-column
+// bottoms long before the sentinel comes into view, and load-more
+// fires too late. 2400px ≈ 2-3 screen-heights of pre-emptive trigger,
+// comfortably covering typical tall-image heights. Operator-flagged
+// 2026-05-30.
 useInfiniteScroll(sentinelEl, () => {
   if (props.hasMore && !props.loading) emit('load-more')
-})
+}, { rootMargin: '2400px' })
 </script>
 
 <style scoped>

frontend/src/components/subscriptions/DownloadsTab.vue

@@ -171,10 +171,12 @@ async function refresh() {
 }
 
 // Retry a single failing source (re-runs its whole feed) then refresh the
-// rollup + stats so the operator sees it move.
+// rollup + stats so the operator sees it move. Passes force=true so the
+// platform cooldown is bypassed — single-source click is an explicit
+// operator override, useful for rapid auth-fix or fixture testing.
 async function onRetrySource(source) {
   try {
-    await sourcesStore.checkNow(source.id)
+    await sourcesStore.checkNow(source.id, { force: true })
     toast({ text: `Retry queued for ${source.artist_name || source.platform}`, type: 'success' })
   } catch (e) {
     if (e?.body?.download_event_id) {
@@ -187,15 +189,24 @@ async function onRetrySource(source) {
   }
 }
 
+// Bulk retry — leaves cooldown enforcement ON so N failing sources on
+// the same platform don't all retry into the rate limit the cooldown is
+// preventing. Sources deferred by cooldown will be picked up by the
+// next scan tick after the AppSetting expires. Toast tallies the three
+// outcomes so the operator can quickly read whether cooldown is the
+// dominant failure mode ("12 deferred (cooldown)" → yes, rate limit is
+// the issue).
 async function onRetryAll(sources) {
   retryingAll.value = true
   let ok = 0
   let conflict = 0
+  let deferred = 0
   try {
     for (const s of sources) {
       try {
-        await sourcesStore.checkNow(s.id)
-        ok += 1
+        const body = await sourcesStore.checkNow(s.id)
+        if (body?.status === 'deferred') deferred += 1
+        else ok += 1
       } catch (e) {
         if (e?.body?.download_event_id) conflict += 1
       }
@@ -206,6 +217,7 @@ async function onRetryAll(sources) {
   }
   const parts = []
   if (ok) parts.push(`${ok} queued`)
+  if (deferred) parts.push(`${deferred} deferred (cooldown)`)
   if (conflict) parts.push(`${conflict} already running`)
   toast({ text: parts.join(', ') || 'Nothing to retry', type: 'info' })
 }

frontend/src/stores/showcase.js

@@ -1,44 +1,108 @@
 import { defineStore } from 'pinia'
 import { ref, computed } from 'vue'
 import { useApi } from '../composables/useApi.js'
-import { useAsyncAction } from '../composables/useAsyncAction.js'
 
-// Operator-confirmed 2026-05-30: instead of one 60-item request, fetch
-// PAGE-sized chunks sequentially so items render as each batch lands
-// rather than blocking on the full 60-item response. Total initial count
-// is unchanged (PAGE * INITIAL_BATCHES = 60). Infinite-scroll also pulls
-// PAGE items per trigger so subsequent appends stay progressive too.
-const PAGE = 5
-const INITIAL_BATCHES = 12
+// Operator-flagged 2026-05-30 (round 3): the all-parallel fetch was fast
+// but risked later chunks arriving first — undesirable even when each
+// chunk is a random sample. Switched to a PIPELINE: only one fetch in
+// flight at any moment, but the next fetch kicks off as soon as the
+// previous one resolves (NOT after its trickle finishes). The next RTT
+// overlaps with the current batch's trickle, hiding the per-batch
+// round-trip behind the visible animation cadence. Responses arrive in
+// fire-order, so no out-of-order rendering surprises.
+//
+// Smaller PAGE (3 vs 5) → first chunk's items appear sooner: a chunk of
+// 3 trickles in 240 ms, well within one RTT, so by the time chunk 2 is
+// in-hand the trickle is just finishing. Total wall-clock is roughly
+// RTT + N × max(trickle_time, RTT); APPEND_DELAY_MS keeps the visible
+// cadence smooth throughout.
+const PAGE = 3
+const INITIAL_BATCHES = 20
+const APPEND_DELAY_MS = 80  // ≈ the MasonryGrid stagger animation (70 ms)
+
+
+function _sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
+
 
 export const useShowcaseStore = defineStore('showcase', () => {
   const api = useApi()
   const images = ref([])
-  const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
+  const loading = ref(false)
+  const error = ref(null)
   const exhausted = ref(false)
   const seen = new Set()
 
+  // Sequence token: every call to loadInitial bumps this. _trickleAppend
+  // bails between items if its captured seq is no longer current — guards
+  // against a fast shuffle / mount-then-shuffle from interleaving two
+  // trickles into the same images.value.
+  let _seq = 0
+
+  async function _trickleAppend(items, mySeq) {
+    for (const item of items) {
+      if (mySeq !== _seq) return
+      if (seen.has(item.id)) continue
+      seen.add(item.id)
+      images.value.push(item)
+      await _sleep(APPEND_DELAY_MS)
+    }
+  }
+
+  // Single batch — used by infinite-scroll appends. Trickles its 5 items
+  // in for the same one-at-a-time cadence as the initial load.
   async function fetchPage() {
     if (loading.value) return
-    await run(async () => {
+    loading.value = true
+    error.value = null
+    try {
       const body = await api.get('/api/showcase', { params: { limit: PAGE } })
-      const fresh = body.images.filter(i => !seen.has(i.id))
+      const fresh = (body.images || []).filter(i => !seen.has(i.id))
       if (fresh.length === 0) { exhausted.value = true; return }
-      for (const i of fresh) seen.add(i.id)
-      images.value.push(...fresh)
+      await _trickleAppend(fresh, _seq)
+    } catch (e) {
+      error.value = e.message || String(e)
+    } finally {
+      loading.value = false
+    }
+  }
+
+  function _fetchOne() {
+    return api.get('/api/showcase', { params: { limit: PAGE } }).catch(e => {
+      error.value = error.value || (e.message || String(e))
+      return null
     })
   }
 
-  // Reset state and fetch INITIAL_BATCHES chunks in sequence. Used by
-  // mount, the Shuffle button, and the R-key handler — all want the
-  // same progressive-cascade behavior.
+  // Reset state and pipeline INITIAL_BATCHES fetches: only one in flight
+  // at a time, but kick off the next one as soon as the previous resolves
+  // (NOT after its trickle finishes), so the next RTT runs concurrently
+  // with the current batch's trickle. Responses arrive in fire-order, so
+  // items always render in the order they were fetched — no out-of-order
+  // surprises from parallel races.
   async function loadInitial() {
+    _seq += 1
+    const mySeq = _seq
     images.value = []
     seen.clear()
     exhausted.value = false
-    for (let i = 0; i < INITIAL_BATCHES; i++) {
-      if (exhausted.value) break
-      await fetchPage()
+    error.value = null
+    loading.value = true
+    try {
+      let nextFetch = _fetchOne()
+      for (let i = 0; i < INITIAL_BATCHES; i++) {
+        if (mySeq !== _seq) return
+        const body = await nextFetch
+        // Fire the NEXT fetch immediately so its RTT overlaps the trickle.
+        if (i + 1 < INITIAL_BATCHES) nextFetch = _fetchOne()
+        if (!body || !body.images || body.images.length === 0) {
+          exhausted.value = true
+          break
+        }
+        await _trickleAppend(body.images, mySeq)
+      }
+      if (mySeq === _seq && images.value.length === 0) exhausted.value = true
+    } finally {
+      if (mySeq === _seq) loading.value = false
     }
   }
 

frontend/src/stores/sources.js

@@ -67,10 +67,17 @@ export const useSourcesStore = defineStore('sources', () => {
   // FC-3c: trigger a download for one source. Returns {download_event_id,status}.
   const checkingIds = ref(new Set())
 
-  async function checkNow(id) {
+  // force=true bypasses the platform-rate-limit cooldown gate. Single-
+  // source RETRY clicks pass it (operator-explicit override, useful for
+  // rapid auth-fix testing); bulk RETRY ALL / MaintenanceMenu retries
+  // leave it off so the cooldown does its preventive job.
+  async function checkNow(id, { force = false } = {}) {
     checkingIds.value = new Set(checkingIds.value).add(id)
     try {
-      return await api.post(`/api/sources/${id}/check`)
+      const url = force
+        ? `/api/sources/${id}/check?force=true`
+        : `/api/sources/${id}/check`
+      return await api.post(url)
     } finally {
       const next = new Set(checkingIds.value)
       next.delete(id)

tests/test_api_sources_check.py

@@ -63,3 +63,55 @@ async def test_check_409_when_already_running(client, db, seed, monkeypatch):
     assert resp.status_code == 409
     body = await resp.get_json()
     assert body["download_event_id"] == existing.id
+
+
+@pytest.mark.asyncio
+async def test_check_defers_when_platform_in_cooldown(client, db, seed, monkeypatch):
+    """Bulk retries land here without ?force — when the source's platform
+    has an active cooldown, return 202 with status='deferred' instead of
+    creating an event. Lets the bulk path tally deferred-vs-queued in
+    the toast so the operator can see rate-limit-induced failures at a
+    glance."""
+    from backend.app.services.scheduler_service import set_platform_cooldown
+
+    delays: list = []
+    monkeypatch.setattr(
+        "backend.app.tasks.download.download_source.delay",
+        lambda *a, **k: delays.append(a),
+    )
+
+    await set_platform_cooldown(db, seed.platform, seconds=900)
+    await db.commit()
+
+    resp = await client.post(f"/api/sources/{seed.id}/check")
+    assert resp.status_code == 202
+    body = await resp.get_json()
+    assert body["status"] == "deferred"
+    assert body["platform"] == seed.platform
+    assert body["cooldown_until"]
+    assert delays == []  # no dispatch when deferred
+
+
+@pytest.mark.asyncio
+async def test_check_force_overrides_cooldown(client, db, seed, monkeypatch):
+    """Single-source RETRY click sends ?force=true — explicit operator
+    override, used for rapid auth-fix testing without waiting on the
+    cooldown."""
+    from backend.app.services.scheduler_service import set_platform_cooldown
+
+    delays: list = []
+    monkeypatch.setattr(
+        "backend.app.tasks.download.download_source.delay",
+        lambda *a, **k: delays.append(a),
+    )
+
+    await set_platform_cooldown(db, seed.platform, seconds=900)
+    await db.commit()
+
+    resp = await client.post(f"/api/sources/{seed.id}/check?force=true")
+    assert resp.status_code == 202
+    body = await resp.get_json()
+    assert body["status"] == "pending"
+    assert isinstance(body["download_event_id"], int)
+    assert len(delays) == 1
+    assert delays[0] == (seed.id,)