Release: dev → main (first public release) #258
@@ -17,6 +17,11 @@
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
|
||||
<!-- The dial for the pane above (milestone 422). Directly beneath it
|
||||
because the two are read together: you see a queue growing and give
|
||||
that lane more slots without leaving the pane. -->
|
||||
<WorkerLanesCard />
|
||||
|
||||
<!-- The non-Celery halves of the app (2026-07-02): the GPU agent does the
|
||||
majority of processing and downloads feed the library — Activity is
|
||||
the whole-app pulse, not just the worker queues. -->
|
||||
@@ -172,6 +177,7 @@ import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
import { formatRelative as fmtRelative } from '../../utils/date.js'
|
||||
import ErrorDetailModal from '../common/ErrorDetailModal.vue'
|
||||
import QueuesTable from './QueuesTable.vue'
|
||||
import WorkerLanesCard from './WorkerLanesCard.vue'
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
import GpuActivityPanel from './GpuActivityPanel.vue'
|
||||
import DownloadsActivityPanel from './DownloadsActivityPanel.vue'
|
||||
@@ -198,6 +204,16 @@ const filterErrorType = ref(null)
|
||||
const filterTask = ref(null) // server-side task-name search (All activity)
|
||||
const failureSearch = ref('') // client-side search over loaded failures
|
||||
|
||||
// This filters HISTORY — task_run.queue — which is why it stays a written-out
|
||||
// list rather than being derived from the lanes endpoint like the other queue
|
||||
// lists were in milestone 422. Two reasons, and the second is the real one:
|
||||
// a derived list is empty whenever that endpoint is down, and more importantly
|
||||
// it would HIDE the filter for any queue that has rows but no longer has a
|
||||
// lane serving it, which is exactly when someone is looking.
|
||||
//
|
||||
// It had drifted regardless — `maintenance_long` was missing, so activity on
|
||||
// the long-maintenance lane could not be filtered for at all despite four task
|
||||
// routes pointing there. Added.
|
||||
const queueOptions = [
|
||||
{ title: 'All queues', value: null },
|
||||
{ title: 'import', value: 'import' },
|
||||
@@ -206,6 +222,7 @@ const queueOptions = [
|
||||
{ title: 'download', value: 'download' },
|
||||
{ title: 'scan', value: 'scan' },
|
||||
{ title: 'maintenance', value: 'maintenance' },
|
||||
{ title: 'maintenance_long', value: 'maintenance_long' },
|
||||
{ title: 'default', value: 'default' },
|
||||
]
|
||||
const statusOptions = [
|
||||
@@ -225,6 +242,7 @@ function pollQueues() {
|
||||
store.loadQueues()
|
||||
store.loadWorkers()
|
||||
store.loadRecentRuns()
|
||||
store.loadLanes()
|
||||
}
|
||||
function pollFailures() {
|
||||
if (document.hidden) return
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
<template>
|
||||
<!-- Milestone 422 step 4. The dial for how much work each lane does, beside
|
||||
the "Queues + workers" pane that shows the backlog it is reacting to —
|
||||
you see the queue growing and adjust the lane without leaving the pane.
|
||||
|
||||
Deliberately NOT in the System tab. That one (milestone 365) answers "is
|
||||
everything running", and every control on it would be about a thing that
|
||||
is broken. This is about a thing that is working and should work harder. -->
|
||||
<v-card class="mb-4">
|
||||
<CardHeading icon="mdi-tune-variant" title="Worker lanes">
|
||||
<v-spacer />
|
||||
<span class="text-caption fc-muted">
|
||||
updated {{ formatRelative(store.lanes?.fetched_at) }}
|
||||
</span>
|
||||
</CardHeading>
|
||||
|
||||
<v-card-text>
|
||||
<p class="fc-section__hint mb-3">
|
||||
Slots are how many tasks a lane runs at once. Changes apply to the
|
||||
running worker immediately and survive a restart.
|
||||
</p>
|
||||
|
||||
<v-alert
|
||||
v-if="notice"
|
||||
:type="notice.type" variant="tonal" density="compact"
|
||||
class="mb-3" closable
|
||||
@click:close="notice = null"
|
||||
>
|
||||
{{ notice.text }}
|
||||
</v-alert>
|
||||
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Lane</th>
|
||||
<th>Queues</th>
|
||||
<th class="text-right">Pending</th>
|
||||
<th class="text-right">Busy</th>
|
||||
<th style="width: 200px;">Slots</th>
|
||||
<th class="text-right">On</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="lane in lanesList" :key="lane.name">
|
||||
<td>
|
||||
<div>{{ lane.display_name }}</div>
|
||||
<!-- Not present is NOT zero slots — it is "nothing answered".
|
||||
Saying "stopped" here would be a verdict drawn from an
|
||||
unswept read, and the operator would go looking for a crash
|
||||
that has not happened. -->
|
||||
<div v-if="!lane.live.present" class="text-caption fc-muted">
|
||||
not answering
|
||||
</div>
|
||||
<div v-else-if="lane.live.replicas > 1" class="text-caption fc-muted">
|
||||
{{ lane.live.replicas }} replicas
|
||||
</div>
|
||||
</td>
|
||||
|
||||
<td class="text-caption fc-muted">{{ lane.queues.join(', ') }}</td>
|
||||
|
||||
<!-- depth + reserved. LLEN alone reads 0 while a worker holds
|
||||
prefetched tasks in memory, which is the number that would
|
||||
make someone think a lane was idle while it was buried. -->
|
||||
<td class="text-right">
|
||||
<span v-if="lane.pending === null" class="fc-muted">—</span>
|
||||
<span v-else>{{ lane.pending }}</span>
|
||||
</td>
|
||||
|
||||
<td class="text-right">
|
||||
<span v-if="!lane.live.present" class="fc-muted">—</span>
|
||||
<span v-else>{{ lane.live.active }} / {{ lane.live.pool ?? '?' }}</span>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<div class="d-flex align-center" style="gap: 4px;">
|
||||
<v-btn
|
||||
icon="mdi-minus" size="x-small" variant="text"
|
||||
:disabled="busy === lane.name || lane.slots <= 0"
|
||||
:aria-label="`Fewer slots for ${lane.display_name}`"
|
||||
@click="step(lane, -1)"
|
||||
/>
|
||||
<span class="fc-slots">{{ lane.slots }}</span>
|
||||
<v-btn
|
||||
icon="mdi-plus" size="x-small" variant="text"
|
||||
:disabled="busy === lane.name || lane.slots >= lane.slots_cap"
|
||||
:aria-label="`More slots for ${lane.display_name}`"
|
||||
@click="step(lane, 1)"
|
||||
/>
|
||||
<!-- The REASON a higher value is unavailable, always on screen.
|
||||
A greyed control with no explanation reads as a bug, and
|
||||
the ceiling is the one number here the operator cannot
|
||||
change — so it has to justify itself. -->
|
||||
<span class="text-caption fc-muted ml-1">
|
||||
cap {{ lane.slots_cap }}<template v-if="lane.slots_cap >= lane.ceiling">
|
||||
· max {{ lane.ceiling }}{{ lane.memory_bound ? ' (memory)' : '' }}
|
||||
</template>
|
||||
</span>
|
||||
</div>
|
||||
</td>
|
||||
|
||||
<td class="text-right">
|
||||
<v-switch
|
||||
:model-value="lane.enabled"
|
||||
density="compact" hide-details color="accent"
|
||||
:disabled="busy === lane.name"
|
||||
:aria-label="`Enable ${lane.display_name}`"
|
||||
@update:model-value="toggle(lane, $event)"
|
||||
/>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
|
||||
<p v-if="mlOff" class="fc-section__hint mt-3">
|
||||
ML tagging is off. Turning it on downloads the tagging model the first
|
||||
time it runs — a few GB, once.
|
||||
</p>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref } from 'vue'
|
||||
|
||||
import { useSystemActivityStore } from '../../stores/systemActivity.js'
|
||||
import { formatRelative } from '../../utils/date.js'
|
||||
import CardHeading from '../common/CardHeading.vue'
|
||||
|
||||
const store = useSystemActivityStore()
|
||||
|
||||
// Which lane is mid-write. Per-lane rather than a global flag so adjusting
|
||||
// one lane does not freeze the others.
|
||||
const busy = ref(null)
|
||||
const notice = ref(null)
|
||||
|
||||
const lanesList = computed(() => store.lanes?.lanes ?? [])
|
||||
const mlOff = computed(() =>
|
||||
lanesList.value.some((l) => l.name === 'ml' && !l.enabled),
|
||||
)
|
||||
|
||||
async function apply(lane, fields) {
|
||||
busy.value = lane.name
|
||||
notice.value = null
|
||||
try {
|
||||
const reply = await store.setLane(lane.name, fields)
|
||||
// Saved but not pushed — the lane is restarting, or the broker blipped.
|
||||
// NOT an error: the reconcile carries it when the lane answers again, and
|
||||
// saying "failed" would invite the operator to set it a second time.
|
||||
if (reply && reply.applied === false) {
|
||||
notice.value = {
|
||||
type: 'info',
|
||||
text: `Saved. ${lane.display_name} is not answering right now — `
|
||||
+ 'it will pick this up within a few minutes.',
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// The endpoint's `detail` is written to be read by a person ("cap 10000 is
|
||||
// above what this container can hold"). Surface it rather than a status
|
||||
// code, and never swallow it — a control that silently does nothing is
|
||||
// worse than one that refuses out loud.
|
||||
//
|
||||
// `e.body`, not `e.detail`: ApiError puts the parsed response on `.body`
|
||||
// and sets `.message` to the short `error` key. Reading the wrong one
|
||||
// falls back to that key and shows the operator the word "refused" with
|
||||
// no reason — which is exactly the greyed-control-with-no-explanation
|
||||
// failure this line exists to prevent.
|
||||
notice.value = { type: 'error', text: e.body?.detail || e.message }
|
||||
} finally {
|
||||
busy.value = null
|
||||
}
|
||||
}
|
||||
|
||||
function step(lane, delta) {
|
||||
return apply(lane, { slots: lane.slots + delta })
|
||||
}
|
||||
|
||||
function toggle(lane, value) {
|
||||
return apply(lane, { enabled: Boolean(value) })
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
/* Fixed width so the number does not shift the +/- buttons as it changes
|
||||
between one and two digits. */
|
||||
.fc-slots {
|
||||
display: inline-block;
|
||||
min-width: 1.75em;
|
||||
text-align: center;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
</style>
|
||||
@@ -12,13 +12,21 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
const recentRuns = ref([]) // last-60s rows (for Overview summary)
|
||||
const failures = ref(null) // { recent, count_by_type, since }
|
||||
|
||||
// Worker lanes (milestone 422): the configured slots joined to the live
|
||||
// pool. Lives here rather than in its own store because it is the same
|
||||
// domain the queues and workers above describe — a second store polling
|
||||
// /api/system/* would be two things to keep in step.
|
||||
const lanes = ref(null) // { lanes: [...], fetched_at }
|
||||
|
||||
// Paginated runs (Activity tab "All recent activity" pane).
|
||||
const runs = ref([])
|
||||
const runsCursor = ref(null)
|
||||
const runsHasMore = ref(false)
|
||||
const runsFilter = ref({ queue: null, status: null, task: null, limit: 50 })
|
||||
|
||||
const loading = ref({ queues: false, workers: false, runs: false, failures: false })
|
||||
const loading = ref({
|
||||
queues: false, workers: false, runs: false, failures: false, lanes: false,
|
||||
})
|
||||
const lastError = ref(null)
|
||||
|
||||
async function loadQueues() {
|
||||
@@ -45,6 +53,32 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
}
|
||||
}
|
||||
|
||||
async function loadLanes() {
|
||||
loading.value.lanes = true
|
||||
lastError.value = null
|
||||
try {
|
||||
lanes.value = await api.get('/api/system/workers')
|
||||
} catch (e) {
|
||||
lastError.value = e.message
|
||||
} finally {
|
||||
loading.value.lanes = false
|
||||
}
|
||||
}
|
||||
|
||||
// Change one lane. Returns the endpoint's reply so the caller can tell a
|
||||
// stored-but-not-yet-live change (`applied: false`) from a live one — the
|
||||
// difference between "saved, the lane is restarting" and "that failed",
|
||||
// which the UI must not collapse into one message.
|
||||
//
|
||||
// Deliberately NOT swallowing the error: a refused value (400) carries the
|
||||
// sentence explaining why, and the card shows it. Returning null on failure
|
||||
// would leave the operator with a control that silently did nothing.
|
||||
async function setLane(name, fields) {
|
||||
const reply = await api.post(`/api/system/workers/${name}`, { body: fields })
|
||||
await loadLanes()
|
||||
return reply
|
||||
}
|
||||
|
||||
async function loadRecentRuns() {
|
||||
// Used by the Overview summary card: pull last 60s of runs to compute
|
||||
// per-queue ok/err counts. One call covers all queues; UI groups.
|
||||
@@ -107,10 +141,10 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
||||
}
|
||||
|
||||
return {
|
||||
queues, workers, recentRuns, failures, summary,
|
||||
queues, workers, recentRuns, failures, summary, lanes,
|
||||
runs, runsCursor, runsHasMore, runsFilter,
|
||||
loading, lastError,
|
||||
loadQueues, loadWorkers, loadRecentRuns,
|
||||
loadQueues, loadWorkers, loadRecentRuns, loadLanes, setLane,
|
||||
loadRuns, loadFailures, loadSummary, setFilter,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useSystemActivityStore } from '../src/stores/systemActivity.js'
|
||||
|
||||
// Milestone 422 step 4. Covers the store half of the worker-lane dial — the
|
||||
// part that decides what the card can tell the operator.
|
||||
//
|
||||
// The distinction being protected: a change that was STORED but not pushed
|
||||
// (`applied: false`, because the lane is restarting) is not a failure, and a
|
||||
// REFUSED value (400) is. Collapsing those two into one message is how a
|
||||
// control stops being trustworthy — one invites waiting, the other invites
|
||||
// changing what you asked for.
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const LANES_BODY = {
|
||||
lanes: [
|
||||
{
|
||||
name: 'worker', display_name: 'Worker',
|
||||
queues: ['default', 'import', 'thumbnail', 'download'],
|
||||
slots: 1, slots_cap: 4, ceiling: 8, enabled: true, memory_bound: false,
|
||||
live: { present: true, replicas: 1, pool: 1, active: 0, reserved: 3 },
|
||||
queue_depth: 5, pending: 8,
|
||||
},
|
||||
{
|
||||
name: 'ml', display_name: 'ML tagging', queues: ['ml'],
|
||||
slots: 0, slots_cap: 1, ceiling: 2, enabled: false, memory_bound: true,
|
||||
live: { present: false, replicas: 0, pool: null, active: 0, reserved: 0 },
|
||||
queue_depth: null, pending: null,
|
||||
},
|
||||
],
|
||||
fetched_at: '2026-09-22T12:00:00Z',
|
||||
}
|
||||
|
||||
describe('worker lanes store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('loads the lanes', async () => {
|
||||
stubFetch(() => ({ status: 200, body: LANES_BODY }))
|
||||
const s = useSystemActivityStore()
|
||||
await s.loadLanes()
|
||||
expect(s.lanes.lanes.map((l) => l.name)).toEqual(['worker', 'ml'])
|
||||
})
|
||||
|
||||
it('a load failure records the error rather than throwing at the caller', async () => {
|
||||
// The card polls this every 3s. An unhandled rejection per tick would
|
||||
// drown the console and stop the other pollers in the same function.
|
||||
stubFetch(() => ({ status: 500, body: { error: 'boom' } }))
|
||||
const s = useSystemActivityStore()
|
||||
await expect(s.loadLanes()).resolves.toBeUndefined()
|
||||
expect(s.lastError).toBeTruthy()
|
||||
})
|
||||
|
||||
it('setLane posts only the fields it was given', async () => {
|
||||
// Partial update: the stepper sends slots without restating a cap it did
|
||||
// not touch. Sending the whole row back would make two operators editing
|
||||
// different fields clobber each other.
|
||||
const calls = []
|
||||
stubFetch((url, init) => {
|
||||
calls.push({ url, init })
|
||||
if (init?.method === 'POST') {
|
||||
return { status: 200, body: { name: 'worker', slots: 2, applied: true } }
|
||||
}
|
||||
return { status: 200, body: LANES_BODY }
|
||||
})
|
||||
const s = useSystemActivityStore()
|
||||
await s.setLane('worker', { slots: 2 })
|
||||
|
||||
const post = calls.find((c) => c.init?.method === 'POST')
|
||||
expect(post.url).toContain('/api/system/workers/worker')
|
||||
expect(JSON.parse(post.init.body)).toEqual({ slots: 2 })
|
||||
})
|
||||
|
||||
it('setLane refetches so the card shows the server truth, not the guess', async () => {
|
||||
// The reply is one lane; the card renders all of them plus live pool and
|
||||
// pending. Patching the local row from the reply would leave every other
|
||||
// column stale and eventually wrong.
|
||||
let gets = 0
|
||||
stubFetch((url, init) => {
|
||||
if (init?.method === 'POST') return { status: 200, body: { applied: true } }
|
||||
gets += 1
|
||||
return { status: 200, body: LANES_BODY }
|
||||
})
|
||||
const s = useSystemActivityStore()
|
||||
await s.setLane('worker', { slots: 2 })
|
||||
expect(gets).toBe(1)
|
||||
})
|
||||
|
||||
it('a stored-but-unapplied change comes back as applied:false, not an error', async () => {
|
||||
// The lane is restarting. The value IS saved and the reconcile will carry
|
||||
// it — so this must reach the card as information, not as a failure that
|
||||
// invites the operator to set it again.
|
||||
stubFetch((url, init) => {
|
||||
if (init?.method === 'POST') {
|
||||
return {
|
||||
status: 200,
|
||||
body: { applied: false, apply_error: 'lane is not running', slots: 2 },
|
||||
}
|
||||
}
|
||||
return { status: 200, body: LANES_BODY }
|
||||
})
|
||||
const s = useSystemActivityStore()
|
||||
const reply = await s.setLane('worker', { slots: 2 })
|
||||
expect(reply.applied).toBe(false)
|
||||
expect(reply.apply_error).toContain('not running')
|
||||
})
|
||||
|
||||
it('a refused value throws so the card can show the reason', async () => {
|
||||
// Deliberately NOT swallowed. The detail is written to be read by a person
|
||||
// ("above what this container can hold"), and a control that silently does
|
||||
// nothing is worse than one that refuses out loud.
|
||||
stubFetch((url, init) => {
|
||||
if (init?.method === 'POST') {
|
||||
return {
|
||||
status: 400,
|
||||
body: { error: 'refused', detail: 'cap 10000 is above what this container can hold (2 for ML tagging)' },
|
||||
}
|
||||
}
|
||||
return { status: 200, body: LANES_BODY }
|
||||
})
|
||||
const s = useSystemActivityStore()
|
||||
// Assert the REASON is reachable, not merely that it threw. `toThrow()`
|
||||
// alone passes whether the card can read the sentence or not — which is
|
||||
// how the first version of the card shipped reading `e.detail` (always
|
||||
// undefined) and would have shown the operator the bare word "refused".
|
||||
const err = await s.setLane('ml', { slots_cap: 10000 }).catch((e) => e)
|
||||
expect(err.status).toBe(400)
|
||||
expect(err.body.detail).toContain('container can hold')
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user