bvandeusen/FabledCurator
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 19 Wiki Activity

Release: dev → main (first public release) #258

Merged
bvandeusen merged 94 commits from dev into main 2026-09-25 10:02:40 -04:00
Conversation 0 Commits 94 Files Changed 224
+761 -470
8 changed files with 761 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit abe16aa382 - Show all commits
backend/app/services/worker_control.py
+40 -8
View File
@@ -534,9 +534,33 @@ async def set_lane(
new_cap = row.slots_cap if slots_cap is None else slots_cap
new_slots = row.slots if slots is None else slots
new_enabled = row.enabled if enabled is None else enabled
new_autoscale = row.autoscale if autoscale is None else autoscale
# THE DIAL IS THE SWITCH. A lane at zero slots is a lane that is off, and
# there is no second control saying so.
#
# Operator, 2026-09-23, on the card that had both: *"there's nothing to
# describe what 'auto' means or why their needs to be or should be on/off
# toggles. almost all of it always needs to run there's only one optional
# piece and it is killed by moving the 'cap' to zero."* They are right. Of
# four lanes, three must run for the application to work at all, so a
# switch beside each of them offered a choice that was never real — and
# for the one lane that IS optional, "off" and "zero slots" were two ways
# of saying the same thing that could disagree with each other.
#
# `enabled` stays in the model and on the API. It is still the mechanism:
# a disabled lane keeps its process and cancels its consumers, which is
# what makes it visible in the roster instead of looking like a crash. It
# is now DERIVED from the number the operator actually sets, rather than
# being a second thing for them to keep in agreement with it.
was_enabled = row.enabled
if enabled is not None:
new_enabled = enabled
elif slots is not None:
new_enabled = new_slots > 0
else:
new_enabled = row.enabled
ceiling = derived_ceiling(lane)
if new_cap < 0 or new_slots < 0:
raise LaneUpdateRefused("slots and cap cannot be negative")
@@ -555,7 +579,11 @@ async def set_lane(
await session.commit()
applied, error = True, None
if enabled is not None:
# On the CHANGE, not on the field being present. Now that `enabled` is
# derived, every slots write would otherwise re-send a consumer command
# that changes nothing — the churn lesson #4183 keeps producing, arriving
# here through the new derivation.
if new_enabled != was_enabled:
applied, error = await asyncio.to_thread(
set_lane_enabled_sync, lane, new_enabled,
)
@@ -567,13 +595,17 @@ async def set_lane(
# HuggingFace for ~3.5GB, and rule 164 permits a runtime fetch only for a
# feature that is optional and clearly OFF.
#
# Only when the lane actually came on — `enabled is True` rather than
# `new_enabled`, so re-saving slots on an already-enabled lane does not
# re-enqueue. And only when the consumer change landed: enqueueing a task
# onto a queue nothing is consuming would leave it pending with no
# explanation until the lane returns.
# Only on the TRANSITION from off to on, so re-saving slots on a lane that
# is already running does not re-enqueue. This used to test `enabled is
# True` — the field having been sent — which stopped meaning "came on" the
# moment the dial became the switch: the UI no longer sends `enabled` at
# all, so the fetch that makes the ML lane usable would never have fired.
#
# And only when the consumer change landed: enqueueing onto a queue
# nothing is consuming would leave the task pending with no explanation
# until the lane returns.
fetching = False
if enabled is True and lane.models and applied:
if new_enabled and not was_enabled and lane.models and applied:
fetching = _enqueue_model_fetch()
return {
frontend/src/components/settings/SystemHealthTab.vue
+376 -68
View File
@@ -1,8 +1,22 @@
<template>
<!-- A Settings tab, not a page of its own (operator 2026-09-02): the first
cut hung this off the health dot alone, which is a target you have to
already suspect something to look for. Settings is where someone goes
to ask the instance about itself, so it lives beside Activity. -->
already suspect something to look for. Settings is where someone goes to
ask the instance about itself.
ONE table, not two panes (operator 2026-09-23: "I feel that we can
probably combine the two sections into a single table and to format it
in such a way that it appears more bounded and less free-form or open").
They were the roster (milestone 365, what is running) above the worker
lanes (milestone 422, how hard) — and the four celery lanes appeared in
BOTH, once as a status row and again as a control row. The page said
"ML tagging is running" and then, four hundred pixels later, "ML tagging
1 / 1 busy". Two answers to one question, from two endpoints, free to
disagree with each other on screen.
So it is one row per part now, and a lane's row simply has controls
where a datastore's has none. -->
<div>
<div class="d-flex align-center mb-1">
<v-spacer />
@@ -11,95 +25,389 @@
</span>
</div>
<p class="fc-sys__lede text-body-2 mb-5">
Every moving part of FabledCurator and whether it is still checking in.
Parts are learned as they appear, so anything that has run at least once
stays listed — that is what lets a stopped one be noticed rather than
simply vanishing.
<p class="fc-sys__lede text-body-2 mb-4">
Every moving part of FabledCurator: whether it is still checking in, and —
for the worker lanes — how much work it is allowed to take at once. Parts
are learned as they appear, so anything that has run at least once stays
listed. That is what lets a stopped one be noticed rather than simply
vanishing.
</p>
<v-alert
v-if="store.lastError" type="error" variant="tonal" density="compact" class="mb-4"
v-if="store.lastError" type="error" variant="tonal" density="compact"
class="mb-4"
>
Could not reach FabledCurator: {{ store.lastError }}
</v-alert>
<v-card v-else variant="flat" class="fc-sys__card">
<div v-if="!store.parts.length" class="pa-6 text-center fc-sys__muted">
Still gathering — this fills in on the first check.
</div>
<v-alert
v-if="notice"
:type="notice.type" variant="tonal" density="compact"
class="mb-4" closable
@click:close="notice = null"
>
{{ notice.text }}
</v-alert>
<div
v-for="part in store.parts" :key="part.key"
class="fc-sys__row" :class="`fc-sys__row--${part.state}`"
>
<span class="fc-sys__dot" :class="`fc-sys__dot--${part.state}`" />
<v-card v-if="!store.lastError" variant="flat" class="fc-parts">
<v-table density="compact" class="fc-parts__table">
<thead>
<tr>
<th>Part</th>
<th>Queues</th>
<th class="text-right">Pending</th>
<th class="text-right">Busy</th>
<th class="text-center" style="width: 150px;">Slots</th>
<th class="text-center" style="width: 84px;">Auto</th>
</tr>
</thead>
<tbody>
<tr v-if="!rows.length">
<td colspan="6" class="text-center py-6 fc-sys__muted">
Still gathering — this fills in on the first check.
</td>
</tr>
<div class="fc-sys__body">
<div class="fc-sys__name">
{{ part.name }}
<span class="fc-sys__kind">{{ kindLabel(part.kind) }}</span>
</div>
<!-- The sentence, not just a chip. At the moment someone is deciding
whether to go and open Portainer, "has not checked in for 6 min"
is the thing that answers them. -->
<div class="fc-sys__detail">{{ part.detail }}</div>
</div>
<tr v-for="row in rows" :key="row.key">
<td>
<div class="fc-parts__name">
<span class="fc-sys__dot" :class="`fc-sys__dot--${row.state}`" />
{{ row.name }}
<span class="fc-sys__kind">{{ row.kindLabel }}</span>
</div>
<!-- The sentence, not just a chip. At the moment someone is
deciding whether to go and open Portainer, "has not checked
in for 6 min" is the thing that answers them. -->
<div class="fc-parts__detail">{{ row.detail }}</div>
<!-- The operator asked for a lane held at full for a long time to
TRIGGER growth. It reports instead: extra slots do not make a
long task finish sooner, so scaling on this would spend
memory to change nothing. Seeing a lane wedged on one slow
job is the useful half. -->
<div v-if="row.stuckFor" class="fc-parts__detail text-warning">
all slots busy for {{ row.stuckFor }}
</div>
</td>
<div class="fc-sys__meta">
<div v-if="part.last_seen_at" :title="part.last_seen_at">
seen {{ formatRelative(part.last_seen_at) }}
</div>
<div v-if="part.latency_ms != null">{{ part.latency_ms }} ms</div>
<div v-if="part.queues?.length" class="fc-sys__queues">{{ part.queues.join(', ') }}</div>
</div>
</div>
<td class="fc-parts__queues">{{ row.queues || '—' }}</td>
<!-- depth + reserved. LLEN alone reads 0 while a worker holds
prefetched tasks in memory, which is the number that would make
someone think a buried lane was idle. -->
<td class="text-right fc-parts__num">
{{ row.lane ? (row.lane.pending ?? '—') : '—' }}
</td>
<td class="text-right fc-parts__num">
<template v-if="row.lane && row.lane.live.present">
{{ row.lane.live.active }} / {{ row.lane.live.pool ?? '?' }}
</template>
<template v-else>—</template>
</td>
<td class="text-center">
<template v-if="row.lane">
<div class="d-flex align-center justify-center">
<v-btn
icon="mdi-minus" size="x-small" variant="text"
:disabled="busy === row.lane.name || row.lane.slots <= 0"
:aria-label="`Fewer slots for ${row.name}`"
@click="step(row.lane, -1)"
/>
<span class="fc-parts__slots">{{ row.lane.slots }}</span>
<v-btn
icon="mdi-plus" size="x-small" variant="text"
:disabled="busy === row.lane.name
|| row.lane.slots >= row.lane.slots_cap"
:aria-label="`More slots for ${row.name}`"
@click="step(row.lane, 1)"
/>
</div>
<!-- The REASON a higher number is unavailable, always on
screen. A greyed control with no explanation reads as a
bug, and this ceiling is the one number here the operator
cannot change from this table. -->
<div class="fc-parts__sub">of {{ row.lane.slots_cap }}</div>
</template>
<span v-else class="fc-sys__muted">—</span>
</td>
<td class="text-center">
<v-switch
v-if="row.lane"
:model-value="row.lane.autoscale"
density="compact" hide-details color="accent"
class="d-inline-flex"
:disabled="busy === row.lane.name
|| (!row.lane.autoscale && !canGrow(row.lane))"
:aria-label="`Let ${row.name} add slots by itself`"
@update:model-value="setAutoscale(row.lane, $event)"
/>
<span v-else class="fc-sys__muted">—</span>
</td>
</tr>
</tbody>
</v-table>
</v-card>
<p v-if="store.thresholds" class="fc-sys__foot text-caption mt-4">
A part is called stale after
{{ Math.round(store.thresholds.stale_after_seconds / 60) }} min without a
check-in and treated as stopped after
{{ Math.round(store.thresholds.down_after_seconds / 60) }} min. The window
is deliberately wide: a rolling deploy briefly runs two of a service and
then neither, and an indicator that reddened on every update would stop
being read.
</p>
<!-- What the two columns that are not self-explanatory actually mean.
Operator, 2026-09-23: *"there's nothing to describe what 'auto' means
or why their needs to be or should be on/off toggles."* It was a bare
switch in a column headed Auto, beside a second switch headed On, with
nothing anywhere saying what either one did. -->
<div class="fc-parts__legend mt-4">
<p class="mb-2">
<strong>Slots</strong> is how many tasks a lane runs at once. Changes
reach the running worker immediately and survive a restart.
<strong>Zero slots turns the lane off</strong> — it keeps its process
and stops taking work, so it stays listed here rather than looking like
a crash. Three of the four lanes need to be running for FabledCurator to
work at all; ML tagging is the one that is genuinely optional. The
<em>of N</em> beneath each dial is the most that lane may have on this
machine.
</p>
<p class="mb-2">
<strong>Auto</strong> lets a lane add slots <em>by itself</em> when its
queue is backed up <em>and</em> every slot it has is busy — one at a
time, never past <em>of N</em> — and hand them back once the backlog
clears. Off means the lane stays at exactly the number you set. It is
off by default, per lane, because this is the only thing on this page
that acts without being asked. A lane already dialled to its maximum has
nowhere to grow, so its switch stays unavailable until you leave it some
room.
</p>
<p v-if="store.thresholds" class="mb-0">
A part is called <strong>stale</strong> after
{{ Math.round(store.thresholds.stale_after_seconds / 60) }} min without a
check-in and treated as <strong>stopped</strong> after
{{ Math.round(store.thresholds.down_after_seconds / 60) }} min. The window
is deliberately wide: a rolling deploy briefly runs two of a service and
then neither, and an indicator that reddened on every update would stop
being read.
</p>
</div>
<!-- The dial for the roster above (milestone 422). Moved here from
Activity on 2026-09-23 at the operator's ask: they went looking for
"how many workers of each kind" under System, which is where the
question belongs — the roster says what is running, and this says how
hard. Its own first cut argued the opposite, that a control has no
place on a tab about things being broken. That was wrong twice over:
half these lanes are optional and off by DESIGN, so "not running" is
the normal state the operator comes here to change, and a tab that
only reports is a tab you leave to go act somewhere else. -->
<WorkerLanesCard />
<!-- What an optional, off lane will cost BEFORE it is given a slot. Every
number comes from the lane payload, and `measured` travels with them —
an estimate is labelled rather than rounded into something that reads
like a fact. -->
<v-alert
v-for="lane in offOptionalLanes" :key="`advisory-${lane.name}`"
type="info" variant="tonal" density="compact" class="mt-4"
>
<div class="font-weight-medium mb-1">
{{ lane.display_name }} is optional and currently off
</div>
<p class="mb-2">
It computes image embeddings on the CPU — what similarity search,
duplicate grouping and tag suggestions are built on. You only need it if
you are <em>not</em> running the GPU agent, which does the same work
faster.
</p>
<p class="mb-1">Giving it a slot downloads, once:</p>
<ul class="mb-2">
<li v-for="m in lane.models" :key="m.repo">
<code>{{ m.repo }}</code> —
{{ approx(m.measured) }}{{ gb(m.download_bytes) }} to download, and
about {{ gb(m.resident_bytes) }} of RAM for
<strong>each</strong> slot while it runs.
</li>
</ul>
<p class="mb-0 text-caption">
Each slot loads its own copy, which is why this machine allows it at
most {{ lane.ceiling }}.
<template v-if="lane.ceiling === 0">
It has too little memory to run this at all.
</template>
</p>
</v-alert>
</div>
</template>
<script setup>
import { onMounted } from 'vue'
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { laneStuckFor, useSystemActivityStore } from '../../stores/systemActivity.js'
import { useSystemHealthStore } from '../../stores/systemHealth.js'
import { formatRelative } from '../../utils/date.js'
import WorkerLanesCard from './WorkerLanesCard.vue'
import { mergeParts } from '../../utils/systemParts.js'
const store = useSystemHealthStore()
const lanesStore = useSystemActivityStore()
function kindLabel(kind) {
if (kind === 'celery') return 'background worker'
if (kind === 'agent') return 'GPU agent'
if (kind === 'datastore') return 'datastore'
return kind
// Which lane is mid-write. Per-lane rather than a global flag so adjusting one
// lane does not freeze the others.
const busy = ref(null)
const notice = ref(null)
// The roster half needs no timer: TopNav already polls that same store every
// 15s for the health dot, on every route this tab is reachable from. The LANES
// have no such poller, so this tab owns them — 15s and `document.hidden`
// guarded, because `/api/system/workers` costs a broker round trip and
// v-window keeps a visited tab MOUNTED, so a naive interval would keep firing
// behind Maintenance and behind a backgrounded browser tab.
const LANE_POLL_MS = 15000
let pollId = null
onMounted(() => {
store.refresh()
lanesStore.loadLanes()
pollId = setInterval(() => {
if (document.hidden) return
lanesStore.loadLanes()
}, LANE_POLL_MS)
})
onUnmounted(() => { clearInterval(pollId) })
// One row per moving part, lanes joined to their roster entry. The join lives
// in `utils/systemParts.js` because its failure is silent: a lane that stops
// matching its part does not throw, it grows a SECOND row for the same worker
// — the duplication this table was built to remove, returning through the code
// that removed it.
const rows = computed(() => mergeParts(
store.parts, lanesStore.lanes?.lanes ?? [], laneStuckFor,
))
const offOptionalLanes = computed(() =>
(lanesStore.lanes?.lanes ?? []).filter(
(l) => l.optional && l.slots === 0 && l.models?.length,
),
)
function gb(bytes) {
return `${(bytes / 1024 ** 3).toFixed(1)} GB`
}
// No timer of its own. TopNav already polls this same pinia store every 15s
// for the health dot, and it is mounted on every route this tab is reachable
// from — a second interval here would just double the request rate for a 5s
// freshness gain. v-window keeps a visited item MOUNTED (hidden, not
// destroyed), so a local timer would also have kept firing behind Maintenance.
// One refresh on open, so arriving at the tab doesn't wait out the nav's tick.
onMounted(() => { store.refresh() })
// An unmeasured figure says so. It decides whether someone's server survives,
// and presenting an estimate as a measurement is the failure this guards.
function approx(measured) {
return measured ? '' : 'about '
}
async function apply(lane, fields) {
busy.value = lane.name
notice.value = null
try {
const reply = await lanesStore.setLane(lane.name, fields)
if (reply && reply.fetching_models) {
notice.value = {
type: 'info',
text: `${lane.display_name} is on. Downloading its model now — watch `
+ 'progress under Activity. It only happens once.',
}
} else if (reply && reply.applied === false) {
// Saved but not pushed — the lane is restarting, or the broker blipped.
// NOT an error: the reconcile carries it when the lane answers again,
// and saying "failed" would invite setting it a second time.
notice.value = {
type: 'info',
text: `Saved. ${lane.display_name} is not answering right now — `
+ 'it will pick this up within a few minutes.',
}
}
} catch (e) {
// The endpoint's `detail` is written to be read by a person ("cap 10000 is
// above what this container can hold"). Surface it rather than a status
// code, and never swallow it — a control that silently does nothing is
// worse than one that refuses out loud.
//
// `e.body`, not `e.detail`: ApiError puts the parsed response on `.body`
// and sets `.message` to the short `error` key.
notice.value = { type: 'error', text: e.body?.detail || e.message }
} finally {
busy.value = null
}
}
// The dial IS the switch — the API derives `enabled` from the number, so
// stepping to zero turns the lane off and stepping off zero turns it on.
// Nothing here sends `enabled`, and there is no second control that could
// disagree with the number on screen.
function step(lane, delta) {
return apply(lane, { slots: lane.slots + delta })
}
// Room to grow into. The autoscaler moves the LIVE pool, but the floor it
// starts from is the stored value, so a lane already dialled to its cap has
// nowhere to go and turning this on would do nothing at all.
function canGrow(lane) {
return lane.slots_cap > lane.slots
}
function setAutoscale(lane, value) {
return apply(lane, { autoscale: Boolean(value) })
}
</script>
<style scoped>
/* These lived in app.css for one commit, globalised so the roster and the
worker lanes could share them. Merging the two into one table left a single
consumer, so they come back here — a global utility with one caller is a
rule nobody can find from the markup that uses it. */
.fc-sys__lede, .fc-sys__muted, .fc-sys__checked {
color: rgb(var(--v-theme-on-surface) / 0.66);
}
.fc-sys__checked { font-size: 0.78rem; }
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
.fc-sys__kind {
font-weight: 400; font-size: 0.72rem; text-transform: uppercase;
letter-spacing: 0.04em; color: rgb(var(--v-theme-on-surface) / 0.5);
}
/* Bounded rather than free-form (operator 2026-09-23). What was here before was
dotted rows floating on the page background with no header and no edge, so
nothing said where the list began or what any column meant. */
.fc-parts {
background: rgb(var(--v-theme-on-surface) / 0.04);
border: 1px solid rgb(var(--v-theme-on-surface) / 0.12);
border-radius: 6px;
overflow: hidden;
}
.fc-parts__table :deep(thead th) {
font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.05em;
color: rgb(var(--v-theme-on-surface) / 0.55);
border-bottom: 1px solid rgb(var(--v-theme-on-surface) / 0.18) !important;
white-space: nowrap;
}
.fc-parts__table :deep(tbody td) {
border-bottom: 1px solid rgb(var(--v-theme-on-surface) / 0.08) !important;
padding-top: 8px; padding-bottom: 8px;
}
.fc-parts__table :deep(tbody tr:last-child td) { border-bottom: 0 !important; }
.fc-parts__name {
display: flex; align-items: center; gap: 8px; font-weight: 600;
}
/* Clears the dot, so the sentence lines up under the name rather than under
the indicator. */
.fc-parts__detail {
font-size: 0.8rem; color: rgb(var(--v-theme-on-surface) / 0.72);
padding-left: 17px;
}
.fc-parts__queues {
font-size: 0.78rem; color: rgb(var(--v-theme-on-surface) / 0.6);
}
.fc-parts__num { font-variant-numeric: tabular-nums; }
.fc-parts__sub {
font-size: 0.68rem; text-transform: uppercase; letter-spacing: 0.04em;
color: rgb(var(--v-theme-on-surface) / 0.5); white-space: nowrap;
}
/* Fixed width so the number does not shift the +/- buttons as it changes
between one and two digits. */
.fc-parts__slots {
display: inline-block; min-width: 1.75em; text-align: center;
font-variant-numeric: tabular-nums;
}
.fc-parts__legend {
font-size: 0.82rem; line-height: 1.55;
color: rgb(var(--v-theme-on-surface) / 0.66);
}
</style>
frontend/src/components/settings/WorkerLanesCard.vue
-342
View File
@@ -1,342 +0,0 @@
<template>
<!-- Milestone 422 step 4, restyled and moved to the System tab on
2026-09-23 (operator: "move it to system tab and improve the view to be
more inline with other UI elements").
It is the roster's second half. The pane above answers "what is
running"; this answers "how hard" — so it is built out of the SAME row
idiom rather than a bordered card wrapping a v-table, which is what it
was and what made it read as bolted on. Dot, name, detail, numbers on
the right; the controls take the place the roster gives to latency. -->
<div class="fc-lanes">
<div class="d-flex align-center mb-1 mt-8">
<span class="fc-section-h">Worker lanes</span>
<v-spacer />
<span class="fc-sys__checked">
{{ store.lanes ? `updated ${formatRelative(store.lanes.fetched_at)}` : 'reading…' }}
</span>
</div>
<p class="fc-sys__lede text-body-2 mb-5">
Slots are how many tasks a lane runs at once. Changes reach the running
worker immediately and survive a restart, so this is the dial to reach
for when a queue above is growing faster than it drains.
</p>
<v-alert
v-if="notice"
:type="notice.type" variant="tonal" density="compact"
class="mb-4" closable
@click:close="notice = null"
>
{{ notice.text }}
</v-alert>
<v-card variant="flat" class="fc-sys__card">
<div v-if="!lanesList.length" class="pa-6 text-center fc-sys__muted">
Still reading the lanes — this fills in on the first check.
</div>
<div v-for="lane in lanesList" :key="lane.name" class="fc-sys__row">
<span class="fc-sys__dot" :class="`fc-sys__dot--${laneState(lane)}`" />
<div class="fc-sys__body">
<div class="fc-sys__name">
{{ lane.display_name }}
<!-- Said in the row rather than only in the advisory below.
Someone scanning to work out why a lane is off should not have
to turn it on to find out it was never required. -->
<span class="fc-sys__kind">{{ lane.optional ? 'optional' : 'lane' }}</span>
</div>
<!-- "Not answering" is NOT zero slots — it is "nothing replied".
Calling it stopped would be a verdict drawn from an unswept
read, and would send the operator looking for a crash that has
not happened. -->
<div class="fc-sys__detail">{{ laneDetail(lane) }}</div>
<!-- The operator asked for a lane held at full for a long time to
TRIGGER growth. It reports instead: extra slots do not make a
long task finish sooner, so scaling on this would spend memory
to change nothing. Seeing the lane wedged on one slow job is the
useful half, and it cannot mislead a lane into growing for the
wrong reason. -->
<div v-if="laneStuckFor(lane)" class="fc-sys__detail text-warning">
all slots busy for {{ laneStuckFor(lane) }}
</div>
</div>
<div class="fc-sys__meta">
<!-- depth + reserved. LLEN alone reads 0 while a worker holds
prefetched tasks in memory, which is the number that would make
someone think a buried lane was idle. -->
<div>{{ lane.pending === null ? '—' : lane.pending }} pending</div>
<div v-if="lane.live.present">
{{ lane.live.active }} / {{ lane.live.pool ?? '?' }} busy
</div>
<div class="fc-sys__queues">{{ lane.queues.join(', ') }}</div>
</div>
<div class="fc-lanes__controls">
<div class="fc-lanes__control">
<div class="d-flex align-center">
<v-btn
icon="mdi-minus" size="x-small" variant="text"
:disabled="busy === lane.name || lane.slots <= 0"
:aria-label="`Fewer slots for ${lane.display_name}`"
@click="step(lane, -1)"
/>
<span class="fc-lanes__slots">{{ lane.slots }}</span>
<v-btn
icon="mdi-plus" size="x-small" variant="text"
:disabled="busy === lane.name || lane.slots >= lane.slots_cap"
:aria-label="`More slots for ${lane.display_name}`"
@click="step(lane, 1)"
/>
</div>
<!-- The REASON a higher value is unavailable, always on screen. A
greyed control with no explanation reads as a bug, and the
ceiling is the one number here the operator cannot change — so
it has to justify itself. -->
<div class="fc-lanes__label">
slots · cap {{ lane.slots_cap }}<template
v-if="lane.slots_cap >= lane.ceiling"
>
· max {{ lane.ceiling }}{{ lane.memory_bound ? ' (memory)' : '' }}
</template>
</div>
</div>
<!-- The autoscaler is the one control here that acts without being
asked, so it is a separate switch from `On` rather than a mode
folded into it, and it is off until someone opts this particular
lane in.
Turning it ON needs room to grow into; turning it OFF never
does. Disabling the whole switch at cap == slots would strand an
already-autoscaling lane the moment someone raised its floor to
the cap, with the control that would undo it greyed out. -->
<div class="fc-lanes__control">
<v-switch
:model-value="lane.autoscale"
density="compact" hide-details color="accent"
:disabled="busy === lane.name || (!lane.autoscale && !canGrow(lane))"
:aria-label="`Let ${lane.display_name} grow itself`"
@update:model-value="setAutoscale(lane, $event)"
/>
<div class="fc-lanes__label">
{{ canGrow(lane) ? 'auto' : 'raise the cap' }}
</div>
</div>
<div class="fc-lanes__control">
<v-switch
:model-value="lane.enabled"
density="compact" hide-details color="accent"
:disabled="busy === lane.name"
:aria-label="`Enable ${lane.display_name}`"
@update:model-value="toggle(lane, $event)"
/>
<div class="fc-lanes__label">on</div>
</div>
</div>
</div>
</v-card>
<!-- What an optional, off lane will cost BEFORE it is switched on. Every
number comes from the lane payload, and `measured` travels with them —
an estimate is labelled rather than rounded into something that reads
like a fact. An earlier version said "a few GB", which told the
operator nothing they could plan with and was sourced from nothing. -->
<v-alert
v-for="lane in offOptionalLanes" :key="`advisory-${lane.name}`"
type="info" variant="tonal" density="compact" class="mt-4"
>
<div class="font-weight-medium mb-1">
{{ lane.display_name }} is optional and currently off
</div>
<p class="mb-2">
It computes image embeddings on the CPU — what similarity search,
duplicate grouping and tag suggestions are built on. You only need it
if you are <em>not</em> running the GPU agent, which does the same work
faster.
</p>
<p class="mb-1">Turning it on downloads, once:</p>
<ul class="mb-2">
<li v-for="m in lane.models" :key="m.repo">
<code>{{ m.repo }}</code> —
{{ approx(m.measured) }}{{ gb(m.download_bytes) }} to download, and
about {{ gb(m.resident_bytes) }} of RAM for
<strong>each</strong> slot while it runs.
</li>
</ul>
<p class="mb-0 text-caption">
Each slot loads its own copy, which is why this container caps the lane
at {{ lane.ceiling }}.
<template v-if="lane.ceiling === 0">
It has too little memory to run this at all.
</template>
</p>
</v-alert>
</div>
</template>
<script setup>
import { computed, onMounted, onUnmounted, ref } from 'vue'
import { laneStuckFor, useSystemActivityStore } from '../../stores/systemActivity.js'
import { formatRelative } from '../../utils/date.js'
const store = useSystemActivityStore()
// Which lane is mid-write. Per-lane rather than a global flag so adjusting one
// lane does not freeze the others.
const busy = ref(null)
const notice = ref(null)
const lanesList = computed(() => store.lanes?.lanes ?? [])
// Optional lanes that are OFF — the only ones whose cost the operator has not
// already accepted. Keyed on the lane's own `optional` flag rather than on the
// name 'ml', so a second optional lane gets the same treatment without anyone
// remembering to add it here.
const offOptionalLanes = computed(() =>
lanesList.value.filter((l) => l.optional && !l.enabled && l.models?.length),
)
// This card owns its own reads now. On Activity it rode that tab's 3s queue
// poll; the System tab has no timer at all — the roster is fed by TopNav's
// store, which knows nothing about lanes — so moving without this would have
// shown an empty pane to anyone who opened System first.
//
// 15s, not 3s: `/api/system/workers` costs a broker round trip, and this pane
// is a dial rather than a live meter. `document.hidden` because v-window keeps
// a visited tab MOUNTED, so the interval keeps firing behind Maintenance and
// behind a backgrounded browser tab.
const POLL_MS = 15000
let pollId = null
function poll() {
if (document.hidden) return
store.loadLanes()
}
onMounted(() => {
store.loadLanes()
pollId = setInterval(poll, POLL_MS)
})
onUnmounted(() => { clearInterval(pollId) })
// Same four states the roster's dots mean, so one legend covers both panes.
// A lane that is off reads `unknown` (grey), never `down` (red): half of these
// are optional and off by design, and colouring the intended state as a fault
// is how an indicator stops being read.
function laneState(lane) {
if (!lane.live.present) return lane.enabled ? 'down' : 'unknown'
if (!lane.enabled) return 'unknown'
if (laneStuckFor(lane)) return 'stale'
return 'ok'
}
function laneDetail(lane) {
if (!lane.live.present) {
return lane.enabled
? 'not answering — it is switched on, so it should be'
: 'not answering, and switched off'
}
if (!lane.enabled) return 'running but not consuming its queues'
if (lane.live.replicas > 1) return `running on ${lane.live.replicas} replicas`
return 'running'
}
function gb(bytes) {
return `${(bytes / 1024 ** 3).toFixed(1)} GB`
}
// An unmeasured figure says so. It decides whether someone's server survives,
// and presenting an estimate as a measurement is the failure this guards.
function approx(measured) {
return measured ? '' : 'about '
}
async function apply(lane, fields) {
busy.value = lane.name
notice.value = null
try {
const reply = await store.setLane(lane.name, fields)
// Saved but not pushed — the lane is restarting, or the broker blipped.
// NOT an error: the reconcile carries it when the lane answers again, and
// saying "failed" would invite the operator to set it a second time.
if (reply && reply.fetching_models) {
notice.value = {
type: 'info',
text: `${lane.display_name} is on. Downloading its model now — `
+ 'watch progress under Activity. It only happens once.',
}
} else if (reply && reply.applied === false) {
notice.value = {
type: 'info',
text: `Saved. ${lane.display_name} is not answering right now — `
+ 'it will pick this up within a few minutes.',
}
}
} catch (e) {
// The endpoint's `detail` is written to be read by a person ("cap 10000 is
// above what this container can hold"). Surface it rather than a status
// code, and never swallow it — a control that silently does nothing is
// worse than one that refuses out loud.
//
// `e.body`, not `e.detail`: ApiError puts the parsed response on `.body`
// and sets `.message` to the short `error` key. Reading the wrong one
// falls back to that key and shows the operator the word "refused" with no
// reason — which is exactly the greyed-control-with-no-explanation failure
// this line exists to prevent.
notice.value = { type: 'error', text: e.body?.detail || e.message }
} finally {
busy.value = null
}
}
function step(lane, delta) {
return apply(lane, { slots: lane.slots + delta })
}
function toggle(lane, value) {
return apply(lane, { enabled: Boolean(value) })
}
// Room to grow into. The autoscaler moves the LIVE pool, but the floor it
// starts from is the stored value, so a lane whose floor already sits at its
// cap has nowhere to go and enabling it would do nothing at all.
function canGrow(lane) {
return lane.slots_cap > lane.slots
}
function setAutoscale(lane, value) {
return apply(lane, { autoscale: Boolean(value) })
}
</script>
<style scoped>
/* The controls occupy the slot the roster gives to latency and last-seen, so
they are right-aligned and fixed-width — a row whose switches slide left as
a lane's numbers change is the thing that made the old table feel unrelated
to the pane above it. */
.fc-lanes__controls {
display: flex; align-items: flex-start; gap: 4px; flex: 0 0 auto;
}
.fc-lanes__control { text-align: center; min-width: 56px; }
.fc-lanes__control:first-child { min-width: 132px; }
.fc-lanes__label {
font-size: 0.68rem; text-transform: uppercase; letter-spacing: 0.04em;
color: rgb(var(--v-theme-on-surface) / 0.5); white-space: nowrap;
}
/* Fixed width so the number does not shift the +/- buttons as it changes
between one and two digits. */
.fc-lanes__slots {
display: inline-block;
min-width: 1.75em;
text-align: center;
font-variant-numeric: tabular-nums;
}
</style>
frontend/src/styles/app.css
-47
View File
@@ -132,50 +132,3 @@
backdrop-filter: blur(2px);
-webkit-backdrop-filter: blur(2px);
}
/* The System tab's row idiom (operator 2026-09-23: "improve the view to be
more inline with other UI elements").
The roster (milestone 365) and the worker lanes (milestone 422) sit on the
same tab and answer the two halves of one question — what is running, and
how hard. They were built months apart and looked it: one a flat list of
dotted rows, the other a bordered card wrapping a v-table. Lifting these out
of SystemHealthTab's scoped block is what lets the second pane BE the first
one's idiom rather than imitate it — a copy would drift the first time
either was touched.
`fc-sys__` rather than a new prefix because the roster's markup already uses
these names; renaming would have been churn in the file that is not
changing. */
.fc-sys__lede, .fc-sys__muted, .fc-sys__checked, .fc-sys__foot {
color: rgb(var(--v-theme-on-surface) / 0.66);
}
.fc-sys__checked { font-size: 0.78rem; }
.fc-sys__card { background: rgb(var(--v-theme-on-surface) / 0.04); }
.fc-sys__row {
display: flex; align-items: center; gap: 12px;
padding: 12px 16px;
border-bottom: 1px solid rgb(var(--v-theme-on-surface) / 0.08);
}
.fc-sys__row:last-child { border-bottom: 0; }
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
.fc-sys__body { min-width: 0; flex: 1 1 auto; }
.fc-sys__name { font-weight: 600; }
.fc-sys__kind {
margin-left: 8px; font-weight: 400; font-size: 0.72rem; text-transform: uppercase;
letter-spacing: 0.04em; color: rgb(var(--v-theme-on-surface) / 0.5);
}
.fc-sys__detail { font-size: 0.82rem; color: rgb(var(--v-theme-on-surface) / 0.72); }
.fc-sys__meta {
text-align: right; font-size: 0.75rem; flex: 0 0 auto;
font-variant-numeric: tabular-nums; color: rgb(var(--v-theme-on-surface) / 0.6);
}
.fc-sys__queues { opacity: 0.75; }
frontend/src/utils/systemParts.js
+100
View File
@@ -0,0 +1,100 @@
// Joining the roster to the worker lanes, for the System tab's one table.
//
// Extracted from the component rather than left inline because the failure
// this can have is SILENT and is exactly the thing the merge exists to fix: if
// a lane stops matching its roster part, nothing throws — the table simply
// grows a second row for the same worker, one with controls and one without,
// which is the duplication the operator asked to be rid of, returned by the
// code that removed it.
//
// Operator, 2026-09-23: "I feel that we can probably combine the two
// sections into a single table."
// A learned roster part and a lane are the same thing seen from two sides, and
// the QUEUES are what identify it — `service_roster.refresh_celery_roster`
// keys a celery part on exactly `"celery:" + ",".join(sorted(queues))`.
//
// Matched on the sorted set rather than on that string so the join survives a
// change to how the key is spelled, and so neither side has to agree about
// ORDER: the lane table lists a lane's queues in the order the role reads them
// (`default, import, thumbnail, download`) while the roster sorts them
// (`default, download, import, thumbnail`).
export function queueKey(queues) {
return [...(queues || [])].sort().join(',')
}
// Worst first. A stopped datastore is why someone opened this tab.
export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
export function kindLabel(kind) {
if (kind === 'celery') return 'worker lane'
if (kind === 'agent') return 'GPU agent'
if (kind === 'datastore') return 'datastore'
return kind
}
/**
* One row per moving part, with a lane attached where there is one.
*
* @param parts the roster's parts, as /api/system/health returns them
* @param lanes the lane rows, as /api/system/workers returns them
* @param stuckFor a lane -> "40 minutes" | null reporter (laneStuckFor)
*/
export function mergeParts(parts, lanes, stuckFor = () => null) {
const unmatched = {}
for (const lane of lanes || []) unmatched[queueKey(lane.queues)] = lane
const out = []
for (const part of parts || []) {
const key = queueKey(part.queues)
const lane = part.kind === 'celery' ? unmatched[key] : undefined
if (lane) delete unmatched[key]
out.push({
key: part.key,
name: part.name,
kindLabel: lane?.optional ? 'optional lane' : kindLabel(part.kind),
state: part.state,
// A lane dialled to zero is OFF, not broken. Say so, rather than let the
// roster's heartbeat sentence report the operator's own choice as a
// fault — the roster cannot know the difference, and the lane can.
detail: lane && lane.slots === 0 ? 'off — no slots' : part.detail,
queues: (part.queues || []).join(', '),
lane,
stuckFor: lane ? stuckFor(lane) : null,
severity: SEVERITY[part.state] ?? SEVERITY.unknown,
})
}
// A lane the roster has not learned yet. Parts appear only once they have
// checked in, while the lane table is known up front — so without this, the
// lane an operator most needs to find (an optional one, never yet started)
// would be the only one missing from the table.
for (const lane of Object.values(unmatched)) out.push(laneRow(lane, stuckFor))
// Severity leads; then lanes ahead of everything else, because they are the
// rows you can actually do something about; then by name.
return out.sort((a, b) =>
b.severity - a.severity
|| Number(Boolean(b.lane)) - Number(Boolean(a.lane))
|| a.name.localeCompare(b.name))
}
function laneRow(lane, stuckFor) {
const on = lane.slots > 0
let state = 'unknown'
if (lane.live?.present) state = on ? (stuckFor(lane) ? 'stale' : 'ok') : 'unknown'
else if (on) state = 'down'
return {
key: `lane:${lane.name}`,
name: lane.display_name,
kindLabel: lane.optional ? 'optional lane' : 'worker lane',
state,
detail: lane.live?.present
? (on ? 'running' : 'off — no slots')
: 'has not checked in yet',
queues: (lane.queues || []).join(', '),
lane,
stuckFor: stuckFor(lane),
severity: SEVERITY[state],
}
}
frontend/test/systemParts.spec.js
+128
View File
@@ -0,0 +1,128 @@
import { describe, expect, it } from 'vitest'
import { mergeParts, queueKey } from '../src/utils/systemParts.js'
// The System tab's one table (operator 2026-09-23: "combine the two sections
// into a single table"). What is pinned here is the JOIN, because its failure
// is silent and is precisely the thing the merge exists to fix: a lane that
// stops matching its roster part does not throw — the table grows a SECOND row
// for the same worker, one with controls and one without.
const PART = {
key: 'celery:default,download,import,thumbnail',
kind: 'celery',
name: 'Worker',
state: 'ok',
detail: 'Worker is running',
// The roster SORTS a celery part's queues into its key.
queues: ['default', 'download', 'import', 'thumbnail'],
}
const LANE = {
name: 'worker',
display_name: 'Worker',
// The lane table lists them in the order the role reads them, which is NOT
// sorted. If the join ever compares these two lists directly rather than as
// sets, this fixture is what catches it.
queues: ['default', 'import', 'thumbnail', 'download'],
slots: 2,
slots_cap: 4,
autoscale: false,
live: { present: true, replicas: 1, pool: 2, active: 0 },
pending: 0,
}
const POSTGRES = {
key: 'postgres', kind: 'datastore', name: 'PostgreSQL',
state: 'ok', detail: 'answering', latency_ms: 2.5,
}
describe('queueKey', () => {
it('does not care what order either side lists its queues in', () => {
expect(queueKey(LANE.queues)).toBe(queueKey(PART.queues))
})
it('survives a part that has no queues at all', () => {
// A datastore, and also the stale `Worker ()` row a previous deployment
// left in the roster with an empty queue set. Neither must match a lane.
expect(queueKey(undefined)).toBe('')
expect(queueKey([])).toBe('')
})
})
describe('mergeParts', () => {
it('gives a worker ONE row, carrying its controls', () => {
const rows = mergeParts([PART, POSTGRES], [LANE])
expect(rows).toHaveLength(2)
const worker = rows.find((r) => r.name === 'Worker')
expect(worker.lane).toBe(LANE)
expect(rows.filter((r) => r.name === 'Worker')).toHaveLength(1)
})
it('leaves a datastore without a lane rather than guessing one', () => {
const pg = mergeParts([PART, POSTGRES], [LANE]).find((r) => r.key === 'postgres')
expect(pg.lane).toBeUndefined()
})
it('still lists a lane the roster has never seen', () => {
// Parts are learned as they appear; the lane table is known up front. The
// lane an operator most needs to find — an optional one, never started —
// is exactly the one with no roster entry.
const ml = {
...LANE, name: 'ml', display_name: 'ML tagging', queues: ['ml'],
slots: 0, optional: true,
live: { present: false, replicas: 0, pool: null, active: 0 },
}
const rows = mergeParts([POSTGRES], [ml])
const row = rows.find((r) => r.name === 'ML tagging')
expect(row.lane).toBe(ml)
expect(row.kindLabel).toBe('optional lane')
})
it('does not call a lane at zero slots broken', () => {
// The roster only knows a heartbeat age, so it goes on saying "is running"
// for a lane the operator deliberately dialled to nothing. The lane knows
// the difference; reporting the operator's own choice as a fault is how an
// indicator stops being read.
const off = { ...LANE, slots: 0 }
const row = mergeParts([PART], [off])[0]
expect(row.detail).toBe('off — no slots')
})
it('puts the broken thing first, whatever it is', () => {
const down = { ...POSTGRES, state: 'down', detail: 'not answering' }
const rows = mergeParts([PART, down], [LANE])
expect(rows[0].name).toBe('PostgreSQL')
})
it('otherwise puts the rows you can act on first', () => {
const rows = mergeParts([POSTGRES, PART], [LANE])
expect(rows.map((r) => r.name)).toEqual(['Worker', 'PostgreSQL'])
})
it('reports a wedged lane, and only through the reporter it was given', () => {
// `laneStuckFor` is passed in rather than imported, so this file does not
// re-test the store's rule — it tests that the merge asks.
const asked = []
const rows = mergeParts([PART], [LANE], (lane) => {
asked.push(lane.name)
return '40 minutes'
})
expect(asked).toEqual(['worker'])
expect(rows[0].stuckFor).toBe('40 minutes')
// The roster still owns a matched row's state — `stuckFor` is a note
// beside it, not a verdict that overrides the heartbeat.
expect(rows[0].state).toBe('ok')
})
it('handles an empty everything without inventing rows', () => {
expect(mergeParts([], [])).toEqual([])
expect(mergeParts(undefined, undefined)).toEqual([])
})
})
frontend/test/workerLanes.spec.js
+6 -5
View File
@@ -54,8 +54,9 @@ describe('worker lanes store', () => {
})
it('a load failure records the error rather than throwing at the caller', async () => {
// The card polls this every 3s. An unhandled rejection per tick would
// drown the console and stop the other pollers in the same function.
// The System tab polls this every 15s. An unhandled rejection per tick
// would drown the console and stop the other pollers in the same
// function.
stubFetch(() => ({ status: 500, body: { error: 'boom' } }))
const s = useSystemActivityStore()
await expect(s.loadLanes()).resolves.toBeUndefined()
@@ -83,9 +84,9 @@ describe('worker lanes store', () => {
})
it('setLane refetches so the card shows the server truth, not the guess', async () => {
// The reply is one lane; the card renders all of them plus live pool and
// pending. Patching the local row from the reply would leave every other
// column stale and eventually wrong.
// The reply is one lane; the table renders all of them plus live pool
// and pending. Patching the local row from the reply would leave every
// other column stale and eventually wrong.
let gets = 0
stubFetch((url, init) => {
if (init?.method === 'POST') return { status: 200, body: { applied: true } }
tests/test_api_workers.py
+111
View File
@@ -170,3 +170,114 @@ async def test_an_empty_body_is_refused_rather_than_treated_as_a_no_op(
assert resp.status_code == 400
body = await resp.get_json()
assert body["error"] == "invalid_body"
# --- the dial is the switch --------------------------------------------------
#
# Operator, 2026-09-23: *"almost all of it always needs to run there's only one
# optional piece and it is killed by moving the 'cap' to zero."* So `enabled`
# is derived from the number rather than being a second control the operator
# has to keep in agreement with it. It stays on the API — these assert that it
# still does, because it is the mechanism the reconcile and the healthcheck
# read.
@pytest.mark.asyncio
async def test_dialling_a_lane_to_zero_turns_it_off(client, db, no_live_workers):
await client.post("/api/system/workers/worker", json={"slots": 0})
row = await _lane_row(db, "worker")
assert row.slots == 0
assert row.enabled is False
@pytest.mark.asyncio
async def test_dialling_it_back_up_turns_it_on(client, db, no_live_workers):
await client.post("/api/system/workers/ml", json={"slots": 1})
row = await _lane_row(db, "ml")
assert row.slots == 1
assert row.enabled is True, "the lane the operator just asked for work from"
@pytest.mark.asyncio
async def test_an_explicit_enabled_still_wins(client, db, no_live_workers):
"""The field is not removed, only derived when absent. Something that
genuinely wants a lane holding its process with consumers cancelled — a
drain before a restart — must still be able to say so without having to
destroy the operator's slot count to express it."""
await client.post(
"/api/system/workers/worker", json={"slots": 3, "enabled": False},
)
row = await _lane_row(db, "worker")
assert (row.slots, row.enabled) == (3, False)
@pytest.mark.asyncio
async def test_a_cap_only_write_does_not_decide_the_switch(
client, db, no_live_workers,
):
"""Only the SLOTS dial derives it. A cap is a ceiling, not a request for
work, and letting it flip the lane would make raising a ceiling start
something."""
before = await _lane_row(db, "ml")
assert before.enabled is False
await client.post("/api/system/workers/ml", json={"slots_cap": 1})
await db.refresh(before)
assert (before.slots, before.enabled) == (0, False)
@pytest.mark.asyncio
async def test_the_model_fetch_fires_on_the_transition_not_on_the_field(
client, db, no_live_workers, monkeypatch,
):
"""The trap the derivation set, caught here rather than in production.
The fetch used to be conditioned on `enabled is True` — the FIELD having
been sent. The UI no longer sends it at all, so the download that makes
the ML lane usable would simply never have fired, and the lane would have
come on and sat there consuming a queue it had no model for.
"""
fired = []
monkeypatch.setattr(wc, "_enqueue_model_fetch", lambda: fired.append(1) or True)
monkeypatch.setattr(
wc, "set_lane_enabled_sync", lambda lane, enabled, live=None: (True, None),
)
monkeypatch.setattr(
wc, "set_lane_slots_sync", lambda lane, target, live=None: (True, None),
)
body = await (await client.post(
"/api/system/workers/ml", json={"slots": 1},
)).get_json()
assert body["fetching_models"] is True
assert fired == [1]
@pytest.mark.asyncio
async def test_it_does_not_fire_again_on_a_lane_already_running(
client, db, no_live_workers, monkeypatch,
):
"""The other half. A second nudge of the dial on a lane that is already on
must not re-enqueue a multi-GB download."""
monkeypatch.setattr(
wc, "set_lane_enabled_sync", lambda lane, enabled, live=None: (True, None),
)
monkeypatch.setattr(
wc, "set_lane_slots_sync", lambda lane, target, live=None: (True, None),
)
await client.post("/api/system/workers/ml", json={"slots": 1})
fired = []
monkeypatch.setattr(wc, "_enqueue_model_fetch", lambda: fired.append(1) or True)
body = await (await client.post(
"/api/system/workers/ml", json={"slots": 1},
)).get_json()
assert body["fetching_models"] is False
assert fired == []