Files
FabledCurator/frontend/test/workerLanes.spec.js
T
bvandeusenandClaude Opus 5 1353d346b3
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 3s
CI and images / frontend-build (push) Successful in 25s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m13s
CI and images / sign-extension (push) Successful in 3s
CI and images / build-agent (push) Successful in 6s
CI and images / build-web (push) Successful in 1m59s
CI and images / smoke-web (push) Successful in 58s
CI and images / promote (push) Skipped
fix: the cap dial waited out a broker round trip it did not need (4295)
Operator: "when the number is changed the change should be queued so that
it isn't blocking of the webui or the system itself. we shouldn't have to
wait for the validation live."

Two waits, and 5b6f2ba removed neither — it stopped a Postgres connection
being HELD across them, which is what had been stalling the whole site,
and left the press itself as slow as it was.

1. The store refetched after every write. GET /api/system/workers runs a
   celery inspect on an eleven-second budget, so the stepper stayed
   disabled through a round trip the press did not need. It now patches
   the row from the reply — cap, ceiling, enabled, the three fields that
   reply actually decides — and lets the 15s poll bring the live columns,
   which are measurements it must not invent.

2. The endpoint pushed to the broker before answering. Turning a lane off
   is four cancel_consumer messages; lowering a cap reads the live pool
   first. Now it stores the cap, answers `queued`, and hands the push to a
   Quart background task. Raising a cap was already free and stays free.

Nothing is lost by not waiting: the stored cap is what the system obeys
and the sizing pass re-reads it every minute. That sweep was already the
backstop for a push that failed, which under `no_live_workers` is every
push in the suite.

Also closes a hole the move exposed: the model fetch was gated on the
consumer change having landed, so raising ML off zero while the lane was
restarting stored the cap, let the sizing pass start the consumers a
minute later, and left the lane running with no model — nothing else ever
asks for one. It now fires on the transition and waits in the ml queue.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
2026-09-23 15:41:32 -04:00

235 lines
9.1 KiB
JavaScript

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { setActivePinia, createPinia } from 'pinia'
import { laneStuckFor, useSystemActivityStore } from '../src/stores/systemActivity.js'
// Milestone 422 step 4. Covers the store half of the worker-lane dial — the
// part that decides what the card can tell the operator.
//
// What is protected here since 2026-09-23: pressing the dial must not wait on
// a broker round trip. Operator: *"when the number is changed the change
// should be queued so that it isn't blocking of the webui or the system
// itself. we shouldn't have to wait for the validation live."* The endpoint
// answers once the cap is STORED, so the store patches its row from that
// reply and lets the 15s poll bring the live columns — it used to refetch,
// and GET /api/system/workers costs a celery inspect on an eleven-second
// budget.
//
// And still: a REFUSED value (400) is a failure and must reach the operator
// as one. A control that silently does nothing is worse than one that
// refuses out loud.
function stubFetch(handler) {
globalThis.fetch = vi.fn(async (url, init) => {
const { status, body } = handler(url, init)
return {
ok: status >= 200 && status < 300,
status,
statusText: String(status),
text: async () => (body == null ? '' : JSON.stringify(body)),
}
})
}
const LANES_BODY = {
lanes: [
{
name: 'worker', display_name: 'Worker',
queues: ['default', 'import', 'thumbnail', 'download'],
slots_cap: 1, ceiling: 8, enabled: true, memory_bound: false,
live: { present: true, replicas: 1, pool: 1, active: 0, reserved: 3 },
queue_depth: 5, pending: 8,
},
{
name: 'ml', display_name: 'ML tagging', queues: ['ml'],
slots_cap: 0, ceiling: 2, enabled: false, memory_bound: true,
live: { present: false, replicas: 0, pool: null, active: 0, reserved: 0 },
queue_depth: null, pending: null,
},
],
fetched_at: '2026-09-22T12:00:00Z',
}
describe('worker lanes store', () => {
beforeEach(() => setActivePinia(createPinia()))
afterEach(() => vi.restoreAllMocks())
it('loads the lanes', async () => {
stubFetch(() => ({ status: 200, body: LANES_BODY }))
const s = useSystemActivityStore()
await s.loadLanes()
expect(s.lanes.lanes.map((l) => l.name)).toEqual(['worker', 'ml'])
})
it('a load failure records the error rather than throwing at the caller', async () => {
// The System tab polls this every 15s. An unhandled rejection per tick
// would drown the console and stop the other pollers in the same
// function.
stubFetch(() => ({ status: 500, body: { error: 'boom' } }))
const s = useSystemActivityStore()
await expect(s.loadLanes()).resolves.toBeUndefined()
expect(s.lastError).toBeTruthy()
})
it('setLane posts only the fields it was given', async () => {
// One field: the cap. How many workers are running is a measurement the
// sizing pass owns, so there is nothing else for the UI to send.
const calls = []
stubFetch((url, init) => {
calls.push({ url, init })
if (init?.method === 'POST') {
return { status: 200, body: { name: 'worker', slots_cap: 2, queued: true } }
}
return { status: 200, body: LANES_BODY }
})
const s = useSystemActivityStore()
await s.setLane('worker', { slots_cap: 2 })
const post = calls.find((c) => c.init?.method === 'POST')
expect(post.url).toContain('/api/system/workers/worker')
expect(JSON.parse(post.init.body)).toEqual({ slots_cap: 2 })
})
it('setLane does not refetch — that refetch is what blocked the press', async () => {
// It refetched until 2026-09-23, so that the card showed server truth
// rather than a local guess. The cost was the operator's: GET
// /api/system/workers runs a celery inspect, so every press of `+` sat
// with the stepper disabled through a broker round trip the press did not
// need.
let gets = 0
stubFetch((url, init) => {
if (init?.method === 'POST') {
return { status: 200, body: { slots_cap: 2, ceiling: 8, enabled: true } }
}
gets += 1
return { status: 200, body: LANES_BODY }
})
const s = useSystemActivityStore()
await s.loadLanes()
gets = 0
await s.setLane('worker', { slots_cap: 2 })
expect(gets).toBe(0)
})
it('patches the row from the reply, so the new number shows at once', async () => {
stubFetch((url, init) => {
if (init?.method === 'POST') {
return {
status: 200,
body: { name: 'worker', slots_cap: 2, ceiling: 8, enabled: true, queued: true },
}
}
return { status: 200, body: LANES_BODY }
})
const s = useSystemActivityStore()
await s.loadLanes()
await s.setLane('worker', { slots_cap: 2 })
const worker = s.lanes.lanes.find((l) => l.name === 'worker')
expect(worker.slots_cap).toBe(2)
})
it('does not invent the live columns the reply cannot know', async () => {
// The reply is decided from the stored cap and the machine's ceiling
// alone — it never asked a worker anything. Pool, active and pending are
// MEASUREMENTS, and the poll a few seconds later is what carries them.
// Zeroing or guessing them here would make the table lie in the direction
// that reads as "the lane stopped".
stubFetch((url, init) => {
if (init?.method === 'POST') {
return { status: 200, body: { slots_cap: 2, ceiling: 8, enabled: true } }
}
return { status: 200, body: LANES_BODY }
})
const s = useSystemActivityStore()
await s.loadLanes()
const before = { ...s.lanes.lanes[0].live }
await s.setLane('worker', { slots_cap: 2 })
expect(s.lanes.lanes[0].live).toEqual(before)
expect(s.lanes.lanes[0].pending).toBe(8)
})
it('survives a reply arriving for a lane it has not loaded yet', async () => {
// First paint, or a lane added by a newer build. Patching must not be the
// thing that throws inside the click handler.
stubFetch(() => ({ status: 200, body: { slots_cap: 2 } }))
const s = useSystemActivityStore()
await expect(s.setLane('worker', { slots_cap: 2 })).resolves.toBeTruthy()
})
it('a refused value throws so the card can show the reason', async () => {
// Deliberately NOT swallowed. The detail is written to be read by a person
// ("above what this container can hold"), and a control that silently does
// nothing is worse than one that refuses out loud.
stubFetch((url, init) => {
if (init?.method === 'POST') {
return {
status: 400,
body: { error: 'refused', detail: 'cap 10000 is above what this container can hold (2 for ML tagging)' },
}
}
return { status: 200, body: LANES_BODY }
})
const s = useSystemActivityStore()
// Assert the REASON is reachable, not merely that it threw. `toThrow()`
// alone passes whether the card can read the sentence or not — which is
// how the first version of the card shipped reading `e.detail` (always
// undefined) and would have shown the operator the bare word "refused".
const err = await s.setLane('ml', { slots_cap: 10000 }).catch((e) => e)
expect(err.status).toBe(400)
expect(err.body.detail).toContain('container can hold')
})
})
// --- the long-running-task warning (step 7) ----------------------------------
//
// The operator asked for "a task runs for x concurrent time" to TRIGGER
// growth. It became a warning: extra slots do not make a running task finish
// sooner. What is pinned here is the AND — a warning that fires on half its
// condition is one people learn to ignore, and at that point it is worse than
// not having it.
describe('laneStuckFor', () => {
const lane = (over = {}) => ({
oldest_running_minutes: 40,
live: { present: true, pool: 4, active: 4 },
...over,
})
it('warns when every slot is busy and the oldest task is old', () => {
expect(laneStuckFor(lane())).toBe('40 minutes')
})
it('says nothing when the lane has a free slot', () => {
expect(laneStuckFor(lane({ live: { present: true, pool: 4, active: 3 } })))
.toBeNull()
})
it('says nothing for a saturated lane whose tasks are young', () => {
expect(laneStuckFor(lane({ oldest_running_minutes: 2 }))).toBeNull()
})
it('says nothing when nothing is running', () => {
expect(laneStuckFor(lane({ oldest_running_minutes: null }))).toBeNull()
})
it('says nothing about a lane that is not answering', () => {
// Unknown is not busy. A lane nothing answered for has no live reading to
// call saturated, and asserting one would be a verdict from an unswept
// read — the same distinction `present` exists for everywhere else here.
expect(laneStuckFor(lane({ live: { present: false, pool: null, active: 0 } })))
.toBeNull()
})
it('says nothing when the pool is zero', () => {
// A lane with no workers is not "fully busy at zero" — it is off.
expect(laneStuckFor(lane({ live: { present: true, pool: 0, active: 0 } })))
.toBeNull()
})
it('switches to hours once minutes stop being readable', () => {
expect(laneStuckFor(lane({ oldest_running_minutes: 195 }))).toBe('3 hours')
})
})