// Joining the roster to the worker lanes, for the System tab's one table. // // Extracted from the component rather than left inline because the failure // this can have is SILENT and is exactly the thing the merge exists to fix: if // a lane stops matching its roster part, nothing throws — the table simply // grows a second row for the same worker, one with controls and one without, // which is the duplication the operator asked to be rid of, returned by the // code that removed it. // // Operator, 2026-09-23: "I feel that we can probably combine the two // sections into a single table." // A learned roster part and a lane are the same thing seen from two sides, and // the QUEUES are what identify it — `service_roster.refresh_celery_roster` // keys a celery part on exactly `"celery:" + ",".join(sorted(queues))`. // // Matched on the sorted set rather than on that string so the join survives a // change to how the key is spelled, and so neither side has to agree about // ORDER: the lane table lists a lane's queues in the order the role reads them // (`default, import, thumbnail, download`) while the roster sorts them // (`default, download, import, thumbnail`). export function queueKey(queues) { return [...(queues || [])].sort().join(',') } // Worst first. A stopped datastore is why someone opened this tab. export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 } export function kindLabel(kind) { if (kind === 'celery') return 'worker lane' if (kind === 'agent') return 'GPU agent' if (kind === 'datastore') return 'datastore' return kind } /** * One row per moving part, with a lane attached where there is one. * * @param parts the roster's parts, as /api/system/health returns them * @param lanes the lane rows, as /api/system/workers returns them * @param stuckFor a lane -> "40 minutes" | null reporter (laneStuckFor) */ export function mergeParts(parts, lanes, stuckFor = () => null) { const unmatched = {} for (const lane of lanes || []) unmatched[queueKey(lane.queues)] = lane const out = [] for (const part of parts || []) { const key = queueKey(part.queues) const lane = part.kind === 'celery' ? unmatched[key] : undefined if (lane) delete unmatched[key] out.push({ key: part.key, name: part.name, kindLabel: lane?.optional ? 'optional lane' : kindLabel(part.kind), state: part.state, // A lane capped at zero is OFF, not broken. Say so, rather than let the // roster's heartbeat sentence report the operator's own choice as a // fault — the roster cannot know the difference, and the lane can. detail: lane && lane.slots_cap === 0 ? 'off — cap is zero' : part.detail, queues: (part.queues || []).join(', '), lane, stuckFor: lane ? stuckFor(lane) : null, severity: SEVERITY[part.state] ?? SEVERITY.unknown, }) } // A lane the roster has not learned yet. Parts appear only once they have // checked in, while the lane table is known up front — so without this, the // lane an operator most needs to find (an optional one, never yet started) // would be the only one missing from the table. for (const lane of Object.values(unmatched)) out.push(laneRow(lane, stuckFor)) // Severity leads; then lanes ahead of everything else, because they are the // rows you can actually do something about; then by name. return out.sort((a, b) => b.severity - a.severity || Number(Boolean(b.lane)) - Number(Boolean(a.lane)) || a.name.localeCompare(b.name)) } function laneRow(lane, stuckFor) { const on = lane.slots_cap > 0 let state = 'unknown' if (lane.live?.present) state = on ? (stuckFor(lane) ? 'stale' : 'ok') : 'unknown' else if (on) state = 'down' return { key: `lane:${lane.name}`, name: lane.display_name, kindLabel: lane.optional ? 'optional lane' : 'worker lane', state, detail: lane.live?.present ? (on ? 'running' : 'off — cap is zero') : 'has not checked in yet', queues: (lane.queues || []).join(', '), lane, stuckFor: stuckFor(lane), severity: SEVERITY[state], } } // How much has to be waiting before we tell someone to raise a cap. // // Not "anything at all". A lane at its cap with three items queued is working // normally and will be empty in a moment; a notice there is one people learn // to scroll past, and at that point it is worse than not having it. export const ADVISE_BACKLOG = 10 /** * The sentence that tells an operator the cap is now the limiting factor. * * Operator, 2026-09-23: *"there needs to be something that tells you user to * bump those numbers to improve processing rate or they'd never know the * controls exist."* The defaults are deliberately one-of-each, so on a busy * instance the shipped configuration IS the bottleneck — and a conservative * default nobody knows how to raise is just a slow product. * * Only fires when raising the cap would actually help: there is real work * waiting, every worker the cap allows is already running, and the cap is * below what this machine can hold. A lane already at its ceiling gets * nothing, because there is nothing it could be told to do. */ export function laneAdvice(lane) { if (!lane) return null const pending = lane.pending if (pending == null || pending < ADVISE_BACKLOG) return null if (lane.slots_cap === 0) { return `${pending.toLocaleString()} waiting, and this lane is off. ` + 'Raise its cap to start working through them.' } if (lane.ceiling <= lane.slots_cap) { // At the machine's limit, not the operator's. Saying "raise the cap" // here would be advice they cannot take. return null } if (!lane.live?.present || (lane.live.pool ?? 0) < lane.slots_cap) return null return `${pending.toLocaleString()} waiting and all ${lane.slots_cap} ` + `worker${lane.slots_cap === 1 ? '' : 's'} busy. ` + `Raise the cap to run more at once — this machine allows up to ` + `${lane.ceiling}.` } // When the worker numbers in the table were actually read. // // They used to be fetched live on every page load — a celery broadcast per // request, on a page that polls every 15s. Operator, 2026-09-23: *"there is a // repull every time this page loads — is there a reason this info isn't being // tracked in the background and stored in some way?"* It is now: a sweep // measures on a timer and stores it, and the page reads the table. // // So the age has to be ON SCREEN. A number that is up to half a minute old, // presented as current, is how someone ends up watching a queue "not move" // that is in fact moving. The NEWEST across lanes, because they are written // by one sweep in one pass — a lane lagging the others means its row has // never been written, and that lane says so in its own detail line. export function measuredAt(lanes) { const stamps = (lanes || []) .map((l) => l.measured_at) .filter(Boolean) .sort() return stamps.length ? stamps[stamps.length - 1] : null }