FabledCurator/frontend/src/utils/systemParts.js

// Joining the roster to the worker lanes, for the System tab's one table.
//
// Extracted from the component rather than left inline because the failure
// this can have is SILENT and is exactly the thing the merge exists to fix: if
// a lane stops matching its roster part, nothing throws — the table simply
// grows a second row for the same worker, one with controls and one without,
// which is the duplication the operator asked to be rid of, returned by the
// code that removed it.
//
//   Operator, 2026-09-23: "I feel that we can probably combine the two
//   sections into a single table."

// A learned roster part and a lane are the same thing seen from two sides, and
// the QUEUES are what identify it — `service_roster.refresh_celery_roster`
// keys a celery part on exactly `"celery:" + ",".join(sorted(queues))`.
//
// Matched on the sorted set rather than on that string so the join survives a
// change to how the key is spelled, and so neither side has to agree about
// ORDER: the lane table lists a lane's queues in the order the role reads them
// (`default, import, thumbnail, download`) while the roster sorts them
// (`default, download, import, thumbnail`).
export function queueKey(queues) {
  return [...(queues || [])].sort().join(',')
}

// Worst first. A stopped datastore is why someone opened this tab.
export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }

export function kindLabel(kind) {
  if (kind === 'celery') return 'worker lane'
  if (kind === 'agent') return 'GPU agent'
  if (kind === 'datastore') return 'datastore'
  return kind
}

/**
 * One row per moving part, with a lane attached where there is one.
 *
 * @param parts   the roster's parts, as /api/system/health returns them
 * @param lanes   the lane rows, as /api/system/workers returns them
 * @param stuckFor a lane -> "40 minutes" | null reporter (laneStuckFor)
 */
export function mergeParts(parts, lanes, stuckFor = () => null) {
  const unmatched = {}
  for (const lane of lanes || []) unmatched[queueKey(lane.queues)] = lane

  const out = []
  for (const part of parts || []) {
    const key = queueKey(part.queues)
    const lane = part.kind === 'celery' ? unmatched[key] : undefined
    if (lane) delete unmatched[key]
    out.push({
      key: part.key,
      name: part.name,
      kindLabel: lane?.optional ? 'optional lane' : kindLabel(part.kind),
      state: part.state,
      // A lane capped at zero is OFF, not broken. Say so, rather than let the
      // roster's heartbeat sentence report the operator's own choice as a
      // fault — the roster cannot know the difference, and the lane can.
      detail: lane && lane.slots_cap === 0 ? 'off — cap is zero' : part.detail,
      queues: (part.queues || []).join(', '),
      lane,
      stuckFor: lane ? stuckFor(lane) : null,
      severity: SEVERITY[part.state] ?? SEVERITY.unknown,
    })
  }

  // A lane the roster has not learned yet. Parts appear only once they have
  // checked in, while the lane table is known up front — so without this, the
  // lane an operator most needs to find (an optional one, never yet started)
  // would be the only one missing from the table.
  for (const lane of Object.values(unmatched)) out.push(laneRow(lane, stuckFor))

  // Severity leads; then lanes ahead of everything else, because they are the
  // rows you can actually do something about; then by name.
  return out.sort((a, b) =>
    b.severity - a.severity
    || Number(Boolean(b.lane)) - Number(Boolean(a.lane))
    || a.name.localeCompare(b.name))
}

function laneRow(lane, stuckFor) {
  const on = lane.slots_cap > 0
  let state = 'unknown'
  if (lane.live?.present) state = on ? (stuckFor(lane) ? 'stale' : 'ok') : 'unknown'
  else if (on) state = 'down'
  return {
    key: `lane:${lane.name}`,
    name: lane.display_name,
    kindLabel: lane.optional ? 'optional lane' : 'worker lane',
    state,
    detail: lane.live?.present
      ? (on ? 'running' : 'off — cap is zero')
      : 'has not checked in yet',
    queues: (lane.queues || []).join(', '),
    lane,
    stuckFor: stuckFor(lane),
    severity: SEVERITY[state],
  }
}


// How much has to be waiting before we tell someone to raise a cap.
//
// Not "anything at all". A lane at its cap with three items queued is working
// normally and will be empty in a moment; a notice there is one people learn
// to scroll past, and at that point it is worse than not having it.
export const ADVISE_BACKLOG = 10

/**
 * The sentence that tells an operator the cap is now the limiting factor.
 *
 * Operator, 2026-09-23: *"there needs to be something that tells you user to
 * bump those numbers to improve processing rate or they'd never know the
 * controls exist."* The defaults are deliberately one-of-each, so on a busy
 * instance the shipped configuration IS the bottleneck — and a conservative
 * default nobody knows how to raise is just a slow product.
 *
 * Only fires when raising the cap would actually help: there is real work
 * waiting, every worker the cap allows is already running, and the cap is
 * below what this machine can hold. A lane already at its ceiling gets
 * nothing, because there is nothing it could be told to do.
 */
export function laneAdvice(lane) {
  if (!lane) return null
  const pending = lane.pending
  if (pending == null || pending < ADVISE_BACKLOG) return null

  if (lane.slots_cap === 0) {
    return `${pending.toLocaleString()} waiting, and this lane is off. `
      + 'Raise its cap to start working through them.'
  }
  if (lane.ceiling <= lane.slots_cap) {
    // At the machine's limit, not the operator's. Saying "raise the cap"
    // here would be advice they cannot take.
    return null
  }
  if (!lane.live?.present || (lane.live.pool ?? 0) < lane.slots_cap) return null

  return `${pending.toLocaleString()} waiting and all ${lane.slots_cap} `
    + `worker${lane.slots_cap === 1 ? '' : 's'} busy. `
    + `Raise the cap to run more at once — this machine allows up to `
    + `${lane.ceiling}.`
}


// When the worker numbers in the table were actually read.
//
// They used to be fetched live on every page load — a celery broadcast per
// request, on a page that polls every 15s. Operator, 2026-09-23: *"there is a
// repull every time this page loads — is there a reason this info isn't being
// tracked in the background and stored in some way?"* It is now: a sweep
// measures on a timer and stores it, and the page reads the table.
//
// So the age has to be ON SCREEN. A number that is up to half a minute old,
// presented as current, is how someone ends up watching a queue "not move"
// that is in fact moving. The NEWEST across lanes, because they are written
// by one sweep in one pass — a lane lagging the others means its row has
// never been written, and that lane says so in its own detail line.
export function measuredAt(lanes) {
  const stamps = (lanes || [])
    .map((l) => l.measured_at)
    .filter(Boolean)
    .sort()
  return stamps.length ? stamps[stamps.length - 1] : null
}