FabledCurator can now tell you one of its own parts has stopped #250

Merged
bvandeusen merged 3 commits from dev into main 2026-09-02 18:01:39 -04:00
5 changed files with 250 additions and 8 deletions
Showing only changes of commit 5084ba666b - Show all commits
+59 -7
View File
@@ -5,9 +5,12 @@
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
<span class="fc-brand__text">FabledCurator</span>
</RouterLink>
<span class="fc-health" :title="health.label">
<RouterLink
:to="{ name: 'system' }" class="fc-health" :title="health.label"
:aria-label="`System health: ${health.label}`"
>
<v-icon size="x-small" :color="health.color">{{ health.icon }}</v-icon>
</span>
</RouterLink>
<PipelineStatusChip />
</div>
@@ -64,13 +67,15 @@
</template>
<script setup>
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
import { computed, onBeforeUnmount, onMounted, onUnmounted, ref } from 'vue'
import { useRoute } from 'vue-router'
import router, { FRONT_DOOR } from '../router.js'
import { useSystemStore } from '../stores/system.js'
import { useSystemHealthStore } from '../stores/systemHealth.js'
import PipelineStatusChip from './PipelineStatusChip.vue'
const system = useSystemStore()
const healthStore = useSystemHealthStore()
// Publish the nav's REAL height as --fc-nav-h so full-height workspaces
// (Explore/Subscriptions) and sticky sub-headers pin to it exactly instead of a
@@ -116,15 +121,55 @@ const settingsRoute = computed(() =>
navRoutes.value.find(r => r.name === 'settings') || null
)
// The dot beside the brand, and the only ambient signal that something in the
// stack has stopped (milestone 365).
//
// It used to read /api/health — a no-DB liveness check that proves the WEB
// container is serving and nothing else. Green there while the worker was dead
// is exactly what it looked like, and a green dot next to the product name is
// read as "everything is fine". It now reflects the whole-stack verdict.
//
// Deliberately re-using this element rather than adding a second indicator:
// there were already three partial surfaces (this, the pipeline chip, the
// Settings Activity tab) and a fourth would have made the question harder to
// answer, not easier. This is the one that already occupied the slot.
const health = computed(() => {
if (system.healthy === null) {
const overall = healthStore.overall
if (overall === null) {
return { icon: 'mdi-circle-outline', color: 'on-surface', label: 'checking…' }
}
if (system.healthy === true) {
return { icon: 'mdi-circle', color: 'success', label: 'healthy' }
if (overall === 'ok') {
return { icon: 'mdi-circle', color: 'success', label: 'All parts running' }
}
return { icon: 'mdi-alert-circle', color: 'error', label: 'unreachable' }
// Name what is wrong in the tooltip. "Something is unhealthy" sends someone
// hunting; "Scheduler has not checked in for 6 min" does not.
const worst = healthStore.problems[0]
const others = healthStore.problems.length - 1
const suffix = others > 0 ? ` (+${others} more)` : ''
if (overall === 'down') {
return {
icon: 'mdi-alert-circle', color: 'error',
label: (worst?.detail || 'A part has stopped') + suffix,
}
}
if (overall === 'stale') {
return {
icon: 'mdi-alert', color: 'warning',
label: (worst?.detail || 'A part is quiet') + suffix,
}
}
return { icon: 'mdi-help-circle-outline', color: 'on-surface', label: 'Health unknown' }
})
const HEALTH_POLL_MS = 15_000
let healthTimer = null
onMounted(() => {
healthStore.refresh()
healthTimer = setInterval(() => {
if (!document.hidden) healthStore.refresh()
}, HEALTH_POLL_MS)
})
onUnmounted(() => { if (healthTimer) clearInterval(healthTimer) })
</script>
<style scoped>
@@ -237,7 +282,14 @@ const health = computed(() => {
display: flex;
align-items: center;
flex-shrink: 0;
/* A RouterLink since milestone 365 — it is the path to /system, not just an
indicator. Reset the anchor so turning a span into a link changed nothing
about how the nav reads. */
text-decoration: none;
color: inherit;
border-radius: 50%;
}
.fc-health:hover { background: rgb(var(--v-theme-on-surface) / 0.12); }
.fc-nav-right {
flex: 1 1 0;
min-width: 0;
+7
View File
@@ -1,5 +1,6 @@
import { createRouter, createWebHistory, createMemoryHistory } from 'vue-router'
import SettingsView from './views/SettingsView.vue'
import SystemView from './views/SystemView.vue'
import GalleryView from './views/GalleryView.vue'
import ShowcaseView from './views/ShowcaseView.vue'
import ExploreView from './views/ExploreView.vue'
@@ -45,6 +46,12 @@ const routes = [
// Settings — config, pinned to the right of the nav (TopNav special-cases it).
{ path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings', stickyChrome: true } },
// Deliberately NO meta.title: TopNav builds its nav row from routes that
// have one, and this is reached from the health indicator beside the
// brand — the place someone already looks when they suspect something is
// wrong. A sixth top-level tab for a page you visit twice a year would
// cost more attention than it returns.
{ path: '/system', name: 'system', component: SystemView },
// The old standalone paths now redirect into the Browse hub, preserving any
// deep-link query (e.g. /posts?post_id=N → /browse?tab=posts&post_id=N). The
+6 -1
View File
@@ -4,7 +4,12 @@ import { useApi } from '../composables/useApi.js'
export const useSystemStore = defineStore('system', () => {
const api = useApi()
const healthy = ref(null) // null=unknown, true=ok, false=down
// NOT what the nav dot reads any more (milestone 365): that is the
// whole-stack verdict in systemHealth.js. /api/health only proves the web
// container is serving, which is why a green dot here sat happily beside a
// dead worker. refreshHealth() is still called — it is also how build/version
// info arrives — so this stays as its by-product rather than its purpose.
const healthy = ref(null)
// What the instance says it is. Since milestone 318 stopped publishing
// version image tags, this is the only answer to "which build is this?" —
// there is no registry name left to check it against.
+49
View File
@@ -0,0 +1,49 @@
import { defineStore } from 'pinia'
import { computed, ref } from 'vue'
import { useApi } from '../composables/useApi.js'
// Whole-stack health: is every part of FabledCurator running (milestone 365)?
//
// Distinct from `system.js`, which polls /api/health — a no-DB liveness check
// that only proves the web container is serving. That endpoint answers "can I
// reach the API"; this one answers "is anything broken", which is the question
// a green dot beside the brand was already being read as answering.
//
// Also distinct from `systemActivity.js`, which is about what the pipeline is
// DOING — queue depths, running tasks, failures. Running and alive are
// different questions and they fail independently: a perfectly idle stack with
// a dead worker looks identical to a healthy one on the activity surfaces.
export const useSystemHealthStore = defineStore('systemHealth', () => {
const api = useApi()
const overall = ref(null) // null until the first answer: unknown ≠ ok
const parts = ref([])
const checkedAt = ref(null)
const thresholds = ref(null) // server-owned, so the UI keeps no second copy
const lastError = ref(null)
async function refresh() {
try {
const body = await api.get('/api/system/health')
overall.value = body.overall
parts.value = body.parts || []
checkedAt.value = body.checked_at
thresholds.value = body.thresholds || null
lastError.value = null
} catch (e) {
// The endpoint is built never to fail because a dependency failed, so a
// throw here means the API itself is unreachable — which is its own kind
// of unhealthy and must not be shown as "ok".
lastError.value = e.message
overall.value = 'unknown'
}
return overall.value
}
// The parts worth naming in a tooltip — everything that is not ok, worst
// first. The endpoint already sorts that way.
const problems = computed(() => parts.value.filter(p => p.state !== 'ok'))
return { overall, parts, checkedAt, thresholds, lastError, problems, refresh }
})
+129
View File
@@ -0,0 +1,129 @@
<template>
<v-container class="py-6" style="max-width: 900px">
<div class="d-flex align-center mb-1">
<h1 class="text-h5">System</h1>
<v-spacer />
<span class="fc-sys__checked">
{{ store.checkedAt ? `checked ${formatRelative(store.checkedAt)}` : 'checking…' }}
</span>
</div>
<p class="fc-sys__lede text-body-2 mb-5">
Every moving part of FabledCurator and whether it is still checking in.
Parts are learned as they appear, so anything that has run at least once
stays listed that is what lets a stopped one be noticed rather than
simply vanishing.
</p>
<v-alert
v-if="store.lastError" type="error" variant="tonal" density="compact" class="mb-4"
>
Could not reach FabledCurator: {{ store.lastError }}
</v-alert>
<v-card v-else variant="flat" class="fc-sys__card">
<div v-if="!store.parts.length" class="pa-6 text-center fc-sys__muted">
Still gathering this fills in on the first check.
</div>
<div
v-for="part in store.parts" :key="part.key"
class="fc-sys__row" :class="`fc-sys__row--${part.state}`"
>
<span class="fc-sys__dot" :class="`fc-sys__dot--${part.state}`" />
<div class="fc-sys__body">
<div class="fc-sys__name">
{{ part.name }}
<span class="fc-sys__kind">{{ kindLabel(part.kind) }}</span>
</div>
<!-- The sentence, not just a chip. At the moment someone is deciding
whether to go and open Portainer, "has not checked in for 6 min"
is the thing that answers them. -->
<div class="fc-sys__detail">{{ part.detail }}</div>
</div>
<div class="fc-sys__meta">
<div v-if="part.last_seen_at" :title="part.last_seen_at">
seen {{ formatRelative(part.last_seen_at) }}
</div>
<div v-if="part.latency_ms != null">{{ part.latency_ms }} ms</div>
<div v-if="part.queues?.length" class="fc-sys__queues">{{ part.queues.join(', ') }}</div>
</div>
</div>
</v-card>
<p v-if="store.thresholds" class="fc-sys__foot text-caption mt-4">
A part is called stale after
{{ Math.round(store.thresholds.stale_after_seconds / 60) }} min without a
check-in and treated as stopped after
{{ Math.round(store.thresholds.down_after_seconds / 60) }} min. The window
is deliberately wide: a rolling deploy briefly runs two of a service and
then neither, and an indicator that reddened on every update would stop
being read.
</p>
</v-container>
</template>
<script setup>
import { onMounted, onUnmounted } from 'vue'
import { useSystemHealthStore } from '../stores/systemHealth.js'
import { formatRelative } from '../utils/date.js'
const store = useSystemHealthStore()
// Slower than the pipeline chip's 8s: liveness changes on the scale of
// container restarts, not task starts, and this page is open while someone
// watches it.
const POLL_MS = 10_000
let timer = null
function kindLabel(kind) {
if (kind === 'celery') return 'background worker'
if (kind === 'agent') return 'GPU agent'
if (kind === 'datastore') return 'datastore'
return kind
}
onMounted(() => {
store.refresh()
timer = setInterval(() => { if (!document.hidden) store.refresh() }, POLL_MS)
})
onUnmounted(() => { if (timer) clearInterval(timer) })
</script>
<style scoped>
.fc-sys__lede, .fc-sys__muted, .fc-sys__checked, .fc-sys__foot {
color: rgb(var(--v-theme-on-surface) / 0.66);
}
.fc-sys__checked { font-size: 0.78rem; }
.fc-sys__card { background: rgb(var(--v-theme-on-surface) / 0.04); }
.fc-sys__row {
display: flex; align-items: center; gap: 12px;
padding: 12px 16px;
border-bottom: 1px solid rgb(var(--v-theme-on-surface) / 0.08);
}
.fc-sys__row:last-child { border-bottom: 0; }
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
.fc-sys__body { min-width: 0; flex: 1 1 auto; }
.fc-sys__name { font-weight: 600; }
.fc-sys__kind {
margin-left: 8px; font-weight: 400; font-size: 0.72rem; text-transform: uppercase;
letter-spacing: 0.04em; color: rgb(var(--v-theme-on-surface) / 0.5);
}
.fc-sys__detail { font-size: 0.82rem; color: rgb(var(--v-theme-on-surface) / 0.72); }
.fc-sys__meta {
text-align: right; font-size: 0.75rem; flex: 0 0 auto;
font-variant-numeric: tabular-nums; color: rgb(var(--v-theme-on-surface) / 0.6);
}
.fc-sys__queues { opacity: 0.75; }
</style>