feat(ui): host page leads with its alerts; old readings say how old (#5210)

#5210 "Problems first, with since-time, across every list" (milestone 477),
second batch.

- Host page: an "Alerts on this host" strip first (rules on its agent
  series, its containers or its monitors: alert_svc.watches_host); its
  monitors problems first with "Down 12m · reason"; Docker above Ansible.
- Uptime page and widget: worst 24h uptime first, then 7d and 30d; hosts
  with no uptime yet last.
- Agent panel and fleet widget: last seen as "3m ago", not a clock time.
- SNMP: a device is "Reachable" only while it answered within the last 3
  polls; older readings say "No answer" with their age, and every device
  says when it was last read (was: reachable whenever any reading existed).
- Traefik: a router missing from 3 scrapes in a row is listed apart as no
  longer reported, by when it was last seen, instead of keeping its last
  state (a removed router stayed Failing).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -215,6 +215,18 @@ async def active_rows(db, now: datetime) -> list[dict]:
     return rows
 
 
+def watches_host(rule: AlertRule, host_name: str, monitor_keys: set) -> bool:
+    """Whether a rule watches something on this host: its agent series
+    ("<host>" or "<host>:<part>"), one of its containers ("<host>/<name>"),
+    or one of its monitors ((type, name) in monitor_keys)."""
+    source, resource = rule.source_module, rule.resource_name
+    if source == "host_agent":
+        return resource == host_name or resource.startswith(host_name + ":")
+    if source == "docker":
+        return resource.startswith(host_name + "/")
+    return (source, resource) in monitor_keys
+
+
 async def active_count(db) -> int:
     """How many alerts need attention: the sidebar's number."""
     from sqlalchemy import func

steward/hosts/routes.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 import asyncio
 import uuid
+from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from sqlalchemy import select
 from steward.ansible import executor, sources as ansible_src
+from steward.alerts import service as alert_svc
 from steward.auth.middleware import require_role
 from steward.core import renames
 from steward.core.audit import audit_request, log_audit
@@ -78,6 +80,10 @@ async def host_detail(host_id: str):
             return "Not found", 404
         monitors = await host_svc.host_monitors(db, host_id)
         uptime = (await host_svc.compute_uptime(db)).get(host_id)
+        monitor_keys = {(getattr(d["monitor"].type, "value", d["monitor"].type), d["monitor"].name)
+                        for d in monitors}
+        alerts = [r for r in await alert_svc.active_rows(db, datetime.now(timezone.utc))
+                  if alert_svc.watches_host(r["rule"], host.name, monitor_keys)]
         linked_target = (await db.execute(
             select(AnsibleTarget).where(AnsibleTarget.host_id == host_id)
             .options(selectinload(AnsibleTarget.groups)))).scalar_one_or_none()
@@ -88,7 +94,7 @@ async def host_detail(host_id: str):
     from steward.models.monitors import MonitorType
     return await render_template(
         "hosts/detail.html",
-        host=host, monitors=monitors, uptime=uptime,
+        host=host, monitors=monitors, uptime=uptime, alerts=alerts,
         monitor_types=list(MonitorType),
         linked_target=linked_target, linkable_targets=linkable_targets,
         ansible_sources=_ansible_source_names(),
@@ -305,8 +311,8 @@ async def delete_host(host_id: str):
 async def uptime_page():
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).order_by(Host.name))
-        hosts = result.scalars().all()
         uptime = await host_svc.compute_uptime(db)
+        hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
     return await render_template("hosts/uptime.html", hosts=hosts, uptime=uptime)
 
 
@@ -315,8 +321,8 @@ async def uptime_page():
 async def uptime_widget():
     async with current_app.db_sessionmaker() as db:
         result = await db.execute(select(Host).order_by(Host.name))
-        hosts = result.scalars().all()
         uptime = await host_svc.compute_uptime(db)
+        hosts = host_svc.worst_uptime_first(result.scalars().all(), uptime)
     return await render_template(
         "hosts/uptime_widget.html",
         hosts=hosts,

steward/hosts/service.py

@@ -113,6 +113,17 @@ async def host_status(db, host_ids: list[str]) -> dict[str, dict]:
     return out
 
 
+def worst_uptime_first(hosts, uptime: dict) -> list:
+    """Hosts by their 24h uptime, worst first, then 7d and 30d to break ties;
+    hosts with no uptime yet (no monitors) last, by name."""
+    def key(h):
+        u = uptime.get(h.id) or {}
+        windows = [u.get(w) for w in ("24h", "7d", "30d")]
+        known = [w for w in windows if w is not None]
+        return (not known, *[101.0 if w is None else w for w in windows], h.name.lower())
+    return sorted(hosts, key=key)
+
+
 def host_rank(status: dict | None, agent: dict | None) -> int:
     """Where a host sorts in the Hosts list: a down monitor first, then an
     agent gone quiet, then monitors not yet checked, then the healthy; a host
@@ -137,16 +148,23 @@ async def host_monitors(db, host_id: str) -> list[dict]:
     recent = await _last_n_results(db, ids)
     uptime = await _uptime_by_monitor(db, ids)
     data = []
+    from steward.monitors.service import down_reason, down_since
     for m in monitors:
         mrows = recent.get(m.id, [])
         latest = mrows[-1] if mrows else None
+        state = ("paused" if not m.enabled else "pending" if latest is None
+                 else "up" if latest.is_up else "down")
         data.append({
-            "monitor": m, "latest": latest,
+            "monitor": m, "latest": latest, "state": state,
             "heartbeat": [{"state": "up" if r.is_up else "down",
                            "title": _heartbeat_title(m.type, r)} for r in mrows],
             "uptime_pct": uptime.get(m.id, {}).get("24h"),
+            "down_since": await down_since(db, m.id) if state == "down" else None,
+            "down_reason": down_reason(latest) if state == "down" else None,
         })
-    return data
+    # Problems first: down, then not yet checked, then up, paused last.
+    order = {"down": 0, "pending": 1, "up": 2, "paused": 3}
+    return sorted(data, key=lambda d: (order[d["state"]], d["monitor"].name.lower()))
 
 
 async def agent_overview_by_host(

steward/templates/hosts/detail.html

@@ -24,6 +24,27 @@
 </div>
 
 
+{# ── Alerts on this host: what is breaching, before anything else ─────────── #}
+{% if alerts %}
+<section class="card problem-card" aria-labelledby="host-alerts" style="margin-bottom:1rem;">
+  <h2 id="host-alerts" class="section-title" style="margin-bottom:0.6rem;">Alerts on this host</h2>
+  <ul class="problem-list">
+    {% for row in alerts %}
+    <li>
+      {{ status_badge(row.label) }}
+      <span class="problem-detail">
+        {{- row.says.lead -}}
+        {%- if row.url %}<a href="{{ row.url }}">{{ row.says.resource }}</a>{% else %}<strong>{{ row.says.resource }}</strong>{% endif -%}
+        {{- row.says.tail -}}
+        {%- if row.label == 'acknowledged' and row.acked_by %} · acknowledged by {{ row.acked_by }}{% endif -%}
+      </span>
+      <a href="/alerts/history?rule={{ row.rule.id }}" class="field-hint" style="margin-left:0.4rem;">{{ row.rule.name }}</a>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Live vitals strip (host_agent fragment) — full width on top, polled ────── #}
 <div id="hv-strip"
      hx-get="/plugins/host_agent/vitals/{{ host.id }}"
@@ -52,10 +73,15 @@
   <div class="ping-row{% if not m.enabled %} ping-row-paused{% endif %}">
     <div class="ping-meta" style="min-width:150px;max-width:230px;">
       <div style="display:flex;align-items:baseline;gap:0.4rem;">
-        {{ status_dot('paused' if not m.enabled else ('pending' if latest is none else ('up' if latest.is_up else 'down'))) }}
+        {{ status_dot(d.state) }}
         <a class="ping-name" href="/monitors/{{ m.id }}">{{ m.name }}</a>
       </div>
       <span class="ping-addr" title="{{ m.target }}">{{ m.type|monitor_type }} · {{ m.target }}</span>
+      {% if d.state == 'down' %}
+      <span class="tone-crit" style="font-size:0.75rem;overflow-wrap:anywhere;">
+        Down{% if d.down_since %} {{ d.down_since|ago|replace(' ago', '') }}{% endif %}{% if d.down_reason %} · {{ d.down_reason }}{% endif %}
+      </span>
+      {% endif %}
     </div>
     <div class="ping-pills">
       {% for _ in range([30 - d.heartbeat|length, 0]|max) %}<span class="pill pill-empty" title="No data"></span>{% endfor %}
@@ -124,6 +150,12 @@
 </div>
 </div>{# Monitors + Agent grid #}
 
+{# ── Docker (docker plugin fragment; renders nothing if the host has none);
+   above Ansible: containers are state to read, Ansible is something to do ──── #}
+{% if "docker" in enabled_plugins %}
+<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
+{% endif %}
+
 {# ── Ansible (full width) ─────────────────────────────────────────────────── #}
 <div class="card">
   <h3 class="section-title">Ansible</h3>
@@ -211,11 +243,6 @@
   {% endif %}
 </div>{# Ansible card #}
 
-{# ── Docker (docker plugin fragment; renders nothing if the host has none) ──── #}
-{% if "docker" in enabled_plugins %}
-<div hx-get="/plugins/docker/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>
-{% endif %}
-
 {# ── SNMP (renders nothing unless a configured SNMP device maps to this host) ── #}
 {% if "snmp" in enabled_plugins %}
 <div hx-get="/plugins/snmp/host/{{ host.id }}" hx-trigger="load" hx-swap="innerHTML"></div>