feat(syslog): lines beside the readings: around an anomaly, in get_anomaly, and marked on host and SNMP charts (#6002)

get_anomaly's circumstances gain the host's or SNMP device's syslog
warnings and worse (mcp_events / mcp_resource_events). An anomaly row opens
"Syslog around it", read only when opened. A shared Chart.js plugin
(chart-marks.js) draws a dashed line at each syslog error on the host
metrics charts and an SNMP device's charts, with a line saying how many.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/templates/host_detail.html

@@ -42,6 +42,7 @@
   </div>
   {% endif %}
 </div>
+{% if "syslog" in enabled_plugins %}<p id="hm-marks-note" class="field-hint" hidden style="margin:0.5rem 0 0;"></p>{% endif %}
 {# Hidden poller: fetches the history series and applies it to the charts above
    (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
    range change. #}
@@ -112,6 +113,11 @@
       ch.update("none");
     }
     if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
+    {% if "syslog" in enabled_plugins %}
+    // The host's own syslog errors over the same span (#6002).
+    stewardMarkCharts("/plugins/syslog/marks?host_id={{ host.id }}&range=" + encodeURIComponent(rangeKey || ""),
+                      Object.values(charts), document.getElementById("hm-marks-note"));
+    {% endif %}
   };
 })();
 </script>

plugins/snmp/templates/snmp/device.html

@@ -42,6 +42,7 @@
 
 {# The device's own log, when it sends syslog to an added source (#6000). #}
 {% if "syslog" in enabled_plugins %}
+{% if charts %}<p id="snmp-marks-note" class="field-hint" hidden style="margin:0.5rem 0 0;"></p>{% endif %}
 <div hx-get="/plugins/syslog/panel?address={{ device.address|urlencode }}{% if device.host_id %}&amp;host_id={{ device.host_id }}{% endif %}" hx-trigger="load" hx-swap="innerHTML"></div>
 {% endif %}
 
@@ -56,6 +57,7 @@
       ? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
       : d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
   }
+  var drawn = [];
   charts.forEach(function (c, i) {
     var el = document.getElementById("snmp-c-" + i);
     if (!el) return;
@@ -63,7 +65,7 @@
       el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough readings yet for a chart.</p>';
       return;
     }
-    new Chart(el.getContext("2d"), {
+    drawn.push(new Chart(el.getContext("2d"), {
       type: "line",
       data: { labels: c.points.map(function (p) { return p[0]; }),
               datasets: [{ data: c.points.map(function (p) { return p[1]; }), borderColor: line,
@@ -81,8 +83,13 @@
                title: { display: !!c.axis, text: c.axis, font: { size: 9 } } }
         }
       }
-    });
+    }));
   });
+  {% if "syslog" in enabled_plugins %}
+  // The device's own syslog errors over the same span (#6002).
+  if (drawn.length) stewardMarkCharts("/plugins/syslog/marks?address={{ device.address|urlencode }}{% if device.host_id %}&host_id={{ device.host_id }}{% endif %}&hours=" + hours,
+                                      drawn, document.getElementById("snmp-marks-note"));
+  {% endif %}
 })();
 </script>
 {% endblock %}

plugins/syslog/__init__.py

@@ -40,6 +40,16 @@ def register_mcp_tools(register) -> None:
     register_tools(register)
 
 
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    from .mcp_tools import mcp_events as events
+    return await events(db, host_id, host_name, since, until)
+
+
+async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
+    from .mcp_tools import mcp_resource_events as events
+    return await events(db, source, resource, since, until)
+
+
 def get_alert_metrics() -> dict[str, list[str]]:
     """A pattern's count is named for the pattern (lines_matching_<key>), so
     the list is empty here and the rule form offers the counts recorded so far."""

plugins/syslog/mcp_tools.py

@@ -14,11 +14,8 @@ from steward.models.users import UserRole
 
 from .models import SyslogLine, SyslogSource
 from .parse import facility_name, severity_name
-from .service import LineFilter, filtered, rates, severity_level, sources_for
-
-
-def _source_name(s: SyslogSource) -> str:
-    return s.name or s.hostname_seen or s.address
+from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, filtered, lines_between, place_for, rates,
+                      severity_level, shown_on, source_name, sources_for)
 
 
 async def list_syslog_sources(state: str = "") -> dict:
@@ -43,7 +40,7 @@ async def list_syslog_sources(state: str = "") -> dict:
         per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now)
         hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()}
     return {"sources": [{
-        "source_id": s.id, "name": _source_name(s), "address": s.address, "state": s.state,
+        "source_id": s.id, "name": source_name(s), "address": s.address, "state": s.state,
         "host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None,
         "lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None,
         "last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now),
@@ -94,14 +91,14 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi
             want = source.strip()
             by_address = [s for s in accepted if s.address == want]
             chosen = by_address or [pick(want, accepted, kind="syslog source",
-                                         list_tool="list_syslog_sources", name=_source_name)]
+                                         list_tool="list_syslog_sources", name=source_name)]
         if host:
             h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts")
             chosen = sources_for(chosen, h.id, h.address)
             if not chosen:
                 return {"range": key, "matches": [], "returned": 0, "truncated": False,
                         "note": f"No added syslog source is shown on {h.name}."}
-        names = {s.id: _source_name(s) for s in chosen}
+        names = {s.id: source_name(s) for s in chosen}
         stmt = filtered(select(SyslogLine).where(SyslogLine.source_id.in_(list(names))), f)
         if before_dt is not None:
             stmt = stmt.where(SyslogLine.received_at < before_dt)
@@ -122,6 +119,33 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi
     return out
 
 
+def _as_events(lines: list[SyslogLine], sources: list[SyslogSource]) -> list[dict]:
+    from steward.mcp.tools._events import event
+    names = {s.id: source_name(s) for s in sources}
+    return [event(ln.received_at, "syslog",
+                  f"{severity_name(ln.severity)}{' ' + ln.app if ln.app else ''}: {cut_middle(ln.message, 300)}",
+                  subject=names[ln.source_id]) for ln in reversed(lines)]
+
+
+async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
+    """Warnings and worse in a window (get_anomaly) from the syslog sources
+    shown on a host: bound to it, or at its address."""
+    sources = await shown_on(db, host_id, None)
+    return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources)
+
+
+async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
+    """The same for an SNMP device, which has no host to read events by: the
+    sources at its address, or bound to its host."""
+    if source != "snmp":
+        return []
+    host_id, address = await place_for(db, source, resource)
+    if not (host_id or address):
+        return []
+    sources = await shown_on(db, host_id, address)
+    return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources)
+
+
 def register_tools(register) -> None:
     for fn in (list_syslog_sources, search_syslog):
         register(fn, role=UserRole.viewer, read_only=True)

plugins/syslog/routes.py

@@ -6,7 +6,7 @@ and the listener's state in memory; nothing here contacts a device (System
 and audited."""
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 from quart import Blueprint, abort, current_app, redirect, render_template, request, session
 from sqlalchemy import delete, select
@@ -23,7 +23,9 @@ from .parse import SEVERITIES, facility_name, severity_name
 from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternError, key_for, parse_form,
                        resource_of)
 from .receiver import LISTENER, MAX_SENDERS, settings
-from .service import PANEL_LINES, PANEL_SPAN, SEVERITY_CHOICES, apps_of, filtered, line_filter, rates, sources_for
+from .service import (AROUND_LINES, AROUND_SEVERITY, MARK_SEVERITY, MAX_MARKS, PANEL_LINES, PANEL_SPAN,
+                      SEVERITY_CHOICES, apps_of, filtered, line_filter, lines_between, place_for, rates, shown_on,
+                      source_name)
 
 syslog_bp = Blueprint("syslog", __name__, template_folder="templates")
 
@@ -273,19 +275,11 @@ async def lines_panel():
     """The syslog panel on a host's page (?host_id=) or an SNMP device's
     (?address=, and ?host_id= when it is bound): the newest lines of the
     sources that map to it. Nothing when no added source does."""
-    from steward.models.hosts import Host
-
-    host_id = request.args.get("host_id") or None
-    address = request.args.get("address") or None
     async with current_app.db_sessionmaker() as db:
-        if host_id and not address:
-            host = await db.get(Host, host_id)
-            address = host.address if host is not None else None
-        matched = sources_for(list((await db.execute(
-            select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars()), host_id, address)
+        matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None)
         if not matched:
             return ""
-        names = {s.id: s.name or s.hostname_seen or s.address for s in matched}
+        names = {s.id: source_name(s) for s in matched}
         since = datetime.now(timezone.utc) - PANEL_SPAN
         lines = (await db.execute(
             select(SyslogLine).where(SyslogLine.source_id.in_(list(names)), SyslogLine.received_at >= since)
@@ -457,3 +451,61 @@ async def delete_pattern(pattern_id: str):
     await log_audit(current_app, *_actor(), "syslog.pattern_removed", entity_type="syslog_pattern",
                     entity_id=pattern_id, detail={"name": name})
     return redirect(PATTERNS)
+
+
+# ── Lines beside the readings (#6002) ─────────────────────────────────────────
+
+AROUND = timedelta(minutes=30)
+AROUND_SHOWN = 15
+
+
+def _iso_arg(name: str) -> datetime | None:
+    value = request.args.get(name) or ""
+    try:
+        at = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
+
+
+@syslog_bp.get("/around")
+@require_role(UserRole.viewer)
+async def lines_around():
+    """An anomaly's syslog: warnings and worse from the sources shown with
+    its resource (?source=&resource=), from half an hour before ?since= to
+    half an hour after ?until=, the lines nearest its start first."""
+    since = _iso_arg("since")
+    if since is None:
+        abort(400)
+    until = _iso_arg("until") or datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        host_id, address = await place_for(db, request.args.get("source", ""), request.args.get("resource", ""))
+        matched = await shown_on(db, host_id, address) if host_id or address else []
+        lines = await lines_between(db, matched, since - AROUND, until + AROUND, AROUND_SEVERITY, AROUND_LINES)
+    names = {s.id: source_name(s) for s in matched}
+    nearest = sorted(lines, key=lambda ln: abs((ln.received_at - since).total_seconds()))[:AROUND_SHOWN]
+    rows = [{**line_view(ln), "source": names[ln.source_id], "source_id": ln.source_id}
+            for ln in sorted(nearest, key=lambda ln: ln.received_at)]
+    return await render_template("syslog/_around.html", lines=rows, more=len(lines) - len(rows),
+                                 sources=[{"id": sid, "name": name} for sid, name in names.items()])
+
+
+@syslog_bp.get("/marks")
+@require_role(UserRole.viewer)
+async def chart_marks():
+    """Errors and worse from the sources a host's (?host_id=) or a device's
+    (?address=) page shows, over its chart's span (?range= or ?hours=), for
+    the chart to mark: {"marks": [{"at": epoch ms, "level", "text"}], "more"}."""
+    from steward.core.time_range import parse_range
+    now = datetime.now(timezone.utc)
+    hours = request.args.get("hours", type=int)
+    since = now - timedelta(hours=min(max(hours, 1), 24 * 30)) if hours else parse_range(request.args.get("range"))[0]
+    async with current_app.db_sessionmaker() as db:
+        matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None)
+        lines = await lines_between(db, matched, since, now, MARK_SEVERITY, MAX_MARKS + 1)
+    names = {s.id: source_name(s) for s in matched}
+    return {"marks": [{"at": int(ln.received_at.timestamp() * 1000), "level": ln.severity,
+                       "text": f"{names[ln.source_id]} {severity_name(ln.severity)}"
+                               f"{' ' + ln.app if ln.app else ''}: {ln.message[:200]}"}
+                      for ln in lines[:MAX_MARKS]],
+            "more": len(lines) > MAX_MARKS}

plugins/syslog/service.py

@@ -110,3 +110,59 @@ def sources_for(sources: list[SyslogSource], host_id: str | None, address: str |
     address = (address or "").strip().lower()
     return [s for s in sources if s.state == "accepted" and (
         s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)]
+
+
+# ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002)
+
+# What an anomaly's circumstances take from the log: warnings and worse, so a
+# chatty device's info lines don't crowd out the line that explains it.
+AROUND_SEVERITY = 4
+AROUND_LINES = 200
+# What a chart marks: errors and worse. Past MAX_MARKS the newest are kept.
+MARK_SEVERITY = 3
+MAX_MARKS = 300
+
+
+def source_name(s: SyslogSource) -> str:
+    return s.name or s.hostname_seen or s.address
+
+
+async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogSource]:
+    """The added sources a host's page (and its address) or a device's page
+    shows; the host's own address when only the host is given."""
+    from steward.models.hosts import Host
+    if host_id and not address:
+        host = await db.get(Host, host_id)
+        address = host.address if host is not None else None
+    accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars())
+    return sources_for(accepted, host_id, address)
+
+
+async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]:
+    """(host id, address) whose sources a metric's resource is shown with:
+    an SNMP device's own binding and address, else the host the resource
+    lives on, as get_anomaly finds it."""
+    if source == "snmp":
+        from steward.core.plugin_manager import loaded_plugin_submodule
+        snmp = loaded_plugin_submodule("snmp", "models")
+        if snmp is None:
+            return None, None
+        device = (await db.execute(select(snmp.SnmpDevice).where(
+            snmp.SnmpDevice.name == resource).limit(1))).scalar_one_or_none()
+        return (device.host_id, device.address) if device is not None else (None, None)
+    from steward.mcp.tools._events import place_of
+    place = await place_of(db, source, resource)
+    return place.host_id, None
+
+
+async def lines_between(db, sources: list[SyslogSource], since: datetime, until: datetime,
+                        worst: int, limit: int) -> list[SyslogLine]:
+    """Lines of these sources received in [since, until] at `worst` or worse,
+    newest first, at most `limit`."""
+    if not sources:
+        return []
+    return list((await db.execute(
+        select(SyslogLine).where(SyslogLine.source_id.in_([s.id for s in sources]),
+                                 SyslogLine.received_at >= since, SyslogLine.received_at <= until,
+                                 SyslogLine.severity <= worst)
+        .order_by(SyslogLine.received_at.desc(), SyslogLine.id.desc()).limit(limit))).scalars())

plugins/syslog/templates/syslog/_around.html

@@ -0,0 +1,25 @@
+{# plugins/syslog/templates/syslog/_around.html — an anomaly's syslog: the
+   warnings and worse nearest its start, from the sources shown with its
+   resource (#6002). Loaded when its "Syslog around it" is opened. #}
+{% from "syslog/_tone.html" import tone %}
+{% if not sources %}
+<p class="field-hint" style="margin:0.3rem 0 0;">No added syslog source is shown with this resource.</p>
+{% elif not lines %}
+<p class="field-hint" style="margin:0.3rem 0 0;">No warnings or worse from {% for s in sources %}{% if not loop.first %}, {% endif %}<a href="/plugins/syslog/sources/{{ s.id }}">{{ s.name }}</a>{% endfor %} from half an hour before to half an hour after.</p>
+{% else %}
+<table class="table" style="font-size:0.78rem;margin-top:0.3rem;">
+  <thead><tr><th class="nowrap">Received ({{ tz_label() }})</th>{% if sources|length > 1 %}<th>Source</th>{% endif %}<th>Severity</th><th>App</th><th>Message</th></tr></thead>
+  <tbody>
+  {% for ln in lines %}
+  <tr>
+    <td class="nowrap">{{ ln.received_at|when(seconds=true) }}</td>
+    {% if sources|length > 1 %}<td class="nowrap"><a href="/plugins/syslog/sources/{{ ln.source_id }}">{{ ln.source }}</a></td>{% endif %}
+    <td class="nowrap"><span class="{{ tone(ln.level) }}">{{ ln.severity }}</span></td>
+    <td class="nowrap">{{ ln.app or "—" }}</td>
+    <td class="mono" style="max-width:32rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ ln.message }}">{{ ln.message }}</td>
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
+{% if more > 0 %}<p class="field-hint" style="margin:0.3rem 0 0;">{{ more }} more in that hour either side, further from its start: open the source to read them.</p>{% endif %}
+{% endif %}