diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index 1e7b447..b8985a3 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -42,6 +42,7 @@ {% endif %} +{% if "syslog" in enabled_plugins %}{% endif %} {# Hidden poller: fetches the history series and applies it to the charts above (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on range change. #} @@ -112,6 +113,11 @@ ch.update("none"); } if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; }); + {% if "syslog" in enabled_plugins %} + // The host's own syslog errors over the same span (#6002). + stewardMarkCharts("/plugins/syslog/marks?host_id={{ host.id }}&range=" + encodeURIComponent(rangeKey || ""), + Object.values(charts), document.getElementById("hm-marks-note")); + {% endif %} }; })(); diff --git a/plugins/snmp/templates/snmp/device.html b/plugins/snmp/templates/snmp/device.html index ac6cfe8..6b9b305 100644 --- a/plugins/snmp/templates/snmp/device.html +++ b/plugins/snmp/templates/snmp/device.html @@ -42,6 +42,7 @@ {# The device's own log, when it sends syslog to an added source (#6000). #} {% if "syslog" in enabled_plugins %} +{% if charts %}{% endif %}
{% endif %} @@ -56,6 +57,7 @@ ? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false }) : d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false }); } + var drawn = []; charts.forEach(function (c, i) { var el = document.getElementById("snmp-c-" + i); if (!el) return; @@ -63,7 +65,7 @@ el.parentElement.innerHTML = '

Not enough readings yet for a chart.

'; return; } - new Chart(el.getContext("2d"), { + drawn.push(new Chart(el.getContext("2d"), { type: "line", data: { labels: c.points.map(function (p) { return p[0]; }), datasets: [{ data: c.points.map(function (p) { return p[1]; }), borderColor: line, @@ -81,8 +83,13 @@ title: { display: !!c.axis, text: c.axis, font: { size: 9 } } } } } - }); + })); }); + {% if "syslog" in enabled_plugins %} + // The device's own syslog errors over the same span (#6002). + if (drawn.length) stewardMarkCharts("/plugins/syslog/marks?address={{ device.address|urlencode }}{% if device.host_id %}&host_id={{ device.host_id }}{% endif %}&hours=" + hours, + drawn, document.getElementById("snmp-marks-note")); + {% endif %} })(); {% endblock %} diff --git a/plugins/syslog/__init__.py b/plugins/syslog/__init__.py index 23003ae..ef5d23b 100644 --- a/plugins/syslog/__init__.py +++ b/plugins/syslog/__init__.py @@ -40,6 +40,16 @@ def register_mcp_tools(register) -> None: register_tools(register) +async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]: + from .mcp_tools import mcp_events as events + return await events(db, host_id, host_name, since, until) + + +async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]: + from .mcp_tools import mcp_resource_events as events + return await events(db, source, resource, since, until) + + def get_alert_metrics() -> dict[str, list[str]]: """A pattern's count is named for the pattern (lines_matching_), so the list is empty here and the rule form offers the counts recorded so far.""" diff --git a/plugins/syslog/mcp_tools.py b/plugins/syslog/mcp_tools.py index bc5c76b..51db478 100644 --- a/plugins/syslog/mcp_tools.py +++ b/plugins/syslog/mcp_tools.py @@ -14,11 +14,8 @@ from steward.models.users import UserRole from .models import SyslogLine, SyslogSource from .parse import facility_name, severity_name -from .service import LineFilter, filtered, rates, severity_level, sources_for - - -def _source_name(s: SyslogSource) -> str: - return s.name or s.hostname_seen or s.address +from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, filtered, lines_between, place_for, rates, + severity_level, shown_on, source_name, sources_for) async def list_syslog_sources(state: str = "") -> dict: @@ -43,7 +40,7 @@ async def list_syslog_sources(state: str = "") -> dict: per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now) hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()} return {"sources": [{ - "source_id": s.id, "name": _source_name(s), "address": s.address, "state": s.state, + "source_id": s.id, "name": source_name(s), "address": s.address, "state": s.state, "host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None, "lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None, "last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now), @@ -94,14 +91,14 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi want = source.strip() by_address = [s for s in accepted if s.address == want] chosen = by_address or [pick(want, accepted, kind="syslog source", - list_tool="list_syslog_sources", name=_source_name)] + list_tool="list_syslog_sources", name=source_name)] if host: h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts") chosen = sources_for(chosen, h.id, h.address) if not chosen: return {"range": key, "matches": [], "returned": 0, "truncated": False, "note": f"No added syslog source is shown on {h.name}."} - names = {s.id: _source_name(s) for s in chosen} + names = {s.id: source_name(s) for s in chosen} stmt = filtered(select(SyslogLine).where(SyslogLine.source_id.in_(list(names))), f) if before_dt is not None: stmt = stmt.where(SyslogLine.received_at < before_dt) @@ -122,6 +119,33 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi return out +def _as_events(lines: list[SyslogLine], sources: list[SyslogSource]) -> list[dict]: + from steward.mcp.tools._events import event + names = {s.id: source_name(s) for s in sources} + return [event(ln.received_at, "syslog", + f"{severity_name(ln.severity)}{' ' + ln.app if ln.app else ''}: {cut_middle(ln.message, 300)}", + subject=names[ln.source_id]) for ln in reversed(lines)] + + +async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]: + """Warnings and worse in a window (get_anomaly) from the syslog sources + shown on a host: bound to it, or at its address.""" + sources = await shown_on(db, host_id, None) + return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources) + + +async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]: + """The same for an SNMP device, which has no host to read events by: the + sources at its address, or bound to its host.""" + if source != "snmp": + return [] + host_id, address = await place_for(db, source, resource) + if not (host_id or address): + return [] + sources = await shown_on(db, host_id, address) + return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources) + + def register_tools(register) -> None: for fn in (list_syslog_sources, search_syslog): register(fn, role=UserRole.viewer, read_only=True) diff --git a/plugins/syslog/routes.py b/plugins/syslog/routes.py index 74a3a85..0a02dd3 100644 --- a/plugins/syslog/routes.py +++ b/plugins/syslog/routes.py @@ -6,7 +6,7 @@ and the listener's state in memory; nothing here contacts a device (System and audited.""" from __future__ import annotations -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import Blueprint, abort, current_app, redirect, render_template, request, session from sqlalchemy import delete, select @@ -23,7 +23,9 @@ from .parse import SEVERITIES, facility_name, severity_name from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternError, key_for, parse_form, resource_of) from .receiver import LISTENER, MAX_SENDERS, settings -from .service import PANEL_LINES, PANEL_SPAN, SEVERITY_CHOICES, apps_of, filtered, line_filter, rates, sources_for +from .service import (AROUND_LINES, AROUND_SEVERITY, MARK_SEVERITY, MAX_MARKS, PANEL_LINES, PANEL_SPAN, + SEVERITY_CHOICES, apps_of, filtered, line_filter, lines_between, place_for, rates, shown_on, + source_name) syslog_bp = Blueprint("syslog", __name__, template_folder="templates") @@ -273,19 +275,11 @@ async def lines_panel(): """The syslog panel on a host's page (?host_id=) or an SNMP device's (?address=, and ?host_id= when it is bound): the newest lines of the sources that map to it. Nothing when no added source does.""" - from steward.models.hosts import Host - - host_id = request.args.get("host_id") or None - address = request.args.get("address") or None async with current_app.db_sessionmaker() as db: - if host_id and not address: - host = await db.get(Host, host_id) - address = host.address if host is not None else None - matched = sources_for(list((await db.execute( - select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars()), host_id, address) + matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None) if not matched: return "" - names = {s.id: s.name or s.hostname_seen or s.address for s in matched} + names = {s.id: source_name(s) for s in matched} since = datetime.now(timezone.utc) - PANEL_SPAN lines = (await db.execute( select(SyslogLine).where(SyslogLine.source_id.in_(list(names)), SyslogLine.received_at >= since) @@ -457,3 +451,61 @@ async def delete_pattern(pattern_id: str): await log_audit(current_app, *_actor(), "syslog.pattern_removed", entity_type="syslog_pattern", entity_id=pattern_id, detail={"name": name}) return redirect(PATTERNS) + + +# ── Lines beside the readings (#6002) ───────────────────────────────────────── + +AROUND = timedelta(minutes=30) +AROUND_SHOWN = 15 + + +def _iso_arg(name: str) -> datetime | None: + value = request.args.get(name) or "" + try: + at = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return at if at.tzinfo else at.replace(tzinfo=timezone.utc) + + +@syslog_bp.get("/around") +@require_role(UserRole.viewer) +async def lines_around(): + """An anomaly's syslog: warnings and worse from the sources shown with + its resource (?source=&resource=), from half an hour before ?since= to + half an hour after ?until=, the lines nearest its start first.""" + since = _iso_arg("since") + if since is None: + abort(400) + until = _iso_arg("until") or datetime.now(timezone.utc) + async with current_app.db_sessionmaker() as db: + host_id, address = await place_for(db, request.args.get("source", ""), request.args.get("resource", "")) + matched = await shown_on(db, host_id, address) if host_id or address else [] + lines = await lines_between(db, matched, since - AROUND, until + AROUND, AROUND_SEVERITY, AROUND_LINES) + names = {s.id: source_name(s) for s in matched} + nearest = sorted(lines, key=lambda ln: abs((ln.received_at - since).total_seconds()))[:AROUND_SHOWN] + rows = [{**line_view(ln), "source": names[ln.source_id], "source_id": ln.source_id} + for ln in sorted(nearest, key=lambda ln: ln.received_at)] + return await render_template("syslog/_around.html", lines=rows, more=len(lines) - len(rows), + sources=[{"id": sid, "name": name} for sid, name in names.items()]) + + +@syslog_bp.get("/marks") +@require_role(UserRole.viewer) +async def chart_marks(): + """Errors and worse from the sources a host's (?host_id=) or a device's + (?address=) page shows, over its chart's span (?range= or ?hours=), for + the chart to mark: {"marks": [{"at": epoch ms, "level", "text"}], "more"}.""" + from steward.core.time_range import parse_range + now = datetime.now(timezone.utc) + hours = request.args.get("hours", type=int) + since = now - timedelta(hours=min(max(hours, 1), 24 * 30)) if hours else parse_range(request.args.get("range"))[0] + async with current_app.db_sessionmaker() as db: + matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None) + lines = await lines_between(db, matched, since, now, MARK_SEVERITY, MAX_MARKS + 1) + names = {s.id: source_name(s) for s in matched} + return {"marks": [{"at": int(ln.received_at.timestamp() * 1000), "level": ln.severity, + "text": f"{names[ln.source_id]} {severity_name(ln.severity)}" + f"{' ' + ln.app if ln.app else ''}: {ln.message[:200]}"} + for ln in lines[:MAX_MARKS]], + "more": len(lines) > MAX_MARKS} diff --git a/plugins/syslog/service.py b/plugins/syslog/service.py index 715d51a..92bf1f1 100644 --- a/plugins/syslog/service.py +++ b/plugins/syslog/service.py @@ -110,3 +110,59 @@ def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | address = (address or "").strip().lower() return [s for s in sources if s.state == "accepted" and ( s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)] + + +# ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002) + +# What an anomaly's circumstances take from the log: warnings and worse, so a +# chatty device's info lines don't crowd out the line that explains it. +AROUND_SEVERITY = 4 +AROUND_LINES = 200 +# What a chart marks: errors and worse. Past MAX_MARKS the newest are kept. +MARK_SEVERITY = 3 +MAX_MARKS = 300 + + +def source_name(s: SyslogSource) -> str: + return s.name or s.hostname_seen or s.address + + +async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogSource]: + """The added sources a host's page (and its address) or a device's page + shows; the host's own address when only the host is given.""" + from steward.models.hosts import Host + if host_id and not address: + host = await db.get(Host, host_id) + address = host.address if host is not None else None + accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars()) + return sources_for(accepted, host_id, address) + + +async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]: + """(host id, address) whose sources a metric's resource is shown with: + an SNMP device's own binding and address, else the host the resource + lives on, as get_anomaly finds it.""" + if source == "snmp": + from steward.core.plugin_manager import loaded_plugin_submodule + snmp = loaded_plugin_submodule("snmp", "models") + if snmp is None: + return None, None + device = (await db.execute(select(snmp.SnmpDevice).where( + snmp.SnmpDevice.name == resource).limit(1))).scalar_one_or_none() + return (device.host_id, device.address) if device is not None else (None, None) + from steward.mcp.tools._events import place_of + place = await place_of(db, source, resource) + return place.host_id, None + + +async def lines_between(db, sources: list[SyslogSource], since: datetime, until: datetime, + worst: int, limit: int) -> list[SyslogLine]: + """Lines of these sources received in [since, until] at `worst` or worse, + newest first, at most `limit`.""" + if not sources: + return [] + return list((await db.execute( + select(SyslogLine).where(SyslogLine.source_id.in_([s.id for s in sources]), + SyslogLine.received_at >= since, SyslogLine.received_at <= until, + SyslogLine.severity <= worst) + .order_by(SyslogLine.received_at.desc(), SyslogLine.id.desc()).limit(limit))).scalars()) diff --git a/plugins/syslog/templates/syslog/_around.html b/plugins/syslog/templates/syslog/_around.html new file mode 100644 index 0000000..c7c8271 --- /dev/null +++ b/plugins/syslog/templates/syslog/_around.html @@ -0,0 +1,25 @@ +{# plugins/syslog/templates/syslog/_around.html — an anomaly's syslog: the + warnings and worse nearest its start, from the sources shown with its + resource (#6002). Loaded when its "Syslog around it" is opened. #} +{% from "syslog/_tone.html" import tone %} +{% if not sources %} +

No added syslog source is shown with this resource.

+{% elif not lines %} +

No warnings or worse from {% for s in sources %}{% if not loop.first %}, {% endif %}{{ s.name }}{% endfor %} from half an hour before to half an hour after.

+{% else %} + + {% if sources|length > 1 %}{% endif %} + + {% for ln in lines %} + + + {% if sources|length > 1 %}{% endif %} + + + + + {% endfor %} + +
Received ({{ tz_label() }})SourceSeverityAppMessage
{{ ln.received_at|when(seconds=true) }}{{ ln.source }}{{ ln.severity }}{{ ln.app or "—" }}{{ ln.message }}
+{% if more > 0 %}

{{ more }} more in that hour either side, further from its start: open the source to read them.

{% endif %} +{% endif %} diff --git a/steward/mcp/tools/anomalies.py b/steward/mcp/tools/anomalies.py index 3651447..2c3cd31 100644 --- a/steward/mcp/tools/anomalies.py +++ b/steward/mcp/tools/anomalies.py @@ -161,7 +161,8 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict: runs, other anomalies on the host, what plugins saw (container starts, deaths and OOM kills; maintenance jobs; reboots and agent stops; for a Traefik router, its swarm service's containers on any - host), and kind "steward": a stretch Steward itself wasn't recording + host; syslog warnings and worse from the host's or the SNMP device's + own log), and kind "steward": a stretch Steward itself wasn't recording (a restart or deploy), around which any reading can be wrong. From 30 minutes before to 30 minutes after. Each kind keeps the 40 events nearest the occurrence's start; `around_left_out` counts the rest diff --git a/steward/static/chart-marks.js b/steward/static/chart-marks.js new file mode 100644 index 0000000..aa4568d --- /dev/null +++ b/steward/static/chart-marks.js @@ -0,0 +1,82 @@ +// Marks on a time chart: a thin line at each moment something was logged +// (#6002, the syslog errors of a host or a device), so a reading's jump can be +// read against what the device said at the time. +// +// A chart opts in by holding marks in options.plugins.stewardMarks.marks, as +// [{at: epoch ms, level, text}]. A linear x axis takes the time itself; a +// category axis of ISO times takes the nearest label. Hovering near a mark +// shows its text in the chart's title attribute. +(function () { + if (!window.Chart) return; + function xOf(chart, at) { + var x = chart.scales.x; + if (!x) return null; + if (x.type !== 'category') return x.getPixelForValue(at); + var labels = chart.data.labels || []; + if (!labels.length) return null; + var first = Date.parse(labels[0]), last = Date.parse(labels[labels.length - 1]); + if (at < first || at > last) return null; + var best = 0, gap = Infinity; + for (var i = 0; i < labels.length; i++) { + var d = Math.abs(Date.parse(labels[i]) - at); + if (d < gap) { gap = d; best = i; } + } + return x.getPixelForValue(best); + } + Chart.register({ + id: 'stewardMarks', + afterDatasetsDraw: function (chart, args, opts) { + var marks = (opts && opts.marks) || []; + if (!marks.length) return; + var area = chart.chartArea, ctx = chart.ctx; + var crit = window.stewardColor ? stewardColor('--red') : 'red'; + chart.$stewardMarks = []; + ctx.save(); + ctx.lineWidth = 1; + ctx.setLineDash([3, 3]); + ctx.strokeStyle = crit; + ctx.globalAlpha = 0.55; + marks.forEach(function (m) { + var px = xOf(chart, m.at); + if (px == null || px < area.left || px > area.right) return; + ctx.beginPath(); + ctx.moveTo(px, area.top); + ctx.lineTo(px, area.bottom); + ctx.stroke(); + chart.$stewardMarks.push({ x: px, text: m.text, at: m.at }); + }); + ctx.restore(); + }, + afterEvent: function (chart, args) { + var near = (chart.$stewardMarks || []).filter(function (m) { + return Math.abs(m.x - args.event.x) <= 3; + }); + chart.canvas.title = near.map(function (m) { + return new Date(m.at).toLocaleString([], { hour12: false }) + ' ' + m.text; + }).join('\n'); + }, + }); + + // Fetch marks from `url` and put them on each chart, redrawing it; `note` + // (an element, optional) says in words how many there are. + window.stewardMarkCharts = function (url, charts, note) { + fetch(url, { credentials: 'same-origin' }) + .then(function (r) { return r.ok ? r.json() : { marks: [] }; }) + .then(function (body) { + var marks = body.marks || []; + charts.forEach(function (ch) { + if (!ch) return; + ch.options.plugins.stewardMarks = { marks: marks }; + ch.update('none'); + }); + if (note) { + note.hidden = !marks.length; + note.textContent = marks.length + ? (body.more ? 'The newest ' : '') + marks.length + ' syslog error' + (marks.length === 1 ? '' : 's') + + ' in this span marked with dashed red lines; hover one to read it.' + : ''; + } + }) + .catch(function () {}); + }; +})(); diff --git a/steward/templates/_anomaly.html b/steward/templates/_anomaly.html index b233e85..03b5f29 100644 --- a/steward/templates/_anomaly.html +++ b/steward/templates/_anomaly.html @@ -68,5 +68,14 @@ Held {{ metric_value(a.metric_name, a.peak_level) }}; usual for that hour {{ anomaly_when(v) }} · {{ anomaly_level(a) }}
{{ anomaly_review(a, back) }}
+ {%- if "syslog" in enabled_plugins and v.last_span %} + {#- The device's own log around its latest occurrence (#6002), read only when opened. #} +
+ Syslog around it +

Reading…

+
+ {%- endif %} {%- endmacro %} diff --git a/steward/templates/base.html b/steward/templates/base.html index 95d5bd5..f2e04e7 100644 --- a/steward/templates/base.html +++ b/steward/templates/base.html @@ -20,6 +20,7 @@ + {# A form with data-confirm="…" asks in #confirm-dialog below before it submits (#5314). #} diff --git a/tests/integration/test_syslog.py b/tests/integration/test_syslog.py index e52d039..9293b6d 100644 --- a/tests/integration/test_syslog.py +++ b/tests/integration/test_syslog.py @@ -364,3 +364,42 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app): await _clear_patterns(app) asyncio.run(go()) + + +def test_lines_beside_the_readings_around_an_anomaly_and_as_chart_marks(app): + now = datetime.now(timezone.utc) + mcp_tools = _mod("mcp_tools") + + async def go(): + host_id, host_name, _, src_name = await _seed(app, now) + client = await _client(app, "viewer") + since, until = (now - timedelta(minutes=35)).isoformat(), (now - timedelta(minutes=25)).isoformat() + around = await (await client.get("/plugins/syslog/around", query_string={ + "source": "host_agent", "resource": f"{host_name}:/", "since": since, "until": until})).get_data(as_text=True) + # Warnings and worse within half an hour either side; info lines left out. + assert "nfsd: server not responding" in around and "100% of quota used" in around + assert "eth0 link up" not in around and "an old error" not in around + elsewhere = await (await client.get("/plugins/syslog/around", query_string={ + "source": "traefik", "resource": "web@docker", "since": since})).get_data(as_text=True) + assert "No added syslog source" in elsewhere + assert (await client.get("/plugins/syslog/around?source=host_agent")).status_code == 400 + + day = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&range=24h")).get_json() + assert [m["text"] for m in day["marks"]] == [f"{src_name} err kernel: nfsd: server not responding"] + assert day["more"] is False + week = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&hours=168")).get_json() + assert len(week["marks"]) == 2 + assert (await (await client.get("/plugins/syslog/marks?address=10.1.1.1")).get_json())["marks"] == [] + + # get_anomaly's circumstances: the host's lines, and nothing for a + # resource syslog can't place. + async with app.db_sessionmaker() as s: + events = await mcp_tools.mcp_events(s, host_id, host_name, now - timedelta(hours=1), now) + assert [(e["kind"], e["subject"]) for e in events] == [("syslog", src_name)] * 2 + assert events[0]["what"].startswith("err kernel: nfsd") + assert await mcp_tools.mcp_resource_events(s, "traefik", "web@docker", now - timedelta(hours=1), now) == [] + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(delete_sources()) + + asyncio.run(go())