feat(syslog): lines beside the readings: around an anomaly, in get_anomaly, and marked on host and SNMP charts (#6002)
get_anomaly's circumstances gain the host's or SNMP device's syslog warnings and worse (mcp_events / mcp_resource_events). An anomaly row opens "Syslog around it", read only when opened. A shared Chart.js plugin (chart-marks.js) draws a dashed line at each syslog error on the host metrics charts and an SNMP device's charts, with a line saying how many. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -42,6 +42,7 @@
|
|||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
|
{% if "syslog" in enabled_plugins %}<p id="hm-marks-note" class="field-hint" hidden style="margin:0.5rem 0 0;"></p>{% endif %}
|
||||||
{# Hidden poller: fetches the history series and applies it to the charts above
|
{# Hidden poller: fetches the history series and applies it to the charts above
|
||||||
(no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
|
(no visible swap). Lazy on load → never blocks paint; refreshed slowly + on
|
||||||
range change. #}
|
range change. #}
|
||||||
@@ -112,6 +113,11 @@
|
|||||||
ch.update("none");
|
ch.update("none");
|
||||||
}
|
}
|
||||||
if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
|
if (rangeKey) document.querySelectorAll(".hm-chart-range").forEach((s) => { s.textContent = rangeKey; });
|
||||||
|
{% if "syslog" in enabled_plugins %}
|
||||||
|
// The host's own syslog errors over the same span (#6002).
|
||||||
|
stewardMarkCharts("/plugins/syslog/marks?host_id={{ host.id }}&range=" + encodeURIComponent(rangeKey || ""),
|
||||||
|
Object.values(charts), document.getElementById("hm-marks-note"));
|
||||||
|
{% endif %}
|
||||||
};
|
};
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -42,6 +42,7 @@
|
|||||||
|
|
||||||
{# The device's own log, when it sends syslog to an added source (#6000). #}
|
{# The device's own log, when it sends syslog to an added source (#6000). #}
|
||||||
{% if "syslog" in enabled_plugins %}
|
{% if "syslog" in enabled_plugins %}
|
||||||
|
{% if charts %}<p id="snmp-marks-note" class="field-hint" hidden style="margin:0.5rem 0 0;"></p>{% endif %}
|
||||||
<div hx-get="/plugins/syslog/panel?address={{ device.address|urlencode }}{% if device.host_id %}&host_id={{ device.host_id }}{% endif %}" hx-trigger="load" hx-swap="innerHTML"></div>
|
<div hx-get="/plugins/syslog/panel?address={{ device.address|urlencode }}{% if device.host_id %}&host_id={{ device.host_id }}{% endif %}" hx-trigger="load" hx-swap="innerHTML"></div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
@@ -56,6 +57,7 @@
|
|||||||
? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
|
? d.toLocaleDateString([], { weekday: "short", day: "numeric" }) + " " + d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
|
||||||
: d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
|
: d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false });
|
||||||
}
|
}
|
||||||
|
var drawn = [];
|
||||||
charts.forEach(function (c, i) {
|
charts.forEach(function (c, i) {
|
||||||
var el = document.getElementById("snmp-c-" + i);
|
var el = document.getElementById("snmp-c-" + i);
|
||||||
if (!el) return;
|
if (!el) return;
|
||||||
@@ -63,7 +65,7 @@
|
|||||||
el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough readings yet for a chart.</p>';
|
el.parentElement.innerHTML = '<p class="field-hint" style="margin:0;">Not enough readings yet for a chart.</p>';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
new Chart(el.getContext("2d"), {
|
drawn.push(new Chart(el.getContext("2d"), {
|
||||||
type: "line",
|
type: "line",
|
||||||
data: { labels: c.points.map(function (p) { return p[0]; }),
|
data: { labels: c.points.map(function (p) { return p[0]; }),
|
||||||
datasets: [{ data: c.points.map(function (p) { return p[1]; }), borderColor: line,
|
datasets: [{ data: c.points.map(function (p) { return p[1]; }), borderColor: line,
|
||||||
@@ -81,8 +83,13 @@
|
|||||||
title: { display: !!c.axis, text: c.axis, font: { size: 9 } } }
|
title: { display: !!c.axis, text: c.axis, font: { size: 9 } } }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
}));
|
||||||
});
|
});
|
||||||
|
{% if "syslog" in enabled_plugins %}
|
||||||
|
// The device's own syslog errors over the same span (#6002).
|
||||||
|
if (drawn.length) stewardMarkCharts("/plugins/syslog/marks?address={{ device.address|urlencode }}{% if device.host_id %}&host_id={{ device.host_id }}{% endif %}&hours=" + hours,
|
||||||
|
drawn, document.getElementById("snmp-marks-note"));
|
||||||
|
{% endif %}
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
@@ -40,6 +40,16 @@ def register_mcp_tools(register) -> None:
|
|||||||
register_tools(register)
|
register_tools(register)
|
||||||
|
|
||||||
|
|
||||||
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
||||||
|
from .mcp_tools import mcp_events as events
|
||||||
|
return await events(db, host_id, host_name, since, until)
|
||||||
|
|
||||||
|
|
||||||
|
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
|
||||||
|
from .mcp_tools import mcp_resource_events as events
|
||||||
|
return await events(db, source, resource, since, until)
|
||||||
|
|
||||||
|
|
||||||
def get_alert_metrics() -> dict[str, list[str]]:
|
def get_alert_metrics() -> dict[str, list[str]]:
|
||||||
"""A pattern's count is named for the pattern (lines_matching_<key>), so
|
"""A pattern's count is named for the pattern (lines_matching_<key>), so
|
||||||
the list is empty here and the rule form offers the counts recorded so far."""
|
the list is empty here and the rule form offers the counts recorded so far."""
|
||||||
|
|||||||
@@ -14,11 +14,8 @@ from steward.models.users import UserRole
|
|||||||
|
|
||||||
from .models import SyslogLine, SyslogSource
|
from .models import SyslogLine, SyslogSource
|
||||||
from .parse import facility_name, severity_name
|
from .parse import facility_name, severity_name
|
||||||
from .service import LineFilter, filtered, rates, severity_level, sources_for
|
from .service import (AROUND_LINES, AROUND_SEVERITY, LineFilter, filtered, lines_between, place_for, rates,
|
||||||
|
severity_level, shown_on, source_name, sources_for)
|
||||||
|
|
||||||
def _source_name(s: SyslogSource) -> str:
|
|
||||||
return s.name or s.hostname_seen or s.address
|
|
||||||
|
|
||||||
|
|
||||||
async def list_syslog_sources(state: str = "") -> dict:
|
async def list_syslog_sources(state: str = "") -> dict:
|
||||||
@@ -43,7 +40,7 @@ async def list_syslog_sources(state: str = "") -> dict:
|
|||||||
per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now)
|
per_hour = await rates(db, [s.id for s in sources if s.state == "accepted"], now)
|
||||||
hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()}
|
hosts = {h.id: h.name for h in (await db.execute(select(Host))).scalars()}
|
||||||
return {"sources": [{
|
return {"sources": [{
|
||||||
"source_id": s.id, "name": _source_name(s), "address": s.address, "state": s.state,
|
"source_id": s.id, "name": source_name(s), "address": s.address, "state": s.state,
|
||||||
"host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None,
|
"host_id": s.host_id, "host_name": hosts.get(s.host_id), "calls_itself": s.hostname_seen or None,
|
||||||
"lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None,
|
"lines_total": s.lines_total, "lines_last_hour": per_hour.get(s.id, 0) if s.state == "accepted" else None,
|
||||||
"last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now),
|
"last_heard": iso(s.last_seen_at), "last_heard_age_s": age_s(s.last_seen_at, now),
|
||||||
@@ -94,14 +91,14 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi
|
|||||||
want = source.strip()
|
want = source.strip()
|
||||||
by_address = [s for s in accepted if s.address == want]
|
by_address = [s for s in accepted if s.address == want]
|
||||||
chosen = by_address or [pick(want, accepted, kind="syslog source",
|
chosen = by_address or [pick(want, accepted, kind="syslog source",
|
||||||
list_tool="list_syslog_sources", name=_source_name)]
|
list_tool="list_syslog_sources", name=source_name)]
|
||||||
if host:
|
if host:
|
||||||
h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts")
|
h = pick(host, list((await db.execute(select(Host))).scalars()), kind="host", list_tool="list_hosts")
|
||||||
chosen = sources_for(chosen, h.id, h.address)
|
chosen = sources_for(chosen, h.id, h.address)
|
||||||
if not chosen:
|
if not chosen:
|
||||||
return {"range": key, "matches": [], "returned": 0, "truncated": False,
|
return {"range": key, "matches": [], "returned": 0, "truncated": False,
|
||||||
"note": f"No added syslog source is shown on {h.name}."}
|
"note": f"No added syslog source is shown on {h.name}."}
|
||||||
names = {s.id: _source_name(s) for s in chosen}
|
names = {s.id: source_name(s) for s in chosen}
|
||||||
stmt = filtered(select(SyslogLine).where(SyslogLine.source_id.in_(list(names))), f)
|
stmt = filtered(select(SyslogLine).where(SyslogLine.source_id.in_(list(names))), f)
|
||||||
if before_dt is not None:
|
if before_dt is not None:
|
||||||
stmt = stmt.where(SyslogLine.received_at < before_dt)
|
stmt = stmt.where(SyslogLine.received_at < before_dt)
|
||||||
@@ -122,6 +119,33 @@ async def search_syslog(source: str = "", host: str = "", text: str = "", severi
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _as_events(lines: list[SyslogLine], sources: list[SyslogSource]) -> list[dict]:
|
||||||
|
from steward.mcp.tools._events import event
|
||||||
|
names = {s.id: source_name(s) for s in sources}
|
||||||
|
return [event(ln.received_at, "syslog",
|
||||||
|
f"{severity_name(ln.severity)}{' ' + ln.app if ln.app else ''}: {cut_middle(ln.message, 300)}",
|
||||||
|
subject=names[ln.source_id]) for ln in reversed(lines)]
|
||||||
|
|
||||||
|
|
||||||
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
||||||
|
"""Warnings and worse in a window (get_anomaly) from the syslog sources
|
||||||
|
shown on a host: bound to it, or at its address."""
|
||||||
|
sources = await shown_on(db, host_id, None)
|
||||||
|
return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources)
|
||||||
|
|
||||||
|
|
||||||
|
async def mcp_resource_events(db, source: str, resource: str, since, until) -> list[dict]:
|
||||||
|
"""The same for an SNMP device, which has no host to read events by: the
|
||||||
|
sources at its address, or bound to its host."""
|
||||||
|
if source != "snmp":
|
||||||
|
return []
|
||||||
|
host_id, address = await place_for(db, source, resource)
|
||||||
|
if not (host_id or address):
|
||||||
|
return []
|
||||||
|
sources = await shown_on(db, host_id, address)
|
||||||
|
return _as_events(await lines_between(db, sources, since, until, AROUND_SEVERITY, AROUND_LINES), sources)
|
||||||
|
|
||||||
|
|
||||||
def register_tools(register) -> None:
|
def register_tools(register) -> None:
|
||||||
for fn in (list_syslog_sources, search_syslog):
|
for fn in (list_syslog_sources, search_syslog):
|
||||||
register(fn, role=UserRole.viewer, read_only=True)
|
register(fn, role=UserRole.viewer, read_only=True)
|
||||||
|
|||||||
+64
-12
@@ -6,7 +6,7 @@ and the listener's state in memory; nothing here contacts a device (System
|
|||||||
and audited."""
|
and audited."""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
|
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
|
||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select
|
||||||
@@ -23,7 +23,9 @@ from .parse import SEVERITIES, facility_name, severity_name
|
|||||||
from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternError, key_for, parse_form,
|
from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternError, key_for, parse_form,
|
||||||
resource_of)
|
resource_of)
|
||||||
from .receiver import LISTENER, MAX_SENDERS, settings
|
from .receiver import LISTENER, MAX_SENDERS, settings
|
||||||
from .service import PANEL_LINES, PANEL_SPAN, SEVERITY_CHOICES, apps_of, filtered, line_filter, rates, sources_for
|
from .service import (AROUND_LINES, AROUND_SEVERITY, MARK_SEVERITY, MAX_MARKS, PANEL_LINES, PANEL_SPAN,
|
||||||
|
SEVERITY_CHOICES, apps_of, filtered, line_filter, lines_between, place_for, rates, shown_on,
|
||||||
|
source_name)
|
||||||
|
|
||||||
syslog_bp = Blueprint("syslog", __name__, template_folder="templates")
|
syslog_bp = Blueprint("syslog", __name__, template_folder="templates")
|
||||||
|
|
||||||
@@ -273,19 +275,11 @@ async def lines_panel():
|
|||||||
"""The syslog panel on a host's page (?host_id=) or an SNMP device's
|
"""The syslog panel on a host's page (?host_id=) or an SNMP device's
|
||||||
(?address=, and ?host_id= when it is bound): the newest lines of the
|
(?address=, and ?host_id= when it is bound): the newest lines of the
|
||||||
sources that map to it. Nothing when no added source does."""
|
sources that map to it. Nothing when no added source does."""
|
||||||
from steward.models.hosts import Host
|
|
||||||
|
|
||||||
host_id = request.args.get("host_id") or None
|
|
||||||
address = request.args.get("address") or None
|
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
if host_id and not address:
|
matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None)
|
||||||
host = await db.get(Host, host_id)
|
|
||||||
address = host.address if host is not None else None
|
|
||||||
matched = sources_for(list((await db.execute(
|
|
||||||
select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars()), host_id, address)
|
|
||||||
if not matched:
|
if not matched:
|
||||||
return ""
|
return ""
|
||||||
names = {s.id: s.name or s.hostname_seen or s.address for s in matched}
|
names = {s.id: source_name(s) for s in matched}
|
||||||
since = datetime.now(timezone.utc) - PANEL_SPAN
|
since = datetime.now(timezone.utc) - PANEL_SPAN
|
||||||
lines = (await db.execute(
|
lines = (await db.execute(
|
||||||
select(SyslogLine).where(SyslogLine.source_id.in_(list(names)), SyslogLine.received_at >= since)
|
select(SyslogLine).where(SyslogLine.source_id.in_(list(names)), SyslogLine.received_at >= since)
|
||||||
@@ -457,3 +451,61 @@ async def delete_pattern(pattern_id: str):
|
|||||||
await log_audit(current_app, *_actor(), "syslog.pattern_removed", entity_type="syslog_pattern",
|
await log_audit(current_app, *_actor(), "syslog.pattern_removed", entity_type="syslog_pattern",
|
||||||
entity_id=pattern_id, detail={"name": name})
|
entity_id=pattern_id, detail={"name": name})
|
||||||
return redirect(PATTERNS)
|
return redirect(PATTERNS)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Lines beside the readings (#6002) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
AROUND = timedelta(minutes=30)
|
||||||
|
AROUND_SHOWN = 15
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_arg(name: str) -> datetime | None:
|
||||||
|
value = request.args.get(name) or ""
|
||||||
|
try:
|
||||||
|
at = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
@syslog_bp.get("/around")
|
||||||
|
@require_role(UserRole.viewer)
|
||||||
|
async def lines_around():
|
||||||
|
"""An anomaly's syslog: warnings and worse from the sources shown with
|
||||||
|
its resource (?source=&resource=), from half an hour before ?since= to
|
||||||
|
half an hour after ?until=, the lines nearest its start first."""
|
||||||
|
since = _iso_arg("since")
|
||||||
|
if since is None:
|
||||||
|
abort(400)
|
||||||
|
until = _iso_arg("until") or datetime.now(timezone.utc)
|
||||||
|
async with current_app.db_sessionmaker() as db:
|
||||||
|
host_id, address = await place_for(db, request.args.get("source", ""), request.args.get("resource", ""))
|
||||||
|
matched = await shown_on(db, host_id, address) if host_id or address else []
|
||||||
|
lines = await lines_between(db, matched, since - AROUND, until + AROUND, AROUND_SEVERITY, AROUND_LINES)
|
||||||
|
names = {s.id: source_name(s) for s in matched}
|
||||||
|
nearest = sorted(lines, key=lambda ln: abs((ln.received_at - since).total_seconds()))[:AROUND_SHOWN]
|
||||||
|
rows = [{**line_view(ln), "source": names[ln.source_id], "source_id": ln.source_id}
|
||||||
|
for ln in sorted(nearest, key=lambda ln: ln.received_at)]
|
||||||
|
return await render_template("syslog/_around.html", lines=rows, more=len(lines) - len(rows),
|
||||||
|
sources=[{"id": sid, "name": name} for sid, name in names.items()])
|
||||||
|
|
||||||
|
|
||||||
|
@syslog_bp.get("/marks")
|
||||||
|
@require_role(UserRole.viewer)
|
||||||
|
async def chart_marks():
|
||||||
|
"""Errors and worse from the sources a host's (?host_id=) or a device's
|
||||||
|
(?address=) page shows, over its chart's span (?range= or ?hours=), for
|
||||||
|
the chart to mark: {"marks": [{"at": epoch ms, "level", "text"}], "more"}."""
|
||||||
|
from steward.core.time_range import parse_range
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
hours = request.args.get("hours", type=int)
|
||||||
|
since = now - timedelta(hours=min(max(hours, 1), 24 * 30)) if hours else parse_range(request.args.get("range"))[0]
|
||||||
|
async with current_app.db_sessionmaker() as db:
|
||||||
|
matched = await shown_on(db, request.args.get("host_id") or None, request.args.get("address") or None)
|
||||||
|
lines = await lines_between(db, matched, since, now, MARK_SEVERITY, MAX_MARKS + 1)
|
||||||
|
names = {s.id: source_name(s) for s in matched}
|
||||||
|
return {"marks": [{"at": int(ln.received_at.timestamp() * 1000), "level": ln.severity,
|
||||||
|
"text": f"{names[ln.source_id]} {severity_name(ln.severity)}"
|
||||||
|
f"{' ' + ln.app if ln.app else ''}: {ln.message[:200]}"}
|
||||||
|
for ln in lines[:MAX_MARKS]],
|
||||||
|
"more": len(lines) > MAX_MARKS}
|
||||||
|
|||||||
@@ -110,3 +110,59 @@ def sources_for(sources: list[SyslogSource], host_id: str | None, address: str |
|
|||||||
address = (address or "").strip().lower()
|
address = (address or "").strip().lower()
|
||||||
return [s for s in sources if s.state == "accepted" and (
|
return [s for s in sources if s.state == "accepted" and (
|
||||||
s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)]
|
s.host_id == host_id if s.host_id else bool(address) and s.address.lower() == address)]
|
||||||
|
|
||||||
|
|
||||||
|
# ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002)
|
||||||
|
|
||||||
|
# What an anomaly's circumstances take from the log: warnings and worse, so a
|
||||||
|
# chatty device's info lines don't crowd out the line that explains it.
|
||||||
|
AROUND_SEVERITY = 4
|
||||||
|
AROUND_LINES = 200
|
||||||
|
# What a chart marks: errors and worse. Past MAX_MARKS the newest are kept.
|
||||||
|
MARK_SEVERITY = 3
|
||||||
|
MAX_MARKS = 300
|
||||||
|
|
||||||
|
|
||||||
|
def source_name(s: SyslogSource) -> str:
|
||||||
|
return s.name or s.hostname_seen or s.address
|
||||||
|
|
||||||
|
|
||||||
|
async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogSource]:
|
||||||
|
"""The added sources a host's page (and its address) or a device's page
|
||||||
|
shows; the host's own address when only the host is given."""
|
||||||
|
from steward.models.hosts import Host
|
||||||
|
if host_id and not address:
|
||||||
|
host = await db.get(Host, host_id)
|
||||||
|
address = host.address if host is not None else None
|
||||||
|
accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars())
|
||||||
|
return sources_for(accepted, host_id, address)
|
||||||
|
|
||||||
|
|
||||||
|
async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]:
|
||||||
|
"""(host id, address) whose sources a metric's resource is shown with:
|
||||||
|
an SNMP device's own binding and address, else the host the resource
|
||||||
|
lives on, as get_anomaly finds it."""
|
||||||
|
if source == "snmp":
|
||||||
|
from steward.core.plugin_manager import loaded_plugin_submodule
|
||||||
|
snmp = loaded_plugin_submodule("snmp", "models")
|
||||||
|
if snmp is None:
|
||||||
|
return None, None
|
||||||
|
device = (await db.execute(select(snmp.SnmpDevice).where(
|
||||||
|
snmp.SnmpDevice.name == resource).limit(1))).scalar_one_or_none()
|
||||||
|
return (device.host_id, device.address) if device is not None else (None, None)
|
||||||
|
from steward.mcp.tools._events import place_of
|
||||||
|
place = await place_of(db, source, resource)
|
||||||
|
return place.host_id, None
|
||||||
|
|
||||||
|
|
||||||
|
async def lines_between(db, sources: list[SyslogSource], since: datetime, until: datetime,
|
||||||
|
worst: int, limit: int) -> list[SyslogLine]:
|
||||||
|
"""Lines of these sources received in [since, until] at `worst` or worse,
|
||||||
|
newest first, at most `limit`."""
|
||||||
|
if not sources:
|
||||||
|
return []
|
||||||
|
return list((await db.execute(
|
||||||
|
select(SyslogLine).where(SyslogLine.source_id.in_([s.id for s in sources]),
|
||||||
|
SyslogLine.received_at >= since, SyslogLine.received_at <= until,
|
||||||
|
SyslogLine.severity <= worst)
|
||||||
|
.order_by(SyslogLine.received_at.desc(), SyslogLine.id.desc()).limit(limit))).scalars())
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
{# plugins/syslog/templates/syslog/_around.html — an anomaly's syslog: the
|
||||||
|
warnings and worse nearest its start, from the sources shown with its
|
||||||
|
resource (#6002). Loaded when its "Syslog around it" is opened. #}
|
||||||
|
{% from "syslog/_tone.html" import tone %}
|
||||||
|
{% if not sources %}
|
||||||
|
<p class="field-hint" style="margin:0.3rem 0 0;">No added syslog source is shown with this resource.</p>
|
||||||
|
{% elif not lines %}
|
||||||
|
<p class="field-hint" style="margin:0.3rem 0 0;">No warnings or worse from {% for s in sources %}{% if not loop.first %}, {% endif %}<a href="/plugins/syslog/sources/{{ s.id }}">{{ s.name }}</a>{% endfor %} from half an hour before to half an hour after.</p>
|
||||||
|
{% else %}
|
||||||
|
<table class="table" style="font-size:0.78rem;margin-top:0.3rem;">
|
||||||
|
<thead><tr><th class="nowrap">Received ({{ tz_label() }})</th>{% if sources|length > 1 %}<th>Source</th>{% endif %}<th>Severity</th><th>App</th><th>Message</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
{% for ln in lines %}
|
||||||
|
<tr>
|
||||||
|
<td class="nowrap">{{ ln.received_at|when(seconds=true) }}</td>
|
||||||
|
{% if sources|length > 1 %}<td class="nowrap"><a href="/plugins/syslog/sources/{{ ln.source_id }}">{{ ln.source }}</a></td>{% endif %}
|
||||||
|
<td class="nowrap"><span class="{{ tone(ln.level) }}">{{ ln.severity }}</span></td>
|
||||||
|
<td class="nowrap">{{ ln.app or "—" }}</td>
|
||||||
|
<td class="mono" style="max-width:32rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ ln.message }}">{{ ln.message }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% if more > 0 %}<p class="field-hint" style="margin:0.3rem 0 0;">{{ more }} more in that hour either side, further from its start: open the source to read them.</p>{% endif %}
|
||||||
|
{% endif %}
|
||||||
@@ -161,7 +161,8 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
|
|||||||
runs, other anomalies on the host, what plugins saw (container
|
runs, other anomalies on the host, what plugins saw (container
|
||||||
starts, deaths and OOM kills; maintenance jobs; reboots and agent
|
starts, deaths and OOM kills; maintenance jobs; reboots and agent
|
||||||
stops; for a Traefik router, its swarm service's containers on any
|
stops; for a Traefik router, its swarm service's containers on any
|
||||||
host), and kind "steward": a stretch Steward itself wasn't recording
|
host; syslog warnings and worse from the host's or the SNMP device's
|
||||||
|
own log), and kind "steward": a stretch Steward itself wasn't recording
|
||||||
(a restart or deploy), around which any reading can be wrong. From 30
|
(a restart or deploy), around which any reading can be wrong. From 30
|
||||||
minutes before to 30 minutes after. Each kind keeps the 40 events
|
minutes before to 30 minutes after. Each kind keeps the 40 events
|
||||||
nearest the occurrence's start; `around_left_out` counts the rest
|
nearest the occurrence's start; `around_left_out` counts the rest
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
// Marks on a time chart: a thin line at each moment something was logged
|
||||||
|
// (#6002, the syslog errors of a host or a device), so a reading's jump can be
|
||||||
|
// read against what the device said at the time.
|
||||||
|
//
|
||||||
|
// A chart opts in by holding marks in options.plugins.stewardMarks.marks, as
|
||||||
|
// [{at: epoch ms, level, text}]. A linear x axis takes the time itself; a
|
||||||
|
// category axis of ISO times takes the nearest label. Hovering near a mark
|
||||||
|
// shows its text in the chart's title attribute.
|
||||||
|
(function () {
|
||||||
|
if (!window.Chart) return;
|
||||||
|
function xOf(chart, at) {
|
||||||
|
var x = chart.scales.x;
|
||||||
|
if (!x) return null;
|
||||||
|
if (x.type !== 'category') return x.getPixelForValue(at);
|
||||||
|
var labels = chart.data.labels || [];
|
||||||
|
if (!labels.length) return null;
|
||||||
|
var first = Date.parse(labels[0]), last = Date.parse(labels[labels.length - 1]);
|
||||||
|
if (at < first || at > last) return null;
|
||||||
|
var best = 0, gap = Infinity;
|
||||||
|
for (var i = 0; i < labels.length; i++) {
|
||||||
|
var d = Math.abs(Date.parse(labels[i]) - at);
|
||||||
|
if (d < gap) { gap = d; best = i; }
|
||||||
|
}
|
||||||
|
return x.getPixelForValue(best);
|
||||||
|
}
|
||||||
|
Chart.register({
|
||||||
|
id: 'stewardMarks',
|
||||||
|
afterDatasetsDraw: function (chart, args, opts) {
|
||||||
|
var marks = (opts && opts.marks) || [];
|
||||||
|
if (!marks.length) return;
|
||||||
|
var area = chart.chartArea, ctx = chart.ctx;
|
||||||
|
var crit = window.stewardColor ? stewardColor('--red') : 'red';
|
||||||
|
chart.$stewardMarks = [];
|
||||||
|
ctx.save();
|
||||||
|
ctx.lineWidth = 1;
|
||||||
|
ctx.setLineDash([3, 3]);
|
||||||
|
ctx.strokeStyle = crit;
|
||||||
|
ctx.globalAlpha = 0.55;
|
||||||
|
marks.forEach(function (m) {
|
||||||
|
var px = xOf(chart, m.at);
|
||||||
|
if (px == null || px < area.left || px > area.right) return;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.moveTo(px, area.top);
|
||||||
|
ctx.lineTo(px, area.bottom);
|
||||||
|
ctx.stroke();
|
||||||
|
chart.$stewardMarks.push({ x: px, text: m.text, at: m.at });
|
||||||
|
});
|
||||||
|
ctx.restore();
|
||||||
|
},
|
||||||
|
afterEvent: function (chart, args) {
|
||||||
|
var near = (chart.$stewardMarks || []).filter(function (m) {
|
||||||
|
return Math.abs(m.x - args.event.x) <= 3;
|
||||||
|
});
|
||||||
|
chart.canvas.title = near.map(function (m) {
|
||||||
|
return new Date(m.at).toLocaleString([], { hour12: false }) + ' ' + m.text;
|
||||||
|
}).join('\n');
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Fetch marks from `url` and put them on each chart, redrawing it; `note`
|
||||||
|
// (an element, optional) says in words how many there are.
|
||||||
|
window.stewardMarkCharts = function (url, charts, note) {
|
||||||
|
fetch(url, { credentials: 'same-origin' })
|
||||||
|
.then(function (r) { return r.ok ? r.json() : { marks: [] }; })
|
||||||
|
.then(function (body) {
|
||||||
|
var marks = body.marks || [];
|
||||||
|
charts.forEach(function (ch) {
|
||||||
|
if (!ch) return;
|
||||||
|
ch.options.plugins.stewardMarks = { marks: marks };
|
||||||
|
ch.update('none');
|
||||||
|
});
|
||||||
|
if (note) {
|
||||||
|
note.hidden = !marks.length;
|
||||||
|
note.textContent = marks.length
|
||||||
|
? (body.more ? 'The newest ' : '') + marks.length + ' syslog error' + (marks.length === 1 ? '' : 's') +
|
||||||
|
' in this span marked with dashed red lines; hover one to read it.'
|
||||||
|
: '';
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(function () {});
|
||||||
|
};
|
||||||
|
})();
|
||||||
@@ -68,5 +68,14 @@ Held {{ metric_value(a.metric_name, a.peak_level) }}; usual for that hour
|
|||||||
{{ anomaly_when(v) }} · {{ anomaly_level(a) }}
|
{{ anomaly_when(v) }} · {{ anomaly_level(a) }}
|
||||||
</div>
|
</div>
|
||||||
<div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">{{ anomaly_review(a, back) }}</div>
|
<div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">{{ anomaly_review(a, back) }}</div>
|
||||||
|
{%- if "syslog" in enabled_plugins and v.last_span %}
|
||||||
|
{#- The device's own log around its latest occurrence (#6002), read only when opened. #}
|
||||||
|
<details style="font-size:0.8rem;margin-top:0.3rem;"
|
||||||
|
hx-get="/plugins/syslog/around?source={{ a.source_module|urlencode }}&resource={{ a.resource_name|urlencode }}&since={{ v.last_span.started_at.isoformat()|urlencode }}{% if v.last_span.ended_at %}&until={{ v.last_span.ended_at.isoformat()|urlencode }}{% endif %}"
|
||||||
|
hx-trigger="toggle once" hx-target="find .anomaly-syslog" hx-swap="innerHTML">
|
||||||
|
<summary style="cursor:pointer;color:var(--text-muted);">Syslog around it</summary>
|
||||||
|
<div class="anomaly-syslog"><p class="field-hint" style="margin:0.3rem 0 0;">Reading…</p></div>
|
||||||
|
</details>
|
||||||
|
{%- endif %}
|
||||||
</div>
|
</div>
|
||||||
{%- endmacro %}
|
{%- endmacro %}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
<link rel="stylesheet" href="{{ url_for('static', filename='theme.css') }}">
|
<link rel="stylesheet" href="{{ url_for('static', filename='theme.css') }}">
|
||||||
<link rel="stylesheet" href="{{ url_for('static', filename='steward.css') }}">
|
<link rel="stylesheet" href="{{ url_for('static', filename='steward.css') }}">
|
||||||
<script src="{{ url_for('static', filename='chart-colors.js') }}"></script>
|
<script src="{{ url_for('static', filename='chart-colors.js') }}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='chart-marks.js') }}"></script>
|
||||||
<script src="{{ url_for('static', filename='scroll-more.js') }}" defer></script>
|
<script src="{{ url_for('static', filename='scroll-more.js') }}" defer></script>
|
||||||
{# A form with data-confirm="…" asks in #confirm-dialog below before it submits (#5314). #}
|
{# A form with data-confirm="…" asks in #confirm-dialog below before it submits (#5314). #}
|
||||||
<script src="{{ url_for('static', filename='confirm.js') }}" defer></script>
|
<script src="{{ url_for('static', filename='confirm.js') }}" defer></script>
|
||||||
|
|||||||
@@ -364,3 +364,42 @@ def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
|
|||||||
await _clear_patterns(app)
|
await _clear_patterns(app)
|
||||||
|
|
||||||
asyncio.run(go())
|
asyncio.run(go())
|
||||||
|
|
||||||
|
|
||||||
|
def test_lines_beside_the_readings_around_an_anomaly_and_as_chart_marks(app):
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
mcp_tools = _mod("mcp_tools")
|
||||||
|
|
||||||
|
async def go():
|
||||||
|
host_id, host_name, _, src_name = await _seed(app, now)
|
||||||
|
client = await _client(app, "viewer")
|
||||||
|
since, until = (now - timedelta(minutes=35)).isoformat(), (now - timedelta(minutes=25)).isoformat()
|
||||||
|
around = await (await client.get("/plugins/syslog/around", query_string={
|
||||||
|
"source": "host_agent", "resource": f"{host_name}:/", "since": since, "until": until})).get_data(as_text=True)
|
||||||
|
# Warnings and worse within half an hour either side; info lines left out.
|
||||||
|
assert "nfsd: server not responding" in around and "100% of quota used" in around
|
||||||
|
assert "eth0 link up" not in around and "an old error" not in around
|
||||||
|
elsewhere = await (await client.get("/plugins/syslog/around", query_string={
|
||||||
|
"source": "traefik", "resource": "web@docker", "since": since})).get_data(as_text=True)
|
||||||
|
assert "No added syslog source" in elsewhere
|
||||||
|
assert (await client.get("/plugins/syslog/around?source=host_agent")).status_code == 400
|
||||||
|
|
||||||
|
day = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&range=24h")).get_json()
|
||||||
|
assert [m["text"] for m in day["marks"]] == [f"{src_name} err kernel: nfsd: server not responding"]
|
||||||
|
assert day["more"] is False
|
||||||
|
week = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&hours=168")).get_json()
|
||||||
|
assert len(week["marks"]) == 2
|
||||||
|
assert (await (await client.get("/plugins/syslog/marks?address=10.1.1.1")).get_json())["marks"] == []
|
||||||
|
|
||||||
|
# get_anomaly's circumstances: the host's lines, and nothing for a
|
||||||
|
# resource syslog can't place.
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
events = await mcp_tools.mcp_events(s, host_id, host_name, now - timedelta(hours=1), now)
|
||||||
|
assert [(e["kind"], e["subject"]) for e in events] == [("syslog", src_name)] * 2
|
||||||
|
assert events[0]["what"].startswith("err kernel: nfsd")
|
||||||
|
assert await mcp_tools.mcp_resource_events(s, "traefik", "web@docker", now - timedelta(hours=1), now) == []
|
||||||
|
async with app.db_sessionmaker() as s:
|
||||||
|
async with s.begin():
|
||||||
|
await s.execute(delete_sources())
|
||||||
|
|
||||||
|
asyncio.run(go())
|
||||||
|
|||||||
Reference in New Issue
Block a user