FabledSteward/plugins/syslog/service.py

# plugins/syslog/service.py
"""Reading the kept syslog lines: a source's lines filtered, the sources a
host's or an SNMP device's page shows, and each source's rate. Shared by the
pages and the MCP tool, so both filter the same way (#6000)."""
from __future__ import annotations

from dataclasses import dataclass
from datetime import datetime, timedelta, timezone

from sqlalchemy import func, select

from steward.core.like import escape_like
from steward.core.time_range import RANGES

from .models import SyslogLine, SyslogSource
from .parse import SEVERITIES

# The severity filter's choices: a level and everything worse than it.
SEVERITY_CHOICES = (("3", "Errors and worse"), ("4", "Warnings and worse"),
                    ("5", "Notices and worse"), ("6", "Info and worse"))
DEFAULT_RANGE = "24h"
# Lines a host's or a device's page shows, from how far back.
PANEL_LINES = 15
PANEL_SPAN = timedelta(hours=24)
# How far back the rate on the Syslog page counts.
RATE_WINDOW = timedelta(hours=1)


@dataclass
class LineFilter:
    severity: int | None = None      # this level and worse (lower numbers)
    app: str = ""
    text: str = ""
    range: str = DEFAULT_RANGE

    def since(self, now: datetime | None = None) -> datetime:
        return (now or datetime.now(timezone.utc)) - RANGES[self.range]

    def args(self) -> dict:
        """The filter as query arguments, the empty ones left out."""
        out = {"severity": "" if self.severity is None else str(self.severity), "app": self.app,
               "q": self.text, "range": self.range if self.range != DEFAULT_RANGE else ""}
        return {k: v for k, v in out.items() if v}


def severity_level(value: str | int | None) -> int | None:
    """A severity as its level: a number 0-7 or a name ("err", "warning")."""
    if value is None or value == "":
        return None
    text = str(value).strip().lower()
    if text.isdigit() and 0 <= int(text) <= 7:
        return int(text)
    aliases = {"error": "err", "warn": "warning", "critical": "crit", "emergency": "emerg"}
    text = aliases.get(text, text)
    if text in SEVERITIES:
        return SEVERITIES.index(text)
    raise ValueError(f"severity {value!r} is not 0-7 or one of {', '.join(SEVERITIES)}.")


def line_filter(args) -> LineFilter:
    """A filter from a page's query arguments; a value that doesn't read is
    left out rather than failing the page."""
    try:
        severity = severity_level(args.get("severity"))
    except ValueError:
        severity = None
    range_ = args.get("range") or DEFAULT_RANGE
    return LineFilter(severity=severity, app=(args.get("app") or "").strip()[:128],
                      text=(args.get("q") or "").strip()[:200],
                      range=range_ if range_ in RANGES else DEFAULT_RANGE)


def filtered(stmt, f: LineFilter, now: datetime | None = None):
    stmt = stmt.where(SyslogLine.received_at >= f.since(now))
    if f.severity is not None:
        stmt = stmt.where(SyslogLine.severity <= f.severity)
    if f.app:
        stmt = stmt.where(SyslogLine.app == f.app)
    if f.text:
        stmt = stmt.where(SyslogLine.message.ilike(f"%{escape_like(f.text)}%", escape="\\"))
    return stmt


async def apps_of(db, source_id: str, since: datetime) -> list[str]:
    """The apps a source's lines name since `since`, for the filter's choices."""
    rows = await db.execute(
        select(SyslogLine.app).where(SyslogLine.source_id == source_id, SyslogLine.received_at >= since,
                                     SyslogLine.app != "")
        .group_by(SyslogLine.app).order_by(SyslogLine.app).limit(100))
    return [a for (a,) in rows]


async def rates(db, source_ids: list[str], now: datetime | None = None) -> dict[str, int]:
    """{source id: lines received in the last hour} for kept sources."""
    if not source_ids:
        return {}
    since = (now or datetime.now(timezone.utc)) - RATE_WINDOW
    rows = await db.execute(
        select(SyslogLine.source_id, func.count()).where(
            SyslogLine.source_id.in_(source_ids), SyslogLine.received_at >= since)
        .group_by(SyslogLine.source_id))
    return {sid: n for sid, n in rows}


def device_address(s: SyslogSource, named: dict[str, str] | None = None) -> str:
    """The device's own address: the one set on the source; else, for a
    device whose lines come through Docker's address (#6096), the address of
    the SNMP device or host its name is; else the address its lines come from."""
    if s.device_address:
        return s.device_address
    if s.sender and named:
        found = named.get(s.sender.lower())
        if found:
            return found
    return s.address


async def addresses_by_name(db) -> dict[str, str]:
    """{name: address} of the SNMP devices (by the name each reports, and the
    one it was given) and hosts Steward knows, names lower-cased: how a device
    behind Docker's address is placed by the name its lines carry."""
    from steward.core.plugin_manager import loaded_plugin_submodule
    from steward.models.hosts import Host
    named: dict[str, str] = {}
    for name, address in (await db.execute(select(Host.name, Host.address))).all():
        if name and address:
            named.setdefault(name.lower(), address)
    snmp = loaded_plugin_submodule("snmp", "models")
    if snmp is not None:
        rows = await db.execute(select(snmp.SnmpDevice.sys_name, snmp.SnmpDevice.name, snmp.SnmpDevice.address))
        for sys_name, name, address in rows.all():
            for n in (sys_name, name):
                if n:
                    named[n.lower()] = address       # a device's own name over a host's
    return named


def sources_for(sources: list[SyslogSource], host_id: str | None, address: str | None,
                named: dict[str, str] | None = None) -> list[SyslogSource]:
    """The added sources a host's or a device's page shows: those bound to
    the host, and those bound to no host whose device address is the one
    given. A binding is exclusive, as for SNMP devices: a source bound
    elsewhere never matches by address."""
    address = (address or "").strip().lower()
    return [s for s in sources if s.state == "accepted" and (
        s.host_id == host_id if s.host_id else bool(address) and device_address(s, named).lower() == address)]


# ── Lines beside the readings: an anomaly's circumstances, a chart's marks (#6002)

# What an anomaly's circumstances take from the log: warnings and worse, so a
# chatty device's info lines don't crowd out the line that explains it.
AROUND_SEVERITY = 4
AROUND_LINES = 200
# What a chart marks: errors and worse. Past MAX_MARKS the newest are kept.
MARK_SEVERITY = 3
MAX_MARKS = 300


def source_name(s: SyslogSource) -> str:
    return s.name or s.hostname_seen or s.address


async def shown_on(db, host_id: str | None, address: str | None) -> list[SyslogSource]:
    """The added sources a host's page (and its address) or a device's page
    shows; the host's own address when only the host is given."""
    from steward.models.hosts import Host
    if host_id and not address:
        host = await db.get(Host, host_id)
        address = host.address if host is not None else None
    accepted = list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted"))).scalars())
    named = await addresses_by_name(db) if any(s.sender for s in accepted) else None
    return sources_for(accepted, host_id, address, named)


async def place_for(db, source: str, resource: str) -> tuple[str | None, str | None]:
    """(host id, address) whose sources a metric's resource is shown with:
    an SNMP device's own binding and address, else the host the resource
    lives on, as get_anomaly finds it."""
    if source == "snmp":
        from steward.core.plugin_manager import loaded_plugin_submodule
        snmp = loaded_plugin_submodule("snmp", "models")
        if snmp is None:
            return None, None
        device = (await db.execute(select(snmp.SnmpDevice).where(
            snmp.SnmpDevice.name == resource).limit(1))).scalar_one_or_none()
        return (device.host_id, device.address) if device is not None else (None, None)
    from steward.mcp.tools._events import place_of
    place = await place_of(db, source, resource)
    return place.host_id, None


async def lines_between(db, sources: list[SyslogSource], since: datetime, until: datetime,
                        worst: int, limit: int) -> list[SyslogLine]:
    """Lines of these sources received in [since, until] at `worst` or worse,
    newest first, at most `limit`."""
    if not sources:
        return []
    return list((await db.execute(
        select(SyslogLine).where(SyslogLine.source_id.in_([s.id for s in sources]),
                                 SyslogLine.received_at >= since, SyslogLine.received_at <= until,
                                 SyslogLine.severity <= worst)
        .order_by(SyslogLine.received_at.desc(), SyslogLine.id.desc()).limit(limit))).scalars())