FabledSteward/plugins/syslog/parse.py

# plugins/syslog/parse.py
"""One syslog message read into its parts: RFC 5424 (the IETF format) and
RFC 3164 (the older BSD format), and whatever a device sends that is neither.

Pure: no I/O and no clock, so every format a device has been seen to send
is a unit test.

A message that doesn't parse is still kept, whole, as its own text: a line
the device sent is never dropped for its shape.
"""
from __future__ import annotations

import re
from dataclasses import dataclass
from datetime import datetime

SEVERITIES = ("emerg", "alert", "crit", "err", "warning", "notice", "info", "debug")
FACILITIES = ("kern", "user", "mail", "daemon", "auth", "syslog", "lpr", "news", "uucp", "cron",
              "authpriv", "ftp", "ntp", "audit", "alert", "clock",
              "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7")

# The longest message kept; a device that sends more is cut here.
MAX_MESSAGE = 8192
# A message with no <PRI> is user.notice (RFC 3164 §4.3.3).
_DEFAULT_PRI = 13

_PRI = re.compile(r"<(\d{1,3})>")
# <PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID [SD] MSG
_RFC5424 = re.compile(r"(\d{1,2}) (\S+) (\S+) (\S+) (\S+) (\S+) ?(.*)", re.S)
# Mmm dd hh:mm:ss (the day space-padded)
_BSD_TIME = re.compile(r"([A-Z][a-z]{2}) {1,2}(\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ")
# TAG[pid]: or TAG: at the start of a BSD message
_TAG = re.compile(r"([^\s:\[\]]{1,128})(?:\[[^\]]*\])?: ?")
_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")


@dataclass
class Message:
    facility: int
    severity: int
    sent_at: datetime | None     # only when the message says its own zone (RFC 5424)
    hostname: str
    app: str
    text: str


def parse(raw: bytes | str) -> Message:
    """Read one message. A BSD timestamp has no year and no zone, so it is
    read past and never kept as the sent time: the time Steward received the
    line is the one to trust."""
    if isinstance(raw, bytes):
        raw = raw.decode("utf-8", errors="replace")
    raw = raw.lstrip("\ufeff").rstrip("\r\n\x00 ")
    pri, rest = _DEFAULT_PRI, raw
    m = _PRI.match(raw)
    if m and int(m.group(1)) <= 191:
        pri, rest = int(m.group(1)), raw[m.end():]
    facility, severity = divmod(pri, 8)

    m = _RFC5424.match(rest)
    if m and m.group(1) == "1":
        return _rfc5424(facility, severity, m)
    return _bsd(facility, severity, rest)


def _nil(value: str) -> str:
    return "" if value == "-" else value


def _rfc5424(facility: int, severity: int, m: re.Match) -> Message:
    _, stamp, host, app, _procid, _msgid, tail = m.groups()
    text = _strip_structured_data(tail)
    return Message(facility, severity, _iso(stamp), _nil(host)[:255], _nil(app)[:128],
                   text.lstrip("\ufeff")[:MAX_MESSAGE])


def _strip_structured_data(tail: str) -> str:
    """Drop RFC 5424's STRUCTURED-DATA ("-" or one or more [id k="v"]
    elements) from in front of the message."""
    if tail.startswith("- ") or tail == "-":
        return tail[2:]
    i = 0
    while i < len(tail) and tail[i] == "[":
        quoted, j = False, i + 1
        while j < len(tail):
            c = tail[j]
            if c == "\\":
                j += 2
                continue
            if c == '"':
                quoted = not quoted
            elif c == "]" and not quoted:
                break
            j += 1
        i = j + 1
    return tail[i:].lstrip(" ")


def _iso(stamp: str) -> datetime | None:
    if stamp == "-":
        return None
    try:
        at = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
    except ValueError:
        return None
    return at if at.tzinfo else None


def _bsd(facility: int, severity: int, rest: str) -> Message:
    hostname = ""
    m = _BSD_TIME.match(rest)
    if m and m.group(1) in _MONTHS:
        rest = rest[m.end():]
        # The HOSTNAME follows the time, unless the sender left it out and the
        # tag comes straight after (some devices do).
        word, _, after = rest.partition(" ")
        if word and not word.endswith(":") and "[" not in word:
            hostname, rest = word[:255], after
    app = ""
    t = _TAG.match(rest)
    if t:
        app, rest = t.group(1)[:128], rest[t.end():]
    return Message(facility, severity, None, hostname, app, rest[:MAX_MESSAGE])


def severity_name(level: int) -> str:
    return SEVERITIES[level] if 0 <= level < len(SEVERITIES) else str(level)


def facility_name(code: int) -> str:
    return FACILITIES[code] if 0 <= code < len(FACILITIES) else str(code)


def frames(buffer: bytearray) -> list[bytes]:
    """Take every complete message out of a TCP stream's buffer (RFC 6587):
    octet-counted ("<len> <msg>") or newline-ended. What is left is a message
    still arriving. A count past MAX_MESSAGE × 2 is not a count: the stream is
    read as newline-ended from there."""
    out: list[bytes] = []
    while buffer:
        if buffer[:1].isdigit():
            space = buffer.find(b" ", 0, 8)
            if space > 0 and buffer[:space].isdigit() and int(buffer[:space]) <= MAX_MESSAGE * 2:
                size = int(buffer[:space])
                end = space + 1 + size
                if len(buffer) < end:
                    break
                out.append(bytes(buffer[space + 1:end]))
                del buffer[:end]
                continue
        nl = buffer.find(b"\n")
        if nl < 0:
            if len(buffer) > MAX_MESSAGE * 2:       # no end in sight: take what's there
                out.append(bytes(buffer))
                buffer.clear()
            break
        line = bytes(buffer[:nl]).rstrip(b"\r\x00")
        del buffer[:nl + 1]
        if line:
            out.append(line)
    return out