feat(syslog): a syslog receiver: UDP and TCP, senders offered before they're kept, each source's lines in a ring (#5999)
- plugins/syslog: RFC 5424 and RFC 3164 parsing, RFC 6587 TCP framing. - Lines are queued as they arrive and stored every 5 s through a Holding, so a storage stall delays them rather than losing them. - A new sender is an offered source, counted and not kept, until an admin adds it on the Syslog page (name, host); it can be ignored instead. - The listener follows the plugin's settings (port, UDP, TCP), rebinding within seconds of a save; a port it can't open is said on the page. - Hourly ring per source: age window and byte cap from Settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,161 @@
|
||||
# plugins/syslog/parse.py
|
||||
"""One syslog message read into its parts: RFC 5424 (the IETF format) and
|
||||
RFC 3164 (the older BSD format), and whatever a device sends that is neither.
|
||||
|
||||
Pure: no I/O and no clock, so every format a device has been seen to send
|
||||
is a unit test.
|
||||
|
||||
A message that doesn't parse is still kept, whole, as its own text: a line
|
||||
the device sent is never dropped for its shape.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
SEVERITIES = ("emerg", "alert", "crit", "err", "warning", "notice", "info", "debug")
|
||||
FACILITIES = ("kern", "user", "mail", "daemon", "auth", "syslog", "lpr", "news", "uucp", "cron",
|
||||
"authpriv", "ftp", "ntp", "audit", "alert", "clock",
|
||||
"local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7")
|
||||
|
||||
# The longest message kept; a device that sends more is cut here.
|
||||
MAX_MESSAGE = 8192
|
||||
# A message with no <PRI> is user.notice (RFC 3164 §4.3.3).
|
||||
_DEFAULT_PRI = 13
|
||||
|
||||
_PRI = re.compile(r"<(\d{1,3})>")
|
||||
# <PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID [SD] MSG
|
||||
_RFC5424 = re.compile(r"(\d{1,2}) (\S+) (\S+) (\S+) (\S+) (\S+) ?(.*)", re.S)
|
||||
# Mmm dd hh:mm:ss (the day space-padded)
|
||||
_BSD_TIME = re.compile(r"([A-Z][a-z]{2}) {1,2}(\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ")
|
||||
# TAG[pid]: or TAG: at the start of a BSD message
|
||||
_TAG = re.compile(r"([^\s:\[\]]{1,128})(?:\[[^\]]*\])?: ?")
|
||||
_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
facility: int
|
||||
severity: int
|
||||
sent_at: datetime | None # only when the message says its own zone (RFC 5424)
|
||||
hostname: str
|
||||
app: str
|
||||
text: str
|
||||
|
||||
|
||||
def parse(raw: bytes | str) -> Message:
|
||||
"""Read one message. A BSD timestamp has no year and no zone, so it is
|
||||
read past and never kept as the sent time: the time Steward received the
|
||||
line is the one to trust."""
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8", errors="replace")
|
||||
raw = raw.lstrip("\ufeff").rstrip("\r\n\x00 ")
|
||||
pri, rest = _DEFAULT_PRI, raw
|
||||
m = _PRI.match(raw)
|
||||
if m and int(m.group(1)) <= 191:
|
||||
pri, rest = int(m.group(1)), raw[m.end():]
|
||||
facility, severity = divmod(pri, 8)
|
||||
|
||||
m = _RFC5424.match(rest)
|
||||
if m and m.group(1) == "1":
|
||||
return _rfc5424(facility, severity, m)
|
||||
return _bsd(facility, severity, rest)
|
||||
|
||||
|
||||
def _nil(value: str) -> str:
|
||||
return "" if value == "-" else value
|
||||
|
||||
|
||||
def _rfc5424(facility: int, severity: int, m: re.Match) -> Message:
|
||||
_, stamp, host, app, _procid, _msgid, tail = m.groups()
|
||||
text = _strip_structured_data(tail)
|
||||
return Message(facility, severity, _iso(stamp), _nil(host)[:255], _nil(app)[:128],
|
||||
text.lstrip("\ufeff")[:MAX_MESSAGE])
|
||||
|
||||
|
||||
def _strip_structured_data(tail: str) -> str:
|
||||
"""Drop RFC 5424's STRUCTURED-DATA ("-" or one or more [id k="v"]
|
||||
elements) from in front of the message."""
|
||||
if tail.startswith("- ") or tail == "-":
|
||||
return tail[2:]
|
||||
i = 0
|
||||
while i < len(tail) and tail[i] == "[":
|
||||
quoted, j = False, i + 1
|
||||
while j < len(tail):
|
||||
c = tail[j]
|
||||
if c == "\\":
|
||||
j += 2
|
||||
continue
|
||||
if c == '"':
|
||||
quoted = not quoted
|
||||
elif c == "]" and not quoted:
|
||||
break
|
||||
j += 1
|
||||
i = j + 1
|
||||
return tail[i:].lstrip(" ")
|
||||
|
||||
|
||||
def _iso(stamp: str) -> datetime | None:
|
||||
if stamp == "-":
|
||||
return None
|
||||
try:
|
||||
at = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo else None
|
||||
|
||||
|
||||
def _bsd(facility: int, severity: int, rest: str) -> Message:
|
||||
hostname = ""
|
||||
m = _BSD_TIME.match(rest)
|
||||
if m and m.group(1) in _MONTHS:
|
||||
rest = rest[m.end():]
|
||||
# The HOSTNAME follows the time, unless the sender left it out and the
|
||||
# tag comes straight after (some devices do).
|
||||
word, _, after = rest.partition(" ")
|
||||
if word and not word.endswith(":") and "[" not in word:
|
||||
hostname, rest = word[:255], after
|
||||
app = ""
|
||||
t = _TAG.match(rest)
|
||||
if t:
|
||||
app, rest = t.group(1)[:128], rest[t.end():]
|
||||
return Message(facility, severity, None, hostname, app, rest[:MAX_MESSAGE])
|
||||
|
||||
|
||||
def severity_name(level: int) -> str:
|
||||
return SEVERITIES[level] if 0 <= level < len(SEVERITIES) else str(level)
|
||||
|
||||
|
||||
def facility_name(code: int) -> str:
|
||||
return FACILITIES[code] if 0 <= code < len(FACILITIES) else str(code)
|
||||
|
||||
|
||||
def frames(buffer: bytearray) -> list[bytes]:
|
||||
"""Take every complete message out of a TCP stream's buffer (RFC 6587):
|
||||
octet-counted ("<len> <msg>") or newline-ended. What is left is a message
|
||||
still arriving. A count past MAX_MESSAGE × 2 is not a count: the stream is
|
||||
read as newline-ended from there."""
|
||||
out: list[bytes] = []
|
||||
while buffer:
|
||||
if buffer[:1].isdigit():
|
||||
space = buffer.find(b" ", 0, 8)
|
||||
if space > 0 and buffer[:space].isdigit() and int(buffer[:space]) <= MAX_MESSAGE * 2:
|
||||
size = int(buffer[:space])
|
||||
end = space + 1 + size
|
||||
if len(buffer) < end:
|
||||
break
|
||||
out.append(bytes(buffer[space + 1:end]))
|
||||
del buffer[:end]
|
||||
continue
|
||||
nl = buffer.find(b"\n")
|
||||
if nl < 0:
|
||||
if len(buffer) > MAX_MESSAGE * 2: # no end in sight: take what's there
|
||||
out.append(bytes(buffer))
|
||||
buffer.clear()
|
||||
break
|
||||
line = bytes(buffer[:nl]).rstrip(b"\r\x00")
|
||||
del buffer[:nl + 1]
|
||||
if line:
|
||||
out.append(line)
|
||||
return out
|
||||
Reference in New Issue
Block a user