feat(syslog): a syslog receiver: UDP and TCP, senders offered before they're kept, each source's lines in a ring (#5999)
- plugins/syslog: RFC 5424 and RFC 3164 parsing, RFC 6587 TCP framing. - Lines are queued as they arrive and stored every 5 s through a Holding, so a storage stall delays them rather than losing them. - A new sender is an offered source, counted and not kept, until an admin adds it on the Syslog page (name, host); it can be ignored instead. - The listener follows the plugin's settings (port, UDP, TCP), rebinding within seconds of a save; a port it can't open is said on the page. - Hourly ring per source: age window and byte cap from Settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,40 @@
|
||||
# plugins/syslog/__init__.py
|
||||
from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
_app: "Quart | None" = None
|
||||
|
||||
|
||||
def setup(app: "Quart") -> None:
|
||||
global _app
|
||||
_app = app
|
||||
from .models import SyslogLine, SyslogSource # noqa: F401 — registers the tables
|
||||
from .receiver import close
|
||||
|
||||
# The listener is bound by the receive task, so it follows the settings
|
||||
# and starts on a live enable too; it is closed with the app.
|
||||
app.after_serving(close)
|
||||
|
||||
|
||||
def get_scheduled_tasks() -> list:
|
||||
from .receiver import make_receive_task
|
||||
from .retention import make_prune_task
|
||||
return [make_receive_task(_app), make_prune_task(_app)]
|
||||
|
||||
|
||||
def get_blueprint():
|
||||
from .routes import syslog_bp
|
||||
return syslog_bp
|
||||
|
||||
|
||||
def get_nav() -> list[dict]:
|
||||
return [{"label": "Syslog", "href": "/plugins/syslog/"}]
|
||||
|
||||
|
||||
def get_audit_entities() -> dict:
|
||||
"""Audit rows about a source name it and link to the Syslog page."""
|
||||
from .models import SyslogSource
|
||||
return {"syslog_source": (SyslogSource, "/plugins/syslog/")}
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Syslog: the devices that send it, and the lines kept from those added
|
||||
|
||||
A sender is a row whether or not it has been added: one not yet added is
|
||||
counted and offered, never stored (milestone 532, #5999).
|
||||
|
||||
Revision ID: syslog_001_sources_lines
|
||||
Revises: (none — a branch off core via depends_on)
|
||||
Create Date: 2026-10-19
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "syslog_001_sources_lines"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = "syslog"
|
||||
depends_on: Union[str, Sequence[str], None] = ("0004_app_settings",)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"syslog_sources",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("address", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("state", sa.String(10), nullable=False, server_default="offered"),
|
||||
sa.Column("name", sa.String(100), nullable=False, server_default=""),
|
||||
sa.Column("host_id", sa.String(36), sa.ForeignKey("hosts.id", ondelete="SET NULL"),
|
||||
nullable=True),
|
||||
sa.Column("hostname_seen", sa.String(255), nullable=False, server_default=""),
|
||||
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("lines_total", sa.BigInteger, nullable=False, server_default="0"),
|
||||
sa.Column("last_severity", sa.SmallInteger, nullable=True),
|
||||
sa.Column("last_line", sa.Text, nullable=False, server_default=""),
|
||||
)
|
||||
op.create_table(
|
||||
"syslog_lines",
|
||||
sa.Column("id", sa.String(36), primary_key=True),
|
||||
sa.Column("source_id", sa.String(36),
|
||||
sa.ForeignKey("syslog_sources.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("received_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("sent_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("facility", sa.SmallInteger, nullable=False),
|
||||
sa.Column("severity", sa.SmallInteger, nullable=False),
|
||||
sa.Column("hostname", sa.String(255), nullable=False, server_default=""),
|
||||
sa.Column("app", sa.String(128), nullable=False, server_default=""),
|
||||
sa.Column("message", sa.Text, nullable=False, server_default=""),
|
||||
)
|
||||
op.create_index("ix_syslog_lines_source_time", "syslog_lines", ["source_id", "received_at"])
|
||||
op.create_index("ix_syslog_lines_received", "syslog_lines", ["received_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("syslog_lines")
|
||||
op.drop_table("syslog_sources")
|
||||
@@ -0,0 +1,67 @@
|
||||
# plugins/syslog/models.py
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, SmallInteger, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from steward.core.ids import time_ordered_id
|
||||
from steward.models.base import Base
|
||||
|
||||
|
||||
class SyslogSource(Base):
|
||||
"""A device that sends syslog to Steward, one row per sending address.
|
||||
|
||||
A sender Steward hasn't been told about is a row too, `state="offered"`:
|
||||
its lines are counted and its newest kept as a sample, so the page can
|
||||
offer it ("1,204 lines from 192.168.0.7: add as a source?"), but none is
|
||||
stored. Adding it makes it `"accepted"` and its lines are kept from then
|
||||
on; `"ignored"` stops the offer and keeps counting.
|
||||
"""
|
||||
__tablename__ = "syslog_sources"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
address: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
state: Mapped[str] = mapped_column(String(10), nullable=False, default="offered")
|
||||
name: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
host_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
||||
# Steward's name for the device, from what the lines say they come from.
|
||||
hostname_seen: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
first_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
last_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
lines_total: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
last_severity: Mapped[int | None] = mapped_column(SmallInteger, nullable=True)
|
||||
last_line: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
|
||||
|
||||
class SyslogLine(Base):
|
||||
"""One line an accepted source sent. A state table, not a time series
|
||||
(System 29): bounded by a per-source size and age ring, like docker_logs.
|
||||
`received_at` is when Steward got it, the time to trust; `sent_at` is the
|
||||
device's own, kept only when the line says its zone (RFC 5424)."""
|
||||
__tablename__ = "syslog_lines"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
source_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("syslog_sources.id", ondelete="CASCADE"), nullable=False)
|
||||
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
facility: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
||||
severity: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
||||
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
app: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
|
||||
# A source's lines newest first, and the ring's age cut; the twin-index
|
||||
# idiom of docker_logs.
|
||||
__table_args__ = (
|
||||
Index("ix_syslog_lines_source_time", "source_id", "received_at"),
|
||||
Index("ix_syslog_lines_received", "received_at"),
|
||||
)
|
||||
|
||||
|
||||
STATES = ("offered", "accepted", "ignored")
|
||||
@@ -0,0 +1,161 @@
|
||||
# plugins/syslog/parse.py
|
||||
"""One syslog message read into its parts: RFC 5424 (the IETF format) and
|
||||
RFC 3164 (the older BSD format), and whatever a device sends that is neither.
|
||||
|
||||
Pure: no I/O and no clock, so every format a device has been seen to send
|
||||
is a unit test.
|
||||
|
||||
A message that doesn't parse is still kept, whole, as its own text: a line
|
||||
the device sent is never dropped for its shape.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
SEVERITIES = ("emerg", "alert", "crit", "err", "warning", "notice", "info", "debug")
|
||||
FACILITIES = ("kern", "user", "mail", "daemon", "auth", "syslog", "lpr", "news", "uucp", "cron",
|
||||
"authpriv", "ftp", "ntp", "audit", "alert", "clock",
|
||||
"local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7")
|
||||
|
||||
# The longest message kept; a device that sends more is cut here.
|
||||
MAX_MESSAGE = 8192
|
||||
# A message with no <PRI> is user.notice (RFC 3164 §4.3.3).
|
||||
_DEFAULT_PRI = 13
|
||||
|
||||
_PRI = re.compile(r"<(\d{1,3})>")
|
||||
# <PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID MSGID [SD] MSG
|
||||
_RFC5424 = re.compile(r"(\d{1,2}) (\S+) (\S+) (\S+) (\S+) (\S+) ?(.*)", re.S)
|
||||
# Mmm dd hh:mm:ss (the day space-padded)
|
||||
_BSD_TIME = re.compile(r"([A-Z][a-z]{2}) {1,2}(\d{1,2}) (\d{2}):(\d{2}):(\d{2}) ")
|
||||
# TAG[pid]: or TAG: at the start of a BSD message
|
||||
_TAG = re.compile(r"([^\s:\[\]]{1,128})(?:\[[^\]]*\])?: ?")
|
||||
_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
facility: int
|
||||
severity: int
|
||||
sent_at: datetime | None # only when the message says its own zone (RFC 5424)
|
||||
hostname: str
|
||||
app: str
|
||||
text: str
|
||||
|
||||
|
||||
def parse(raw: bytes | str) -> Message:
|
||||
"""Read one message. A BSD timestamp has no year and no zone, so it is
|
||||
read past and never kept as the sent time: the time Steward received the
|
||||
line is the one to trust."""
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8", errors="replace")
|
||||
raw = raw.lstrip("\ufeff").rstrip("\r\n\x00 ")
|
||||
pri, rest = _DEFAULT_PRI, raw
|
||||
m = _PRI.match(raw)
|
||||
if m and int(m.group(1)) <= 191:
|
||||
pri, rest = int(m.group(1)), raw[m.end():]
|
||||
facility, severity = divmod(pri, 8)
|
||||
|
||||
m = _RFC5424.match(rest)
|
||||
if m and m.group(1) == "1":
|
||||
return _rfc5424(facility, severity, m)
|
||||
return _bsd(facility, severity, rest)
|
||||
|
||||
|
||||
def _nil(value: str) -> str:
|
||||
return "" if value == "-" else value
|
||||
|
||||
|
||||
def _rfc5424(facility: int, severity: int, m: re.Match) -> Message:
|
||||
_, stamp, host, app, _procid, _msgid, tail = m.groups()
|
||||
text = _strip_structured_data(tail)
|
||||
return Message(facility, severity, _iso(stamp), _nil(host)[:255], _nil(app)[:128],
|
||||
text.lstrip("\ufeff")[:MAX_MESSAGE])
|
||||
|
||||
|
||||
def _strip_structured_data(tail: str) -> str:
|
||||
"""Drop RFC 5424's STRUCTURED-DATA ("-" or one or more [id k="v"]
|
||||
elements) from in front of the message."""
|
||||
if tail.startswith("- ") or tail == "-":
|
||||
return tail[2:]
|
||||
i = 0
|
||||
while i < len(tail) and tail[i] == "[":
|
||||
quoted, j = False, i + 1
|
||||
while j < len(tail):
|
||||
c = tail[j]
|
||||
if c == "\\":
|
||||
j += 2
|
||||
continue
|
||||
if c == '"':
|
||||
quoted = not quoted
|
||||
elif c == "]" and not quoted:
|
||||
break
|
||||
j += 1
|
||||
i = j + 1
|
||||
return tail[i:].lstrip(" ")
|
||||
|
||||
|
||||
def _iso(stamp: str) -> datetime | None:
|
||||
if stamp == "-":
|
||||
return None
|
||||
try:
|
||||
at = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo else None
|
||||
|
||||
|
||||
def _bsd(facility: int, severity: int, rest: str) -> Message:
|
||||
hostname = ""
|
||||
m = _BSD_TIME.match(rest)
|
||||
if m and m.group(1) in _MONTHS:
|
||||
rest = rest[m.end():]
|
||||
# The HOSTNAME follows the time, unless the sender left it out and the
|
||||
# tag comes straight after (some devices do).
|
||||
word, _, after = rest.partition(" ")
|
||||
if word and not word.endswith(":") and "[" not in word:
|
||||
hostname, rest = word[:255], after
|
||||
app = ""
|
||||
t = _TAG.match(rest)
|
||||
if t:
|
||||
app, rest = t.group(1)[:128], rest[t.end():]
|
||||
return Message(facility, severity, None, hostname, app, rest[:MAX_MESSAGE])
|
||||
|
||||
|
||||
def severity_name(level: int) -> str:
|
||||
return SEVERITIES[level] if 0 <= level < len(SEVERITIES) else str(level)
|
||||
|
||||
|
||||
def facility_name(code: int) -> str:
|
||||
return FACILITIES[code] if 0 <= code < len(FACILITIES) else str(code)
|
||||
|
||||
|
||||
def frames(buffer: bytearray) -> list[bytes]:
|
||||
"""Take every complete message out of a TCP stream's buffer (RFC 6587):
|
||||
octet-counted ("<len> <msg>") or newline-ended. What is left is a message
|
||||
still arriving. A count past MAX_MESSAGE × 2 is not a count: the stream is
|
||||
read as newline-ended from there."""
|
||||
out: list[bytes] = []
|
||||
while buffer:
|
||||
if buffer[:1].isdigit():
|
||||
space = buffer.find(b" ", 0, 8)
|
||||
if space > 0 and buffer[:space].isdigit() and int(buffer[:space]) <= MAX_MESSAGE * 2:
|
||||
size = int(buffer[:space])
|
||||
end = space + 1 + size
|
||||
if len(buffer) < end:
|
||||
break
|
||||
out.append(bytes(buffer[space + 1:end]))
|
||||
del buffer[:end]
|
||||
continue
|
||||
nl = buffer.find(b"\n")
|
||||
if nl < 0:
|
||||
if len(buffer) > MAX_MESSAGE * 2: # no end in sight: take what's there
|
||||
out.append(bytes(buffer))
|
||||
buffer.clear()
|
||||
break
|
||||
line = bytes(buffer[:nl]).rstrip(b"\r\x00")
|
||||
del buffer[:nl + 1]
|
||||
if line:
|
||||
out.append(line)
|
||||
return out
|
||||
@@ -0,0 +1,20 @@
|
||||
# plugins/syslog/plugin.yaml
|
||||
name: syslog
|
||||
version: "1.0.0"
|
||||
description: "Receives syslog from NASes, routers and other devices, and keeps the lines of the sources you add."
|
||||
author: "Steward"
|
||||
license: "MIT"
|
||||
min_app_version: "0.1.0"
|
||||
tags:
|
||||
- syslog
|
||||
- logs
|
||||
|
||||
# Sources are added from the Syslog page (/plugins/syslog/), where a device
|
||||
# that starts sending is offered. Steward runs as an unprivileged user, so
|
||||
# the port is above 1024: point devices at it, or map 514 to it.
|
||||
config:
|
||||
port: 5514
|
||||
udp: true
|
||||
tcp: true
|
||||
retention_days: 14
|
||||
max_bytes_per_source: 50000000
|
||||
@@ -0,0 +1,260 @@
|
||||
# plugins/syslog/receiver.py
|
||||
"""The syslog listener: UDP and TCP on one port, every line taken as it
|
||||
arrives and stored a few seconds later.
|
||||
|
||||
A line is never written from inside the socket's callback. It is queued with
|
||||
the time it arrived and the address it came from, and the receive task, every
|
||||
few seconds, reads what is queued into a round and hands it to a Holding
|
||||
(steward/core/holding.py). The NAS that stalls the storage under Postgres is
|
||||
one of the devices sending, so its lines about the stall must wait out the
|
||||
stall in memory, not be lost to it.
|
||||
|
||||
The listener follows the plugin's settings: the receive task binds it, and
|
||||
binds it again when the port or a protocol is changed, so a save in Settings
|
||||
takes effect within seconds with no restart (rule 25). A port that can't be
|
||||
bound is said on the Syslog page, and tried again every minute.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
|
||||
from .parse import MAX_MESSAGE, Message, frames, parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Lines received and not yet read into a round. Past this the oldest go, and
|
||||
# the page says how many: a flood is bounded in memory.
|
||||
MAX_QUEUED = 20_000
|
||||
# The most lines one round takes; the rest wait for the next.
|
||||
ROUND_LINES = 2_000
|
||||
# Rounds held while the database doesn't take them: half an hour at the
|
||||
# five-second receive.
|
||||
HELD_ROUNDS = 360
|
||||
RECEIVE_S = 5
|
||||
# How long a receive waits for its own store before the next one may start.
|
||||
STORE_WAIT_S = 4.0
|
||||
# How long after a failed bind it is tried again.
|
||||
RETRY_BIND = timedelta(minutes=1)
|
||||
# Senders known at most, added or offered: a new one past them is counted,
|
||||
# not kept, so a spray of forged UDP sources can't fill the table.
|
||||
MAX_SENDERS = 200
|
||||
# TCP connections at once, and how long one may say nothing before it is
|
||||
# closed (a sender reconnects when it next has a line).
|
||||
MAX_CONNECTIONS = 64
|
||||
TCP_IDLE_S = 3600.0
|
||||
# The sample of a source's newest line kept on its row.
|
||||
SAMPLE_CHARS = 500
|
||||
|
||||
|
||||
@dataclass
|
||||
class Listener:
|
||||
"""What the listener is bound to, and what it has taken since it started.
|
||||
The page reads it as it is; nothing here is stored."""
|
||||
port: int | None = None
|
||||
udp: bool = False
|
||||
tcp: bool = False
|
||||
error: str | None = None
|
||||
tried_at: datetime | None = None
|
||||
bound_at: datetime | None = None
|
||||
received: int = 0
|
||||
dropped: int = 0 # queued lines lost to MAX_QUEUED
|
||||
unoffered: int = 0 # lines from senders past MAX_SENDERS
|
||||
connections: int = 0
|
||||
_udp_transport: object = None
|
||||
_tcp_server: object = None
|
||||
|
||||
@property
|
||||
def listening(self) -> bool:
|
||||
return self._udp_transport is not None or self._tcp_server is not None
|
||||
|
||||
|
||||
LISTENER = Listener()
|
||||
# (received_at, address, raw)
|
||||
QUEUE: deque[tuple[datetime, str, bytes]] = deque()
|
||||
|
||||
|
||||
def take(address: str, raw: bytes, now: datetime | None = None) -> None:
|
||||
"""Queue one received message."""
|
||||
if not raw:
|
||||
return
|
||||
LISTENER.received += 1
|
||||
QUEUE.append((now or datetime.now(timezone.utc), address, raw[:MAX_MESSAGE * 2]))
|
||||
while len(QUEUE) > MAX_QUEUED:
|
||||
QUEUE.popleft()
|
||||
LISTENER.dropped += 1
|
||||
|
||||
|
||||
class _Udp(asyncio.DatagramProtocol):
|
||||
def datagram_received(self, data: bytes, addr) -> None:
|
||||
take(addr[0], data)
|
||||
|
||||
|
||||
async def _tcp_client(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
|
||||
peer = writer.get_extra_info("peername")
|
||||
address = peer[0] if peer else "?"
|
||||
if LISTENER.connections >= MAX_CONNECTIONS:
|
||||
writer.close()
|
||||
return
|
||||
LISTENER.connections += 1
|
||||
buffer = bytearray()
|
||||
try:
|
||||
while chunk := await asyncio.wait_for(reader.read(65536), TCP_IDLE_S):
|
||||
buffer += chunk
|
||||
for message in frames(buffer):
|
||||
take(address, message)
|
||||
for message in frames(buffer + b"\n"): # the last line, unended
|
||||
take(address, message)
|
||||
except (ConnectionError, asyncio.IncompleteReadError, TimeoutError):
|
||||
pass
|
||||
finally:
|
||||
LISTENER.connections -= 1
|
||||
writer.close()
|
||||
|
||||
|
||||
def settings(app: "Quart") -> tuple[int, bool, bool]:
|
||||
cfg = (app.config.get("PLUGINS") or {}).get("syslog") or {}
|
||||
return int(cfg.get("port", 5514)), bool(cfg.get("udp", True)), bool(cfg.get("tcp", True))
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
if LISTENER._udp_transport is not None:
|
||||
LISTENER._udp_transport.close()
|
||||
if LISTENER._tcp_server is not None:
|
||||
LISTENER._tcp_server.close()
|
||||
await LISTENER._tcp_server.wait_closed()
|
||||
LISTENER._udp_transport = LISTENER._tcp_server = None
|
||||
LISTENER.bound_at = None
|
||||
|
||||
|
||||
async def ensure_listening(app: "Quart", now: datetime | None = None) -> None:
|
||||
"""Bind the listener as the settings say, when it isn't already: at
|
||||
start, after a change of port or protocol, and a minute after a failure."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
port, udp, tcp = settings(app)
|
||||
same = (port, udp, tcp) == (LISTENER.port, LISTENER.udp, LISTENER.tcp)
|
||||
if same and LISTENER.error is None and (LISTENER.listening or not (udp or tcp)):
|
||||
return
|
||||
if same and LISTENER.error and LISTENER.tried_at and now - LISTENER.tried_at < RETRY_BIND:
|
||||
return
|
||||
await close()
|
||||
LISTENER.port, LISTENER.udp, LISTENER.tcp = port, udp, tcp
|
||||
LISTENER.error, LISTENER.tried_at = None, now
|
||||
loop = asyncio.get_running_loop()
|
||||
failed = []
|
||||
if udp:
|
||||
try:
|
||||
LISTENER._udp_transport, _ = await loop.create_datagram_endpoint(
|
||||
_Udp, local_addr=("0.0.0.0", port))
|
||||
except OSError as exc:
|
||||
failed.append(f"UDP: {exc.strerror or exc}")
|
||||
if tcp:
|
||||
try:
|
||||
LISTENER._tcp_server = await asyncio.start_server(_tcp_client, "0.0.0.0", port)
|
||||
except OSError as exc:
|
||||
failed.append(f"TCP: {exc.strerror or exc}")
|
||||
if failed:
|
||||
LISTENER.error = f"Port {port} couldn't be opened ({'; '.join(failed)})."
|
||||
logger.warning("Syslog: %s", LISTENER.error)
|
||||
if LISTENER.listening:
|
||||
LISTENER.bound_at = now
|
||||
bound = (("UDP", LISTENER._udp_transport), ("TCP", LISTENER._tcp_server))
|
||||
logger.info("Syslog: listening on %s port %d", " and ".join(p for p, on in bound if on), port)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Received:
|
||||
at: datetime
|
||||
address: str
|
||||
message: Message
|
||||
|
||||
|
||||
def read_round(limit: int = ROUND_LINES) -> list[Received]:
|
||||
"""Take up to `limit` queued lines, oldest first, each read into its parts."""
|
||||
out = []
|
||||
while QUEUE and len(out) < limit:
|
||||
at, address, raw = QUEUE.popleft()
|
||||
out.append(Received(at, address, parse(raw)))
|
||||
return out
|
||||
|
||||
|
||||
def make_receive_task(app: "Quart") -> ScheduledTask:
|
||||
async def receive() -> None:
|
||||
await _receive(app)
|
||||
|
||||
return ScheduledTask(name="syslog_receive", coro_factory=receive,
|
||||
interval_seconds=RECEIVE_S, run_on_startup=True)
|
||||
|
||||
|
||||
async def _receive(app: "Quart") -> None:
|
||||
from steward.core.holding import holding
|
||||
|
||||
await ensure_listening(app)
|
||||
held = holding(app, "Syslog", store=lambda rounds: _store(app, rounds), limit=HELD_ROUNDS)
|
||||
lines = read_round()
|
||||
if lines:
|
||||
held.hold(lines)
|
||||
await held.flush(STORE_WAIT_S)
|
||||
|
||||
|
||||
async def _store(app: "Quart", rounds: list[list[Received]]) -> None:
|
||||
"""Write held rounds, oldest first, in one transaction: each sender's
|
||||
count, newest line and when it was last heard, and the lines of the
|
||||
sources that have been added. A sender not seen before becomes an
|
||||
offered source; its lines are counted and not kept."""
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
await store_lines(session, [line for lines in rounds for line in lines])
|
||||
|
||||
|
||||
async def store_lines(session, lines: list[Received]) -> None:
|
||||
from .models import SyslogLine, SyslogSource
|
||||
|
||||
if not lines:
|
||||
return
|
||||
by_address: dict[str, list[Received]] = {}
|
||||
for line in lines:
|
||||
by_address.setdefault(line.address, []).append(line)
|
||||
sources = {s.address: s for s in (await session.execute(
|
||||
select(SyslogSource).where(SyslogSource.address.in_(list(by_address))))).scalars()}
|
||||
new = [a for a in by_address if a not in sources]
|
||||
if new:
|
||||
known = (await session.execute(select(func.count()).select_from(SyslogSource))).scalar_one()
|
||||
for address in new:
|
||||
if known >= MAX_SENDERS:
|
||||
LISTENER.unoffered += len(by_address.pop(address))
|
||||
continue
|
||||
first = by_address[address][0]
|
||||
source = SyslogSource(address=address, first_seen_at=first.at, last_seen_at=first.at)
|
||||
session.add(source)
|
||||
sources[address] = source
|
||||
known += 1
|
||||
await session.flush()
|
||||
|
||||
for address, received in by_address.items():
|
||||
source = sources[address]
|
||||
newest = received[-1]
|
||||
source.lines_total = (source.lines_total or 0) + len(received)
|
||||
source.last_seen_at = max(source.last_seen_at, newest.at) if source.last_seen_at else newest.at
|
||||
source.last_severity = newest.message.severity
|
||||
source.last_line = newest.message.text[:SAMPLE_CHARS]
|
||||
named = next((r.message.hostname for r in reversed(received) if r.message.hostname), "")
|
||||
if named:
|
||||
source.hostname_seen = named[:255]
|
||||
if source.state == "accepted":
|
||||
session.add_all([SyslogLine(
|
||||
source_id=source.id, received_at=r.at, sent_at=r.message.sent_at,
|
||||
facility=r.message.facility, severity=r.message.severity,
|
||||
hostname=r.message.hostname, app=r.message.app, message=r.message.text,
|
||||
) for r in received])
|
||||
@@ -0,0 +1,59 @@
|
||||
# plugins/syslog/retention.py
|
||||
"""The syslog_lines ring: each source's lines kept for the age window and
|
||||
within its byte cap, whichever is less, the same shape as the docker_logs
|
||||
ring (plugins/docker/retention.py). Run hourly by the plugin's own task, with
|
||||
both bounds read from its settings each run (rule 25)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from steward.core.scheduler import ScheduledTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from quart import Quart
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def prune_syslog_lines(session, *, retention_days: int, max_bytes_per_source: int,
|
||||
now: datetime | None = None) -> dict:
|
||||
"""Drop lines past the age window, then keep only the newest ~cap bytes
|
||||
per source. Only sources at or over the cap are ranked. Runs inside the
|
||||
caller's open transaction."""
|
||||
from .models import SyslogLine
|
||||
|
||||
now = now or datetime.now(timezone.utc)
|
||||
res = await session.execute(
|
||||
delete(SyslogLine).where(SyslogLine.received_at < now - timedelta(days=retention_days)))
|
||||
age_pruned = res.rowcount or 0
|
||||
|
||||
size = func.length(SyslogLine.message)
|
||||
over = (select(SyslogLine.source_id).group_by(SyslogLine.source_id)
|
||||
.having(func.sum(size) >= max_bytes_per_source).subquery())
|
||||
# Newest first, a line goes once the lines strictly newer than it already
|
||||
# fill the cap: the newest line always stays, however long.
|
||||
running = func.sum(size).over(partition_by=SyslogLine.source_id,
|
||||
order_by=[SyslogLine.received_at.desc(), SyslogLine.id.desc()])
|
||||
ranked = (select(SyslogLine.id, (running - size).label("prefix_excl"))
|
||||
.join(over, SyslogLine.source_id == over.c.source_id).subquery())
|
||||
res = await session.execute(delete(SyslogLine).where(SyslogLine.id.in_(
|
||||
select(ranked.c.id).where(ranked.c.prefix_excl >= max_bytes_per_source))))
|
||||
return {"age_pruned": age_pruned, "size_pruned": res.rowcount or 0}
|
||||
|
||||
|
||||
def make_prune_task(app: "Quart") -> ScheduledTask:
|
||||
async def prune() -> None:
|
||||
cfg = (app.config.get("PLUGINS") or {}).get("syslog") or {}
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
counts = await prune_syslog_lines(
|
||||
session, retention_days=int(cfg.get("retention_days", 14)),
|
||||
max_bytes_per_source=int(cfg.get("max_bytes_per_source", 50_000_000)))
|
||||
if any(counts.values()):
|
||||
logger.info("Syslog retention: %s", counts)
|
||||
|
||||
return ScheduledTask(name="syslog_prune", coro_factory=prune, interval_seconds=3600)
|
||||
@@ -0,0 +1,182 @@
|
||||
# plugins/syslog/routes.py
|
||||
"""The Syslog page: whether the listener is open, the sources whose lines are
|
||||
kept, and the senders offered to be added. Every GET reads Steward's own rows
|
||||
and the listener's state in memory; nothing here contacts a device (System
|
||||
178's ruling). Adding, ignoring, editing and removing a source are an admin's,
|
||||
and audited."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from quart import Blueprint, abort, current_app, redirect, render_template, request, session
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.core.audit import log_audit
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from .models import SyslogSource
|
||||
from .parse import severity_name
|
||||
from .receiver import LISTENER, MAX_SENDERS, settings
|
||||
|
||||
syslog_bp = Blueprint("syslog", __name__, template_folder="templates")
|
||||
|
||||
INDEX = "/plugins/syslog/"
|
||||
|
||||
|
||||
def _actor() -> tuple[str | None, str]:
|
||||
return session.get("user_id"), session.get("username", "")
|
||||
|
||||
|
||||
async def _hosts(db) -> list:
|
||||
from steward.models.hosts import Host
|
||||
return list((await db.execute(select(Host.id, Host.name, Host.address).order_by(Host.name))).all())
|
||||
|
||||
|
||||
def listener_view(port: int, udp: bool, tcp: bool) -> dict:
|
||||
"""What the page says about the listener: open on which protocols, or
|
||||
why not."""
|
||||
protocols = [p for p, on in (("UDP", udp), ("TCP", tcp)) if on]
|
||||
return {"port": port, "protocols": protocols, "listening": LISTENER.listening,
|
||||
"error": LISTENER.error, "bound_at": LISTENER.bound_at,
|
||||
"received": LISTENER.received, "dropped": LISTENER.dropped,
|
||||
"unoffered": LISTENER.unoffered, "connections": LISTENER.connections,
|
||||
"senders_full": False, "max_senders": MAX_SENDERS}
|
||||
|
||||
|
||||
def source_view(s: SyslogSource, hosts: dict) -> dict:
|
||||
host = hosts.get(s.host_id) if s.host_id else None
|
||||
return {"id": s.id, "address": s.address, "state": s.state,
|
||||
"name": s.name or s.hostname_seen or s.address,
|
||||
"hostname_seen": s.hostname_seen, "host": host,
|
||||
"first_seen_at": s.first_seen_at, "last_seen_at": s.last_seen_at,
|
||||
"lines_total": s.lines_total or 0,
|
||||
"severity": severity_name(s.last_severity) if s.last_severity is not None else None,
|
||||
"level": s.last_severity, "last_line": s.last_line}
|
||||
|
||||
|
||||
def suggested_host(source: SyslogSource, hosts: list) -> str | None:
|
||||
"""The host whose address is the sender's, if one is."""
|
||||
return next((h.id for h in hosts if h.address == source.address), None)
|
||||
|
||||
|
||||
@syslog_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
rows = list((await db.execute(
|
||||
select(SyslogSource).order_by(SyslogSource.last_seen_at.desc()))).scalars())
|
||||
hosts = await _hosts(db)
|
||||
by_id = {h.id: h for h in hosts}
|
||||
views = {state: [] for state in ("accepted", "offered", "ignored")}
|
||||
for row in rows:
|
||||
v = source_view(row, by_id)
|
||||
if row.state == "offered":
|
||||
v["suggested_host"] = suggested_host(row, hosts)
|
||||
views.setdefault(row.state, []).append(v)
|
||||
views["accepted"].sort(key=lambda v: v["name"].lower())
|
||||
listener = listener_view(*settings(current_app))
|
||||
listener["senders_full"] = len(rows) >= MAX_SENDERS
|
||||
return await render_template("syslog/index.html", listener=listener,
|
||||
sources=views["accepted"], offered=views["offered"],
|
||||
ignored=views["ignored"], hosts=hosts,
|
||||
now=datetime.now(timezone.utc))
|
||||
|
||||
|
||||
async def _source_or_404(db, source_id: str) -> SyslogSource:
|
||||
source = await db.get(SyslogSource, source_id)
|
||||
if source is None:
|
||||
abort(404)
|
||||
return source
|
||||
|
||||
|
||||
async def _host_id(db, value: str | None) -> str | None:
|
||||
from steward.models.hosts import Host
|
||||
if not value:
|
||||
return None
|
||||
return value if await db.get(Host, value) is not None else None
|
||||
|
||||
|
||||
@syslog_bp.post("/sources/<source_id>/add")
|
||||
@require_role(UserRole.admin)
|
||||
async def add_source(source_id: str):
|
||||
"""Keep an offered sender's lines from now on, under the name given."""
|
||||
form = await request.form
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
source = await _source_or_404(db, source_id)
|
||||
source.name = (form.get("name") or "").strip()[:100] or source.hostname_seen or source.address
|
||||
source.host_id = await _host_id(db, form.get("host_id"))
|
||||
source.state = "accepted"
|
||||
detail = {"name": source.name, "address": source.address}
|
||||
await log_audit(current_app, *_actor(), "syslog.source_added", entity_type="syslog_source",
|
||||
entity_id=source_id, detail=detail)
|
||||
return redirect(INDEX)
|
||||
|
||||
|
||||
@syslog_bp.post("/sources/<source_id>/ignore")
|
||||
@require_role(UserRole.admin)
|
||||
async def ignore_source(source_id: str):
|
||||
"""Stop offering a sender: it is still counted, never kept."""
|
||||
return await _set_state(source_id, "ignored", "syslog.source_ignored")
|
||||
|
||||
|
||||
@syslog_bp.post("/sources/<source_id>/offer")
|
||||
@require_role(UserRole.admin)
|
||||
async def offer_source(source_id: str):
|
||||
"""Offer an ignored sender again."""
|
||||
return await _set_state(source_id, "offered", "syslog.source_offered")
|
||||
|
||||
|
||||
async def _set_state(source_id: str, state: str, action: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
source = await _source_or_404(db, source_id)
|
||||
source.state = state
|
||||
detail = {"address": source.address}
|
||||
await log_audit(current_app, *_actor(), action, entity_type="syslog_source",
|
||||
entity_id=source_id, detail=detail)
|
||||
return redirect(INDEX)
|
||||
|
||||
|
||||
@syslog_bp.get("/sources/<source_id>/edit")
|
||||
@require_role(UserRole.admin)
|
||||
async def edit_source(source_id: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
source = await _source_or_404(db, source_id)
|
||||
hosts = await _hosts(db)
|
||||
return await render_template("syslog/edit.html", source=source, hosts=hosts,
|
||||
label=source.name or source.hostname_seen or source.address)
|
||||
|
||||
|
||||
@syslog_bp.post("/sources/<source_id>/edit")
|
||||
@require_role(UserRole.admin)
|
||||
async def save_source(source_id: str):
|
||||
form = await request.form
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
source = await _source_or_404(db, source_id)
|
||||
old_name = source.name
|
||||
source.name = (form.get("name") or "").strip()[:100] or source.hostname_seen or source.address
|
||||
source.host_id = await _host_id(db, form.get("host_id"))
|
||||
detail = {"name": source.name}
|
||||
if old_name != source.name:
|
||||
detail["old_name"] = old_name
|
||||
await log_audit(current_app, *_actor(), "syslog.source_updated", entity_type="syslog_source",
|
||||
entity_id=source_id, detail=detail)
|
||||
return redirect(INDEX)
|
||||
|
||||
|
||||
@syslog_bp.post("/sources/<source_id>/delete")
|
||||
@require_role(UserRole.admin)
|
||||
async def delete_source(source_id: str):
|
||||
"""Remove a source and the lines kept from it. A device still sending is
|
||||
offered again."""
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
source = await _source_or_404(db, source_id)
|
||||
detail = {"name": source.name, "address": source.address}
|
||||
await db.delete(source)
|
||||
await log_audit(current_app, *_actor(), "syslog.source_removed", entity_type="syslog_source",
|
||||
entity_id=source_id, detail=detail)
|
||||
return redirect(INDEX)
|
||||
@@ -0,0 +1,48 @@
|
||||
{# plugins/syslog/templates/syslog/edit.html — a source's name and host, and
|
||||
Remove, which deletes the lines kept from it (#5999). #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import crumbs, page_header %}
|
||||
{% block title %}Edit {{ label }} — Steward{% endblock %}
|
||||
{% block breadcrumb %}{{ crumbs([("Syslog", "/plugins/syslog/"), (label, ""), ("Edit", "")]) }}{% endblock %}
|
||||
{% block content %}
|
||||
{{ page_header("Edit " ~ label) }}
|
||||
|
||||
<div class="form-page">
|
||||
<form method="post" action="/plugins/syslog/sources/{{ source.id }}/edit">{{ csrf_field() }}
|
||||
<section class="card form-section" aria-labelledby="syslog-source">
|
||||
<h2 id="syslog-source" class="section-title">Source</h2>
|
||||
<div class="form-pair">
|
||||
<div class="field">
|
||||
<label class="field-label" for="syslog-name">Name</label>
|
||||
<input class="input" id="syslog-name" name="name" maxlength="100" value="{{ source.name }}" placeholder="{{ source.hostname_seen or source.address }}" autocomplete="off">
|
||||
</div>
|
||||
<div class="field">
|
||||
<span class="field-label">Address</span>
|
||||
<span class="mono">{{ source.address }}</span>
|
||||
<span class="field-hint">Lines are matched to a source by the address they come from.</span>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label class="field-label" for="syslog-host">Show on host</label>
|
||||
<select class="input" id="syslog-host" name="host_id">
|
||||
<option value="">No host</option>
|
||||
{% for h in hosts %}<option value="{{ h.id }}"{% if source.host_id == h.id %} selected{% endif %}>{{ h.name }}</option>{% endfor %}
|
||||
</select>
|
||||
<span class="field-hint">Which host's page shows this source's lines.</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
<div class="page-actions">
|
||||
<button type="submit" class="btn">Save</button>
|
||||
<a href="/plugins/syslog/" class="btn btn-ghost">Cancel</a>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<section class="card form-section" aria-labelledby="syslog-remove" style="margin-top:1.5rem;">
|
||||
<h2 id="syslog-remove" class="section-title">Remove</h2>
|
||||
<p class="field-hint" style="margin:0 0 0.75rem;">Deletes the source and every line kept from it. If the device is still sending, it is offered again.</p>
|
||||
<form method="post" action="/plugins/syslog/sources/{{ source.id }}/delete" data-confirm="Remove {{ label }} and the lines kept from it?" style="margin:0;">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-danger btn-sm">Remove source</button>
|
||||
</form>
|
||||
</section>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,114 @@
|
||||
{# plugins/syslog/templates/syslog/index.html — whether Steward is listening,
|
||||
the sources whose lines are kept, and the devices sending that haven't been
|
||||
added: counted, never stored, until an admin adds one (#5999). #}
|
||||
{% extends "base.html" %}
|
||||
{% from "_macros.html" import page_header, status_badge, empty_state %}
|
||||
{% block title %}Syslog — Steward{% endblock %}
|
||||
{% macro tone(level) -%}{% if level is not none and level <= 3 %}tone-crit{% elif level == 4 %}tone-warn{% endif %}{%- endmacro %}
|
||||
{% block content %}
|
||||
{% set admin = session.user_role == 'admin' %}
|
||||
{{ page_header("Syslog", (sources|length ~ " source" ~ ("s" if sources|length != 1 else "")) if sources else none) }}
|
||||
|
||||
<section class="card" aria-labelledby="syslog-listener">
|
||||
<div class="host-health-head">
|
||||
{% if listener.error %}{{ status_badge('down', 'Not listening') }}
|
||||
{% elif listener.listening %}{{ status_badge('up', 'Listening') }}
|
||||
{% elif not listener.protocols %}{{ status_badge('inactive', 'Off') }}
|
||||
{% else %}{{ status_badge('pending', 'Starting') }}{% endif %}
|
||||
<h2 id="syslog-listener" class="section-title" style="margin:0;">Listener</h2>
|
||||
{% if listener.protocols %}<span class="field-hint mono">{{ listener.protocols|join(" and ") }} port {{ listener.port }}</span>{% endif %}
|
||||
{% if admin %}<span class="page-actions" style="margin-left:auto;"><a href="/settings/plugins/syslog/" class="btn btn-sm btn-ghost">Settings</a></span>{% endif %}
|
||||
</div>
|
||||
{% if listener.error %}
|
||||
<p class="tone-crit" style="margin:0 0 0.4rem;">{{ listener.error }}</p>
|
||||
<p class="field-hint" style="margin:0;">Tried again every minute. Another program may hold the port, or it is below 1024, which Steward can't open: choose another in Settings.</p>
|
||||
{% elif not listener.protocols %}
|
||||
<p class="field-hint" style="margin:0;">UDP and TCP are both off in Settings, so nothing is received.</p>
|
||||
{% else %}
|
||||
<p class="field-hint" style="margin:0;">
|
||||
Point a device's remote syslog at this server, {{ listener.protocols|join(" or ") }} port {{ listener.port }}.
|
||||
{%- if listener.bound_at %} Open since {{ listener.bound_at|ago }}: {{ "{:,}".format(listener.received) }} line{{ "s" if listener.received != 1 }} received{% if listener.connections %}, {{ listener.connections }} TCP sender{{ "s" if listener.connections != 1 }} connected{% endif %}.{% endif %}
|
||||
</p>
|
||||
{% endif %}
|
||||
{% if listener.dropped %}<p class="tone-warn" style="margin:0.4rem 0 0;">{{ "{:,}".format(listener.dropped) }} line{{ "s" if listener.dropped != 1 }} lost: they arrived faster than they could be read.</p>{% endif %}
|
||||
{% if listener.senders_full %}<p class="tone-warn" style="margin:0.4rem 0 0;">Steward knows {{ listener.max_senders }} senders, the most it keeps; a new one is counted, not offered{% if listener.unoffered %} ({{ "{:,}".format(listener.unoffered) }} line{{ "s" if listener.unoffered != 1 }} so far){% endif %}. Remove ones you don't need.</p>{% endif %}
|
||||
</section>
|
||||
|
||||
{% if offered %}
|
||||
<section class="card" aria-labelledby="syslog-offered">
|
||||
<h2 id="syslog-offered" class="section-title">Sending, not added</h2>
|
||||
<p class="field-hint" style="margin:0 0 0.75rem;">These devices are sending syslog. Their lines are counted and not kept until you add them.</p>
|
||||
<table class="table" style="width:100%;font-size:0.82rem;">
|
||||
<thead><tr><th>Sender</th><th>Lines</th><th>Last heard</th><th>Newest line</th>{% if admin %}<th></th>{% endif %}</tr></thead>
|
||||
<tbody>
|
||||
{% for s in offered %}
|
||||
<tr>
|
||||
<td><span class="mono">{{ s.address }}</span>{% if s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}</td>
|
||||
<td>{{ "{:,}".format(s.lines_total) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
<td class="mono" style="max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ s.last_line }}"><span class="{{ tone(s.level) }}">{{ s.severity or "" }}</span> {{ s.last_line or "—" }}</td>
|
||||
{% if admin %}
|
||||
<td style="white-space:nowrap;">
|
||||
<form method="post" action="/plugins/syslog/sources/{{ s.id }}/add" style="display:inline-flex;gap:0.4rem;align-items:center;margin:0;">{{ csrf_field() }}
|
||||
<input class="input" name="name" aria-label="Name for {{ s.address }}" maxlength="100" value="{{ s.hostname_seen or '' }}" placeholder="Name" style="width:9rem;">
|
||||
<select class="input" name="host_id" aria-label="Host for {{ s.address }}" style="width:9rem;">
|
||||
<option value="">No host</option>
|
||||
{% for h in hosts %}<option value="{{ h.id }}"{% if s.suggested_host == h.id %} selected{% endif %}>{{ h.name }}</option>{% endfor %}
|
||||
</select>
|
||||
<button type="submit" class="btn btn-sm">Add</button>
|
||||
</form>
|
||||
<form method="post" action="/plugins/syslog/sources/{{ s.id }}/ignore" style="display:inline;margin:0;">{{ csrf_field() }}
|
||||
<button type="submit" class="btn btn-sm btn-ghost">Ignore</button>
|
||||
</form>
|
||||
</td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="card" aria-labelledby="syslog-sources">
|
||||
<h2 id="syslog-sources" class="section-title">Sources</h2>
|
||||
{% if not sources %}
|
||||
{{ empty_state("No sources yet. Point a NAS, router or other device at this server's syslog port: it shows above as sending, and Add keeps its lines.") }}
|
||||
{% else %}
|
||||
<table class="table" style="width:100%;font-size:0.82rem;">
|
||||
<thead><tr><th>Source</th><th>Host</th><th>Lines</th><th>Last heard</th><th>Newest line</th>{% if admin %}<th></th>{% endif %}</tr></thead>
|
||||
<tbody>
|
||||
{% for s in sources %}
|
||||
<tr>
|
||||
<td>{{ s.name }}<div class="field-hint mono">{{ s.address }}</div></td>
|
||||
<td>{% if s.host %}<a href="/hosts/{{ s.host.id }}">{{ s.host.name }}</a>{% else %}—{% endif %}</td>
|
||||
<td>{{ "{:,}".format(s.lines_total) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
<td class="mono" style="max-width:28rem;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ s.last_line }}"><span class="{{ tone(s.level) }}">{{ s.severity or "" }}</span> {{ s.last_line or "—" }}</td>
|
||||
{% if admin %}<td><a href="/plugins/syslog/sources/{{ s.id }}/edit" class="btn btn-sm btn-ghost">Edit</a></td>{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<p class="field-hint" style="margin:0.6rem 0 0;">Lines counts every line since the device was first heard, added or not.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
{% if ignored %}
|
||||
<details class="card">
|
||||
<summary class="section-title" style="cursor:pointer;">Ignored ({{ ignored|length }})</summary>
|
||||
<table class="table" style="width:100%;font-size:0.82rem;margin-top:0.75rem;">
|
||||
<thead><tr><th>Sender</th><th>Lines</th><th>Last heard</th>{% if admin %}<th></th>{% endif %}</tr></thead>
|
||||
<tbody>
|
||||
{% for s in ignored %}
|
||||
<tr>
|
||||
<td><span class="mono">{{ s.address }}</span>{% if s.hostname_seen %}<div class="field-hint">calls itself {{ s.hostname_seen }}</div>{% endif %}</td>
|
||||
<td>{{ "{:,}".format(s.lines_total) }}</td>
|
||||
<td>{{ s.last_seen_at|ago }}</td>
|
||||
{% if admin %}<td><form method="post" action="/plugins/syslog/sources/{{ s.id }}/offer" style="margin:0;">{{ csrf_field() }}<button type="submit" class="btn btn-sm btn-ghost">Offer again</button></form></td>{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -162,6 +162,7 @@ DEFAULTS: dict[str, Any] = {
|
||||
"plugin.host_agent": {"enabled": True},
|
||||
"plugin.snmp": {"enabled": True},
|
||||
"plugin.prometheus": {"enabled": True},
|
||||
"plugin.syslog": {"enabled": True},
|
||||
# OIDC single-sign-on
|
||||
"oidc.enabled": False,
|
||||
"oidc.discovery_url": "",
|
||||
|
||||
@@ -975,7 +975,7 @@ def _build_plugin_cfg_from_form(plugin: dict, form) -> dict:
|
||||
# of a host), not discrete vendor integrations. Presentation-only split — they
|
||||
# still load through the normal plugin mechanism. A plugin.yaml may set
|
||||
# kind: capability|integration to override.
|
||||
CAPABILITY_PLUGINS = {"host_agent", "snmp", "docker"}
|
||||
CAPABILITY_PLUGINS = {"host_agent", "snmp", "docker", "syslog"}
|
||||
|
||||
|
||||
@settings_bp.get("/plugins/")
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Integration: syslog lines stored against Postgres (#5999). A new sender is
|
||||
offered and its lines counted, not kept; an added source's lines are kept;
|
||||
the ring keeps each source within its age window and byte cap."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def _received(address, text, at):
|
||||
from plugins.syslog.parse import parse
|
||||
from plugins.syslog.receiver import Received
|
||||
return Received(at, address, parse(f"<11>Oct 19 01:12:03 synnas kernel: {text}"))
|
||||
|
||||
|
||||
def test_an_unknown_sender_is_offered_and_an_added_one_kept(app):
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from plugins.syslog.models import SyslogLine, SyslogSource
|
||||
from plugins.syslog.receiver import store_lines
|
||||
|
||||
now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(delete(SyslogSource))
|
||||
await store_lines(s, [_received("10.9.0.1", "first", now),
|
||||
_received("10.9.0.1", "second", now + timedelta(seconds=1))])
|
||||
async with app.db_sessionmaker() as s:
|
||||
src = (await s.execute(select(SyslogSource))).scalar_one()
|
||||
assert (src.state, src.lines_total, src.hostname_seen) == ("offered", 2, "synnas")
|
||||
assert src.last_line == "second" and src.last_severity == 3
|
||||
assert (await s.execute(select(func.count()).select_from(SyslogLine))).scalar_one() == 0
|
||||
src_id = src.id
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
(await s.get(SyslogSource, src_id)).state = "accepted"
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await store_lines(s, [_received("10.9.0.1", "kept", now + timedelta(seconds=2))])
|
||||
async with app.db_sessionmaker() as s:
|
||||
lines = (await s.execute(select(SyslogLine))).scalars().all()
|
||||
assert [(l.app, l.hostname, l.message, l.severity) for l in lines] == [("kernel", "synnas", "kept", 3)]
|
||||
assert (await s.get(SyslogSource, src_id)).lines_total == 3
|
||||
async with s.begin():
|
||||
await s.execute(delete(SyslogSource))
|
||||
|
||||
asyncio.run(go())
|
||||
|
||||
|
||||
def test_the_ring_keeps_each_source_within_its_window_and_cap(app):
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from plugins.syslog.models import SyslogLine, SyslogSource
|
||||
from plugins.syslog.retention import prune_syslog_lines
|
||||
|
||||
now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
|
||||
line40 = "x" * 40 # cap 100 keeps 3 (prefixes 0, 40, 80)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.execute(delete(SyslogSource))
|
||||
a = SyslogSource(address="10.9.0.2", state="accepted", first_seen_at=now, last_seen_at=now)
|
||||
b = SyslogSource(address="10.9.0.3", state="accepted", first_seen_at=now, last_seen_at=now)
|
||||
s.add_all([a, b])
|
||||
await s.flush()
|
||||
|
||||
def line(src, minutes_ago, text):
|
||||
return SyslogLine(source_id=src.id, received_at=now - timedelta(minutes=minutes_ago),
|
||||
facility=1, severity=6, message=text)
|
||||
|
||||
s.add(line(a, 60 * 24 * 30, "old"))
|
||||
s.add_all([line(a, 6 - i, line40) for i in range(1, 6)])
|
||||
s.add_all([line(b, 2, line40), line(b, 1, line40)])
|
||||
a_id, b_id = a.id, b.id
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
counts = await prune_syslog_lines(s, retention_days=14, max_bytes_per_source=100, now=now)
|
||||
kept = (await s.execute(select(SyslogLine.source_id))).scalars().all()
|
||||
assert counts == {"age_pruned": 1, "size_pruned": 2}
|
||||
assert kept.count(a_id) == 3 and kept.count(b_id) == 2
|
||||
async with s.begin():
|
||||
await s.execute(delete(SyslogSource))
|
||||
|
||||
asyncio.run(go())
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Syslog messages read into their parts (#5999): RFC 5424, RFC 3164 with and
|
||||
without a hostname, a message with no PRI, and TCP framing (RFC 6587)."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from plugins.syslog.parse import MAX_MESSAGE, facility_name, frames, parse, severity_name
|
||||
|
||||
|
||||
def test_rfc5424_with_structured_data_and_a_zone():
|
||||
m = parse(b'<165>1 2026-10-19T01:12:03.5+00:00 synnas kernel 1234 ID47 '
|
||||
b'[exampleSDID@32473 iut="3" eventSource="App\\"lication"] nfsd: server not responding')
|
||||
assert (facility_name(m.facility), severity_name(m.severity)) == ("local4", "notice")
|
||||
assert m.sent_at == datetime(2026, 10, 19, 1, 12, 3, 500000, tzinfo=timezone.utc)
|
||||
assert (m.hostname, m.app, m.text) == ("synnas", "kernel", "nfsd: server not responding")
|
||||
|
||||
|
||||
def test_rfc5424_nil_fields_and_no_zone_keep_no_sent_time():
|
||||
m = parse("<14>1 2026-10-19T01:12:03 - - - - - plain text")
|
||||
assert m.sent_at is None and (m.hostname, m.app, m.text) == ("", "", "plain text")
|
||||
|
||||
|
||||
def test_bsd_with_hostname_and_tag():
|
||||
m = parse(b"<38>Oct 9 22:14:15 adm-nas sshd[4121]: Accepted publickey for admin")
|
||||
assert (m.facility, m.severity) == (4, 6)
|
||||
assert m.sent_at is None # no year, no zone: never trusted
|
||||
assert (m.hostname, m.app, m.text) == ("adm-nas", "sshd", "Accepted publickey for admin")
|
||||
|
||||
|
||||
def test_bsd_without_hostname_reads_the_tag():
|
||||
m = parse("<30>Oct 19 01:12:03 rsyncd[881]: connect from 192.168.0.4")
|
||||
assert (m.hostname, m.app, m.text) == ("", "rsyncd", "connect from 192.168.0.4")
|
||||
|
||||
|
||||
def test_no_pri_is_user_notice_and_an_unknown_shape_is_kept_whole():
|
||||
m = parse(b"something the device made up\n")
|
||||
assert (m.facility, m.severity) == (1, 5)
|
||||
assert m.text == "something the device made up"
|
||||
assert parse(b"<999>not a pri").text == "<999>not a pri"
|
||||
|
||||
|
||||
def test_a_long_message_is_cut_and_bad_bytes_kept_readable():
|
||||
m = parse(b"<13>" + b"x" * (MAX_MESSAGE + 50))
|
||||
assert len(m.text) == MAX_MESSAGE
|
||||
assert "�" in parse(b"<13>caf\xe9").text
|
||||
|
||||
|
||||
def test_frames_octet_counted_and_newline_ended_with_a_partial_left():
|
||||
buf = bytearray(b"11 <13>hello 1<14>second line\n<15>third")
|
||||
assert frames(buf) == [b"<13>hello 1", b"<14>second line"]
|
||||
assert buf == bytearray(b"<15>third")
|
||||
buf += b" part\n"
|
||||
assert frames(buf) == [b"<15>third part"] and not buf
|
||||
|
||||
|
||||
def test_frames_waits_for_the_rest_of_a_counted_message():
|
||||
buf = bytearray(b"20 <13>only part")
|
||||
assert frames(buf) == [] and buf == bytearray(b"20 <13>only part")
|
||||
|
||||
|
||||
def test_a_count_too_large_is_read_as_text():
|
||||
buf = bytearray(b"9999999 is not a count\n")
|
||||
assert frames(buf) == [b"9999999 is not a count"]
|
||||
@@ -0,0 +1,96 @@
|
||||
"""The syslog listener (#5999): lines queued as they arrive, bounded, read in
|
||||
rounds; the listener bound as the settings say, and a port it can't open
|
||||
said, not raised."""
|
||||
import asyncio
|
||||
import socket
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from plugins.syslog import receiver
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh(monkeypatch):
|
||||
monkeypatch.setattr(receiver, "LISTENER", receiver.Listener())
|
||||
monkeypatch.setattr(receiver, "QUEUE", receiver.deque())
|
||||
|
||||
|
||||
def test_the_queue_is_bounded_and_says_what_it_lost(monkeypatch):
|
||||
monkeypatch.setattr(receiver, "MAX_QUEUED", 3)
|
||||
for i in range(5):
|
||||
receiver.take("10.0.0.5", f"<13>line {i}".encode())
|
||||
assert receiver.LISTENER.received == 5 and receiver.LISTENER.dropped == 2
|
||||
assert [r.message.text for r in receiver.read_round()] == ["line 2", "line 3", "line 4"]
|
||||
|
||||
|
||||
def test_a_round_takes_at_most_its_share_and_leaves_the_rest():
|
||||
for i in range(5):
|
||||
receiver.take("10.0.0.5", f"<13>line {i}".encode())
|
||||
first = receiver.read_round(limit=2)
|
||||
assert [r.message.text for r in first] == ["line 0", "line 1"] and len(receiver.QUEUE) == 3
|
||||
assert first[0].address == "10.0.0.5"
|
||||
|
||||
|
||||
def _app(**cfg):
|
||||
return SimpleNamespace(config={"PLUGINS": {"syslog": cfg}})
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as s:
|
||||
s.bind(("0.0.0.0", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def test_it_listens_on_udp_and_tcp_and_takes_what_arrives():
|
||||
async def go():
|
||||
port = _free_port()
|
||||
await receiver.ensure_listening(_app(port=port, udp=True, tcp=True))
|
||||
assert receiver.LISTENER.listening and receiver.LISTENER.error is None
|
||||
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
udp.sendto(b"<13>over udp", ("127.0.0.1", port))
|
||||
udp.close()
|
||||
reader, writer = await asyncio.wait_for(asyncio.open_connection("127.0.0.1", port), 5)
|
||||
writer.write(b"12 <13>over tcp<14>and a line\n")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
for _ in range(50):
|
||||
if len(receiver.QUEUE) >= 3:
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
texts = sorted(r.message.text for r in receiver.read_round())
|
||||
await receiver.close()
|
||||
return texts
|
||||
|
||||
assert asyncio.run(go()) == ["and a line", "over tcp", "over udp"]
|
||||
|
||||
|
||||
def test_a_port_it_cannot_open_is_said_and_tried_again_later():
|
||||
async def go():
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as held:
|
||||
held.bind(("0.0.0.0", 0))
|
||||
port = held.getsockname()[1]
|
||||
now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
|
||||
await receiver.ensure_listening(_app(port=port, udp=True, tcp=False), now)
|
||||
said = receiver.LISTENER.error
|
||||
tried = receiver.LISTENER.tried_at
|
||||
await receiver.ensure_listening(_app(port=port, udp=True, tcp=False), now + timedelta(seconds=10))
|
||||
assert receiver.LISTENER.tried_at == tried # not retried within the minute
|
||||
return said
|
||||
|
||||
said = asyncio.run(go())
|
||||
assert said and said.startswith("Port ") and "UDP" in said
|
||||
|
||||
|
||||
def test_a_change_of_port_moves_the_listener():
|
||||
async def go():
|
||||
a, b = _free_port(), _free_port()
|
||||
await receiver.ensure_listening(_app(port=a, udp=True, tcp=False))
|
||||
await receiver.ensure_listening(_app(port=b, udp=True, tcp=False))
|
||||
port = receiver.LISTENER._udp_transport.get_extra_info("sockname")[1]
|
||||
await receiver.close()
|
||||
return port, b
|
||||
|
||||
port, b = asyncio.run(go())
|
||||
assert port == b
|
||||
@@ -0,0 +1,73 @@
|
||||
"""The Syslog page (#5999): whether the listener is open or why not, the
|
||||
sources kept, and the senders offered to an admin to add or ignore."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace as NS
|
||||
|
||||
from tests._templates import render_plugin_template
|
||||
from tests.test_style_tokens import hex_colours
|
||||
|
||||
|
||||
def _listener(**over):
|
||||
base = {"port": 5514, "protocols": ["UDP", "TCP"], "listening": True, "error": None,
|
||||
"bound_at": datetime.now(timezone.utc) - timedelta(hours=2), "received": 1204,
|
||||
"dropped": 0, "unoffered": 0, "connections": 1, "senders_full": False, "max_senders": 200}
|
||||
return {**base, **over}
|
||||
|
||||
|
||||
def _source(name, state, level=6, line="nfsd: server not responding", host=None):
|
||||
now = datetime.now(timezone.utc)
|
||||
return {"id": f"id-{name}", "address": "192.168.0.7", "state": state, "name": name,
|
||||
"hostname_seen": "synnas", "host": host, "first_seen_at": now - timedelta(days=1),
|
||||
"last_seen_at": now - timedelta(minutes=3), "lines_total": 1204,
|
||||
"severity": "err" if level == 3 else "info", "level": level, "last_line": line,
|
||||
"suggested_host": "h1"}
|
||||
|
||||
|
||||
async def _render(app, role, **ctx):
|
||||
from quart import session
|
||||
ctx = {"listener": _listener(), "sources": [], "offered": [], "ignored": [],
|
||||
"hosts": [NS(id="h1", name="SynNAS", address="192.168.0.7")],
|
||||
"now": datetime.now(timezone.utc), **ctx}
|
||||
async with app.test_request_context("/plugins/syslog/"):
|
||||
session["user_role"] = role
|
||||
page = await render_plugin_template(app, "syslog", "syslog/index.html", **ctx)
|
||||
return " ".join(page.split())
|
||||
|
||||
|
||||
async def test_an_offered_sender_can_be_added_or_ignored_by_an_admin(app):
|
||||
page = await _render(app, "admin", offered=[_source("synnas", "offered", level=3)])
|
||||
assert "Sending, not added" in page and "calls itself synnas" in page
|
||||
assert 'action="/plugins/syslog/sources/id-synnas/add"' in page
|
||||
assert 'action="/plugins/syslog/sources/id-synnas/ignore"' in page
|
||||
assert '<option value="h1" selected>SynNAS</option>' in page # the host at its address
|
||||
assert 'class="tone-crit">err</span> nfsd: server not responding' in page
|
||||
assert "UDP and TCP port 5514" in page and "1,204 lines received" in page
|
||||
main = page[page.index("<main"):]
|
||||
assert "font-weight:600" not in main and not hex_colours(main), hex_colours(main)
|
||||
|
||||
|
||||
async def test_a_viewer_sees_sources_but_no_controls(app):
|
||||
page = await _render(app, "viewer", sources=[_source("SynNAS", "accepted", host=NS(id="h1", name="SynNAS"))],
|
||||
offered=[_source("router", "offered")])
|
||||
assert 'href="/hosts/h1"' in page and "/plugins/syslog/sources/" not in page
|
||||
assert "Sending, not added" in page
|
||||
|
||||
|
||||
async def test_a_port_it_cannot_open_is_said(app):
|
||||
page = await _render(app, "admin", listener=_listener(
|
||||
listening=False, error="Port 514 couldn't be opened (UDP: Permission denied)."))
|
||||
assert "Not listening" in page and "Port 514 couldn't be opened" in page
|
||||
assert "below 1024" in page
|
||||
assert "No sources yet." in page
|
||||
|
||||
|
||||
async def test_the_edit_page_has_its_trail_and_a_remove(app):
|
||||
from quart import session
|
||||
source = NS(id="s1", name="SynNAS", hostname_seen="synnas", address="192.168.0.7", host_id=None)
|
||||
async with app.test_request_context("/plugins/syslog/sources/s1/edit"):
|
||||
session["user_role"] = "admin"
|
||||
page = await render_plugin_template(app, "syslog", "syslog/edit.html", source=source,
|
||||
hosts=[], label="SynNAS")
|
||||
page = " ".join(page.split())
|
||||
assert 'href="/plugins/syslog/">Syslog</a>' in page
|
||||
assert 'action="/plugins/syslog/sources/s1/delete"' in page and "data-confirm=" in page
|
||||
Reference in New Issue
Block a user