feat(syslog): a syslog receiver: UDP and TCP, senders offered before they're kept, each source's lines in a ring (#5999)
- plugins/syslog: RFC 5424 and RFC 3164 parsing, RFC 6587 TCP framing. - Lines are queued as they arrive and stored every 5 s through a Holding, so a storage stall delays them rather than losing them. - A new sender is an offered source, counted and not kept, until an admin adds it on the Syslog page (name, host); it can be ignored instead. - The listener follows the plugin's settings (port, UDP, TCP), rebinding within seconds of a save; a port it can't open is said on the page. - Hourly ring per source: age window and byte cap from Settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
# plugins/syslog/models.py
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, SmallInteger, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from steward.core.ids import time_ordered_id
|
||||
from steward.models.base import Base
|
||||
|
||||
|
||||
class SyslogSource(Base):
|
||||
"""A device that sends syslog to Steward, one row per sending address.
|
||||
|
||||
A sender Steward hasn't been told about is a row too, `state="offered"`:
|
||||
its lines are counted and its newest kept as a sample, so the page can
|
||||
offer it ("1,204 lines from 192.168.0.7: add as a source?"), but none is
|
||||
stored. Adding it makes it `"accepted"` and its lines are kept from then
|
||||
on; `"ignored"` stops the offer and keeps counting.
|
||||
"""
|
||||
__tablename__ = "syslog_sources"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
address: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
state: Mapped[str] = mapped_column(String(10), nullable=False, default="offered")
|
||||
name: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
||||
host_id: Mapped[str | None] = mapped_column(
|
||||
String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
||||
# Steward's name for the device, from what the lines say they come from.
|
||||
hostname_seen: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
first_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
last_seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
||||
lines_total: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
last_severity: Mapped[int | None] = mapped_column(SmallInteger, nullable=True)
|
||||
last_line: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
|
||||
|
||||
class SyslogLine(Base):
|
||||
"""One line an accepted source sent. A state table, not a time series
|
||||
(System 29): bounded by a per-source size and age ring, like docker_logs.
|
||||
`received_at` is when Steward got it, the time to trust; `sent_at` is the
|
||||
device's own, kept only when the line says its zone (RFC 5424)."""
|
||||
__tablename__ = "syslog_lines"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||||
source_id: Mapped[str] = mapped_column(
|
||||
String(36), ForeignKey("syslog_sources.id", ondelete="CASCADE"), nullable=False)
|
||||
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
facility: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
||||
severity: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
||||
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
||||
app: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
|
||||
# A source's lines newest first, and the ring's age cut; the twin-index
|
||||
# idiom of docker_logs.
|
||||
__table_args__ = (
|
||||
Index("ix_syslog_lines_source_time", "source_id", "received_at"),
|
||||
Index("ix_syslog_lines_received", "received_at"),
|
||||
)
|
||||
|
||||
|
||||
STATES = ("offered", "accepted", "ignored")
|
||||
Reference in New Issue
Block a user