bvandeusenandClaude Opus 5.5
8eae3d3825
A pattern (phrases, a severity, a source, a window) is counted every minute into plugin_metrics as lines_matching_<key> per added source, zero included, so an ordinary alert rule fires on a match and clears once the window passes. Each count reads only the lines stored since the last, by line id; a new or changed pattern reads its window once. Patterns page with ready-made storage-trouble presets and "Alert on it", which opens the rule form aimed at the count. Source names are kept unique, and a rename moves its counts and the rules on them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
98 lines
4.9 KiB
Python
98 lines
4.9 KiB
Python
# plugins/syslog/models.py
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import JSON, BigInteger, Boolean, DateTime, ForeignKey, Index, SmallInteger, String, Text
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from steward.core.ids import time_ordered_id
|
|
from steward.models.base import Base
|
|
|
|
|
|
class SyslogSource(Base):
|
|
"""A device that sends syslog to Steward, one row per sending address.
|
|
|
|
A sender Steward hasn't been told about is a row too, `state="offered"`:
|
|
its lines are counted and its newest kept as a sample, so the page can
|
|
offer it ("1,204 lines from 192.168.0.7: add as a source?"), but none is
|
|
stored. Adding it makes it `"accepted"` and its lines are kept from then
|
|
on; `"ignored"` stops the offer and keeps counting.
|
|
"""
|
|
__tablename__ = "syslog_sources"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
address: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
|
state: Mapped[str] = mapped_column(String(10), nullable=False, default="offered")
|
|
name: Mapped[str] = mapped_column(String(100), nullable=False, default="")
|
|
host_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("hosts.id", ondelete="SET NULL"), nullable=True)
|
|
# Steward's name for the device, from what the lines say they come from.
|
|
hostname_seen: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
|
first_seen_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
last_seen_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
lines_total: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
|
last_severity: Mapped[int | None] = mapped_column(SmallInteger, nullable=True)
|
|
last_line: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
|
|
|
|
|
class SyslogLine(Base):
|
|
"""One line an accepted source sent. A state table, not a time series
|
|
(System 29): bounded by a per-source size and age ring, like docker_logs.
|
|
`received_at` is when Steward got it, the time to trust; `sent_at` is the
|
|
device's own, kept only when the line says its zone (RFC 5424)."""
|
|
__tablename__ = "syslog_lines"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
source_id: Mapped[str] = mapped_column(
|
|
String(36), ForeignKey("syslog_sources.id", ondelete="CASCADE"), nullable=False)
|
|
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
sent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
facility: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
|
severity: Mapped[int] = mapped_column(SmallInteger, nullable=False)
|
|
hostname: Mapped[str] = mapped_column(String(255), nullable=False, default="")
|
|
app: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
|
message: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
|
|
|
# A source's lines newest first, and the ring's age cut; the twin-index
|
|
# idiom of docker_logs.
|
|
__table_args__ = (
|
|
Index("ix_syslog_lines_source_time", "source_id", "received_at"),
|
|
Index("ix_syslog_lines_received", "received_at"),
|
|
)
|
|
|
|
|
|
STATES = ("offered", "accepted", "ignored")
|
|
|
|
|
|
class SyslogPattern(Base):
|
|
"""A telling phrase to count (#6001): lines whose message holds any of
|
|
`phrases` (case-insensitive) and, when `max_severity` is set, whose
|
|
severity is that or worse. Every minute each added source it covers (all,
|
|
or `source_id`'s) gets a sample of how many of its lines matched in the
|
|
last `window_minutes`, as metric `lines_matching_<key>`, so an ordinary
|
|
alert rule fires on a match and clears once the window passes it."""
|
|
__tablename__ = "syslog_patterns"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
key: Mapped[str] = mapped_column(String(48), nullable=False, unique=True)
|
|
name: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
phrases: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
|
max_severity: Mapped[int | None] = mapped_column(SmallInteger, nullable=True)
|
|
source_id: Mapped[str | None] = mapped_column(
|
|
String(36), ForeignKey("syslog_sources.id", ondelete="CASCADE"), nullable=True)
|
|
window_minutes: Mapped[int] = mapped_column(SmallInteger, nullable=False, default=15)
|
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
@property
|
|
def metric(self) -> str:
|
|
return metric_for(self.key)
|
|
|
|
|
|
def metric_for(key: str) -> str:
|
|
return f"lines_matching_{key}"
|