bvandeusenandClaude Opus 5.5
51892a02c0
- plugins/syslog: RFC 5424 and RFC 3164 parsing, RFC 6587 TCP framing. - Lines are queued as they arrive and stored every 5 s through a Holding, so a storage stall delays them rather than losing them. - A new sender is an offered source, counted and not kept, until an admin adds it on the Syslog page (name, host); it can be ignored instead. - The listener follows the plugin's settings (port, UDP, TCP), rebinding within seconds of a save; a port it can't open is said on the page. - Hourly ring per source: age window and byte cap from Settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
60 lines
2.7 KiB
Python
60 lines
2.7 KiB
Python
# plugins/syslog/retention.py
|
|
"""The syslog_lines ring: each source's lines kept for the age window and
|
|
within its byte cap, whichever is less, the same shape as the docker_logs
|
|
ring (plugins/docker/retention.py). Run hourly by the plugin's own task, with
|
|
both bounds read from its settings each run (rule 25)."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import delete, func, select
|
|
|
|
from steward.core.scheduler import ScheduledTask
|
|
|
|
if TYPE_CHECKING:
|
|
from quart import Quart
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def prune_syslog_lines(session, *, retention_days: int, max_bytes_per_source: int,
|
|
now: datetime | None = None) -> dict:
|
|
"""Drop lines past the age window, then keep only the newest ~cap bytes
|
|
per source. Only sources at or over the cap are ranked. Runs inside the
|
|
caller's open transaction."""
|
|
from .models import SyslogLine
|
|
|
|
now = now or datetime.now(timezone.utc)
|
|
res = await session.execute(
|
|
delete(SyslogLine).where(SyslogLine.received_at < now - timedelta(days=retention_days)))
|
|
age_pruned = res.rowcount or 0
|
|
|
|
size = func.length(SyslogLine.message)
|
|
over = (select(SyslogLine.source_id).group_by(SyslogLine.source_id)
|
|
.having(func.sum(size) >= max_bytes_per_source).subquery())
|
|
# Newest first, a line goes once the lines strictly newer than it already
|
|
# fill the cap: the newest line always stays, however long.
|
|
running = func.sum(size).over(partition_by=SyslogLine.source_id,
|
|
order_by=[SyslogLine.received_at.desc(), SyslogLine.id.desc()])
|
|
ranked = (select(SyslogLine.id, (running - size).label("prefix_excl"))
|
|
.join(over, SyslogLine.source_id == over.c.source_id).subquery())
|
|
res = await session.execute(delete(SyslogLine).where(SyslogLine.id.in_(
|
|
select(ranked.c.id).where(ranked.c.prefix_excl >= max_bytes_per_source))))
|
|
return {"age_pruned": age_pruned, "size_pruned": res.rowcount or 0}
|
|
|
|
|
|
def make_prune_task(app: "Quart") -> ScheduledTask:
|
|
async def prune() -> None:
|
|
cfg = (app.config.get("PLUGINS") or {}).get("syslog") or {}
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
counts = await prune_syslog_lines(
|
|
session, retention_days=int(cfg.get("retention_days", 14)),
|
|
max_bytes_per_source=int(cfg.get("max_bytes_per_source", 50_000_000)))
|
|
if any(counts.values()):
|
|
logger.info("Syslog retention: %s", counts)
|
|
|
|
return ScheduledTask(name="syslog_prune", coro_factory=prune, interval_seconds=3600)
|