bvandeusenandClaude Opus 5.5
51892a02c0
- plugins/syslog: RFC 5424 and RFC 3164 parsing, RFC 6587 TCP framing. - Lines are queued as they arrive and stored every 5 s through a Holding, so a storage stall delays them rather than losing them. - A new sender is an offered source, counted and not kept, until an admin adds it on the Syslog page (name, host); it can be ignored instead. - The listener follows the plugin's settings (port, UDP, TCP), rebinding within seconds of a save; a port it can't open is said on the page. - Hourly ring per source: age window and byte cap from Settings. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
4.3 KiB
Python
101 lines
4.3 KiB
Python
"""Integration: syslog lines stored against Postgres (#5999). A new sender is
|
|
offered and its lines counted, not kept; an added source's lines are kept;
|
|
the ring keeps each source within its age window and byte cap."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def _received(address, text, at):
|
|
from plugins.syslog.parse import parse
|
|
from plugins.syslog.receiver import Received
|
|
return Received(at, address, parse(f"<11>Oct 19 01:12:03 synnas kernel: {text}"))
|
|
|
|
|
|
def test_an_unknown_sender_is_offered_and_an_added_one_kept(app):
|
|
from sqlalchemy import delete, func, select
|
|
|
|
from plugins.syslog.models import SyslogLine, SyslogSource
|
|
from plugins.syslog.receiver import store_lines
|
|
|
|
now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
|
|
|
|
async def go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(delete(SyslogSource))
|
|
await store_lines(s, [_received("10.9.0.1", "first", now),
|
|
_received("10.9.0.1", "second", now + timedelta(seconds=1))])
|
|
async with app.db_sessionmaker() as s:
|
|
src = (await s.execute(select(SyslogSource))).scalar_one()
|
|
assert (src.state, src.lines_total, src.hostname_seen) == ("offered", 2, "synnas")
|
|
assert src.last_line == "second" and src.last_severity == 3
|
|
assert (await s.execute(select(func.count()).select_from(SyslogLine))).scalar_one() == 0
|
|
src_id = src.id
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
(await s.get(SyslogSource, src_id)).state = "accepted"
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await store_lines(s, [_received("10.9.0.1", "kept", now + timedelta(seconds=2))])
|
|
async with app.db_sessionmaker() as s:
|
|
lines = (await s.execute(select(SyslogLine))).scalars().all()
|
|
assert [(l.app, l.hostname, l.message, l.severity) for l in lines] == [("kernel", "synnas", "kept", 3)]
|
|
assert (await s.get(SyslogSource, src_id)).lines_total == 3
|
|
async with s.begin():
|
|
await s.execute(delete(SyslogSource))
|
|
|
|
asyncio.run(go())
|
|
|
|
|
|
def test_the_ring_keeps_each_source_within_its_window_and_cap(app):
|
|
from sqlalchemy import delete, select
|
|
|
|
from plugins.syslog.models import SyslogLine, SyslogSource
|
|
from plugins.syslog.retention import prune_syslog_lines
|
|
|
|
now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
|
|
line40 = "x" * 40 # cap 100 keeps 3 (prefixes 0, 40, 80)
|
|
|
|
async def go():
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(delete(SyslogSource))
|
|
a = SyslogSource(address="10.9.0.2", state="accepted", first_seen_at=now, last_seen_at=now)
|
|
b = SyslogSource(address="10.9.0.3", state="accepted", first_seen_at=now, last_seen_at=now)
|
|
s.add_all([a, b])
|
|
await s.flush()
|
|
|
|
def line(src, minutes_ago, text):
|
|
return SyslogLine(source_id=src.id, received_at=now - timedelta(minutes=minutes_ago),
|
|
facility=1, severity=6, message=text)
|
|
|
|
s.add(line(a, 60 * 24 * 30, "old"))
|
|
s.add_all([line(a, 6 - i, line40) for i in range(1, 6)])
|
|
s.add_all([line(b, 2, line40), line(b, 1, line40)])
|
|
a_id, b_id = a.id, b.id
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
counts = await prune_syslog_lines(s, retention_days=14, max_bytes_per_source=100, now=now)
|
|
kept = (await s.execute(select(SyslogLine.source_id))).scalars().all()
|
|
assert counts == {"age_pruned": 1, "size_pruned": 2}
|
|
assert kept.count(a_id) == 3 and kept.count(b_id) == 2
|
|
async with s.begin():
|
|
await s.execute(delete(SyslogSource))
|
|
|
|
asyncio.run(go())
|