FabledSteward/tests/integration/test_syslog.py

"""Integration: syslog lines stored against Postgres (#5999). A new sender is
offered and its lines counted, not kept; an added source's lines are kept;
the ring keeps each source within its age window and byte cap."""
from __future__ import annotations

import asyncio
import os
from datetime import datetime, timedelta, timezone

import pytest

pytestmark = pytest.mark.integration


@pytest.fixture
def app():
    if not os.environ.get("STEWARD_DATABASE_URL"):
        pytest.skip("needs Postgres")
    from steward.app import create_app
    return create_app(testing=False)


def _mod(sub):
    """The loaded plugin's module: importing plugins.syslog.models by path
    would define its tables a second time (#5775)."""
    from steward.core.plugin_manager import loaded_plugin_submodule
    module = loaded_plugin_submodule("syslog", sub)
    assert module is not None, "the syslog plugin isn't loaded"
    return module


def _received(address, text, at):
    receiver = _mod("receiver")
    return receiver.Received(at, address, _mod("parse").parse(f"<11>Oct 19 01:12:03 synnas kernel: {text}"))


def test_an_unknown_sender_is_offered_and_an_added_one_kept(app):
    from sqlalchemy import delete, func, select

    SyslogLine, SyslogSource = _mod("models").SyslogLine, _mod("models").SyslogSource
    store_lines = _mod("receiver").store_lines

    now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)

    async def go():
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete(SyslogSource))
                await store_lines(s, [_received("10.9.0.1", "first", now),
                                      _received("10.9.0.1", "second", now + timedelta(seconds=1))])
        async with app.db_sessionmaker() as s:
            src = (await s.execute(select(SyslogSource))).scalar_one()
            assert (src.state, src.lines_total, src.hostname_seen) == ("offered", 2, "synnas")
            assert src.last_line == "second" and src.last_severity == 3
            assert (await s.execute(select(func.count()).select_from(SyslogLine))).scalar_one() == 0
            src_id = src.id
        async with app.db_sessionmaker() as s:
            async with s.begin():
                (await s.get(SyslogSource, src_id)).state = "accepted"
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await store_lines(s, [_received("10.9.0.1", "kept", now + timedelta(seconds=2))])
        async with app.db_sessionmaker() as s:
            lines = (await s.execute(select(SyslogLine))).scalars().all()
            assert [(ln.app, ln.hostname, ln.message, ln.severity) for ln in lines] == [("kernel", "synnas", "kept", 3)]
            assert (await s.get(SyslogSource, src_id)).lines_total == 3
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete(SyslogSource))

    asyncio.run(go())


def test_the_ring_keeps_each_source_within_its_window_and_cap(app):
    from sqlalchemy import delete, select

    SyslogLine, SyslogSource = _mod("models").SyslogLine, _mod("models").SyslogSource
    prune_syslog_lines = _mod("retention").prune_syslog_lines

    now = datetime(2026, 10, 19, 3, 0, tzinfo=timezone.utc)
    line40 = "x" * 40                    # cap 100 keeps 3 (prefixes 0, 40, 80)

    async def go():
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete(SyslogSource))
                a = SyslogSource(address="10.9.0.2", state="accepted", first_seen_at=now, last_seen_at=now)
                b = SyslogSource(address="10.9.0.3", state="accepted", first_seen_at=now, last_seen_at=now)
                s.add_all([a, b])
                await s.flush()

                def line(src, minutes_ago, text):
                    return SyslogLine(source_id=src.id, received_at=now - timedelta(minutes=minutes_ago),
                                      facility=1, severity=6, message=text)

                s.add(line(a, 60 * 24 * 30, "old"))
                s.add_all([line(a, 6 - i, line40) for i in range(1, 6)])
                s.add_all([line(b, 2, line40), line(b, 1, line40)])
                a_id, b_id = a.id, b.id
        async with app.db_sessionmaker() as s:
            async with s.begin():
                counts = await prune_syslog_lines(s, retention_days=14, max_bytes_per_source=100, now=now)
            kept = (await s.execute(select(SyslogLine.source_id))).scalars().all()
            assert counts == {"age_pruned": 1, "size_pruned": 2}
            assert kept.count(a_id) == 3 and kept.count(b_id) == 2
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete(SyslogSource))

    asyncio.run(go())


async def _client(app, role):
    import time
    import uuid

    from steward.models.users import User, UserRole
    uid, tag = str(uuid.uuid4()), uuid.uuid4().hex[:8]
    async with app.db_sessionmaker() as s:
        async with s.begin():
            s.add(User(id=uid, username=f"syslog-{tag}", email=f"syslog-{tag}@test",
                       password_hash="x", role=UserRole(role), is_active=True))
    client = app.test_client()
    async with client.session_transaction() as sess:
        sess.update(user_id=uid, user_role=role, username=f"syslog-{tag}",
                    _csrf="test-token", login_at=time.time())
    return client


async def _seed(app, now):
    """A host, an added source bound to it with five lines, and an offered
    sender. Returns (host_id, host_name, source_id, source_name)."""
    import uuid

    from steward.models.hosts import Host
    SyslogLine, SyslogSource = _mod("models").SyslogLine, _mod("models").SyslogSource
    tag = uuid.uuid4().hex[:8]
    async with app.db_sessionmaker() as s:
        async with s.begin():
            await s.execute(delete_sources())
            host = Host(id=str(uuid.uuid4()), name=f"nas-{tag}", address=f"10.8.{int(tag[:2], 16)}.1")
            s.add(host)
            await s.flush()
            src = SyslogSource(address="10.8.0.9", state="accepted", name=f"SynNAS-{tag}", host_id=host.id,
                               first_seen_at=now, last_seen_at=now, lines_total=5)
            s.add_all([src, SyslogSource(address="10.8.0.10", first_seen_at=now, last_seen_at=now)])
            await s.flush()
            for minutes, sev, app_, text in ((50, 6, "kernel", "eth0 link up"),
                                              (40, 3, "kernel", "nfsd: server not responding"),
                                              (30, 4, "rsyncd", "100% of quota used"),
                                              (20, 6, "sshd", "Accepted publickey for admin"),
                                              (90 * 60, 3, "kernel", "an old error")):
                s.add(SyslogLine(source_id=src.id, received_at=now - timedelta(minutes=minutes),
                                 facility=0, severity=sev, hostname="synnas", app=app_, message=text))
            return host.id, host.name, src.id, src.name


def delete_sources():
    from sqlalchemy import delete
    return delete(_mod("models").SyslogSource)


def test_a_source_page_filters_its_lines_and_hosts_show_them(app):
    now = datetime.now(timezone.utc)

    async def go():
        host_id, _, src_id, src_name = await _seed(app, now)
        client = await _client(app, "viewer")
        page = await (await client.get(f"/plugins/syslog/sources/{src_id}")).get_data(as_text=True)
        # The last day by default: the 90-hour-old error is outside it.
        assert "nfsd: server not responding" in page and "an old error" not in page
        assert "4 lines in the last hour" in page
        errors = await (await client.get(f"/plugins/syslog/sources/{src_id}?severity=3&range=7d",
                                         headers={"HX-Request": "true"})).get_data(as_text=True)
        assert "nfsd: server not responding" in errors and "an old error" in errors
        assert "eth0 link up" not in errors and "<html" not in errors             # the table alone
        # % matches literally; the app filter is exact.
        pct = await (await client.get(f"/plugins/syslog/sources/{src_id}?q=100%25")).get_data(as_text=True)
        assert "100% of quota used" in pct and "eth0 link up" not in pct
        sshd = await (await client.get(f"/plugins/syslog/sources/{src_id}?app=sshd")).get_data(as_text=True)
        assert "Accepted publickey" in sshd and "nfsd" not in sshd.split("<tbody>", 1)[1]

        panel = await (await client.get(f"/plugins/syslog/panel?host_id={host_id}")).get_data(as_text=True)
        assert src_name in panel and "Accepted publickey" in panel and "an old error" not in panel
        by_address = await (await client.get("/plugins/syslog/panel?address=10.8.0.9")).get_data(as_text=True)
        assert by_address == ""                     # bound to a host: never matched by address
        assert await (await client.get("/plugins/syslog/panel?address=10.1.1.1")).get_data(as_text=True) == ""

        index = await (await client.get("/plugins/syslog/")).get_data(as_text=True)
        assert f'href="/plugins/syslog/sources/{src_id}"' in index
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete_sources())

    asyncio.run(go())


def test_search_syslog_filters_and_pages(app):
    from steward.core.api_tokens import Caller
    from steward.mcp import _context
    from steward.models.users import UserRole

    tools = _mod("mcp_tools")
    now = datetime.now(timezone.utc)

    async def go():
        _context._app_ctx.set(app)
        _context._caller_ctx.set(Caller(user_id="u", username="viewer", role=UserRole.viewer, token_id=0))
        _, host_name, _, src_name = await _seed(app, now)
        errs = await tools.search_syslog(source=src_name, severity="err", range="7d")
        assert [m["message"] for m in errs["matches"]] == ["nfsd: server not responding", "an old error"]
        assert errs["matches"][0]["severity"] == "err" and errs["matches"][0]["app"] == "kernel"
        # _ and % match literally: as wildcards "1_0%" would match "100% of quota used".
        assert (await tools.search_syslog(host=host_name, text="1_0%"))["matches"] == []
        assert len((await tools.search_syslog(host=host_name, text="100%"))["matches"]) == 1
        page1 = await tools.search_syslog(source="10.8.0.9", limit=2)
        assert page1["truncated"] and [m["message"] for m in page1["matches"]] == [
            "Accepted publickey for admin", "100% of quota used"]
        page2 = await tools.search_syslog(source="10.8.0.9", limit=2, before=page1["next_before"])
        assert [m["message"] for m in page2["matches"]] == ["nfsd: server not responding", "eth0 link up"]
        listed = await tools.list_syslog_sources()
        states = {s["address"]: (s["state"], s["lines_last_hour"]) for s in listed["sources"]}
        assert states == {"10.8.0.9": ("accepted", 4), "10.8.0.10": ("offered", None)}
        with pytest.raises(ValueError, match="list_syslog_sources"):
            await tools.search_syslog(source="no-such-source")
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete_sources())

    asyncio.run(go())


def _latest(app, metric):
    async def read():
        from sqlalchemy import select

        from steward.models.metrics import PluginMetricLatest
        async with app.db_sessionmaker() as s:
            return {r.resource_name: r.value for r in (await s.execute(select(PluginMetricLatest).where(
                PluginMetricLatest.source_module == "syslog", PluginMetricLatest.metric_name == metric))).scalars()}
    return read()


async def _clear_patterns(app):
    from sqlalchemy import delete

    from steward.models.metrics import PluginMetric, PluginMetricLatest
    async with app.db_sessionmaker() as s:
        async with s.begin():
            await s.execute(delete(_mod("models").SyslogPattern))
            await s.execute(delete_sources())
            for model in (PluginMetric, PluginMetricLatest):
                await s.execute(delete(model).where(model.source_module == "syslog"))


def test_patterns_count_matches_in_their_window_zeros_included(app):
    from sqlalchemy import select

    models, patterns = _mod("models"), _mod("patterns")
    now = datetime.now(timezone.utc)

    async def go():
        await _clear_patterns(app)
        patterns.TALLIES = patterns.Tallies()
        _, _, src_id, src_name = await _seed(app, now)
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add_all([
                    models.SyslogPattern(key="nfs", name="NFS", phrases=["NFSD:"], window_minutes=60),
                    models.SyslogPattern(key="errors", name="Errors", phrases=[], max_severity=3, window_minutes=15),
                    models.SyslogPattern(key="quota", name="Quota", phrases=["100%"], window_minutes=15),
                    models.SyslogPattern(key="off", name="Off", phrases=["eth0"], enabled=False)])
        assert await patterns.record_counts(app, now) == 3
        # Case-insensitive; the err 40 minutes ago is past a 15-minute window;
        # % is literal, and a pattern counting nothing still writes its 0.
        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}
        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
        assert await _latest(app, "lines_matching_quota") == {src_name: 0.0}
        assert await _latest(app, "lines_matching_off") == {}

        # A line stored after the count is counted by the next, which reads
        # only the new lines.
        async with app.db_sessionmaker() as s:
            async with s.begin():
                s.add(models.SyslogLine(source_id=src_id, received_at=now, facility=0, severity=2,
                                        hostname="synnas", app="kernel", message="sd 0:0:0:0: I/O error"))
        later = now + timedelta(minutes=1)
        await patterns.record_counts(app, later)
        assert await _latest(app, "lines_matching_errors") == {src_name: 1.0}
        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}       # not counted twice

        # A changed pattern reads its window again.
        async with app.db_sessionmaker() as s:
            async with s.begin():
                p = (await s.execute(select(models.SyslogPattern).where(models.SyslogPattern.key == "errors"))).scalar_one()
                p.window_minutes = 60
        await patterns.record_counts(app, later)
        assert await _latest(app, "lines_matching_errors") == {src_name: 2.0}
        # The window passes the matches and the count falls back to 0.
        await patterns.record_counts(app, now + timedelta(hours=2))
        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
        await _clear_patterns(app)

    asyncio.run(go())


def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
    from sqlalchemy import select

    models, patterns = _mod("models"), _mod("patterns")
    csrf = {"X-CSRF-Token": "test-token"}
    now = datetime.now(timezone.utc)

    async def go():
        await _clear_patterns(app)
        patterns.TALLIES = patterns.Tallies()
        _, _, src_id, src_name = await _seed(app, now)
        admin = await _client(app, "admin")
        assert (await admin.post("/plugins/syslog/patterns/preset/nfs", headers=csrf)).status_code == 302
        refused = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
                                   form={"name": "Nothing", "phrases": " \n", "window_minutes": "15"})
        assert refused.status_code == 422
        assert "at least one phrase" in await refused.get_data(as_text=True)
        made = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
                                form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4",
                                      "window_minutes": "60", "enabled": "on"})
        assert made.status_code == 302
        async with app.db_sessionmaker() as s:
            rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()}
        assert set(rows) == {"nfs", "quota_full"}
        assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4)
        assert rows["nfs"].phrases[0] == "nfsd:"

        await patterns.record_counts(app, now)
        assert await _latest(app, "lines_matching_nfs") == {src_name: 0.0}
        page = await (await admin.get("/plugins/syslog/patterns")).get_data(as_text=True)
        assert "Quota full" in page and "NFS server trouble" in page
        assert "metric_name=lines_matching_quota_full" in page
        assert 'action="/plugins/syslog/patterns/preset/nfs"' not in page      # added: no longer offered
        rule_form = await (await admin.get(
            f"/alerts/rules/new?source_module=syslog&resource_name={src_name}&metric_name=lines_matching_nfs"
        )).get_data(as_text=True)
        assert '<option value="syslog" selected>' in rule_form          # aimed at the pattern's count

        # Edited: the key, and so the metric, stay.
        edited = await admin.post(f"/plugins/syslog/patterns/{rows['quota_full'].id}/edit", headers=csrf,
                                  form={"name": "Quota", "phrases": "quota", "window_minutes": "15"})
        assert edited.status_code == 302
        async with app.db_sessionmaker() as s:
            p = await s.get(models.SyslogPattern, rows["quota_full"].id)
            assert (p.key, p.name, p.enabled, p.max_severity) == ("quota_full", "Quota", False, None)

        # A renamed source takes its counts with it.
        renamed = await admin.post(f"/plugins/syslog/sources/{src_id}/edit", headers=csrf,
                                   form={"name": "Big NAS", "host_id": ""})
        assert renamed.status_code == 302
        # (0: a preset's window is 15 minutes, and the nfsd line is 40 old;
        # no count has run since, so the row under the new name was moved.)
        assert await _latest(app, "lines_matching_nfs") == {"Big NAS": 0.0}

        removed = await admin.post(f"/plugins/syslog/patterns/{rows['nfs'].id}/delete", headers=csrf)
        assert removed.status_code == 302
        assert await _latest(app, "lines_matching_nfs") == {}
        await _clear_patterns(app)

    asyncio.run(go())


def test_lines_beside_the_readings_around_an_anomaly_and_as_chart_marks(app):
    now = datetime.now(timezone.utc)
    mcp_tools = _mod("mcp_tools")

    async def go():
        host_id, host_name, _, src_name = await _seed(app, now)
        client = await _client(app, "viewer")
        since, until = (now - timedelta(minutes=35)).isoformat(), (now - timedelta(minutes=25)).isoformat()
        around = await (await client.get("/plugins/syslog/around", query_string={
            "source": "host_agent", "resource": f"{host_name}:/", "since": since, "until": until})).get_data(as_text=True)
        # Warnings and worse within half an hour either side; info lines left out.
        assert "nfsd: server not responding" in around and "100% of quota used" in around
        assert "eth0 link up" not in around and "an old error" not in around
        elsewhere = await (await client.get("/plugins/syslog/around", query_string={
            "source": "traefik", "resource": "web@docker", "since": since})).get_data(as_text=True)
        assert "No added syslog source" in elsewhere
        assert (await client.get("/plugins/syslog/around?source=host_agent")).status_code == 400

        day = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&range=24h")).get_json()
        assert [m["text"] for m in day["marks"]] == [f"{src_name} err kernel: nfsd: server not responding"]
        assert day["more"] is False
        week = await (await client.get(f"/plugins/syslog/marks?host_id={host_id}&hours=168")).get_json()
        assert len(week["marks"]) == 2
        assert (await (await client.get("/plugins/syslog/marks?address=10.1.1.1")).get_json())["marks"] == []

        # get_anomaly's circumstances: the host's lines, and nothing for a
        # resource syslog can't place.
        async with app.db_sessionmaker() as s:
            events = await mcp_tools.mcp_events(s, host_id, host_name, now - timedelta(hours=1), now)
            assert [(e["kind"], e["subject"]) for e in events] == [("syslog", src_name)] * 2
            assert events[0]["what"].startswith("err kernel: nfsd")
            assert await mcp_tools.mcp_resource_events(s, "traefik", "web@docker", now - timedelta(hours=1), now) == []
        async with app.db_sessionmaker() as s:
            async with s.begin():
                await s.execute(delete_sources())

    asyncio.run(go())