feat(syslog): pattern alerts: telling phrases counted per source each minute for alert rules, storage-trouble presets (#6001)

A pattern (phrases, a severity, a source, a window) is counted every minute
into plugin_metrics as lines_matching_<key> per added source, zero included,
so an ordinary alert rule fires on a match and clears once the window passes.
Each count reads only the lines stored since the last, by line id; a new or
changed pattern reads its window once.

Patterns page with ready-made storage-trouble presets and "Alert on it",
which opens the rule form aimed at the count. Source names are kept unique,
and a rename moves its counts and the rules on them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_syslog.py

@@ -228,3 +228,136 @@ def test_search_syslog_filters_and_pages(app):
                 await s.execute(delete_sources())
 
     asyncio.run(go())
+
+
+def _latest(app, metric):
+    async def read():
+        from sqlalchemy import select
+
+        from steward.models.metrics import PluginMetricLatest
+        async with app.db_sessionmaker() as s:
+            return {r.resource_name: r.value for r in (await s.execute(select(PluginMetricLatest).where(
+                PluginMetricLatest.source_module == "syslog", PluginMetricLatest.metric_name == metric))).scalars()}
+    return read()
+
+
+async def _clear_patterns(app):
+    from sqlalchemy import delete
+
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            await s.execute(delete(_mod("models").SyslogPattern))
+            await s.execute(delete_sources())
+            for model in (PluginMetric, PluginMetricLatest):
+                await s.execute(delete(model).where(model.source_module == "syslog"))
+
+
+def test_patterns_count_matches_in_their_window_zeros_included(app):
+    from sqlalchemy import select
+
+    models, patterns = _mod("models"), _mod("patterns")
+    now = datetime.now(timezone.utc)
+
+    async def go():
+        await _clear_patterns(app)
+        patterns.TALLIES = patterns.Tallies()
+        _, _, src_id, src_name = await _seed(app, now)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all([
+                    models.SyslogPattern(key="nfs", name="NFS", phrases=["NFSD:"], window_minutes=60),
+                    models.SyslogPattern(key="errors", name="Errors", phrases=[], max_severity=3, window_minutes=15),
+                    models.SyslogPattern(key="quota", name="Quota", phrases=["100%"], window_minutes=15),
+                    models.SyslogPattern(key="off", name="Off", phrases=["eth0"], enabled=False)])
+        assert await patterns.record_counts(app, now) == 3
+        # Case-insensitive; the err 40 minutes ago is past a 15-minute window;
+        # % is literal, and a pattern counting nothing still writes its 0.
+        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}
+        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
+        assert await _latest(app, "lines_matching_quota") == {src_name: 0.0}
+        assert await _latest(app, "lines_matching_off") == {}
+
+        # A line stored after the count is counted by the next, which reads
+        # only the new lines.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(models.SyslogLine(source_id=src_id, received_at=now, facility=0, severity=2,
+                                        hostname="synnas", app="kernel", message="sd 0:0:0:0: I/O error"))
+        later = now + timedelta(minutes=1)
+        await patterns.record_counts(app, later)
+        assert await _latest(app, "lines_matching_errors") == {src_name: 1.0}
+        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}       # not counted twice
+
+        # A changed pattern reads its window again.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                p = (await s.execute(select(models.SyslogPattern).where(models.SyslogPattern.key == "errors"))).scalar_one()
+                p.window_minutes = 60
+        await patterns.record_counts(app, later)
+        assert await _latest(app, "lines_matching_errors") == {src_name: 2.0}
+        # The window passes the matches and the count falls back to 0.
+        await patterns.record_counts(app, now + timedelta(hours=2))
+        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
+        await _clear_patterns(app)
+
+    asyncio.run(go())
+
+
+def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
+    from sqlalchemy import select
+
+    models, patterns = _mod("models"), _mod("patterns")
+    csrf = {"X-CSRF-Token": "test-token"}
+    now = datetime.now(timezone.utc)
+
+    async def go():
+        await _clear_patterns(app)
+        patterns.TALLIES = patterns.Tallies()
+        _, _, src_id, src_name = await _seed(app, now)
+        admin = await _client(app, "admin")
+        assert (await admin.post("/plugins/syslog/patterns/preset/nfs", headers=csrf)).status_code == 302
+        refused = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
+                                   form={"name": "Nothing", "phrases": " \n", "window_minutes": "15"})
+        assert refused.status_code == 422
+        assert "at least one phrase" in await refused.get_data(as_text=True)
+        made = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
+                                form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4",
+                                      "window_minutes": "60", "enabled": "on"})
+        assert made.status_code == 302
+        async with app.db_sessionmaker() as s:
+            rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()}
+        assert set(rows) == {"nfs", "quota_full"}
+        assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4)
+        assert rows["nfs"].phrases[0] == "nfsd:"
+
+        await patterns.record_counts(app, now)
+        page = await (await admin.get("/plugins/syslog/patterns")).get_data(as_text=True)
+        assert "Quota full" in page and "NFS server trouble" in page
+        assert "metric_name=lines_matching_quota_full" in page
+        assert 'action="/plugins/syslog/patterns/preset/nfs"' not in page      # added: no longer offered
+        rule_form = await (await admin.get(
+            f"/alerts/rules/new?source_module=syslog&resource_name={src_name}&metric_name=lines_matching_nfs"
+        )).get_data(as_text=True)
+        assert '<option value="syslog" selected>' in rule_form          # aimed at the pattern's count
+
+        # Edited: the key, and so the metric, stay.
+        edited = await admin.post(f"/plugins/syslog/patterns/{rows['quota_full'].id}/edit", headers=csrf,
+                                  form={"name": "Quota", "phrases": "quota", "window_minutes": "15"})
+        assert edited.status_code == 302
+        async with app.db_sessionmaker() as s:
+            p = await s.get(models.SyslogPattern, rows["quota_full"].id)
+            assert (p.key, p.name, p.enabled, p.max_severity) == ("quota_full", "Quota", False, None)
+
+        # A renamed source takes its counts with it.
+        renamed = await admin.post(f"/plugins/syslog/sources/{src_id}/edit", headers=csrf,
+                                   form={"name": "Big NAS", "host_id": ""})
+        assert renamed.status_code == 302
+        assert await _latest(app, "lines_matching_nfs") == {"Big NAS": 1.0}
+
+        removed = await admin.post(f"/plugins/syslog/patterns/{rows['nfs'].id}/delete", headers=csrf)
+        assert removed.status_code == 302
+        assert await _latest(app, "lines_matching_nfs") == {}
+        await _clear_patterns(app)
+
+    asyncio.run(go())

tests/plugins/syslog/test_patterns.py

@@ -0,0 +1,57 @@
+"""Syslog pattern alerts (#6001): a pattern read from its form, the line
+test it makes, its metric key, and the per-minute tally its count keeps."""
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+from werkzeug.datastructures import MultiDict
+
+from plugins.syslog.patterns import PRESETS, PatternError, Tally, key_for, matches, parse_form
+
+
+def test_a_form_reads_into_a_pattern():
+    v = parse_form(MultiDict({"name": " NFS ", "phrases": "nfsd:\n\n  lockd: \n", "max_severity": "3",
+                              "source_id": "", "window_minutes": "60", "enabled": "on"}))
+    assert v == {"name": "NFS", "phrases": ["nfsd:", "lockd:"], "max_severity": 3, "source_id": None,
+                 "window_minutes": 60, "enabled": True}
+    # A window or severity not offered falls back rather than failing.
+    v = parse_form(MultiDict({"name": "x", "phrases": "a", "max_severity": "9", "window_minutes": "7"}))
+    assert (v["max_severity"], v["window_minutes"], v["enabled"]) == (None, 15, False)
+
+
+@pytest.mark.parametrize("form, says", [
+    ({"phrases": "a"}, "name"),
+    ({"name": "x", "phrases": " \n "}, "at least one phrase"),
+    ({"name": "x", "phrases": "\n".join(str(i) for i in range(21))}, "At most 20"),
+    ({"name": "x", "phrases": "a" * 201}, "200 characters"),
+])
+def test_a_form_that_does_not_read_says_what_to_fix(form, says):
+    with pytest.raises(PatternError, match=says):
+        parse_form(MultiDict(form))
+
+
+def test_a_line_matches_any_phrase_in_any_case_at_the_severity():
+    p = NS(phrases=["I/O error", "medium error"], max_severity=None)
+    assert matches(p, 6, "blk_update_request: i/o ERROR, dev sda")
+    assert not matches(p, 6, "all well")
+    sev = NS(phrases=[], max_severity=3)
+    assert matches(sev, 3, "anything") and matches(sev, 0, "") and not matches(sev, 4, "anything")
+    both = NS(phrases=["nfsd"], max_severity=3)
+    assert matches(both, 2, "nfsd: too many") and not matches(both, 5, "nfsd: too many")
+
+
+def test_a_key_is_the_name_in_metric_form():
+    assert key_for("Disk I/O errors") == "disk_i_o_errors"
+    assert key_for("  ---  ") == "pattern" and len(key_for("x" * 90)) == 48
+    assert len({p.key for p in PRESETS}) == len(PRESETS)
+    assert all(p.phrases or p.max_severity is not None for p in PRESETS)
+
+
+def test_a_tally_counts_its_window_by_the_minute_and_forgets_older():
+    now = datetime(2026, 10, 19, 12, 30, 40, tzinfo=timezone.utc)
+    t = Tally(sig=())
+    for minutes in (0, 5, 14, 16, 120):
+        t.add("s1", now - timedelta(minutes=minutes))
+    assert t.count("s1", now - timedelta(minutes=15)) == 3
+    assert t.count("s1", now - timedelta(hours=3)) == 3           # what was dropped stays dropped
+    assert t.count("s2", now) == 0