feat(syslog): pattern alerts: telling phrases counted per source each minute for alert rules, storage-trouble presets (#6001)

A pattern (phrases, a severity, a source, a window) is counted every minute
into plugin_metrics as lines_matching_<key> per added source, zero included,
so an ordinary alert rule fires on a match and clears once the window passes.
Each count reads only the lines stored since the last, by line id; a new or
changed pattern reads its window once.

Patterns page with ready-made storage-trouble presets and "Alert on it",
which opens the rule form aimed at the count. Source names are kept unique,
and a rename moves its counts and the rules on them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_syslog.py

@@ -228,3 +228,136 @@ def test_search_syslog_filters_and_pages(app):
                 await s.execute(delete_sources())
 
     asyncio.run(go())
+
+
+def _latest(app, metric):
+    async def read():
+        from sqlalchemy import select
+
+        from steward.models.metrics import PluginMetricLatest
+        async with app.db_sessionmaker() as s:
+            return {r.resource_name: r.value for r in (await s.execute(select(PluginMetricLatest).where(
+                PluginMetricLatest.source_module == "syslog", PluginMetricLatest.metric_name == metric))).scalars()}
+    return read()
+
+
+async def _clear_patterns(app):
+    from sqlalchemy import delete
+
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            await s.execute(delete(_mod("models").SyslogPattern))
+            await s.execute(delete_sources())
+            for model in (PluginMetric, PluginMetricLatest):
+                await s.execute(delete(model).where(model.source_module == "syslog"))
+
+
+def test_patterns_count_matches_in_their_window_zeros_included(app):
+    from sqlalchemy import select
+
+    models, patterns = _mod("models"), _mod("patterns")
+    now = datetime.now(timezone.utc)
+
+    async def go():
+        await _clear_patterns(app)
+        patterns.TALLIES = patterns.Tallies()
+        _, _, src_id, src_name = await _seed(app, now)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add_all([
+                    models.SyslogPattern(key="nfs", name="NFS", phrases=["NFSD:"], window_minutes=60),
+                    models.SyslogPattern(key="errors", name="Errors", phrases=[], max_severity=3, window_minutes=15),
+                    models.SyslogPattern(key="quota", name="Quota", phrases=["100%"], window_minutes=15),
+                    models.SyslogPattern(key="off", name="Off", phrases=["eth0"], enabled=False)])
+        assert await patterns.record_counts(app, now) == 3
+        # Case-insensitive; the err 40 minutes ago is past a 15-minute window;
+        # % is literal, and a pattern counting nothing still writes its 0.
+        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}
+        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
+        assert await _latest(app, "lines_matching_quota") == {src_name: 0.0}
+        assert await _latest(app, "lines_matching_off") == {}
+
+        # A line stored after the count is counted by the next, which reads
+        # only the new lines.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                s.add(models.SyslogLine(source_id=src_id, received_at=now, facility=0, severity=2,
+                                        hostname="synnas", app="kernel", message="sd 0:0:0:0: I/O error"))
+        later = now + timedelta(minutes=1)
+        await patterns.record_counts(app, later)
+        assert await _latest(app, "lines_matching_errors") == {src_name: 1.0}
+        assert await _latest(app, "lines_matching_nfs") == {src_name: 1.0}       # not counted twice
+
+        # A changed pattern reads its window again.
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                p = (await s.execute(select(models.SyslogPattern).where(models.SyslogPattern.key == "errors"))).scalar_one()
+                p.window_minutes = 60
+        await patterns.record_counts(app, later)
+        assert await _latest(app, "lines_matching_errors") == {src_name: 2.0}
+        # The window passes the matches and the count falls back to 0.
+        await patterns.record_counts(app, now + timedelta(hours=2))
+        assert await _latest(app, "lines_matching_errors") == {src_name: 0.0}
+        await _clear_patterns(app)
+
+    asyncio.run(go())
+
+
+def test_patterns_page_adds_edits_and_removes_and_a_rename_moves_counts(app):
+    from sqlalchemy import select
+
+    models, patterns = _mod("models"), _mod("patterns")
+    csrf = {"X-CSRF-Token": "test-token"}
+    now = datetime.now(timezone.utc)
+
+    async def go():
+        await _clear_patterns(app)
+        patterns.TALLIES = patterns.Tallies()
+        _, _, src_id, src_name = await _seed(app, now)
+        admin = await _client(app, "admin")
+        assert (await admin.post("/plugins/syslog/patterns/preset/nfs", headers=csrf)).status_code == 302
+        refused = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
+                                   form={"name": "Nothing", "phrases": " \n", "window_minutes": "15"})
+        assert refused.status_code == 422
+        assert "at least one phrase" in await refused.get_data(as_text=True)
+        made = await admin.post("/plugins/syslog/patterns/new", headers=csrf,
+                                form={"name": "Quota full", "phrases": "100%\nquota", "max_severity": "4",
+                                      "window_minutes": "60", "enabled": "on"})
+        assert made.status_code == 302
+        async with app.db_sessionmaker() as s:
+            rows = {p.key: p for p in (await s.execute(select(models.SyslogPattern))).scalars()}
+        assert set(rows) == {"nfs", "quota_full"}
+        assert (rows["quota_full"].phrases, rows["quota_full"].max_severity) == (["100%", "quota"], 4)
+        assert rows["nfs"].phrases[0] == "nfsd:"
+
+        await patterns.record_counts(app, now)
+        page = await (await admin.get("/plugins/syslog/patterns")).get_data(as_text=True)
+        assert "Quota full" in page and "NFS server trouble" in page
+        assert "metric_name=lines_matching_quota_full" in page
+        assert 'action="/plugins/syslog/patterns/preset/nfs"' not in page      # added: no longer offered
+        rule_form = await (await admin.get(
+            f"/alerts/rules/new?source_module=syslog&resource_name={src_name}&metric_name=lines_matching_nfs"
+        )).get_data(as_text=True)
+        assert '<option value="syslog" selected>' in rule_form          # aimed at the pattern's count
+
+        # Edited: the key, and so the metric, stay.
+        edited = await admin.post(f"/plugins/syslog/patterns/{rows['quota_full'].id}/edit", headers=csrf,
+                                  form={"name": "Quota", "phrases": "quota", "window_minutes": "15"})
+        assert edited.status_code == 302
+        async with app.db_sessionmaker() as s:
+            p = await s.get(models.SyslogPattern, rows["quota_full"].id)
+            assert (p.key, p.name, p.enabled, p.max_severity) == ("quota_full", "Quota", False, None)
+
+        # A renamed source takes its counts with it.
+        renamed = await admin.post(f"/plugins/syslog/sources/{src_id}/edit", headers=csrf,
+                                   form={"name": "Big NAS", "host_id": ""})
+        assert renamed.status_code == 302
+        assert await _latest(app, "lines_matching_nfs") == {"Big NAS": 1.0}
+
+        removed = await admin.post(f"/plugins/syslog/patterns/{rows['nfs'].id}/delete", headers=csrf)
+        assert removed.status_code == 302
+        assert await _latest(app, "lines_matching_nfs") == {}
+        await _clear_patterns(app)
+
+    asyncio.run(go())