feat(syslog): pattern alerts: telling phrases counted per source each minute for alert rules, storage-trouble presets (#6001)

A pattern (phrases, a severity, a source, a window) is counted every minute
into plugin_metrics as lines_matching_<key> per added source, zero included,
so an ordinary alert rule fires on a match and clears once the window passes.
Each count reads only the lines stored since the last, by line id; a new or
changed pattern reads its window once.

Patterns page with ready-made storage-trouble presets and "Alert on it",
which opens the rule form aimed at the count. Source names are kept unique,
and a rename moves its counts and the rules on them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/syslog/routes.py

@@ -9,15 +9,19 @@ from __future__ import annotations
 from datetime import datetime, timezone
 
 from quart import Blueprint, abort, current_app, redirect, render_template, request, session
-from sqlalchemy import select
+from sqlalchemy import delete, select
 
 from steward.auth.middleware import require_role
 from steward.core.audit import log_audit
+from steward.core.metric_series import latest_by_resource
 from steward.core.time_range import RANGES
+from steward.models.metrics import PluginMetricLatest
 from steward.models.users import UserRole
 
-from .models import SyslogLine, SyslogSource
-from .parse import facility_name, severity_name
+from .models import SyslogLine, SyslogPattern, SyslogSource
+from .parse import SEVERITIES, facility_name, severity_name
+from .patterns import (PRESETS, PRESETS_BY_KEY, SOURCE_MODULE, WINDOWS, PatternError, key_for, parse_form,
+                       resource_of)
 from .receiver import LISTENER, MAX_SENDERS, settings
 from .service import PANEL_LINES, PANEL_SPAN, SEVERITY_CHOICES, apps_of, filtered, line_filter, rates, sources_for
 
@@ -93,6 +97,27 @@ async def _source_or_404(db, source_id: str) -> SyslogSource:
     return source
 
 
+async def _unique_name(db, source: SyslogSource, wanted: str) -> str:
+    """The name a source is given, made its own: a source's pattern counts
+    and the alert rules on them are filed under its name, so two sources
+    sharing one would read as one. A name taken is kept with the address."""
+    name = wanted.strip()[:100] or source.hostname_seen or source.address
+    taken = (await db.execute(select(SyslogSource.id).where(
+        SyslogSource.name == name, SyslogSource.id != source.id).limit(1))).first()
+    return f"{name[:80]} ({source.address})" if taken else name
+
+
+async def _move_resource(db, old: str, new: str) -> None:
+    """Move a renamed source's counts, and the alert rules on them, to its new name."""
+    from sqlalchemy import update
+
+    from steward.models.alerts import AlertRule
+    from steward.models.metrics import PluginMetric, PluginMetricLatest
+    for model in (PluginMetric, PluginMetricLatest, AlertRule):
+        await db.execute(update(model).where(
+            model.source_module == SOURCE_MODULE, model.resource_name == old).values(resource_name=new))
+
+
 async def _host_id(db, value: str | None) -> str | None:
     from steward.models.hosts import Host
     if not value:
@@ -108,7 +133,7 @@ async def add_source(source_id: str):
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             source = await _source_or_404(db, source_id)
-            source.name = (form.get("name") or "").strip()[:100] or source.hostname_seen or source.address
+            source.name = await _unique_name(db, source, form.get("name") or "")
             source.host_id = await _host_id(db, form.get("host_id"))
             source.state = "accepted"
             detail = {"name": source.name, "address": source.address}
@@ -159,12 +184,15 @@ async def save_source(source_id: str):
     async with current_app.db_sessionmaker() as db:
         async with db.begin():
             source = await _source_or_404(db, source_id)
+            old_resource = resource_of(source)
             old_name = source.name
-            source.name = (form.get("name") or "").strip()[:100] or source.hostname_seen or source.address
+            source.name = await _unique_name(db, source, form.get("name") or "")
             source.host_id = await _host_id(db, form.get("host_id"))
             detail = {"name": source.name}
             if old_name != source.name:
                 detail["old_name"] = old_name
+            if old_resource != resource_of(source):
+                await _move_resource(db, old_resource, resource_of(source))
     await log_audit(current_app, *_actor(), "syslog.source_updated", entity_type="syslog_source",
                     entity_id=source_id, detail=detail)
     return redirect(f"/plugins/syslog/sources/{source_id}")
@@ -179,7 +207,11 @@ async def delete_source(source_id: str):
         async with db.begin():
             source = await _source_or_404(db, source_id)
             detail = {"name": source.name, "address": source.address}
+            resource = resource_of(source)
             await db.delete(source)
+            # Its newest pattern counts go with it; its history ages out.
+            await db.execute(delete(PluginMetricLatest).where(
+                PluginMetricLatest.source_module == SOURCE_MODULE, PluginMetricLatest.resource_name == resource))
     await log_audit(current_app, *_actor(), "syslog.source_removed", entity_type="syslog_source",
                     entity_id=source_id, detail=detail)
     return redirect(INDEX)
@@ -261,3 +293,167 @@ async def lines_panel():
     rows = [{**line_view(ln), "source": names[ln.source_id], "source_id": ln.source_id} for ln in lines]
     return await render_template("syslog/panel.html", lines=rows,
                                  sources=[{"id": sid, "name": name} for sid, name in names.items()])
+
+
+# ── Patterns: telling phrases counted for alert rules (#6001) ─────────────────
+
+PATTERNS = "/plugins/syslog/patterns"
+
+
+def pattern_view(p: SyslogPattern, sources: dict, latest: dict) -> dict:
+    covered = [s for s in sources.values() if s.state == "accepted" and p.source_id in (None, s.id)]
+    return {"id": p.id, "key": p.key, "name": p.name, "metric": p.metric, "phrases": p.phrases or [],
+            "severity": severity_name(p.max_severity) if p.max_severity is not None else None,
+            "scope": sources[p.source_id].name or sources[p.source_id].address
+            if p.source_id in sources else None,
+            "window": p.window_minutes, "enabled": p.enabled,
+            "counts": [{"source": resource_of(s), "value": _value(latest, resource_of(s), p.metric)} for s in covered],
+            "alert_resource": resource_of(covered[0]) if covered else None}
+
+
+def _value(latest: dict, resource: str, metric: str) -> int | None:
+    row = latest.get(resource, {}).get(metric)
+    return int(row.value) if row is not None else None
+
+
+@syslog_bp.get("/patterns")
+@require_role(UserRole.viewer)
+async def patterns_index():
+    async with current_app.db_sessionmaker() as db:
+        patterns = list((await db.execute(select(SyslogPattern).order_by(SyslogPattern.name))).scalars())
+        sources = {s.id: s for s in (await db.execute(select(SyslogSource))).scalars()}
+        latest = await latest_by_resource(db, SOURCE_MODULE, {resource_of(s) for s in sources.values()},
+                                          [p.metric for p in patterns])
+    taken = {p.key for p in patterns}
+    return await render_template("syslog/patterns.html", patterns=[pattern_view(p, sources, latest) for p in patterns],
+                                 presets=[p for p in PRESETS if p.key not in taken],
+                                 any_sources=any(s.state == "accepted" for s in sources.values()))
+
+
+async def _free_key(db, wanted: str) -> str:
+    taken = set((await db.execute(select(SyslogPattern.key))).scalars())
+    key, n = wanted, 2
+    while key in taken:
+        key = f"{wanted[:44]}_{n}"
+        n += 1
+    return key
+
+
+async def _accepted_sources(db) -> list:
+    return list((await db.execute(select(SyslogSource).where(SyslogSource.state == "accepted")
+                                  .order_by(SyslogSource.name))).scalars())
+
+
+async def _render_pattern_form(pattern, values: dict, error: str | None = None, status: int = 200):
+    async with current_app.db_sessionmaker() as db:
+        sources = await _accepted_sources(db)
+    return await render_template("syslog/pattern_form.html", pattern=pattern, v=values, error=error,
+                                 sources=sources, windows=WINDOWS, severities=list(enumerate(SEVERITIES))), status
+
+
+@syslog_bp.post("/patterns/preset/<key>")
+@require_role(UserRole.admin)
+async def add_preset(key: str):
+    preset = PRESETS_BY_KEY.get(key)
+    if preset is None:
+        abort(404)
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            pattern = SyslogPattern(key=await _free_key(db, preset.key), name=preset.name,
+                                    phrases=list(preset.phrases), max_severity=preset.max_severity)
+            db.add(pattern)
+            await db.flush()
+            pattern_id = pattern.id
+    await log_audit(current_app, *_actor(), "syslog.pattern_added", entity_type="syslog_pattern",
+                    entity_id=pattern_id, detail={"name": preset.name, "preset": preset.key})
+    return redirect(PATTERNS)
+
+
+@syslog_bp.get("/patterns/new")
+@require_role(UserRole.admin)
+async def new_pattern():
+    return await _render_pattern_form(None, {"window_minutes": 15, "enabled": True})
+
+
+def _typed_pattern(form) -> dict:
+    return {"name": form.get("name", ""), "phrases": form.get("phrases", ""),
+            "max_severity": form.get("max_severity", ""), "source_id": form.get("source_id", ""),
+            "window_minutes": form.get("window_minutes", 15, type=int), "enabled": "enabled" in form}
+
+
+@syslog_bp.post("/patterns/new")
+@require_role(UserRole.admin)
+async def create_pattern():
+    form = await request.form
+    try:
+        values = parse_form(form)
+    except PatternError as exc:
+        return await _render_pattern_form(None, _typed_pattern(form), str(exc), 422)
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            if values["source_id"] and await db.get(SyslogSource, values["source_id"]) is None:
+                values["source_id"] = None
+            pattern = SyslogPattern(key=await _free_key(db, key_for(values["name"])), **values)
+            db.add(pattern)
+            await db.flush()
+            pattern_id = pattern.id
+    await log_audit(current_app, *_actor(), "syslog.pattern_added", entity_type="syslog_pattern",
+                    entity_id=pattern_id, detail={"name": values["name"]})
+    return redirect(PATTERNS)
+
+
+async def _pattern_or_404(db, pattern_id: str) -> SyslogPattern:
+    pattern = await db.get(SyslogPattern, pattern_id)
+    if pattern is None:
+        abort(404)
+    return pattern
+
+
+@syslog_bp.get("/patterns/<pattern_id>/edit")
+@require_role(UserRole.admin)
+async def edit_pattern(pattern_id: str):
+    async with current_app.db_sessionmaker() as db:
+        p = await _pattern_or_404(db, pattern_id)
+    return await _render_pattern_form(p, {
+        "name": p.name, "phrases": "\n".join(p.phrases or []),
+        "max_severity": "" if p.max_severity is None else str(p.max_severity),
+        "source_id": p.source_id or "", "window_minutes": p.window_minutes, "enabled": p.enabled})
+
+
+@syslog_bp.post("/patterns/<pattern_id>/edit")
+@require_role(UserRole.admin)
+async def save_pattern(pattern_id: str):
+    """Change what a pattern matches. Its key, and so its metric and the
+    alert rules on it, stay as they were."""
+    form = await request.form
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            p = await _pattern_or_404(db, pattern_id)
+            try:
+                values = parse_form(form)
+            except PatternError as exc:
+                return await _render_pattern_form(p, _typed_pattern(form), str(exc), 422)
+            if values["source_id"] and await db.get(SyslogSource, values["source_id"]) is None:
+                values["source_id"] = None
+            for column, value in values.items():
+                setattr(p, column, value)
+    await log_audit(current_app, *_actor(), "syslog.pattern_updated", entity_type="syslog_pattern",
+                    entity_id=pattern_id, detail={"name": values["name"]})
+    return redirect(PATTERNS)
+
+
+@syslog_bp.post("/patterns/<pattern_id>/delete")
+@require_role(UserRole.admin)
+async def delete_pattern(pattern_id: str):
+    """Remove a pattern. Its newest counts go with it, so nothing reads it as
+    a series that went quiet; its history ages out with retention."""
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            p = await _pattern_or_404(db, pattern_id)
+            name, metric = p.name, p.metric
+            await db.delete(p)
+            await db.execute(delete(PluginMetricLatest).where(
+                PluginMetricLatest.source_module == SOURCE_MODULE, PluginMetricLatest.metric_name == metric))
+    await log_audit(current_app, *_actor(), "syslog.pattern_removed", entity_type="syslog_pattern",
+                    entity_id=pattern_id, detail={"name": name})
+    return redirect(PATTERNS)