feat(alerts): the metric catalog is written by what emits it (#5061)

#5061 "Metric catalog belongs to what emits: plugins register their
metrics; names match what is recorded".

- The static METRIC_CATALOG is gone. metric_catalog() is core's own sources
  (the four monitor types and "steward", the app's open_fds/open_fds_pct,
  which were recorded but never offered) plus each loaded plugin's
  get_alert_metrics() export. A disabled plugin's sources leave the picker.
- traefik offers what it records: error_rate_4xx_pct and error_rate_5xx_pct.
  The old "error_rate" was never recorded, so a rule on it could not fire.
- tests/core/test_metric_catalog.py walks every MetricBatch.add /
  record_metric call (traefik's tuple loop included) and fails on a recorded
  metric the catalog lacks; it shows on a synthetic writer that it can fail.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_metric_catalog.py

@@ -0,0 +1,97 @@
+"""The alert catalog names what is actually recorded: every metric a writer
+hands to the alert pipeline (MetricBatch.add / record_metric with a literal
+source and name) is one the catalog offers for that source (#5061)."""
+import ast
+import importlib
+import pathlib
+
+import steward
+from steward.alerts.service import CORE_METRICS
+
+_REPO = pathlib.Path(steward.__file__).parent.parent
+_PLUGINS = ("traefik", "unifi", "docker", "host_agent", "snmp")
+
+
+def catalog() -> dict[str, set[str]]:
+    cat = {src: set(names) for src, names in CORE_METRICS.items()}
+    for name in _PLUGINS:
+        for src, names in importlib.import_module(f"plugins.{name}").get_alert_metrics().items():
+            cat[src] = set(names)
+    return cat
+
+
+def _const(node):
+    return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
+
+
+def _loop_names(func: ast.AST, var: str) -> set[str]:
+    """Literal first elements of `for var, _ in [("a", x), ("b", y)]` loops."""
+    names: set[str] = set()
+    for node in ast.walk(func):
+        if not (isinstance(node, ast.For) and isinstance(node.iter, (ast.List, ast.Tuple))):
+            continue
+        target = node.target
+        first = target.elts[0] if isinstance(target, ast.Tuple) and target.elts else target
+        if isinstance(first, ast.Name) and first.id == var:
+            for elt in node.iter.elts:
+                if isinstance(elt, ast.Tuple) and elt.elts and _const(elt.elts[0]):
+                    names.add(elt.elts[0].value)
+    return names
+
+
+def recorded(source: str) -> set[tuple[str, str]]:
+    """(source, metric) pairs the code hands to the alert pipeline."""
+    tree = ast.parse(source)
+    pairs: set[tuple[str, str]] = set()
+    for func in ast.walk(tree):
+        if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            continue
+        for call in ast.walk(func):
+            if not isinstance(call, ast.Call):
+                continue
+            callee = call.func
+            is_writer = (isinstance(callee, ast.Attribute) and callee.attr == "add"
+                         and isinstance(callee.value, ast.Name) and callee.value.id == "batch") \
+                or (isinstance(callee, ast.Name) and callee.id == "record_metric")
+            if not is_writer:
+                continue
+            kw = {k.arg: k.value for k in call.keywords}
+            src = _const(kw.get("source_module"))
+            metric = kw.get("metric_name")
+            if src is None or metric is None:
+                continue
+            if _const(metric):
+                pairs.add((src, metric.value))
+            elif isinstance(metric, ast.Name):
+                pairs |= {(src, m) for m in _loop_names(func, metric.id)}
+    return pairs
+
+
+def _writers() -> list[pathlib.Path]:
+    return sorted(p for root in ("steward", "plugins") for p in (_REPO / root).rglob("*.py")
+                  if "migrations" not in p.parts and "batch.add(" in p.read_text())
+
+
+def test_the_catalog_guard_can_fail():
+    code = '''
+async def poll(session):
+    batch = MetricBatch()
+    batch.add(source_module="traefik", resource_name="r", metric_name="error_rate", value=1)
+    for metric_name, value in [("request_rate", 1), ("made_up", 2)]:
+        batch.add(source_module="traefik", resource_name="r", metric_name=metric_name, value=value)
+'''
+    pairs = recorded(code)
+    assert pairs == {("traefik", "error_rate"), ("traefik", "request_rate"), ("traefik", "made_up")}
+    cat = catalog()
+    assert {p for p in pairs if p[1] not in cat.get(p[0], set())} == {
+        ("traefik", "error_rate"), ("traefik", "made_up")}
+
+
+def test_every_recorded_metric_is_in_the_catalog():
+    files = _writers()
+    assert len(files) >= 6, files  # docker, traefik, unifi, snmp, monitors, self_monitor
+    pairs = {p for f in files for p in recorded(f.read_text())}
+    assert ("traefik", "error_rate_5xx_pct") in pairs and ("docker", "is_healthy") in pairs
+    cat = catalog()
+    missing = sorted(p for p in pairs if p[1] not in cat.get(p[0], set()))
+    assert not missing, f"recorded but not in the alert catalog: {missing}"

tests/mcp/test_monitoring_writes.py

@@ -26,7 +26,14 @@ ADMIN_TOOLS = {"delete_monitor", "delete_alert_rule"}
 
 @pytest.fixture
 def known(monkeypatch):
-    """A source with one resource and its catalog metrics, without a database."""
+    """A source with one resource and its catalog metrics, without a database:
+    the host_agent plugin is loaded (its get_alert_metrics hook writes the
+    catalog) and has reported one host."""
+    import plugins.host_agent
+    from steward.core import plugin_manager
+    monkeypatch.setattr(plugin_manager, "get_loaded_plugin_modules",
+                        lambda: {"host_agent": plugins.host_agent})
+
     async def resources(db, src):
         return ["nas"] if src == "host_agent" else []
     monkeypatch.setattr(alert_svc, "resources_for", resources)