feat(host_agent): a dropped ingest says which step of the metric write it stalled in (#5149)

The first instrumented drop on dev read "dropped after 10.0s … (completed:
parse=0.1s)": the whole stall is inside record_metrics. It now marks its
three steps (flush of the rows, the rule and state reads, the per-sample
evaluation) on the caller's phases, and the drop line carries the request's
samples and metric row count.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_alert_states.py

@@ -114,3 +114,51 @@ def test_acknowledge_holds_clear_after_and_reset(app):
         assert (await events())[-1] == ("firing", "resolved", "rule disabled")
 
     asyncio.run(_go())
+
+
+def test_record_metrics_marks_each_step(app):
+    """A caller timing an ingest can tell the row write from the rule reads
+    from the evaluation (#5149)."""
+    from steward.alerts import service as alert_svc
+    from steward.core.alerts import record_metrics
+    from steward.models.metrics import PluginMetric
+    from steward.models.users import User, UserRole
+
+    class _Marks:
+        def __init__(self):
+            self.names = []
+
+        def mark(self, name):
+            self.names.append(name)
+
+    def rows(res):
+        return [PluginMetric(source_module="host_agent", resource_name=res,
+                             metric_name="cpu_pct", value=5.0)]
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        watched, unwatched = f"box-{tag}", f"other-{tag}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"mk-{tag}", email=f"mk-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                s.add_all(rows(watched))
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"hot {tag}", source_module="host_agent", resource_name=watched,
+                    metric_name="cpu_pct", operator=">", threshold=90)
+                await alert_svc.add_rule(s, fields, created_by=user.id)
+
+        seen = {}
+        for res in (watched, unwatched):
+            marks = _Marks()
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await record_metrics(s, rows(res), marks)
+            seen[res] = marks.names
+        return seen[watched], seen[unwatched]
+
+    watched, unwatched = asyncio.run(_go())
+    assert watched == ["flush", "rules", "evaluate"]
+    assert unwatched == ["flush", "rules"]   # no rule watches it: nothing to evaluate

tests/plugins/host_agent/test_ingest_timing.py

@@ -51,7 +51,8 @@ async def test_a_dropped_ingest_says_how_far_it_got(app, monkeypatch, caplog):
         phases = routes._Phases()
         clock.t += 0.2
         phases.mark("parse")
+        phases.size = "samples=21 metric_rows=840"
         clock.t += 9.9
         with caplog.at_level(logging.WARNING):
             phases.cancelled("abcd1234")
-    assert "dropped after 10.1s for token=abcd1234… (completed: parse=0.2s)" in caplog.text
+    assert "dropped after 10.1s for token=abcd1234… (completed: parse=0.2s) samples=21 metric_rows=840" in caplog.text