feat(traffic): a host's top senders, on its page and as get_host_senders (#6115)

The host page gains a "Who's sending" panel, and the MCP a tool, that
put the host's outbound (or inbound) traffic down to its containers and
NFS mounts: read in shared time buckets, so the host's busiest bucket
says who was sending in it. A swarm service's tasks add up under its
name; what none of them accounts for is the host itself.

Also fixes CI 9488: per-container net_rx_bps / net_tx_bps join the
docker alert catalog, so a rule can catch a container flooding the
uplink.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -89,10 +89,13 @@ def get_alert_metrics() -> dict[str, list[str]]:
     """What ingest.py records per container (resource "<host>/<container>"):
     restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 /
     0.0 from the container's HEALTHCHECK, only for containers that define one.
+    net_rx_bps / net_tx_bps are the bytes a second it receives and sends
+    (#6115): "a container flooding the uplink" is net_tx_bps over a level.
     Per host (resource "<host>"), what Docker keeps on its disk (#5304).
     Swarm-wide (resource "swarm"), from any manager's report: services short
     of their replicas, nodes down and managers unreachable, drained nodes
     aside (#5788, swarm_health.py)."""
     from .disk_view import DISK_METRICS
     from .swarm_health import SWARM_METRICS
-    return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]}
+    return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", "net_rx_bps", "net_tx_bps",
+                       *DISK_METRICS, *SWARM_METRICS]}

plugins/host_agent/mcp_tools.py

@@ -10,12 +10,12 @@ import json
 from steward.hosts import service as host_svc
 from steward.mcp._context import current_caller, mcp_app
 from steward.mcp.tools._audit import audit_mcp
-from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, now_utc, pick
+from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd
 from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from . import cadence, downtime, maintenance
+from . import cadence, downtime, maintenance, senders
 from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan,
                      HostMaintenanceRun)
 
@@ -234,6 +234,49 @@ async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) ->
     return {"jobs": rows, "total": len(rows), "truncated": len(jobs) > limit}
 
 
+async def get_host_senders(host: str, range: str = "1h", direction: str = "tx", limit: int = 10) -> dict:
+    """Who sent (or received) a host's network traffic: its containers and NFS
+    mounts, ranked against the host's own total.
+
+    host: name or id (list_hosts). range: "1h" (default), "6h", "24h" or "7d"
+    (at most the raw window; the rollup keeps no per-sender detail).
+    direction: "tx" (default, what the host sent) or "rx" (what it received).
+    limit: senders listed (default 10, max 50).
+
+    The window is cut into about 120 buckets (`bucket_s` wide). `host` is the
+    host's physical interfaces: average, peak and the bucket it peaked in.
+    Each sender is a container (a swarm service's tasks added up, by service
+    name) or an NFS mount, with its average over the window, its own peak
+    and when, and `at_host_peak_bps`, what it sent in the host's peak bucket;
+    biggest there first. `unaccounted_*` is what the host carried beyond them
+    all: the host itself (a download, a backup run on the host, swarm ingress
+    passing traffic to another node). A container's traffic to another on
+    the same host counts for it without crossing an interface, so senders
+    can add up to more than the host. Per-container rates need Steward after
+    #6115; NFS bytes need agent 1.25.0. Bytes a second throughout.
+    """
+    since, key = lookback(range)
+    limit = clamp_limit(limit, 10, 50)
+    async with db_session() as db:
+        h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
+        got = await senders.senders(db, h.name, since, direction=direction, limit=limit)
+    if got["host"] is None:
+        raise ValueError(f"no net_{direction}_bps recorded for host {h.name!r} in the last {key}; "
+                         "is its agent reporting? list_agents says.")
+    return {
+        "host_id": h.id, "host_name": h.name, "range": key, "direction": direction,
+        "since": iso(got["since"]), "bucket_s": got["bucket_s"],
+        "host": {"avg_bps": rnd(got["host"]["avg_bps"], 1), "peak_bps": rnd(got["host"]["peak_bps"], 1),
+                 "peak_at": iso(got["host"]["peak_at"])},
+        "senders": [{"kind": r["kind"], "name": r["name"], "at_host_peak_bps": rnd(r["at_host_peak_bps"], 1),
+                     "avg_bps": rnd(r["avg_bps"], 1), "peak_bps": rnd(r["peak_bps"], 1),
+                     "peak_at": iso(r["peak_at"])} for r in got["senders"]],
+        "more": got["more"],
+        "unaccounted_avg_bps": rnd(got["unaccounted_avg_bps"], 1),
+        "unaccounted_at_peak_bps": rnd(got["unaccounted_at_peak_bps"], 1),
+    }
+
+
 async def run_maintenance(host: str, reboot: str = "if_required") -> dict:
     """Update a host now, as the host page's Update now does: the job starts
     with the agent's next post and get_maintenance_job follows it.
@@ -336,4 +379,5 @@ def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
     register(list_maintenance_plans, role=UserRole.viewer, read_only=True)
     register(get_maintenance_job, role=UserRole.viewer, read_only=True)
+    register(get_host_senders, role=UserRole.viewer, read_only=True)
     register(run_maintenance, role=UserRole.admin, read_only=False)

plugins/host_agent/routes.py

@@ -957,6 +957,23 @@ async def host_detail_charts(host_id: str):
     return await render_template("_host_charts.html", series=series, range_key=range_key)
 
 
+@host_agent_bp.get("/<host_id>/senders")
+@require_role(UserRole.viewer)
+async def host_detail_senders(host_id: str):
+    """Who sent the host's traffic over the range (#6115), lazy-loaded below
+    the charts and refreshed with them."""
+    from .senders import senders
+    since, range_key = parse_range(request.args.get("range"))
+    async with current_app.db_sessionmaker() as session:
+        host = (await session.execute(
+            select(Host).where(Host.id == host_id))).scalar_one_or_none()
+        if host is None:
+            return "", 404
+        raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7)
+        got = await senders(session, host.name, since, raw_days=raw_days)
+    return await render_template("_host_senders.html", got=got, range_key=range_key)
+
+
 def _new_token_pair() -> tuple[str, str]:
     raw = secrets.token_urlsafe(32)
     return raw, _hash_token(raw)

plugins/host_agent/senders.py

@@ -0,0 +1,103 @@
+"""Who sends a host's network traffic (#6115): its containers and NFS mounts,
+ranked against the host's own total over a window.
+
+A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each
+container's comes from Docker ingest (resource "<host>/<container>") and each
+NFS mount's from the agent (resource "<host>:<mount>"). All three are read in
+the same time buckets, so the busiest bucket of the host's total can be put
+down to who was sending in it. What no container or mount accounts for is the
+host itself (an apt download, a backup run from the host, a swarm ingress
+hand-off) or a container's traffic that never left the host.
+
+Imports only the core metric model, like metrics_query, so the docker plugin
+being off leaves just the NFS mounts and the host.
+"""
+from __future__ import annotations
+
+import re
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import func, select
+
+from steward.models.metrics import PluginMetric
+
+SOURCE_MODULE = "host_agent"
+DIRECTIONS = ("tx", "rx")
+# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per
+# 12 minutes. Narrow the window to see a spike sharper.
+BUCKETS = 120
+# A swarm task's container, "<service>.<slot or node id>.<task id>", is shown
+# by its service, so its tasks over the window add up to one sender.
+_TASK = re.compile(r"^(?P<svc>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
+
+
+def sender_name(container: str) -> str:
+    m = _TASK.match(container)
+    return m["svc"] if m else container
+
+
+def rank(host: dict, senders: dict, limit: int = 10) -> dict:
+    """`host` is {bucket: bytes a second}; `senders` is {(kind, name):
+    {bucket: bytes a second}}. The host's average, its peak and when, and each
+    sender's average over the host's buckets (a bucket it is missing from
+    counts as 0), its own peak and what it sent in the host's peak bucket,
+    biggest at the host's peak first. `unaccounted_*` is what the host carried
+    beyond them all, never below 0."""
+    if not host:
+        return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None,
+                "unaccounted_at_peak_bps": None}
+    n = len(host)
+    peak_at = max(host, key=lambda b: host[b])
+    rows = []
+    for (kind, name), series in senders.items():
+        top = max(series, key=lambda b: series[b])
+        rows.append({"kind": kind, "name": name,
+                     "avg_bps": sum(v for b, v in series.items() if b in host) / n,
+                     "peak_bps": series[top], "peak_at": top,
+                     "at_host_peak_bps": series.get(peak_at, 0.0)})
+    rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True)
+    host_avg = sum(host.values()) / n
+    return {
+        "host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at},
+        "senders": rows[:limit],
+        "more": max(0, len(rows) - limit),
+        "unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)),
+        "unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)),
+    }
+
+
+async def senders(session, host_name: str, since: datetime, direction: str = "tx",
+                  limit: int = 10, raw_days: int = 7) -> dict:
+    """rank() over the raw samples since `since` (no earlier than the raw
+    window: the hourly rollup keeps no per-bucket sender to rank). Adds the
+    window actually read and its bucket width."""
+    if direction not in DIRECTIONS:
+        raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}")
+    now = datetime.now(timezone.utc)
+    since = max(since, now - timedelta(days=raw_days))
+    width = max(60, int((now - since).total_seconds() / BUCKETS))
+    bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at,
+                           func.to_timestamp(0)).label("bucket")
+    metric = f"net_{direction}_bps"
+
+    async def series(source, metric_name, resource):
+        rows = (await session.execute(
+            select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value))
+            .where(PluginMetric.source_module == source, PluginMetric.metric_name == metric_name,
+                   resource, PluginMetric.recorded_at >= since)
+            .group_by(PluginMetric.resource_name, bucket))).all()
+        return rows
+
+    host: dict = {}
+    for _, b, v in await series(SOURCE_MODULE, metric, PluginMetric.resource_name == host_name):
+        host[b] = float(v)
+    out: dict = {}
+    for res, b, v in await series("docker", metric,
+                                  PluginMetric.resource_name.startswith(host_name + "/", autoescape=True)):
+        key = ("container", sender_name(res[len(host_name) + 1:]))
+        out.setdefault(key, {})
+        out[key][b] = out[key].get(b, 0.0) + float(v)          # a service's tasks add up
+    for res, b, v in await series(SOURCE_MODULE, f"nfs_{direction}_bps",
+                                  PluginMetric.resource_name.startswith(host_name + ":/", autoescape=True)):
+        out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v)
+    return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)}

plugins/host_agent/templates/_host_senders.html

@@ -0,0 +1,38 @@
+{# Who sent the host's traffic over the range (#6115): its containers and NFS
+   mounts at the host's busiest moment, biggest first. Lazy-loaded below the
+   charts and refreshed with them. #}
+<div class="section-title" style="margin-bottom:0.5rem;">Who's sending — last {{ range_key }}</div>
+{% if not got.host %}
+<p class="field-hint" style="margin:0;">No outbound traffic recorded for this host in the last {{ range_key }}.</p>
+{% else %}
+{% set top = got.senders[0] if got.senders else None %}
+<p style="margin:0 0 0.6rem;">
+  Busiest at {{ got.host.peak_at|when }}: <strong>{{ got.host.peak_bps|rate }}</strong> out (average {{ got.host.avg_bps|rate }}).
+  {% if top and top.at_host_peak_bps >= got.unaccounted_at_peak_bps %}Most of it from <strong>{{ top.name }}</strong> ({{ top.at_host_peak_bps|rate }}).
+  {% else %}Most of it from <strong>no container or NFS mount</strong>: the host itself.{% endif %}
+</p>
+<table class="table">
+  <thead><tr><th>Sender</th><th>At the peak</th><th>Average</th><th>Its own peak</th></tr></thead>
+  <tbody>
+  {% for r in got.senders %}
+    <tr>
+      <td><span title="{{ r.name }}">{{ "NFS " if r.kind == "nfs" }}{{ r.name }}</span></td>
+      <td style="font-variant-numeric:tabular-nums;">{{ r.at_host_peak_bps|rate }}</td>
+      <td style="font-variant-numeric:tabular-nums;">{{ r.avg_bps|rate }}</td>
+      <td style="font-variant-numeric:tabular-nums;">{{ r.peak_bps|rate }} · {{ r.peak_at|when }}</td>
+    </tr>
+  {% endfor %}
+    <tr>
+      <td style="color:var(--text-muted);">The host itself, or none of the above</td>
+      <td style="font-variant-numeric:tabular-nums;">{{ got.unaccounted_at_peak_bps|rate }}</td>
+      <td style="font-variant-numeric:tabular-nums;">{{ got.unaccounted_avg_bps|rate }}</td>
+      <td>—</td>
+    </tr>
+  </tbody>
+</table>
+<p class="field-hint" style="margin:0.5rem 0 0;">
+  Read in {{ (got.bucket_s / 60)|round|int }}-minute steps; a shorter range shows a spike sharper.
+  {% if got.more %}{{ got.more }} more sent less. {% endif %}
+  A container's traffic to another on this host counts for it without leaving the host.
+</p>
+{% endif %}

plugins/host_agent/templates/host_detail.html

@@ -42,6 +42,14 @@
   </div>
   {% endif %}
 </div>
+<div id="hm-senders" class="card" style="margin-top:1rem;margin-bottom:0;"
+     hx-get="/plugins/host_agent/{{ host.id }}/senders"
+     hx-trigger="load, every {{ chart_poll_seconds }}s, rangeChange from:body"
+     hx-include="#time-range"
+     hx-swap="innerHTML">
+  <div class="section-title" style="margin-bottom:0.5rem;">Who's sending</div>
+  <p class="field-hint" style="margin:0;">Loading…</p>
+</div>
 {% if "syslog" in enabled_plugins %}<p id="hm-marks-note" class="field-hint" hidden style="margin:0.5rem 0 0;"></p>{% endif %}
 {# Hidden poller: fetches the history series and applies it to the charts above
    (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on