feat(traffic): a host's top senders, on its page and as get_host_senders (#6115)
The host page gains a "Who's sending" panel, and the MCP a tool, that put the host's outbound (or inbound) traffic down to its containers and NFS mounts: read in shared time buckets, so the host's busiest bucket says who was sending in it. A swarm service's tasks add up under its name; what none of them accounts for is the host itself. Also fixes CI 9488: per-container net_rx_bps / net_tx_bps join the docker alert catalog, so a rule can catch a container flooding the uplink. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
"""Who sends a host's network traffic (#6115): its containers and NFS mounts,
|
||||
ranked against the host's own total over a window.
|
||||
|
||||
A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each
|
||||
container's comes from Docker ingest (resource "<host>/<container>") and each
|
||||
NFS mount's from the agent (resource "<host>:<mount>"). All three are read in
|
||||
the same time buckets, so the busiest bucket of the host's total can be put
|
||||
down to who was sending in it. What no container or mount accounts for is the
|
||||
host itself (an apt download, a backup run from the host, a swarm ingress
|
||||
hand-off) or a container's traffic that never left the host.
|
||||
|
||||
Imports only the core metric model, like metrics_query, so the docker plugin
|
||||
being off leaves just the NFS mounts and the host.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from steward.models.metrics import PluginMetric
|
||||
|
||||
SOURCE_MODULE = "host_agent"
|
||||
DIRECTIONS = ("tx", "rx")
|
||||
# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per
|
||||
# 12 minutes. Narrow the window to see a spike sharper.
|
||||
BUCKETS = 120
|
||||
# A swarm task's container, "<service>.<slot or node id>.<task id>", is shown
|
||||
# by its service, so its tasks over the window add up to one sender.
|
||||
_TASK = re.compile(r"^(?P<svc>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
|
||||
|
||||
|
||||
def sender_name(container: str) -> str:
|
||||
m = _TASK.match(container)
|
||||
return m["svc"] if m else container
|
||||
|
||||
|
||||
def rank(host: dict, senders: dict, limit: int = 10) -> dict:
|
||||
"""`host` is {bucket: bytes a second}; `senders` is {(kind, name):
|
||||
{bucket: bytes a second}}. The host's average, its peak and when, and each
|
||||
sender's average over the host's buckets (a bucket it is missing from
|
||||
counts as 0), its own peak and what it sent in the host's peak bucket,
|
||||
biggest at the host's peak first. `unaccounted_*` is what the host carried
|
||||
beyond them all, never below 0."""
|
||||
if not host:
|
||||
return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None,
|
||||
"unaccounted_at_peak_bps": None}
|
||||
n = len(host)
|
||||
peak_at = max(host, key=lambda b: host[b])
|
||||
rows = []
|
||||
for (kind, name), series in senders.items():
|
||||
top = max(series, key=lambda b: series[b])
|
||||
rows.append({"kind": kind, "name": name,
|
||||
"avg_bps": sum(v for b, v in series.items() if b in host) / n,
|
||||
"peak_bps": series[top], "peak_at": top,
|
||||
"at_host_peak_bps": series.get(peak_at, 0.0)})
|
||||
rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True)
|
||||
host_avg = sum(host.values()) / n
|
||||
return {
|
||||
"host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at},
|
||||
"senders": rows[:limit],
|
||||
"more": max(0, len(rows) - limit),
|
||||
"unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)),
|
||||
"unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)),
|
||||
}
|
||||
|
||||
|
||||
async def senders(session, host_name: str, since: datetime, direction: str = "tx",
|
||||
limit: int = 10, raw_days: int = 7) -> dict:
|
||||
"""rank() over the raw samples since `since` (no earlier than the raw
|
||||
window: the hourly rollup keeps no per-bucket sender to rank). Adds the
|
||||
window actually read and its bucket width."""
|
||||
if direction not in DIRECTIONS:
|
||||
raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}")
|
||||
now = datetime.now(timezone.utc)
|
||||
since = max(since, now - timedelta(days=raw_days))
|
||||
width = max(60, int((now - since).total_seconds() / BUCKETS))
|
||||
bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at,
|
||||
func.to_timestamp(0)).label("bucket")
|
||||
metric = f"net_{direction}_bps"
|
||||
|
||||
async def series(source, metric_name, resource):
|
||||
rows = (await session.execute(
|
||||
select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value))
|
||||
.where(PluginMetric.source_module == source, PluginMetric.metric_name == metric_name,
|
||||
resource, PluginMetric.recorded_at >= since)
|
||||
.group_by(PluginMetric.resource_name, bucket))).all()
|
||||
return rows
|
||||
|
||||
host: dict = {}
|
||||
for _, b, v in await series(SOURCE_MODULE, metric, PluginMetric.resource_name == host_name):
|
||||
host[b] = float(v)
|
||||
out: dict = {}
|
||||
for res, b, v in await series("docker", metric,
|
||||
PluginMetric.resource_name.startswith(host_name + "/", autoescape=True)):
|
||||
key = ("container", sender_name(res[len(host_name) + 1:]))
|
||||
out.setdefault(key, {})
|
||||
out[key][b] = out[key].get(b, 0.0) + float(v) # a service's tasks add up
|
||||
for res, b, v in await series(SOURCE_MODULE, f"nfs_{direction}_bps",
|
||||
PluginMetric.resource_name.startswith(host_name + ":/", autoescape=True)):
|
||||
out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v)
|
||||
return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)}
|
||||
Reference in New Issue
Block a user