bvandeusenandClaude Opus 5.5
26bc57ce04
The host page gains a "Who's sending" panel, and the MCP a tool, that put the host's outbound (or inbound) traffic down to its containers and NFS mounts: read in shared time buckets, so the host's busiest bucket says who was sending in it. A swarm service's tasks add up under its name; what none of them accounts for is the host itself. Also fixes CI 9488: per-container net_rx_bps / net_tx_bps join the docker alert catalog, so a rule can catch a container flooding the uplink. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
104 lines
5.1 KiB
Python
104 lines
5.1 KiB
Python
"""Who sends a host's network traffic (#6115): its containers and NFS mounts,
|
|
ranked against the host's own total over a window.
|
|
|
|
A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each
|
|
container's comes from Docker ingest (resource "<host>/<container>") and each
|
|
NFS mount's from the agent (resource "<host>:<mount>"). All three are read in
|
|
the same time buckets, so the busiest bucket of the host's total can be put
|
|
down to who was sending in it. What no container or mount accounts for is the
|
|
host itself (an apt download, a backup run from the host, a swarm ingress
|
|
hand-off) or a container's traffic that never left the host.
|
|
|
|
Imports only the core metric model, like metrics_query, so the docker plugin
|
|
being off leaves just the NFS mounts and the host.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from steward.models.metrics import PluginMetric
|
|
|
|
SOURCE_MODULE = "host_agent"
|
|
DIRECTIONS = ("tx", "rx")
|
|
# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per
|
|
# 12 minutes. Narrow the window to see a spike sharper.
|
|
BUCKETS = 120
|
|
# A swarm task's container, "<service>.<slot or node id>.<task id>", is shown
|
|
# by its service, so its tasks over the window add up to one sender.
|
|
_TASK = re.compile(r"^(?P<svc>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")
|
|
|
|
|
|
def sender_name(container: str) -> str:
|
|
m = _TASK.match(container)
|
|
return m["svc"] if m else container
|
|
|
|
|
|
def rank(host: dict, senders: dict, limit: int = 10) -> dict:
|
|
"""`host` is {bucket: bytes a second}; `senders` is {(kind, name):
|
|
{bucket: bytes a second}}. The host's average, its peak and when, and each
|
|
sender's average over the host's buckets (a bucket it is missing from
|
|
counts as 0), its own peak and what it sent in the host's peak bucket,
|
|
biggest at the host's peak first. `unaccounted_*` is what the host carried
|
|
beyond them all, never below 0."""
|
|
if not host:
|
|
return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None,
|
|
"unaccounted_at_peak_bps": None}
|
|
n = len(host)
|
|
peak_at = max(host, key=lambda b: host[b])
|
|
rows = []
|
|
for (kind, name), series in senders.items():
|
|
top = max(series, key=lambda b: series[b])
|
|
rows.append({"kind": kind, "name": name,
|
|
"avg_bps": sum(v for b, v in series.items() if b in host) / n,
|
|
"peak_bps": series[top], "peak_at": top,
|
|
"at_host_peak_bps": series.get(peak_at, 0.0)})
|
|
rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True)
|
|
host_avg = sum(host.values()) / n
|
|
return {
|
|
"host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at},
|
|
"senders": rows[:limit],
|
|
"more": max(0, len(rows) - limit),
|
|
"unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)),
|
|
"unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)),
|
|
}
|
|
|
|
|
|
async def senders(session, host_name: str, since: datetime, direction: str = "tx",
|
|
limit: int = 10, raw_days: int = 7) -> dict:
|
|
"""rank() over the raw samples since `since` (no earlier than the raw
|
|
window: the hourly rollup keeps no per-bucket sender to rank). Adds the
|
|
window actually read and its bucket width."""
|
|
if direction not in DIRECTIONS:
|
|
raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}")
|
|
now = datetime.now(timezone.utc)
|
|
since = max(since, now - timedelta(days=raw_days))
|
|
width = max(60, int((now - since).total_seconds() / BUCKETS))
|
|
bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at,
|
|
func.to_timestamp(0)).label("bucket")
|
|
metric = f"net_{direction}_bps"
|
|
|
|
async def series(source, metric_name, resource):
|
|
rows = (await session.execute(
|
|
select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value))
|
|
.where(PluginMetric.source_module == source, PluginMetric.metric_name == metric_name,
|
|
resource, PluginMetric.recorded_at >= since)
|
|
.group_by(PluginMetric.resource_name, bucket))).all()
|
|
return rows
|
|
|
|
host: dict = {}
|
|
for _, b, v in await series(SOURCE_MODULE, metric, PluginMetric.resource_name == host_name):
|
|
host[b] = float(v)
|
|
out: dict = {}
|
|
for res, b, v in await series("docker", metric,
|
|
PluginMetric.resource_name.startswith(host_name + "/", autoescape=True)):
|
|
key = ("container", sender_name(res[len(host_name) + 1:]))
|
|
out.setdefault(key, {})
|
|
out[key][b] = out[key].get(b, 0.0) + float(v) # a service's tasks add up
|
|
for res, b, v in await series(SOURCE_MODULE, f"nfs_{direction}_bps",
|
|
PluginMetric.resource_name.startswith(host_name + ":/", autoescape=True)):
|
|
out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v)
|
|
return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)}
|