diff --git a/docs/core/mcp.md b/docs/core/mcp.md index ae9aedb..de0a5a4 100644 --- a/docs/core/mcp.md +++ b/docs/core/mcp.md @@ -79,7 +79,7 @@ Each role includes the tools of the roles above it in this list. | Automation | `list_playbooks`, `get_playbook`, `list_inventory`, `list_ansible_runs`, `get_ansible_run`, `wait_for_run`, `list_schedules` | | Audit | `list_audit_events` | | Docker plugin | `list_containers`, `get_container`, `list_swarm_services`, `search_container_logs`, `get_docker_disk_usage` | -| Host agent (built in, always on) | `list_agents`, `list_maintenance_plans` (each plan's group, weekly slot, next run and latest run: how far it got, who it skipped and why), `get_maintenance_job` (a host's update jobs or one job: state, packages updated, whether it rebooted, why it failed) | +| Host agent (built in, always on) | `list_agents`, `list_maintenance_plans` (each plan's group, weekly slot, next run and latest run: how far it got, who it skipped and why), `get_maintenance_job` (a host's update jobs or one job: state, packages updated, whether it rebooted, why it failed), `get_host_senders` (who sent or received a host's network traffic over a window: its containers and NFS mounts ranked at the host's peak, and what none of them accounts for) | | Traefik plugin | `list_routers` (each router graded as the Traefik page grades it, with the status codes behind a problem, what they mean and the swarm service behind it), `get_router` (one router's history, codes, normal and anomalies), `list_certificates` (served certificates, days left, the routers they cover), `get_traffic` (the access log's clients, countries and routers, and Traefik's own figures) | | UniFi plugin | `get_network` (the WAN, clients, Wi-Fi networks, speed tests and devices by status), `list_devices`, `list_clients` (everyone seen, or the newest poll's top talkers), `list_network_events`, `list_alarms`, `get_network_config` (networks and VLANs, port forwards, firewall rules) | | SNMP plugin | `list_snmp_devices` (each device's state, newest readings in their units and the readings it didn't return), `get_snmp_device` (one device's readings over a range). Never the community or v3 keys | diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index d667eda..025c514 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -89,10 +89,13 @@ def get_alert_metrics() -> dict[str, list[str]]: """What ingest.py records per container (resource "/"): restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 / 0.0 from the container's HEALTHCHECK, only for containers that define one. + net_rx_bps / net_tx_bps are the bytes a second it receives and sends + (#6115): "a container flooding the uplink" is net_tx_bps over a level. Per host (resource ""), what Docker keeps on its disk (#5304). Swarm-wide (resource "swarm"), from any manager's report: services short of their replicas, nodes down and managers unreachable, drained nodes aside (#5788, swarm_health.py).""" from .disk_view import DISK_METRICS from .swarm_health import SWARM_METRICS - return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]} + return {"docker": ["cpu_pct", "mem_used_pct", "restart_count", "is_healthy", "net_rx_bps", "net_tx_bps", + *DISK_METRICS, *SWARM_METRICS]} diff --git a/plugins/host_agent/mcp_tools.py b/plugins/host_agent/mcp_tools.py index fb4540e..46e8c83 100644 --- a/plugins/host_agent/mcp_tools.py +++ b/plugins/host_agent/mcp_tools.py @@ -10,12 +10,12 @@ import json from steward.hosts import service as host_svc from steward.mcp._context import current_caller, mcp_app from steward.mcp.tools._audit import audit_mcp -from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, now_utc, pick +from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd from steward.mcp.tools.overview import finding from steward.models.hosts import Host from steward.models.users import UserRole -from . import cadence, downtime, maintenance +from . import cadence, downtime, maintenance, senders from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun) @@ -234,6 +234,49 @@ async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) -> return {"jobs": rows, "total": len(rows), "truncated": len(jobs) > limit} +async def get_host_senders(host: str, range: str = "1h", direction: str = "tx", limit: int = 10) -> dict: + """Who sent (or received) a host's network traffic: its containers and NFS + mounts, ranked against the host's own total. + + host: name or id (list_hosts). range: "1h" (default), "6h", "24h" or "7d" + (at most the raw window; the rollup keeps no per-sender detail). + direction: "tx" (default, what the host sent) or "rx" (what it received). + limit: senders listed (default 10, max 50). + + The window is cut into about 120 buckets (`bucket_s` wide). `host` is the + host's physical interfaces: average, peak and the bucket it peaked in. + Each sender is a container (a swarm service's tasks added up, by service + name) or an NFS mount, with its average over the window, its own peak + and when, and `at_host_peak_bps`, what it sent in the host's peak bucket; + biggest there first. `unaccounted_*` is what the host carried beyond them + all: the host itself (a download, a backup run on the host, swarm ingress + passing traffic to another node). A container's traffic to another on + the same host counts for it without crossing an interface, so senders + can add up to more than the host. Per-container rates need Steward after + #6115; NFS bytes need agent 1.25.0. Bytes a second throughout. + """ + since, key = lookback(range) + limit = clamp_limit(limit, 10, 50) + async with db_session() as db: + h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts") + got = await senders.senders(db, h.name, since, direction=direction, limit=limit) + if got["host"] is None: + raise ValueError(f"no net_{direction}_bps recorded for host {h.name!r} in the last {key}; " + "is its agent reporting? list_agents says.") + return { + "host_id": h.id, "host_name": h.name, "range": key, "direction": direction, + "since": iso(got["since"]), "bucket_s": got["bucket_s"], + "host": {"avg_bps": rnd(got["host"]["avg_bps"], 1), "peak_bps": rnd(got["host"]["peak_bps"], 1), + "peak_at": iso(got["host"]["peak_at"])}, + "senders": [{"kind": r["kind"], "name": r["name"], "at_host_peak_bps": rnd(r["at_host_peak_bps"], 1), + "avg_bps": rnd(r["avg_bps"], 1), "peak_bps": rnd(r["peak_bps"], 1), + "peak_at": iso(r["peak_at"])} for r in got["senders"]], + "more": got["more"], + "unaccounted_avg_bps": rnd(got["unaccounted_avg_bps"], 1), + "unaccounted_at_peak_bps": rnd(got["unaccounted_at_peak_bps"], 1), + } + + async def run_maintenance(host: str, reboot: str = "if_required") -> dict: """Update a host now, as the host page's Update now does: the job starts with the agent's next post and get_maintenance_job follows it. @@ -336,4 +379,5 @@ def register_tools(register) -> None: register(list_agents, role=UserRole.viewer, read_only=True) register(list_maintenance_plans, role=UserRole.viewer, read_only=True) register(get_maintenance_job, role=UserRole.viewer, read_only=True) + register(get_host_senders, role=UserRole.viewer, read_only=True) register(run_maintenance, role=UserRole.admin, read_only=False) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 1ab07aa..3bbbc0e 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -957,6 +957,23 @@ async def host_detail_charts(host_id: str): return await render_template("_host_charts.html", series=series, range_key=range_key) +@host_agent_bp.get("//senders") +@require_role(UserRole.viewer) +async def host_detail_senders(host_id: str): + """Who sent the host's traffic over the range (#6115), lazy-loaded below + the charts and refreshed with them.""" + from .senders import senders + since, range_key = parse_range(request.args.get("range")) + async with current_app.db_sessionmaker() as session: + host = (await session.execute( + select(Host).where(Host.id == host_id))).scalar_one_or_none() + if host is None: + return "", 404 + raw_days = int(await get_setting(session, "metrics.retention.raw_days") or 7) + got = await senders(session, host.name, since, raw_days=raw_days) + return await render_template("_host_senders.html", got=got, range_key=range_key) + + def _new_token_pair() -> tuple[str, str]: raw = secrets.token_urlsafe(32) return raw, _hash_token(raw) diff --git a/plugins/host_agent/senders.py b/plugins/host_agent/senders.py new file mode 100644 index 0000000..9efd758 --- /dev/null +++ b/plugins/host_agent/senders.py @@ -0,0 +1,103 @@ +"""Who sends a host's network traffic (#6115): its containers and NFS mounts, +ranked against the host's own total over a window. + +A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each +container's comes from Docker ingest (resource "/") and each +NFS mount's from the agent (resource ":"). All three are read in +the same time buckets, so the busiest bucket of the host's total can be put +down to who was sending in it. What no container or mount accounts for is the +host itself (an apt download, a backup run from the host, a swarm ingress +hand-off) or a container's traffic that never left the host. + +Imports only the core metric model, like metrics_query, so the docker plugin +being off leaves just the NFS mounts and the host. +""" +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone + +from sqlalchemy import func, select + +from steward.models.metrics import PluginMetric + +SOURCE_MODULE = "host_agent" +DIRECTIONS = ("tx", "rx") +# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per +# 12 minutes. Narrow the window to see a spike sharper. +BUCKETS = 120 +# A swarm task's container, "..", is shown +# by its service, so its tasks over the window add up to one sender. +_TASK = re.compile(r"^(?P.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$") + + +def sender_name(container: str) -> str: + m = _TASK.match(container) + return m["svc"] if m else container + + +def rank(host: dict, senders: dict, limit: int = 10) -> dict: + """`host` is {bucket: bytes a second}; `senders` is {(kind, name): + {bucket: bytes a second}}. The host's average, its peak and when, and each + sender's average over the host's buckets (a bucket it is missing from + counts as 0), its own peak and what it sent in the host's peak bucket, + biggest at the host's peak first. `unaccounted_*` is what the host carried + beyond them all, never below 0.""" + if not host: + return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None, + "unaccounted_at_peak_bps": None} + n = len(host) + peak_at = max(host, key=lambda b: host[b]) + rows = [] + for (kind, name), series in senders.items(): + top = max(series, key=lambda b: series[b]) + rows.append({"kind": kind, "name": name, + "avg_bps": sum(v for b, v in series.items() if b in host) / n, + "peak_bps": series[top], "peak_at": top, + "at_host_peak_bps": series.get(peak_at, 0.0)}) + rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True) + host_avg = sum(host.values()) / n + return { + "host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at}, + "senders": rows[:limit], + "more": max(0, len(rows) - limit), + "unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)), + "unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)), + } + + +async def senders(session, host_name: str, since: datetime, direction: str = "tx", + limit: int = 10, raw_days: int = 7) -> dict: + """rank() over the raw samples since `since` (no earlier than the raw + window: the hourly rollup keeps no per-bucket sender to rank). Adds the + window actually read and its bucket width.""" + if direction not in DIRECTIONS: + raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}") + now = datetime.now(timezone.utc) + since = max(since, now - timedelta(days=raw_days)) + width = max(60, int((now - since).total_seconds() / BUCKETS)) + bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at, + func.to_timestamp(0)).label("bucket") + metric = f"net_{direction}_bps" + + async def series(source, metric_name, resource): + rows = (await session.execute( + select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value)) + .where(PluginMetric.source_module == source, PluginMetric.metric_name == metric_name, + resource, PluginMetric.recorded_at >= since) + .group_by(PluginMetric.resource_name, bucket))).all() + return rows + + host: dict = {} + for _, b, v in await series(SOURCE_MODULE, metric, PluginMetric.resource_name == host_name): + host[b] = float(v) + out: dict = {} + for res, b, v in await series("docker", metric, + PluginMetric.resource_name.startswith(host_name + "/", autoescape=True)): + key = ("container", sender_name(res[len(host_name) + 1:])) + out.setdefault(key, {}) + out[key][b] = out[key].get(b, 0.0) + float(v) # a service's tasks add up + for res, b, v in await series(SOURCE_MODULE, f"nfs_{direction}_bps", + PluginMetric.resource_name.startswith(host_name + ":/", autoescape=True)): + out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v) + return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)} diff --git a/plugins/host_agent/templates/_host_senders.html b/plugins/host_agent/templates/_host_senders.html new file mode 100644 index 0000000..2ef5d33 --- /dev/null +++ b/plugins/host_agent/templates/_host_senders.html @@ -0,0 +1,38 @@ +{# Who sent the host's traffic over the range (#6115): its containers and NFS + mounts at the host's busiest moment, biggest first. Lazy-loaded below the + charts and refreshed with them. #} +
Who's sending — last {{ range_key }}
+{% if not got.host %} +

No outbound traffic recorded for this host in the last {{ range_key }}.

+{% else %} +{% set top = got.senders[0] if got.senders else None %} +

+ Busiest at {{ got.host.peak_at|when }}: {{ got.host.peak_bps|rate }} out (average {{ got.host.avg_bps|rate }}). + {% if top and top.at_host_peak_bps >= got.unaccounted_at_peak_bps %}Most of it from {{ top.name }} ({{ top.at_host_peak_bps|rate }}). + {% else %}Most of it from no container or NFS mount: the host itself.{% endif %} +

+ + + + {% for r in got.senders %} + + + + + + + {% endfor %} + + + + + + + +
SenderAt the peakAverageIts own peak
{{ "NFS " if r.kind == "nfs" }}{{ r.name }}{{ r.at_host_peak_bps|rate }}{{ r.avg_bps|rate }}{{ r.peak_bps|rate }} · {{ r.peak_at|when }}
The host itself, or none of the above{{ got.unaccounted_at_peak_bps|rate }}{{ got.unaccounted_avg_bps|rate }}—
+

+ Read in {{ (got.bucket_s / 60)|round|int }}-minute steps; a shorter range shows a spike sharper. + {% if got.more %}{{ got.more }} more sent less. {% endif %} + A container's traffic to another on this host counts for it without leaving the host. +

+{% endif %} diff --git a/plugins/host_agent/templates/host_detail.html b/plugins/host_agent/templates/host_detail.html index b8985a3..d38cc19 100644 --- a/plugins/host_agent/templates/host_detail.html +++ b/plugins/host_agent/templates/host_detail.html @@ -42,6 +42,14 @@ {% endif %} +
+
Who's sending
+

Loading…

+
{% if "syslog" in enabled_plugins %}{% endif %} {# Hidden poller: fetches the history series and applies it to the charts above (no visible swap). Lazy on load → never blocks paint; refreshed slowly + on diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 8c0468a..ce762c7 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -361,8 +361,8 @@ METRICS: dict[str, tuple[str, str]] = { "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed or timed out, or the host didn't come back into service and still isn't"), "temp_c_max": ("Hottest sensor", "the host's hottest real thermometer; inputs with nothing wired to them are left out"), "temp_c": ("Sensor temperature", "one temperature sensor (resource host:temp:label)"), - "net_rx_bps": ("Network in", "bytes per second received, by the whole host or one interface"), - "net_tx_bps": ("Network out", "bytes per second sent, by the whole host or one interface"), + "net_rx_bps": ("Network in", "bytes per second received, by the whole host, one interface or one container"), + "net_tx_bps": ("Network out", "bytes per second sent, by the whole host, one interface or one container"), "disk_read_bps": ("Disk reads", "bytes per second read, from all disks or one"), "disk_write_bps": ("Disk writes", "bytes per second written, to all disks or one"), "drives_failing": ("Drives failing", "drives on the host failing their own health check, raising a critical warning or out of spare blocks"), diff --git a/tests/integration/test_host_senders.py b/tests/integration/test_host_senders.py new file mode 100644 index 0000000..35212e7 --- /dev/null +++ b/tests/integration/test_host_senders.py @@ -0,0 +1,57 @@ +"""Integration (#6115): a host's traffic put down to its containers and NFS +mounts, read from plugin_metrics in shared time buckets on Postgres.""" +from __future__ import annotations + +import asyncio +import os +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def test_the_host_s_peak_is_put_down_to_its_senders(app): + from steward.models.metrics import PluginMetric + from plugins.host_agent.senders import senders + + now = datetime.now(timezone.utc) + host = "senders-host-" + uuid.uuid4().hex[:8] + peak, later = now - timedelta(minutes=30), now - timedelta(minutes=10) + task = "a" * 25 + + def m(source, resource, metric, value, at): + return PluginMetric(source_module=source, resource_name=resource, metric_name=metric, + value=value, recorded_at=at) + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add_all([ + m("host_agent", host, "net_tx_bps", 1000.0, peak), + m("host_agent", host, "net_tx_bps", 100.0, later), + # one service's two tasks add up; another host's don't count + m("docker", f"{host}/web.1.{task}", "net_tx_bps", 600.0, peak), + m("docker", f"{host}/web.2.{task}", "net_tx_bps", 200.0, peak), + m("docker", f"{host}/web.1.{task}", "net_tx_bps", 50.0, later), + m("docker", f"{host}x/web.1.{task}", "net_tx_bps", 9999.0, peak), + m("host_agent", f"{host}:/nfs/data", "nfs_tx_bps", 100.0, peak), + m("host_agent", f"{host}:net:eno1", "net_tx_bps", 1000.0, peak), + ]) + async with app.db_sessionmaker() as s: + return await senders(s, host, now - timedelta(hours=1)) + + got = asyncio.run(_go()) + assert got["host"]["peak_bps"] == 1000.0 and got["bucket_s"] == 60 + assert [(r["kind"], r["name"], r["at_host_peak_bps"]) for r in got["senders"]] == [ + ("container", "web", 800.0), ("nfs", "/nfs/data", 100.0)] + assert got["unaccounted_at_peak_bps"] == 100.0 diff --git a/tests/plugins/host_agent/test_mcp_tools.py b/tests/plugins/host_agent/test_mcp_tools.py index 8f52a83..97e4713 100644 --- a/tests/plugins/host_agent/test_mcp_tools.py +++ b/tests/plugins/host_agent/test_mcp_tools.py @@ -19,10 +19,12 @@ def test_plugin_exposes_the_mcp_hooks(): reg = MagicMock() agent_plugin.register_mcp_tools(reg) assert [c.args[0].__name__ for c in reg.call_args_list] == [ - "list_agents", "list_maintenance_plans", "get_maintenance_job", "run_maintenance"] + "list_agents", "list_maintenance_plans", "get_maintenance_job", "get_host_senders", + "run_maintenance"] roles = {c.args[0].__name__: c.kwargs["role"] for c in reg.call_args_list} assert roles["run_maintenance"].value == "admin" # as Update now on the page assert roles["get_maintenance_job"].value == "viewer" + assert roles["get_host_senders"].value == "viewer" assert callable(agent_plugin.mcp_overview) diff --git a/tests/plugins/host_agent/test_senders.py b/tests/plugins/host_agent/test_senders.py new file mode 100644 index 0000000..0c43c41 --- /dev/null +++ b/tests/plugins/host_agent/test_senders.py @@ -0,0 +1,40 @@ +"""senders.rank: a host's traffic put down to its containers and NFS mounts +(#6115). Pure; no DB.""" +from datetime import datetime, timedelta, timezone + +from plugins.host_agent.senders import rank, sender_name + +T = [datetime(2026, 10, 19, 18, 0, tzinfo=timezone.utc) + timedelta(minutes=i) for i in range(4)] + + +def test_a_swarm_task_is_its_service_and_anything_else_its_own_name(): + assert sender_name("traefik_router.inlbfs5bk5want3q2pw95x9jw.dja1psmx7k2d59o45e1tqyku6") == "traefik_router" + assert sender_name("steward_app.1.ww9m1povzpkx0dcei3p4zh8dm") == "steward_app" + assert sender_name("ak-outpost-fabled_ldap") == "ak-outpost-fabled_ldap" + + +def test_senders_are_ranked_by_what_they_sent_in_the_host_s_peak(): + host = {T[0]: 100.0, T[1]: 1000.0, T[2]: 100.0, T[3]: 200.0} + got = rank(host, { + ("container", "web"): {T[0]: 80.0, T[1]: 100.0, T[2]: 80.0, T[3]: 80.0}, + ("container", "backup"): {T[1]: 700.0}, + ("nfs", "/nfs/data"): {T[1]: 50.0, T[3]: 100.0}, + }) + assert got["host"] == {"avg_bps": 350.0, "peak_bps": 1000.0, "peak_at": T[1]} + assert [r["name"] for r in got["senders"]] == ["backup", "web", "/nfs/data"] + backup = got["senders"][0] + assert backup == {"kind": "container", "name": "backup", "avg_bps": 175.0, "peak_bps": 700.0, + "peak_at": T[1], "at_host_peak_bps": 700.0} + assert got["unaccounted_at_peak_bps"] == 150.0 # 1000 - 700 - 100 - 50 + assert got["unaccounted_avg_bps"] == 350.0 - 175.0 - 85.0 - 37.5 + + +def test_senders_beyond_the_host_leave_nothing_unaccounted_and_the_list_is_cut(): + host = {T[0]: 10.0} + got = rank(host, {("container", f"c{i}"): {T[0]: 20.0} for i in range(3)}, limit=2) + assert len(got["senders"]) == 2 and got["more"] == 1 + assert got["unaccounted_at_peak_bps"] == 0.0 and got["unaccounted_avg_bps"] == 0.0 + + +def test_no_host_samples_is_no_answer(): + assert rank({}, {("container", "web"): {T[0]: 1.0}})["host"] is None