FabledSteward/plugins/host_agent/senders.py

"""Who sends a host's network traffic (#6115): its containers and NFS mounts,
ranked against the host's own total over a window.

A host's net_tx_bps (net_rx_bps) is what its physical interfaces carried; each
container's comes from Docker ingest (resource "<host>/<container>") and each
NFS mount's from the agent (resource "<host>:<mount>"). All three are read in
the same time buckets, so the busiest bucket of the host's total can be put
down to who was sending in it. What no container or mount accounts for is the
host itself (an apt download, a backup run from the host, a swarm ingress
hand-off) or a container's traffic that never left the host.

Imports only the core metric model, like metrics_query, so the docker plugin
being off leaves just the NFS mounts and the host.
"""
from __future__ import annotations

import re
from datetime import datetime, timedelta, timezone

from sqlalchemy import func, select

from steward.models.metrics import PluginMetric, PluginMetricLatest

SOURCE_MODULE = "host_agent"
DIRECTIONS = ("tx", "rx")
# Buckets a window is cut into: a 1 h window reads per 60 s sample, 24 h per
# 12 minutes. Narrow the window to see a spike sharper.
BUCKETS = 120
# A swarm task's container, "<service>.<slot or node id>.<task id>", is shown
# by its service, so its tasks over the window add up to one sender.
_TASK = re.compile(r"^(?P<svc>.+?)\.(?:\d+|[a-z0-9]{25})\.[a-z0-9]{25}$")


def sender_name(container: str) -> str:
    m = _TASK.match(container)
    return m["svc"] if m else container


def rank(host: dict, senders: dict, limit: int = 10) -> dict:
    """`host` is {bucket: bytes a second}; `senders` is {(kind, name):
    {bucket: bytes a second}}. The host's average, its peak and when, and each
    sender's average over the host's buckets (a bucket it is missing from
    counts as 0), its own peak and what it sent in the host's peak bucket,
    biggest at the host's peak first. `unaccounted_*` is what the host carried
    beyond them all, never below 0."""
    if not host:
        return {"host": None, "senders": [], "more": 0, "unaccounted_avg_bps": None,
                "unaccounted_at_peak_bps": None}
    n = len(host)
    peak_at = max(host, key=lambda b: host[b])
    rows = []
    for (kind, name), series in senders.items():
        top = max(series, key=lambda b: series[b])
        rows.append({"kind": kind, "name": name,
                     "avg_bps": sum(v for b, v in series.items() if b in host) / n,
                     "peak_bps": series[top], "peak_at": top,
                     "at_host_peak_bps": series.get(peak_at, 0.0)})
    rows.sort(key=lambda r: (r["at_host_peak_bps"], r["avg_bps"]), reverse=True)
    host_avg = sum(host.values()) / n
    return {
        "host": {"avg_bps": host_avg, "peak_bps": host[peak_at], "peak_at": peak_at},
        "senders": rows[:limit],
        "more": max(0, len(rows) - limit),
        "unaccounted_avg_bps": max(0.0, host_avg - sum(r["avg_bps"] for r in rows)),
        "unaccounted_at_peak_bps": max(0.0, host[peak_at] - sum(r["at_host_peak_bps"] for r in rows)),
    }


async def senders(session, host_name: str, since: datetime, direction: str = "tx",
                  limit: int = 10, raw_days: int = 7) -> dict:
    """rank() over the raw samples since `since` (no earlier than the raw
    window: the hourly rollup keeps no per-bucket sender to rank). Adds the
    window actually read and its bucket width."""
    if direction not in DIRECTIONS:
        raise ValueError(f"direction is one of {', '.join(DIRECTIONS)}")
    now = datetime.now(timezone.utc)
    since = max(since, now - timedelta(days=raw_days))
    width = max(60, int((now - since).total_seconds() / BUCKETS))
    bucket = func.date_bin(func.make_interval(0, 0, 0, 0, 0, 0, width), PluginMetric.recorded_at,
                           func.to_timestamp(0)).label("bucket")
    metric = f"net_{direction}_bps"

    async def named(source, metric_name, prefix):
        # The resources under the host, from the small latest-value table: a
        # prefix LIKE on plugin_metrics itself can't use its index (the
        # column's collation isn't C) and reads the whole table.
        return list((await session.execute(
            select(PluginMetricLatest.resource_name).where(
                PluginMetricLatest.source_module == source, PluginMetricLatest.metric_name == metric_name,
                PluginMetricLatest.resource_name.startswith(prefix, autoescape=True),
                PluginMetricLatest.recorded_at >= since))).scalars())

    async def series(source, metric_name, resources):
        if not resources:
            return []
        return (await session.execute(
            select(PluginMetric.resource_name, bucket, func.avg(PluginMetric.value))
            .where(PluginMetric.source_module == source, PluginMetric.resource_name.in_(resources),
                   PluginMetric.metric_name == metric_name, PluginMetric.recorded_at >= since)
            .group_by(PluginMetric.resource_name, bucket))).all()

    host: dict = {}
    for _, b, v in await series(SOURCE_MODULE, metric, [host_name]):
        host[b] = float(v)
    out: dict = {}
    for res, b, v in await series("docker", metric, await named("docker", metric, host_name + "/")):
        key = ("container", sender_name(res[len(host_name) + 1:]))
        out.setdefault(key, {})
        out[key][b] = out[key].get(b, 0.0) + float(v)          # a service's tasks add up
    nfs = f"nfs_{direction}_bps"
    for res, b, v in await series(SOURCE_MODULE, nfs, await named(SOURCE_MODULE, nfs, host_name + ":/")):
        out.setdefault(("nfs", res[len(host_name) + 1:]), {})[b] = float(v)
    return {"since": since, "bucket_s": width, "direction": direction, **rank(host, out, limit)}