feat(host_agent): collect network, disk I/O, per-core CPU, temps, memory PSI

Extends the push agent (v1.2.0) with the Netdata-style signals the operator wants: per-core CPU, per-interface network throughput and per-disk I/O (rates derived in-agent from monotonic /proc counter deltas, with counter-reset clamping), hardware temperatures (/sys/class/hwmon), memory-pressure PSI (/proc/pressure), and cached/buffers memory breakdown. All stdlib-only. Server side needs no migration — these land as additional rows in the shared PluginMetric table via _expand_sample_to_metrics. Per-resource series use a ':' in resource_name (host:net:eth0, host:core0, host:temp:Package) so the existing fleet widget's ':' filter ignores them; host-level totals/max are emitted at the bare host resource. New sample keys are optional, so older agents keep ingesting unchanged. Unit tests for the new parsers, rate/reset logic, and the expander contract. Data foundation for the Netdata-style host view (task #867). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 10:11:30 -04:00
parent 3b6e005ed8
commit a9e7baee6a
5 changed files with 447 additions and 24 deletions
@@ -0,0 +1,96 @@
+"""Unit tests for the host agent's new metric collectors / parsers.
+
+Pure parsers and the rate helper are tested directly; the file-reading
+collectors are tested by monkeypatching agent._read_file with fixtures.
+"""
+import plugins.host_agent.agent as agent
+
+
+def test_parse_cpu_lines_aggregate_and_per_core():
+    text = (
+        "cpu  100 0 100 800 0 0 0\n"
+        "cpu0 50 0 50 400 0\n"
+        "cpu1 50 0 50 400 0\n"
+        "intr 12345\n"
+    )
+    parsed = agent._parse_cpu_lines(text)
+    assert set(parsed) == {"cpu", "cpu0", "cpu1"}
+    assert parsed["cpu"] == (1000, 800)  # total, idle(=idle+iowait)
+
+
+def test_collect_cpu_returns_aggregate_and_cores(monkeypatch):
+    snapshots = iter([
+        "cpu 100 0 100 800\ncpu0 50 0 50 400\ncpu1 50 0 50 400\n",
+        "cpu 200 0 200 1200\ncpu0 100 0 100 600\ncpu1 100 0 100 600\n",
+    ])
+    monkeypatch.setattr(agent, "_read_file", lambda p: next(snapshots))
+    monkeypatch.setattr(agent.time, "sleep", lambda s: None)
+    agg, cores = agent.collect_cpu()
+    assert isinstance(agg, float)
+    assert len(cores) == 2
+
+
+def test_parse_psi():
+    text = (
+        "some avg10=1.50 avg60=2.00 avg300=3.00 total=12345\n"
+        "full avg10=0.50 avg60=0.10 avg300=0.00 total=678\n"
+    )
+    r = agent._parse_psi(text)
+    assert r["some_avg10"] == 1.5
+    assert r["some_avg60"] == 2.0
+    assert r["full_avg10"] == 0.5
+
+
+def test_rates_basic_and_counter_reset():
+    cur = {"eth0": (2000, 1000), "sda": (5000, 0)}
+    prev = {"eth0": (1000, 500), "sda": (10000, 0)}  # sda counter went backwards
+    out = agent._rates(cur, prev, dt=10.0)
+    assert out["eth0"] == (100.0, 50.0)
+    assert "sda" not in out                    # negative delta dropped
+    assert agent._rates(cur, prev, dt=0.0) == {}   # no elapsed time → no rates
+
+
+def test_rates_skips_names_missing_from_prev():
+    assert agent._rates({"new": (5, 5)}, {}, dt=1.0) == {}
+
+
+def test_collect_net_raw_skips_lo_and_veth(monkeypatch):
+    fixture = (
+        "Inter-|   Receive\n"
+        " face |bytes\n"
+        "    lo: 100 1 0 0 0 0 0 0 200 2 0 0 0 0 0 0\n"
+        "  eth0: 1000 5 0 0 0 0 0 0 500 4 0 0 0 0 0 0\n"
+        " veth9: 9 0 0 0 0 0 0 0 9 0 0 0 0 0 0 0\n"
+    )
+    monkeypatch.setattr(agent, "_read_file", lambda p: fixture)
+    assert agent.collect_net_raw() == {"eth0": (1000, 500)}
+
+
+def test_collect_diskio_raw_whole_disks_only(monkeypatch):
+    # fields: major minor name reads rd_merged sectors_read ms ... sectors_written ...
+    fixture = (
+        "   8       0 sda 1 0 10 0 1 0 20 0 0 0 0\n"
+        "   8       1 sda1 1 0 5 0 1 0 5 0 0 0 0\n"
+        " 259       0 nvme0n1 1 0 100 0 1 0 200 0 0 0 0\n"
+        "   7       0 loop0 1 0 1 0 1 0 1 0 0 0 0\n"
+    )
+    monkeypatch.setattr(agent, "_read_file", lambda p: fixture)
+    out = agent.collect_diskio_raw()
+    assert out == {"sda": (10 * 512, 20 * 512), "nvme0n1": (100 * 512, 200 * 512)}
+
+
+def test_build_sample_first_call_has_no_rates_but_keeps_state(monkeypatch):
+    monkeypatch.setattr(agent, "collect_cpu", lambda: (5.0, [5.0]))
+    monkeypatch.setattr(agent, "collect_memory", lambda: {"total_bytes": 10, "available_bytes": 5})
+    monkeypatch.setattr(agent, "collect_load", lambda: {"1m": 0.0, "5m": 0.0, "15m": 0.0})
+    monkeypatch.setattr(agent, "collect_uptime", lambda: 1)
+    monkeypatch.setattr(agent, "collect_storage", lambda m: [])
+    monkeypatch.setattr(agent, "collect_temps", lambda: [])
+    monkeypatch.setattr(agent, "collect_psi", lambda: {})
+    monkeypatch.setattr(agent, "collect_net_raw", lambda: {"eth0": (1000, 500)})
+    monkeypatch.setattr(agent, "collect_diskio_raw", lambda: {"sda": (10, 20)})
+    state: dict = {}
+    s1 = agent.build_sample(["/"], state)
+    assert s1["net"] == [] and s1["diskio"] == []   # no prior counters yet
+    assert state["mono"] is not None
+    assert state["net"] == {"eth0": (1000, 500)}
@@ -0,0 +1,60 @@
+"""Unit tests for _expand_sample_to_metrics — sample dict → PluginMetric rows.
+
+No DB: PluginMetric objects are constructed in-memory, so we assert the
+(metric_name, resource_name) → value contract directly.
+"""
+from datetime import datetime, timezone
+
+from plugins.host_agent.routes import _expand_sample_to_metrics
+
+
+def _index(rows):
+    return {(r.metric_name, r.resource_name): r.value for r in rows}
+
+
+def test_expand_emits_all_new_metric_families():
+    sample = {
+        "ts": "2026-06-16T00:00:00+00:00",
+        "cpu_pct": 12.5,
+        "cpu_cores": [10.0, 15.0],
+        "mem": {"total_bytes": 100, "available_bytes": 40, "swap_used_bytes": 5,
+                "cached_bytes": 20, "buffers_bytes": 3},
+        "load": {"1m": 0.1, "5m": 0.2, "15m": 0.3},
+        "uptime_secs": 1000,
+        "storage": [{"mount": "/", "total_bytes": 100, "used_bytes": 50}],
+        "net": [{"iface": "eth0", "rx_bps": 1000.0, "tx_bps": 500.0}],
+        "diskio": [{"device": "sda", "read_bps": 2000.0, "write_bps": 1000.0}],
+        "temps": [{"label": "Package", "celsius": 45.0}, {"label": "Core0", "celsius": 50.0}],
+        "psi": {"mem_some_avg10": 1.5, "cpu_some_avg10": 0.0},
+    }
+    idx = _index(_expand_sample_to_metrics(sample, "alpha", datetime.now(timezone.utc)))
+
+    # per-core CPU as sub-resources
+    assert idx[("cpu_pct", "alpha:core0")] == 10.0
+    assert idx[("cpu_pct", "alpha:core1")] == 15.0
+    # richer memory
+    assert idx[("mem_cached_bytes", "alpha")] == 20
+    assert idx[("mem_buffers_bytes", "alpha")] == 3
+    # network: per-iface + host total
+    assert idx[("net_rx_bps", "alpha:net:eth0")] == 1000.0
+    assert idx[("net_rx_bps", "alpha")] == 1000.0
+    # disk I/O: per-device + host total
+    assert idx[("disk_read_bps", "alpha:diskio:sda")] == 2000.0
+    assert idx[("disk_write_bps", "alpha")] == 1000.0
+    # temps: per-sensor + host max
+    assert idx[("temp_c", "alpha:temp:Package")] == 45.0
+    assert idx[("temp_c_max", "alpha")] == 50.0
+    # PSI
+    assert idx[("psi_mem_some_avg10", "alpha")] == 1.5
+
+
+def test_expand_is_backcompat_with_old_agent_sample():
+    """An old agent (no new keys) must still expand cleanly to legacy metrics."""
+    sample = {"ts": "x", "cpu_pct": 5.0, "mem": {"total_bytes": 10, "available_bytes": 5}}
+    rows = _expand_sample_to_metrics(sample, "h", datetime.now(timezone.utc))
+    names = {r.metric_name for r in rows}
+    assert "cpu_pct" in names
+    assert not any(
+        n.startswith(("net_", "disk_read", "disk_write", "psi_", "temp_", "mem_cached"))
+        for n in names
+    )