bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(snmp): a NAS's disk and volume I/O, CPU waiting on disk, available memory, interface errors (#5997)
CI / lint (push) Successful in 2s
Details
CI / integration (push) Successful in 1m31s
Details
CI / unit (push) Failing after 1m32s
Details
CI / publish (push) Skipped
Details

Browse Source 
- Synology: what each disk and volume reads and writes, bytes and requests
  a second and the load DSM gives it (SYNOLOGY-STORAGEIO- and -SPACEIO-MIB,
  64-bit byte counters), and each disk's reconnects and sectors not found.
- Linux server: CPU waiting on disk, the wait tick counter's share of the
  CPU's, and memory available (free + buffers + cache). A reading may now be
  worked out from others: the poll asks for its inputs, rates each counter,
  and stores only the result.
- Interface traffic: errors and discards in and out, per second.
- byte_count unit: a byte counter shown as bytes a second.

Asustor's MIB has nothing more for this; its Linux server readings gain the
same CPU wait and available memory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-17 09:50:52 -04:00
co-authored by Claude Opus 5.5
parent 3f9aae7f46
commit ad8bbd8d69
4 changed files with 198 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/snmp/readings.py
+82 -16
View File
@@ -14,11 +14,17 @@ and stored on its row, so every reader builds the same readings from it.
A vendor's status (a code, or a word) is a "state" reading: stored as a level
(0 OK, 1 busy, 2 warning, 3 failed), so one alert rule reads any vendor's,
and shown in the device's own word.
A reading may be worked out from others (`inputs`, combined by `combine`):
the device is asked for its inputs, each read in its own unit (a counter as
its rate), and only the result is stored and shown. CPU waiting on disk is
one counter's share of the CPU's.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Callable
from steward.core.formatting import human_bytes, human_duration
@@ -32,6 +38,8 @@ class Reading:
scale: float = 1.0
offset: float = 0.0
states: str = "" # a state reading's codes: a STATE_CODES name, or "words"
inputs: tuple[Reading, ...] = ()
combine: Callable[[dict], float | None] | None = field(default=None, compare=False)
@property
def counter(self) -> bool:
@@ -40,7 +48,8 @@ class Reading:
# A counter's unit names what it counts; it is stored as the rate unit it maps
# to, multiplied by the factor (a byte counter is stored as bits per second).
COUNTER_UNITS = {"octets": ("bits_per_second", 8.0), "count": ("per_second", 1.0)}
COUNTER_UNITS = {"octets": ("bits_per_second", 8.0), "byte_count": ("bytes_per_second", 1.0),
"count": ("per_second", 1.0)}
STATUS_WORDS = {
"if_status": {1: "up", 2: "down", 3: "testing", 4: "unknown", 5: "dormant",
@@ -105,6 +114,7 @@ UNITS = {
"kilobytes": "a size in KiB",
"rpm": "revolutions per minute (a fan)",
"octets": "a byte counter, shown as traffic (bit/s)",
"byte_count": "a byte counter, shown as bytes a second (disk I/O)",
"count": "any other counter, shown per second",
}
@@ -184,6 +194,32 @@ def _asus(oid: str) -> str:
_GIB = float(2 ** 30)
def _ucd(oid: str) -> str:
return f"1.3.6.1.4.1.2021.{oid}"
# UCD-SNMP-MIB's CPU tick counters. On Linux ssCpuRawSystem already holds the
# interrupt time, so these five are the whole.
_CPU_TICKS = tuple(Reading(_ucd(f"11.{n}.0"), f"cpu_ticks_{part}", f"CPU ticks {part}", "count")
for n, part in ((50, "user"), (51, "nice"), (52, "system"), (53, "idle"), (54, "wait")))
def _wait_share(v: dict) -> float | None:
total = sum(v.values())
return 100.0 * v["cpu_ticks_wait"] / total if total > 0 else None
_CPU_WAIT = Reading("", "cpu_iowait", "CPU waiting on disk", "percent", inputs=_CPU_TICKS,
combine=_wait_share)
# What the kernel could hand a program without swapping: free, plus the
# buffers and page cache it gives back on demand.
_MEMORY_AVAILABLE = Reading(
"", "memory_available", "Memory available", "kilobytes",
inputs=tuple(Reading(_ucd(f"4.{n}.0"), f"memory_{part}_kb", f"Memory {part}", "kilobytes")
for n, part in ((6, "free"), (14, "buffers"), (15, "cached"))),
combine=lambda v: sum(v.values()))
# id: (label, what it is for, readings). Interface traffic is per interface
# and built by interface_readings from the numbers on the form.
PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = {
@@ -197,16 +233,19 @@ PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = {
Reading("1.3.6.1.2.1.33.1.4.4.1.5.1", "load", "Load", "percent"),
Reading("1.3.6.1.2.1.33.1.3.3.1.3.1", "input_voltage", "Input", "volts"),
Reading("1.3.6.1.2.1.33.1.2.2.0", "on_battery_for", "On battery", "battery_seconds"))),
"linux": ("Linux server", "CPU, memory and load from net-snmp (snmpd's UCD-SNMP-MIB), and how busy "
"each disk has been over the last minute (UCD-DISKIO-MIB). Most NAS boxes "
"answer it too.",
(Reading("1.3.6.1.4.1.2021.11.11.0", "cpu_used", "CPU used", "percent", scale=-1.0, offset=100.0),
Reading("1.3.6.1.4.1.2021.10.1.5.1", "load_1m", "Load (1m)", "number", scale=0.01),
Reading("1.3.6.1.4.1.2021.4.6.0", "memory_free", "Memory free", "kilobytes"),
Reading("1.3.6.1.4.1.2021.4.5.0", "memory_total", "Memory total", "kilobytes"))),
"linux": ("Linux server", "CPU, how much of it waits on disk, memory and load from net-snmp (snmpd's "
"UCD-SNMP-MIB), and how busy each disk has been over the last minute "
"(UCD-DISKIO-MIB). Most NAS boxes answer it too.",
(Reading(_ucd("11.11.0"), "cpu_used", "CPU used", "percent", scale=-1.0, offset=100.0),
_CPU_WAIT,
Reading(_ucd("10.1.5.1"), "load_1m", "Load (1m)", "number", scale=0.01),
Reading(_ucd("4.6.0"), "memory_free", "Memory free", "kilobytes"),
_MEMORY_AVAILABLE,
Reading(_ucd("4.5.0"), "memory_total", "Memory total", "kilobytes"))),
"synology": ("Synology NAS", "System, power and fan status and temperature; each disk's status, health, "
"temperature and bad sectors; each volume's state (scrubbing, rebuilding, "
"degraded) and space. Tick Linux server too for CPU, memory and disk load.",
"temperature, bad sectors and reconnects, and what it reads and writes; "
"each volume's state (scrubbing, rebuilding, degraded), space, reads and "
"writes. Tick Linux server too for CPU, memory and disk load.",
(Reading(_syn("1.1.0"), "system_status", "System", "state", states="synology_ok"),
Reading(_syn("1.2.0"), "temperature", "Temperature", "celsius"),
Reading(_syn("1.3.0"), "power_status", "Power", "state", states="synology_ok"),
@@ -218,6 +257,14 @@ PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = {
()),
}
def _io_columns(mib: int) -> tuple[Column, ...]:
return (Column(_syn(f"{mib}.1.1.12"), "read", "read", "byte_count"),
Column(_syn(f"{mib}.1.1.13"), "written", "written", "byte_count"),
Column(_syn(f"{mib}.1.1.5"), "reads", "reads", "count"),
Column(_syn(f"{mib}.1.1.6"), "writes", "writes", "count"),
Column(_syn(f"{mib}.1.1.8"), "load", "load", "percent"))
# A preset's vendor tables, read as one row per disk, volume, fan or sensor.
TABLES: dict[str, tuple[Table, ...]] = {
"linux": (Table("ucd_diskio", "diskio", "1.3.6.1.4.1.2021.13.15.1.1.2", "Disk",
@@ -228,11 +275,21 @@ TABLES: dict[str, tuple[Table, ...]] = {
(Column(_syn("2.1.1.5"), "status", "status", "state", states="synology_disk"),
Column(_syn("2.1.1.13"), "health", "health", "state", states="synology_health"),
Column(_syn("2.1.1.6"), "temp", "temperature", "celsius"),
Column(_syn("2.1.1.9"), "bad_sectors", "bad sectors"))),
Column(_syn("2.1.1.9"), "bad_sectors", "bad sectors"),
# How often DSM has had to reconnect to the disk, and how often
# the disk couldn't find a sector it was asked for (IDNF).
Column(_syn("2.1.1.8"), "retries", "reconnects"),
Column(_syn("2.1.1.10"), "identify_fails", "sectors not found"))),
Table("synology_volumes", "raid", _syn("3.1.1.2"), "Volume",
(Column(_syn("3.1.1.3"), "status", "status", "state", states="synology_raid"),
Column(_syn("3.1.1.4"), "free", "free", "bytes"),
Column(_syn("3.1.1.5"), "size", "size", "bytes"))),
# SYNOLOGY-STORAGEIO-MIB, a row per disk, and -SPACEIO-MIB, a row per
# volume: bytes from the 64-bit counters, requests a second, and the
# load DSM gives it (%). A volume slow to answer NFS shows here first.
Table("synology_disk_io", "storageio", _syn("101.1.1.2"), "Disk", _io_columns(101),
rows_like=r"(sd[a-z]+|sata\d+|nvme\d+n\d+)"),
Table("synology_volume_io", "spaceio", _syn("102.1.1.2"), "Volume", _io_columns(102)),
),
"asustor": (
Table("asustor_disks", "disk", _asus("4.1.1.2"), "Disk",
@@ -280,9 +337,10 @@ def parse_interfaces(text: str) -> list[int]:
def interface_readings(numbers: list[int], version: str) -> list[Reading]:
"""In, out and link state per interface. v2c and v3 read the 64-bit
counters (ifHCInOctets), which don't wrap on a fast link; v1 has only the
32-bit ones (ifInOctets)."""
"""In, out and link state per interface, and the packets it got wrong or
had to throw away. v2c and v3 read the 64-bit counters (ifHCInOctets),
which don't wrap on a fast link; v1 has only the 32-bit ones
(ifInOctets)."""
hc = version != "1"
out: list[Reading] = []
for n in numbers:
@@ -290,7 +348,12 @@ def interface_readings(numbers: list[int], version: str) -> list[Reading]:
tx = f"1.3.6.1.2.1.31.1.1.1.10.{n}" if hc else f"1.3.6.1.2.1.2.2.1.16.{n}"
out += [Reading(f"1.3.6.1.2.1.2.2.1.8.{n}", f"if{n}_status", f"Interface {n}", "if_status"),
Reading(rx, f"if{n}_in", f"Interface {n} in", "octets"),
Reading(tx, f"if{n}_out", f"Interface {n} out", "octets")]
Reading(tx, f"if{n}_out", f"Interface {n} out", "octets"),
Reading(f"1.3.6.1.2.1.2.2.1.14.{n}", f"if{n}_in_errors", f"Interface {n} errors in", "count"),
Reading(f"1.3.6.1.2.1.2.2.1.20.{n}", f"if{n}_out_errors", f"Interface {n} errors out", "count"),
Reading(f"1.3.6.1.2.1.2.2.1.13.{n}", f"if{n}_in_discards", f"Interface {n} discards in", "count"),
Reading(f"1.3.6.1.2.1.2.2.1.19.{n}", f"if{n}_out_discards", f"Interface {n} discards out",
"count")]
return out
@@ -396,6 +459,8 @@ def format_value(unit: str, value: float | None) -> str:
return human_bytes(value * 1024)
if unit == "bits_per_second":
return _bits(value)
if unit == "bytes_per_second":
return f"{human_bytes(value)}/s"
if unit == "per_second":
return f"{value:.3g}/s"
if unit == "rpm":
@@ -410,6 +475,7 @@ def chart_axis(unit: str) -> tuple[str, float]:
"minutes": ("minutes", 1.0), "percent": ("%", 1.0), "celsius": ("°C", 1.0),
"volts": ("V", 1.0), "decivolts": ("V", 0.1), "bytes": ("MiB", 1 / 1_048_576),
"kilobytes": ("MiB", 1 / 1024), "bits_per_second": ("Mbit/s", 1e-6),
"bytes_per_second": ("MiB/s", 1 / 1_048_576),
"per_second": ("per second", 1.0), "rpm": ("rpm", 1.0),
"state": ("0 OK · 1 busy · 2 warning · 3 failed", 1.0)}.get(unit, ("", 1.0))
plugins/snmp/scheduler.py
+26 -8
View File
@@ -139,7 +139,7 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
scans[d.id] = await walk(d.address, d.port, creds, tables_for(d.presets or []))
found = scans.get(d.id)
asked[d.id] = readings_for(d, found.rows if found is not None and found.error is None else None)
return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in asked[d.id]])
return await poll(d.address, d.port, creds, [(r.oid, r.key) for r in _asks(asked[d.id])])
results = await asyncio.gather(*(one(d) for d in devices), return_exceptions=True)
@@ -158,23 +158,35 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
device.tables, device.tables_at = p.scan.rows, now
device.last_error = result.error
missing = set(result.missing)
for reading in asked[device.id]:
def read(reading):
value = result.readings.get(reading.key)
if reading.unit == "state":
# Its word is shown as answered; its level is stored,
# unless Steward doesn't know the word (level None).
raw = value if value is not None else result.texts.get(reading.key)
if raw is None:
continue
return None
p.texts[reading.key], value = state_of(reading, raw)
elif value is None:
return value
if value is None:
if reading.key in result.texts: # a word where a number was asked for
missing.add(reading.key)
continue
elif reading.counter:
value = RATES.rate(device.id, reading, value, now)
return None
if reading.counter:
return RATES.rate(device.id, reading, value, now)
return value * reading.scale + reading.offset
for reading in asked[device.id]:
if reading.inputs:
# Every input is read, so each counter keeps its last sample.
got = {i.key: read(i) for i in reading.inputs}
if any(i.key in missing for i in reading.inputs):
missing -= {i.key for i in reading.inputs}
missing.add(reading.key)
value = None if None in got.values() else reading.combine(got)
else:
value = value * reading.scale + reading.offset
value = read(reading)
if value is not None:
p.values.append((reading.key, value))
p.missing = sorted(missing)
@@ -183,6 +195,12 @@ async def _do_poll(app: "Quart", poll=None, walk=None) -> None:
await held.flush(STORE_WAIT_S)
def _asks(readings: list) -> list:
"""What the device is asked for: each reading, or the inputs of one
worked out from others."""
return [i for r in readings for i in (r.inputs or (r,))]
async def _store(app: "Quart", rounds: list[list[Polled]]) -> None:
"""Write held polls, oldest first, in one transaction."""
from steward.core.alerts import MetricBatch