bvandeusenandClaude Opus 5.5
bd1c32d2a4
The table walk stopped after 64 raw entries, counting the RAM disks, loops,
partitions and md arrays that rows_like then drops, so a Synology's eighth
disk was never reached. The walk is now bounded at 1,024 entries per column;
64 still caps the rows kept.
ADM labels its disks "Disk0".."Disk4" while their indexes run 1..5; the
Asustor disk table now names each row by its index ("Disk 1"). Keys are
unchanged, so history carries on.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
423 lines
20 KiB
Python
423 lines
20 KiB
Python
# plugins/snmp/readings.py
|
|
"""What an SNMP device is asked for, and how each answer reads.
|
|
|
|
A device's readings come from the presets ticked on its form plus any custom
|
|
lines. Each reading has a key (the metric name it is stored and alerted on),
|
|
a name a person reads, and a unit. A counter (an interface's byte count, any
|
|
"total so far") is stored as a rate: the poller turns two samples into a
|
|
per-second figure, so a chart and an alert see traffic, not a lifetime total.
|
|
|
|
A preset may also read a vendor's tables, a row per disk or volume (Table).
|
|
Which rows a device has is discovered from the device by the scheduled poll
|
|
and stored on its row, so every reader builds the same readings from it.
|
|
|
|
A vendor's status (a code, or a word) is a "state" reading: stored as a level
|
|
(0 OK, 1 busy, 2 warning, 3 failed), so one alert rule reads any vendor's,
|
|
and shown in the device's own word.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
from steward.core.formatting import human_bytes, human_duration
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Reading:
|
|
oid: str
|
|
key: str
|
|
name: str
|
|
unit: str = "number"
|
|
scale: float = 1.0
|
|
offset: float = 0.0
|
|
states: str = "" # a state reading's codes: a STATE_CODES name, or "words"
|
|
|
|
@property
|
|
def counter(self) -> bool:
|
|
return self.unit in COUNTER_UNITS
|
|
|
|
|
|
# A counter's unit names what it counts; it is stored as the rate unit it maps
|
|
# to, multiplied by the factor (a byte counter is stored as bits per second).
|
|
COUNTER_UNITS = {"octets": ("bits_per_second", 8.0), "count": ("per_second", 1.0)}
|
|
|
|
STATUS_WORDS = {
|
|
"if_status": {1: "up", 2: "down", 3: "testing", 4: "unknown", 5: "dormant",
|
|
6: "not present", 7: "lower layer down"},
|
|
"ups_battery": {1: "unknown", 2: "normal", 3: "low", 4: "depleted"},
|
|
}
|
|
|
|
# A state reading is stored as one of these levels; an alert rule on "level
|
|
# at least 2" catches any vendor's degraded or failed.
|
|
LEVELS = {0: "OK", 1: "Busy", 2: "Warning", 3: "Failed"}
|
|
|
|
# A vendor's status codes: code -> (the vendor's word, level). Synology's are
|
|
# from its MIB guide (SYNOLOGY-SYSTEM-, -DISK- and -RAID-MIB).
|
|
STATE_CODES: dict[str, dict[int, tuple[str, int]]] = {
|
|
"synology_ok": {1: ("Normal", 0), 2: ("Failed", 3)},
|
|
"synology_disk": {1: ("Normal", 0), 2: ("Initialized", 0), 3: ("Not initialized", 2),
|
|
4: ("System partition failed", 3), 5: ("Crashed", 3)},
|
|
"synology_health": {1: ("Normal", 0), 2: ("Warning", 2), 3: ("Critical", 3), 4: ("Failing", 3)},
|
|
"synology_raid": {1: ("Normal", 0), 2: ("Repairing", 1), 3: ("Migrating", 1), 4: ("Expanding", 1),
|
|
5: ("Deleting", 1), 6: ("Creating", 1), 7: ("Syncing", 1), 8: ("Parity checking", 1),
|
|
9: ("Assembling", 1), 10: ("Canceling", 1), 11: ("Degraded", 2), 12: ("Crashed", 3),
|
|
13: ("Data scrubbing", 1), 14: ("Deploying", 1), 15: ("Undeploying", 1),
|
|
16: ("Mounting cache", 1), 17: ("Unmounting cache", 1), 18: ("Expanding", 1),
|
|
19: ("Converting to a storage pool", 1), 20: ("Migrating SHR1 to SHR2", 1),
|
|
21: ("Unknown", 2)},
|
|
}
|
|
|
|
# A vendor that answers in words (Asustor) is read by the words' beginnings,
|
|
# worst first: "Unhealthy" is failed before "healthy" can call it OK.
|
|
_WORD_LEVELS = (
|
|
(3, re.compile(r"\b(fail|crash|critical|bad|error|abnormal|unhealthy|broken|removed|missing|offline)", re.I)),
|
|
(2, re.compile(r"\b(degrad|warn|unknown)", re.I)),
|
|
(1, re.compile(r"\b(sync|resync|rebuild|repair|recover|scrub|check|migrat|expand|creat|initiali|build)",
|
|
re.I)),
|
|
(0, re.compile(r"\b(normal|good|healthy|ok|ready|active|online|clean|pass)", re.I)),
|
|
)
|
|
|
|
|
|
def state_of(reading: Reading, raw: float | str) -> tuple[str, int | None]:
|
|
"""A state reading's answer as (the device's word, level). The level is
|
|
None for a code or word Steward doesn't know: it is shown, not stored."""
|
|
if reading.states == "words":
|
|
word = str(raw).strip()
|
|
return word, next((level for level, rx in _WORD_LEVELS if rx.search(word)), None)
|
|
try:
|
|
code = int(float(raw))
|
|
except (TypeError, ValueError):
|
|
return str(raw).strip(), None
|
|
return STATE_CODES.get(reading.states, {}).get(code, (f"code {code}", None))
|
|
|
|
# The units a custom line may name, with how each one reads, for the form's hint.
|
|
UNITS = {
|
|
"number": "a plain number",
|
|
"percent": "0 to 100",
|
|
"celsius": "degrees Celsius",
|
|
"volts": "volts",
|
|
"decivolts": "tenths of a volt",
|
|
"seconds": "a span in seconds",
|
|
"minutes": "a span in minutes",
|
|
"centiseconds": "a span in hundredths of a second (sysUpTime)",
|
|
"bytes": "a size in bytes",
|
|
"kilobytes": "a size in KiB",
|
|
"rpm": "revolutions per minute (a fan)",
|
|
"octets": "a byte counter, shown as traffic (bit/s)",
|
|
"count": "any other counter, shown per second",
|
|
}
|
|
|
|
_UPTIME = Reading("1.3.6.1.2.1.1.3.0", "uptime", "Uptime", "centiseconds")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Column:
|
|
"""One column of a vendor table: a row's reading is this OID plus the
|
|
row's index."""
|
|
oid: str
|
|
key: str
|
|
name: str
|
|
unit: str = "number"
|
|
scale: float = 1.0
|
|
states: str = ""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Table:
|
|
"""A vendor table with a row per disk, volume, fan or sensor.
|
|
|
|
`label` is the column naming each row ("Disk 1", "Volume 1"); a row's
|
|
readings are keyed by prefix, index and column ("disk0_status") and named
|
|
by its label and column ("Disk 1 status"). A table with no label column
|
|
(one temperature sensor, one memory) is keyed and named by its columns
|
|
alone while it has one row. `rows_like` keeps only rows whose label
|
|
matches it (physical disks, not partitions and loop devices).
|
|
`by_index` names each row by its noun and index instead ("Disk 1"), for
|
|
a device whose own label doesn't say which bay it is (ADM's "Disk0")."""
|
|
id: str
|
|
prefix: str
|
|
label: str
|
|
noun: str
|
|
columns: tuple[Column, ...]
|
|
rows_like: str = ""
|
|
by_index: bool = False
|
|
|
|
def column_oids(self) -> list[str]:
|
|
return ([self.label] if self.label else []) + [c.oid for c in self.columns]
|
|
|
|
def keeps(self, label: str) -> bool:
|
|
return not self.rows_like or re.fullmatch(self.rows_like, label) is not None
|
|
|
|
|
|
def table_readings(table: Table, rows: list[dict]) -> list[Reading]:
|
|
"""The readings of a table's discovered rows: [{"index", "label",
|
|
"columns"}], each asked only for the columns it answered."""
|
|
out: list[Reading] = []
|
|
alone = not table.label and len(rows) == 1
|
|
for row in rows:
|
|
index, label = str(row["index"]), str(row.get("label") or "")
|
|
if table.by_index:
|
|
label = f"{table.noun} {index}"
|
|
elif label.isdigit() or (table.label and not label):
|
|
label = f"{table.noun} {label or index}"
|
|
for c in table.columns:
|
|
if c.key not in row.get("columns", ()):
|
|
continue
|
|
if table.label:
|
|
key, name = f"{table.prefix}{index.replace('.', '_')}_{c.key}", f"{label} {c.name}"
|
|
elif alone:
|
|
key, name = c.key, c.name
|
|
else:
|
|
key, name = f"{c.key}{index.replace('.', '_')}", f"{c.name} ({table.noun.lower()} {index})"
|
|
out.append(Reading(f"{c.oid}.{index}", key, name, c.unit, scale=c.scale, states=c.states))
|
|
return out
|
|
|
|
|
|
def _syn(oid: str) -> str:
|
|
return f"1.3.6.1.4.1.6574.{oid}"
|
|
|
|
|
|
def _asus(oid: str) -> str:
|
|
return f"1.3.6.1.4.1.44738.{oid}"
|
|
|
|
|
|
_GIB = float(2 ** 30)
|
|
|
|
# id: (label, what it is for, readings). Interface traffic is per interface
|
|
# and built by interface_readings from the numbers on the form.
|
|
PRESETS: dict[str, tuple[str, str, tuple[Reading, ...]]] = {
|
|
"system": ("System", "How long the device has been up. Nearly every SNMP device answers it.",
|
|
(_UPTIME,)),
|
|
"ups": ("UPS", "Battery state, charge, runtime left, load and input voltage (UPS-MIB, RFC 1628; "
|
|
"APC, Eaton and CyberPower management cards).",
|
|
(Reading("1.3.6.1.2.1.33.1.2.1.0", "battery_status", "Battery", "ups_battery"),
|
|
Reading("1.3.6.1.2.1.33.1.2.4.0", "battery_charge", "Charge", "percent"),
|
|
Reading("1.3.6.1.2.1.33.1.2.3.0", "runtime_left", "Runtime left", "minutes"),
|
|
Reading("1.3.6.1.2.1.33.1.4.4.1.5.1", "load", "Load", "percent"),
|
|
Reading("1.3.6.1.2.1.33.1.3.3.1.3.1", "input_voltage", "Input", "volts"),
|
|
Reading("1.3.6.1.2.1.33.1.2.2.0", "on_battery_for", "On battery", "battery_seconds"))),
|
|
"linux": ("Linux server", "CPU, memory and load from net-snmp (snmpd's UCD-SNMP-MIB), and how busy "
|
|
"each disk has been over the last minute (UCD-DISKIO-MIB). Most NAS boxes "
|
|
"answer it too.",
|
|
(Reading("1.3.6.1.4.1.2021.11.11.0", "cpu_used", "CPU used", "percent", scale=-1.0, offset=100.0),
|
|
Reading("1.3.6.1.4.1.2021.10.1.5.1", "load_1m", "Load (1m)", "number", scale=0.01),
|
|
Reading("1.3.6.1.4.1.2021.4.6.0", "memory_free", "Memory free", "kilobytes"),
|
|
Reading("1.3.6.1.4.1.2021.4.5.0", "memory_total", "Memory total", "kilobytes"))),
|
|
"synology": ("Synology NAS", "System, power and fan status and temperature; each disk's status, health, "
|
|
"temperature and bad sectors; each volume's state (scrubbing, rebuilding, "
|
|
"degraded) and space. Tick Linux server too for CPU, memory and disk load.",
|
|
(Reading(_syn("1.1.0"), "system_status", "System", "state", states="synology_ok"),
|
|
Reading(_syn("1.2.0"), "temperature", "Temperature", "celsius"),
|
|
Reading(_syn("1.3.0"), "power_status", "Power", "state", states="synology_ok"),
|
|
Reading(_syn("1.4.1.0"), "system_fan", "System fan", "state", states="synology_ok"),
|
|
Reading(_syn("1.4.2.0"), "cpu_fan", "CPU fan", "state", states="synology_ok"))),
|
|
"asustor": ("Asustor NAS", "Each disk's status, SMART verdict and temperature; each volume's status and "
|
|
"space; CPU and system temperature, free memory and fan speed (ADM's "
|
|
"ASUSTOR-DISK- and -SYSTEM-MIB). Tick Linux server too for CPU, load and disk load.",
|
|
()),
|
|
}
|
|
|
|
# A preset's vendor tables, read as one row per disk, volume, fan or sensor.
|
|
TABLES: dict[str, tuple[Table, ...]] = {
|
|
"linux": (Table("ucd_diskio", "diskio", "1.3.6.1.4.1.2021.13.15.1.1.2", "Disk",
|
|
(Column("1.3.6.1.4.1.2021.13.15.1.1.9", "busy", "busy", "percent"),),
|
|
rows_like=r"(sd[a-z]+|hd[a-z]+|vd[a-z]+|xvd[a-z]+|nvme\d+n\d+|sata\d+)"),),
|
|
"synology": (
|
|
Table("synology_disks", "disk", _syn("2.1.1.2"), "Disk",
|
|
(Column(_syn("2.1.1.5"), "status", "status", "state", states="synology_disk"),
|
|
Column(_syn("2.1.1.13"), "health", "health", "state", states="synology_health"),
|
|
Column(_syn("2.1.1.6"), "temp", "temperature", "celsius"),
|
|
Column(_syn("2.1.1.9"), "bad_sectors", "bad sectors"))),
|
|
Table("synology_volumes", "raid", _syn("3.1.1.2"), "Volume",
|
|
(Column(_syn("3.1.1.3"), "status", "status", "state", states="synology_raid"),
|
|
Column(_syn("3.1.1.4"), "free", "free", "bytes"),
|
|
Column(_syn("3.1.1.5"), "size", "size", "bytes"))),
|
|
),
|
|
"asustor": (
|
|
Table("asustor_disks", "disk", _asus("4.1.1.2"), "Disk",
|
|
(Column(_asus("4.1.1.5"), "status", "status", "state", states="words"),
|
|
Column(_asus("4.1.1.6"), "smart", "SMART", "state", states="words"),
|
|
Column(_asus("4.1.1.7"), "temp", "temperature", "celsius")), by_index=True),
|
|
Table("asustor_volumes", "volume", _asus("5.1.1.2"), "Volume",
|
|
(Column(_asus("5.1.1.4"), "status", "status", "state", states="words"),
|
|
Column(_asus("5.1.1.7"), "free", "free", "bytes", scale=_GIB),
|
|
Column(_asus("5.1.1.6"), "size", "size", "bytes", scale=_GIB))),
|
|
Table("asustor_temperatures", "", "", "Sensor",
|
|
(Column(_asus("2.5.1.2"), "cpu_temp", "CPU temperature", "celsius"),
|
|
Column(_asus("2.5.1.3"), "system_temp", "System temperature", "celsius"))),
|
|
Table("asustor_memory", "", "", "Memory",
|
|
(Column(_asus("2.6.1.4"), "memory_free_pct", "Memory free", "percent"),)),
|
|
Table("asustor_fans", "", "", "Fan",
|
|
(Column(_asus("2.4.1.2"), "fan", "Fan speed", "rpm"),)),
|
|
),
|
|
}
|
|
|
|
PRESET_ORDER = ("system", "interfaces", "ups", "linux", "synology", "asustor")
|
|
INTERFACES_LABEL = ("Interface traffic", "Traffic in and out and link state for the interfaces numbered "
|
|
"below. Test the device to see its interfaces and their numbers.")
|
|
|
|
|
|
def tables_for(presets: list[str]) -> list[Table]:
|
|
"""The vendor tables the ticked presets read, in preset order."""
|
|
return [t for pid in PRESET_ORDER if pid in (presets or []) for t in TABLES.get(pid, ())]
|
|
|
|
|
|
def parse_interfaces(text: str) -> list[int]:
|
|
"""'1, 2, 5-7' → [1, 2, 5, 6, 7]. Raises ValueError naming what it can't read."""
|
|
out: list[int] = []
|
|
for part in re.split(r"[,\s]+", (text or "").strip()):
|
|
if not part:
|
|
continue
|
|
m = re.fullmatch(r"(\d+)(?:-(\d+))?", part)
|
|
if not m:
|
|
raise ValueError(f"“{part}” is not an interface number or range")
|
|
lo, hi = int(m.group(1)), int(m.group(2) or m.group(1))
|
|
if hi < lo or hi - lo > 64:
|
|
raise ValueError(f"“{part}” is not a range of up to 64 interfaces")
|
|
out.extend(n for n in range(lo, hi + 1) if n not in out)
|
|
return out
|
|
|
|
|
|
def interface_readings(numbers: list[int], version: str) -> list[Reading]:
|
|
"""In, out and link state per interface. v2c and v3 read the 64-bit
|
|
counters (ifHCInOctets), which don't wrap on a fast link; v1 has only the
|
|
32-bit ones (ifInOctets)."""
|
|
hc = version != "1"
|
|
out: list[Reading] = []
|
|
for n in numbers:
|
|
rx = f"1.3.6.1.2.1.31.1.1.1.6.{n}" if hc else f"1.3.6.1.2.1.2.2.1.10.{n}"
|
|
tx = f"1.3.6.1.2.1.31.1.1.1.10.{n}" if hc else f"1.3.6.1.2.1.2.2.1.16.{n}"
|
|
out += [Reading(f"1.3.6.1.2.1.2.2.1.8.{n}", f"if{n}_status", f"Interface {n}", "if_status"),
|
|
Reading(rx, f"if{n}_in", f"Interface {n} in", "octets"),
|
|
Reading(tx, f"if{n}_out", f"Interface {n} out", "octets")]
|
|
return out
|
|
|
|
|
|
_OID = re.compile(r"\.?\d+(\.\d+)+")
|
|
_KEY = re.compile(r"[a-z][a-z0-9_]{0,63}")
|
|
|
|
|
|
def parse_custom(text: str) -> list[Reading]:
|
|
"""One reading a line: `OID name [unit]`. The name is a single word
|
|
(letters, digits, underscores) and becomes the metric name; the unit is
|
|
one of UNITS and defaults to number. Raises ValueError naming the line."""
|
|
out: list[Reading] = []
|
|
for i, raw in enumerate((text or "").splitlines(), start=1):
|
|
line = raw.split("#", 1)[0].strip()
|
|
if not line:
|
|
continue
|
|
parts = line.split()
|
|
if len(parts) not in (2, 3):
|
|
raise ValueError(f"Line {i}: write it as “OID name” or “OID name unit”")
|
|
oid, key = parts[0].lstrip("."), parts[1].lower()
|
|
unit = parts[2].lower() if len(parts) == 3 else "number"
|
|
if not _OID.fullmatch(oid):
|
|
raise ValueError(f"Line {i}: “{parts[0]}” is not an OID (numbers separated by dots)")
|
|
if not _KEY.fullmatch(key):
|
|
raise ValueError(f"Line {i}: the name “{parts[1]}” must start with a letter and use "
|
|
"only letters, digits and underscores")
|
|
if unit not in UNITS:
|
|
raise ValueError(f"Line {i}: “{parts[2]}” is not a unit; use one of {', '.join(UNITS)}")
|
|
out.append(Reading(oid, key, key.replace("_", " ").capitalize(), unit))
|
|
return out
|
|
|
|
|
|
def device_readings(presets: list[str], interfaces: list[int], custom: str, version: str,
|
|
tables: dict | None = None) -> list[Reading]:
|
|
"""Every reading a device is asked for: presets in their order, the rows
|
|
discovered in their tables (`tables`, {table id: rows}), then custom
|
|
lines. Raises ValueError when a custom line repeats a preset's name; a
|
|
table row a custom line already names is left to the custom line, since
|
|
rows appear after the form was saved."""
|
|
out: list[Reading] = []
|
|
for pid in PRESET_ORDER:
|
|
if pid == "interfaces":
|
|
if "interfaces" in presets:
|
|
out += interface_readings(interfaces, version)
|
|
elif pid in presets:
|
|
out += PRESETS[pid][2]
|
|
custom_readings = parse_custom(custom)
|
|
taken = {r.key for r in out} | {r.key for r in custom_readings}
|
|
for table in tables_for(presets):
|
|
for r in table_readings(table, (tables or {}).get(table.id, [])):
|
|
if r.key not in taken:
|
|
out.append(r)
|
|
taken.add(r.key)
|
|
out += custom_readings
|
|
seen: set[str] = set()
|
|
for r in out:
|
|
if r.key in seen:
|
|
raise ValueError(f"Two readings are named “{r.key}”; give the custom one another name")
|
|
seen.add(r.key)
|
|
return out
|
|
|
|
|
|
def stored_unit(reading: Reading) -> str:
|
|
"""The unit a reading is stored in: a counter's rate unit, else its own."""
|
|
return COUNTER_UNITS[reading.unit][0] if reading.counter else reading.unit
|
|
|
|
|
|
def _bits(bps: float) -> str:
|
|
for unit, size in (("Gbit/s", 1e9), ("Mbit/s", 1e6), ("kbit/s", 1e3)):
|
|
if abs(bps) >= size:
|
|
return f"{bps / size:.3g} {unit}"
|
|
return f"{bps:.0f} bit/s"
|
|
|
|
|
|
def format_value(unit: str, value: float | None) -> str:
|
|
"""A stored value as a person reads it, in its unit."""
|
|
if value is None:
|
|
return "—"
|
|
if unit == "state":
|
|
return LEVELS.get(int(value), f"level {value:.0f}")
|
|
if unit in STATUS_WORDS:
|
|
return STATUS_WORDS[unit].get(int(value), f"code {value:.0f}")
|
|
if unit == "centiseconds":
|
|
return human_duration(value / 100)
|
|
if unit == "seconds":
|
|
return human_duration(value)
|
|
if unit == "battery_seconds":
|
|
# UPS-MIB upsSecondsOnBattery: zero whenever the UPS is on mains power.
|
|
return f"for {human_duration(value)}" if value > 0 else "No, on mains"
|
|
if unit == "minutes":
|
|
return human_duration(value * 60)
|
|
if unit == "percent":
|
|
return f"{value:.0f}%"
|
|
if unit == "celsius":
|
|
return f"{value:.1f} °C"
|
|
if unit == "volts":
|
|
return f"{value:.0f} V"
|
|
if unit == "decivolts":
|
|
return f"{value / 10:.1f} V"
|
|
if unit == "bytes":
|
|
return human_bytes(value)
|
|
if unit == "kilobytes":
|
|
return human_bytes(value * 1024)
|
|
if unit == "bits_per_second":
|
|
return _bits(value)
|
|
if unit == "per_second":
|
|
return f"{value:.3g}/s"
|
|
if unit == "rpm":
|
|
return f"{value:,.0f} rpm"
|
|
return f"{value:,.0f}" if float(value).is_integer() else f"{value:.4g}"
|
|
|
|
|
|
# What a chart's axis says for a stored unit, and how its values scale for it.
|
|
def chart_axis(unit: str) -> tuple[str, float]:
|
|
return {"centiseconds": ("days", 1 / 8_640_000), "seconds": ("seconds", 1.0),
|
|
"battery_seconds": ("seconds on battery", 1.0),
|
|
"minutes": ("minutes", 1.0), "percent": ("%", 1.0), "celsius": ("°C", 1.0),
|
|
"volts": ("V", 1.0), "decivolts": ("V", 0.1), "bytes": ("MiB", 1 / 1_048_576),
|
|
"kilobytes": ("MiB", 1 / 1024), "bits_per_second": ("Mbit/s", 1e-6),
|
|
"per_second": ("per second", 1.0), "rpm": ("rpm", 1.0),
|
|
"state": ("0 OK · 1 busy · 2 warning · 3 failed", 1.0)}.get(unit, ("", 1.0))
|
|
|
|
|
|
def reading_text(reading: Reading, value: float | None, texts: dict | None) -> str:
|
|
"""What a page shows for a reading: a state in the device's own word, as
|
|
its newest poll answered it; anything else in its unit."""
|
|
if reading.unit == "state" and (texts or {}).get(reading.key):
|
|
return texts[reading.key]
|
|
return format_value(stored_unit(reading), value)
|